mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-19 14:32:28 +00:00
Compare commits
17 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 739699426b | |||
| 0d2a0ea345 | |||
| c1f97d27f6 | |||
| 1066ab4d1d | |||
| b0e824437b | |||
| 913a5df515 | |||
| f483cfedc8 | |||
| f2177505da | |||
| a758b58bf2 | |||
| 9075131b23 | |||
| 050be16dc7 | |||
| 8efa1a96c0 | |||
| 444dbd9956 | |||
| 33e0fe8c6f | |||
| 205227b441 | |||
| 3e5d37742d | |||
| 8e6265907a |
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=24a8cfaa1b9125bd69d7d4a7ff752a8ab8ed350c761d90ae8472ef52393e2686
|
||||
sha256-linux=014257197fbbfbb48cab2864da9e10d96ee877d478fa29d119f8186532ea293d
|
||||
sha256-darwin=8fef326a3fddd8a548e2cbcc6ea8159f831113595fb003eeebbd9aeb4ac23e48
|
||||
sha256-linux=8b4381578377704ac8b1b5156128b83f036ec4aedc9d863fff25df15112420fc
|
||||
|
||||
@@ -1 +1 @@
|
||||
sha256=fd5683273bba1eae3b4402a4e854a5d168ace845bd0e5fc801f8e336485a4394
|
||||
sha256=061d0a697967d2dc8f472ff885886e069b1cf4aeaa5f230e177b7236017c23b6
|
||||
|
||||
@@ -39,11 +39,9 @@ script-tests: ## Run shell script tests
|
||||
./scripts/test_entrypoint_directories.sh
|
||||
./scripts/test_internode_grpc_ab_bench.sh
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_internode_grpc_window.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_internode_rpc_timeout_boundary.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_internode_msgpack_peer_canary.py
|
||||
./scripts/test_internode_msgpack_peer_canary_runner.sh
|
||||
./scripts/test_object_batch_bench_enhanced.sh
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_inline_docker_lab.py
|
||||
./scripts/test_package_service_scripts.sh
|
||||
./scripts/test_hotpath_warp_ab_gate.sh
|
||||
./scripts/test_hotpath_warp_abba.sh
|
||||
@@ -103,6 +101,8 @@ script-tests: ## Run shell script tests
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_retain_validation_evidence.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_timing_report.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_junit_diff.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_dual_stack.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_mint_stack_diff.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_conformance_rulings.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/compat_matrix_report.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
|
||||
|
||||
@@ -141,9 +141,8 @@ setup = 'ecstore-large-stack'
|
||||
filter = '(package(rustfs) & test(/^(admin::handlers::replication::target_repair_tests::[a-z_]+|admin::handlers::plugin_attempt::v2_tests::delegation_v2_persists_and_vends_with_real_parent_intersection|admin::handlers::site_replication::tests::durable_iam_state_apply_iam_item_rejects_a_stale_recreate_after_a_replicated_delete|admin::service::federated_identity::tests::(binding_keeps_explicit_iam_mapping_strict|markerless_legacy_oidc_session_crosses_auth_and_iam_deletion_boundaries)|storage::scanner_dirty_journal::tests::(clear_callback_keeps_unknown_coverage_until_all_pending_work_is_confirmed|owned_journals_keep_peer_records_after_interleaved_writes_and_clear))$/)) | (package(rustfs-heal) & ((binary(heal_b5_versioned_regression_test) & test(/^serial_tests::(test_heal_unversioned_bucket_e2e|test_historical_null_recursive_heal_restores_physical_versions|test_recursive_deep_heal_reports_stale_delete_marker_repair)$/)) | (binary(usage_observation_read_repair_test) & test(=usage_observation_read_repair_survives_snapshot_cleanup)))) | (package(rustfs-test-utils) & test(=data_usage_snapshot_tests::data_usage_snapshot_recovery_wires_primary_backup_and_negative_cache))'
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
# The multipart refusal fixture composes real-store setup, abort, quota and legal-hold futures.
|
||||
[[profile.default.scripts]]
|
||||
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::complete::tests::completion_absent_binding_refusal_order_baseline|app::multipart_usecase::complete::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
|
||||
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::complete::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
# These handler/store fixtures and the relocated-pool GET compose deep storage
|
||||
@@ -466,7 +465,7 @@ filter = '(package(rustfs) & test(/^(admin::handlers::replication::target_repair
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
[[profile.ci.scripts]]
|
||||
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::complete::tests::completion_absent_binding_refusal_order_baseline|app::multipart_usecase::complete::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
|
||||
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::complete::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
[[profile.ci.scripts]]
|
||||
@@ -723,8 +722,9 @@ test-group = 'ecstore-serial-flaky'
|
||||
# fixture as the existing smoke modules.
|
||||
#
|
||||
# Dual-stack differential (rustfs/backlog#2740): `dual_stack_differential::`
|
||||
# starts two single-node servers on random ports with isolated temp
|
||||
# dirs, and runs one SDK script covering every S3 operation against both. Its
|
||||
# starts two single-node servers from the same binary, one per
|
||||
# RUSTFS_S3_STACK value, on random ports with isolated temp dirs, and runs one
|
||||
# SDK script covering every operation of `impl S3 for FS` against both. Its
|
||||
# comparator unit tests need no server. It meets the admission criteria
|
||||
# unchanged; the module clause admits a new differential test without another
|
||||
# filter edit.
|
||||
|
||||
@@ -9,9 +9,9 @@
|
||||
# Adding a target is one more [[targets]] table, for example:
|
||||
#
|
||||
# [[targets]]
|
||||
# name = "candidate-tuned"
|
||||
# name = "candidate-gateway"
|
||||
# binary = "candidate"
|
||||
# env = { RUSTFS_EXAMPLE_SETTING = "on" }
|
||||
# env = { RUSTFS_S3_STACK = "gateway" }
|
||||
|
||||
schema = 1
|
||||
|
||||
|
||||
@@ -28,6 +28,8 @@ rustfs/src/app/storage_api.rs
|
||||
rustfs/src/admin/router.rs
|
||||
rustfs/src/app/metadata_route.rs
|
||||
rustfs/src/server/http.rs
|
||||
rustfs/src/server/layer.rs
|
||||
rustfs/src/server/hybrid.rs
|
||||
rustfs/src/protocols/client.rs
|
||||
|
||||
# e2e oracle: the test target that verifies S3 behaviour against s3s
|
||||
|
||||
@@ -13,6 +13,6 @@
|
||||
# s3_error_lines lines holding s3_error!, same scope, minus those the script
|
||||
# header's rule clears (rustfs_s3_types's macro); fails closed
|
||||
# ecstore_files Rust files referencing s3s paths under crates/ecstore/src
|
||||
files=50
|
||||
s3_error_lines=103
|
||||
files=63
|
||||
s3_error_lines=209
|
||||
ecstore_files=0
|
||||
|
||||
@@ -22,7 +22,7 @@ rustfs-connect-symbol-catalog 6 0
|
||||
rustfs-credentials 24 0
|
||||
rustfs-crypto 67 0
|
||||
rustfs-data-usage 58 0
|
||||
rustfs-ecstore 7858 3
|
||||
rustfs-ecstore 7858 2
|
||||
rustfs-extension-schema 23 0
|
||||
rustfs-filemeta 361 0
|
||||
rustfs-gateway-adapter 74 0
|
||||
|
||||
@@ -35,7 +35,6 @@ services:
|
||||
- RUSTFS_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_LOCK_ACQUIRE_TIMEOUT:-5}
|
||||
- RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
|
||||
- RUSTFS_DURABILITY_MODE=${RUSTFS_DURABILITY_MODE:-strict}
|
||||
- RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE=${RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS:-0}
|
||||
# Internode gRPC optimization knobs (grpc-optimization P0-P3). Defaults match the binary
|
||||
@@ -47,8 +46,6 @@ services:
|
||||
- RUSTFS_INTERNODE_CHANNEL_ISOLATION=${RUSTFS_INTERNODE_CHANNEL_ISOLATION:-false}
|
||||
- RUSTFS_INTERNODE_BULK_CHANNELS=${RUSTFS_INTERNODE_BULK_CHANNELS:-2}
|
||||
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY:-false}
|
||||
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED:-false}
|
||||
- RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED=${RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED:-false}
|
||||
- RUSTFS_INTERNODE_PREWARM=${RUSTFS_INTERNODE_PREWARM:-false}
|
||||
- RUSTFS_INTERNODE_OFFLINE_BYPASS=${RUSTFS_INTERNODE_OFFLINE_BYPASS:-false}
|
||||
- RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS=${RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS:-5}
|
||||
@@ -62,7 +59,7 @@ services:
|
||||
- node1_data_2:/data/rustfs2
|
||||
- node1_data_3:/data/rustfs3
|
||||
ports:
|
||||
- "${RUSTFS_NODE1_PORT:-9000}:9000"
|
||||
- "9000:9000"
|
||||
networks:
|
||||
- rustfs-cluster-net
|
||||
|
||||
@@ -88,7 +85,6 @@ services:
|
||||
- RUSTFS_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_LOCK_ACQUIRE_TIMEOUT:-5}
|
||||
- RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
|
||||
- RUSTFS_DURABILITY_MODE=${RUSTFS_DURABILITY_MODE:-strict}
|
||||
- RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE=${RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS:-0}
|
||||
# Internode gRPC optimization knobs (grpc-optimization P0-P3). Defaults match the binary
|
||||
@@ -100,8 +96,6 @@ services:
|
||||
- RUSTFS_INTERNODE_CHANNEL_ISOLATION=${RUSTFS_INTERNODE_CHANNEL_ISOLATION:-false}
|
||||
- RUSTFS_INTERNODE_BULK_CHANNELS=${RUSTFS_INTERNODE_BULK_CHANNELS:-2}
|
||||
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY:-false}
|
||||
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED:-false}
|
||||
- RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED=${RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED:-false}
|
||||
- RUSTFS_INTERNODE_PREWARM=${RUSTFS_INTERNODE_PREWARM:-false}
|
||||
- RUSTFS_INTERNODE_OFFLINE_BYPASS=${RUSTFS_INTERNODE_OFFLINE_BYPASS:-false}
|
||||
- RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS=${RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS:-5}
|
||||
@@ -115,7 +109,7 @@ services:
|
||||
- node2_data_2:/data/rustfs2
|
||||
- node2_data_3:/data/rustfs3
|
||||
ports:
|
||||
- "${RUSTFS_NODE2_PORT:-9001}:9000"
|
||||
- "9001:9000"
|
||||
networks:
|
||||
- rustfs-cluster-net
|
||||
|
||||
@@ -141,7 +135,6 @@ services:
|
||||
- RUSTFS_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_LOCK_ACQUIRE_TIMEOUT:-5}
|
||||
- RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
|
||||
- RUSTFS_DURABILITY_MODE=${RUSTFS_DURABILITY_MODE:-strict}
|
||||
- RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE=${RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS:-0}
|
||||
# Internode gRPC optimization knobs (grpc-optimization P0-P3). Defaults match the binary
|
||||
@@ -153,8 +146,6 @@ services:
|
||||
- RUSTFS_INTERNODE_CHANNEL_ISOLATION=${RUSTFS_INTERNODE_CHANNEL_ISOLATION:-false}
|
||||
- RUSTFS_INTERNODE_BULK_CHANNELS=${RUSTFS_INTERNODE_BULK_CHANNELS:-2}
|
||||
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY:-false}
|
||||
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED:-false}
|
||||
- RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED=${RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED:-false}
|
||||
- RUSTFS_INTERNODE_PREWARM=${RUSTFS_INTERNODE_PREWARM:-false}
|
||||
- RUSTFS_INTERNODE_OFFLINE_BYPASS=${RUSTFS_INTERNODE_OFFLINE_BYPASS:-false}
|
||||
- RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS=${RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS:-5}
|
||||
@@ -168,7 +159,7 @@ services:
|
||||
- node3_data_2:/data/rustfs2
|
||||
- node3_data_3:/data/rustfs3
|
||||
ports:
|
||||
- "${RUSTFS_NODE3_PORT:-9002}:9000"
|
||||
- "9002:9000"
|
||||
networks:
|
||||
- rustfs-cluster-net
|
||||
|
||||
@@ -194,7 +185,6 @@ services:
|
||||
- RUSTFS_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_LOCK_ACQUIRE_TIMEOUT:-5}
|
||||
- RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
|
||||
- RUSTFS_DURABILITY_MODE=${RUSTFS_DURABILITY_MODE:-strict}
|
||||
- RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE=${RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS:-0}
|
||||
# Internode gRPC optimization knobs (grpc-optimization P0-P3). Defaults match the binary
|
||||
@@ -206,8 +196,6 @@ services:
|
||||
- RUSTFS_INTERNODE_CHANNEL_ISOLATION=${RUSTFS_INTERNODE_CHANNEL_ISOLATION:-false}
|
||||
- RUSTFS_INTERNODE_BULK_CHANNELS=${RUSTFS_INTERNODE_BULK_CHANNELS:-2}
|
||||
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY:-false}
|
||||
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED:-false}
|
||||
- RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED=${RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED:-false}
|
||||
- RUSTFS_INTERNODE_PREWARM=${RUSTFS_INTERNODE_PREWARM:-false}
|
||||
- RUSTFS_INTERNODE_OFFLINE_BYPASS=${RUSTFS_INTERNODE_OFFLINE_BYPASS:-false}
|
||||
- RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS=${RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS:-5}
|
||||
@@ -221,7 +209,7 @@ services:
|
||||
- node4_data_2:/data/rustfs2
|
||||
- node4_data_3:/data/rustfs3
|
||||
ports:
|
||||
- "${RUSTFS_NODE4_PORT:-9003}:9000"
|
||||
- "9003:9000"
|
||||
networks:
|
||||
- rustfs-cluster-net
|
||||
|
||||
|
||||
@@ -1,58 +0,0 @@
|
||||
# syntax=docker/dockerfile:1.6
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Server-only Linux validation image. The build context supplies a Git bundle.
|
||||
FROM rust:1.100.0-trixie AS build-tools
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates clang lld libssl-dev pkg-config protobuf-compiler python3 python3-boto3 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
ENV RUSTUP_TOOLCHAIN=1.100.0 \
|
||||
CARGO_BUILD_JOBS=4 \
|
||||
CARGO_NET_GIT_FETCH_WITH_CLI=true \
|
||||
CARGO_TARGET_DIR=/work/rustfs/target \
|
||||
RUSTFLAGS="-C link-arg=-fuse-ld=lld"
|
||||
|
||||
FROM build-tools AS build
|
||||
ARG SOURCE_REVISION
|
||||
COPY repository.bundle /tmp/repository.bundle
|
||||
COPY inspect_geometry.rs /tmp/inspect_geometry.rs
|
||||
RUN git clone /tmp/repository.bundle /work/rustfs \
|
||||
&& git -C /work/rustfs checkout --detach "$SOURCE_REVISION" \
|
||||
&& test "$(git -C /work/rustfs rev-parse HEAD)" = "$SOURCE_REVISION"
|
||||
WORKDIR /work/rustfs
|
||||
RUN --mount=type=cache,id=rustfs-inline-registry,target=/usr/local/cargo/registry \
|
||||
--mount=type=cache,id=rustfs-inline-git,target=/usr/local/cargo/git \
|
||||
--mount=type=cache,id=rustfs-inline-linux-target,target=/work/rustfs/target \
|
||||
cargo build --locked --release --bin rustfs \
|
||||
&& install -Dm755 target/release/rustfs /out/rustfs
|
||||
|
||||
# Build the diagnostic tool after preserving the clean-source server binary.
|
||||
RUN cp /tmp/inspect_geometry.rs crates/filemeta/examples/inline_lab_inspect.rs
|
||||
RUN --mount=type=cache,id=rustfs-inline-registry,target=/usr/local/cargo/registry \
|
||||
--mount=type=cache,id=rustfs-inline-git,target=/usr/local/cargo/git \
|
||||
--mount=type=cache,id=rustfs-inline-linux-target,target=/work/rustfs/target \
|
||||
cargo build --locked --release -p rustfs-filemeta --example inline_lab_inspect \
|
||||
&& install -Dm755 target/release/examples/inline_lab_inspect /out/inline_lab_inspect
|
||||
|
||||
FROM debian:trixie-slim AS runtime
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates curl libssl3t64 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
ARG SOURCE_REVISION
|
||||
LABEL org.opencontainers.image.revision="$SOURCE_REVISION" \
|
||||
rustfs.validation="inline"
|
||||
COPY --from=build /out/rustfs /usr/local/bin/rustfs
|
||||
COPY --from=build /out/inline_lab_inspect /usr/local/bin/inline_lab_inspect
|
||||
ENTRYPOINT ["/usr/local/bin/rustfs"]
|
||||
@@ -1,5 +0,0 @@
|
||||
FROM debian:trixie-slim
|
||||
COPY mc /usr/bin/mc
|
||||
RUN chmod 0755 /usr/bin/mc && /usr/bin/mc --version
|
||||
LABEL rustfs.validation="inline"
|
||||
ENTRYPOINT ["/usr/bin/mc"]
|
||||
@@ -1,133 +0,0 @@
|
||||
# Inline geometry Linux lab
|
||||
|
||||
**Use this when** validating inline placement, block geometry, Range reads and reader rollback in a task-owned local Docker cluster.
|
||||
|
||||
The adapter reuses the repository's four-node compose topology with 16 logical Docker volumes and EC12+4. It binds one S3 port to loopback, keeps strict durability and checks running binary hashes and persisted object geometry. Docker volumes share the host storage substrate. Performance observations describe this local virtualized lab.
|
||||
|
||||
## Build a frozen server image
|
||||
|
||||
Use the current stable toolchain pinned in the lab Dockerfile. Supply a Git bundle and the metadata inspector source as the build context. The server is compiled from a clean checkout before the diagnostic example is added.
|
||||
|
||||
```bash
|
||||
set -euo pipefail
|
||||
LAB_BUILD_DIR="$(mktemp -d)"
|
||||
LAB_REVISION="$(git rev-parse HEAD)"
|
||||
git bundle create "$LAB_BUILD_DIR/repository.bundle" HEAD
|
||||
cp crates/filemeta/examples/inline_lab_inspect.rs "$LAB_BUILD_DIR/inspect_geometry.rs"
|
||||
docker --context "$LAB_CONTEXT" build --target build-tools \
|
||||
-t rustfs-inline-build-tools:1.100 \
|
||||
-f .docker/inline-lab/Dockerfile .docker/inline-lab
|
||||
docker --context "$LAB_CONTEXT" build \
|
||||
--build-arg SOURCE_REVISION="$LAB_REVISION" \
|
||||
-t rustfs-inline-validation:local \
|
||||
-f .docker/inline-lab/Dockerfile "$LAB_BUILD_DIR"
|
||||
LAB_EXPORT_ID="$(docker --context "$LAB_CONTEXT" create \
|
||||
--label rustfs.validation=inline rustfs-inline-validation:local)"
|
||||
docker --context "$LAB_CONTEXT" cp \
|
||||
"$LAB_EXPORT_ID:/usr/local/bin/rustfs" "$LAB_BUILD_DIR/rustfs-linux"
|
||||
docker --context "$LAB_CONTEXT" rm "$LAB_EXPORT_ID"
|
||||
```
|
||||
|
||||
## Configure and probe
|
||||
|
||||
Select a new compose project and a private state directory. The adapter generates test credentials there. Existing resources require matching ownership labels. Configure checks the image's source revision and pins immutable image IDs. Each operation takes an exclusive state lock. Deployment waits for `/health/ready` on every node, covering storage, IAM, lock quorum and peer dependencies.
|
||||
|
||||
```bash
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" configure \
|
||||
--context "$LAB_CONTEXT" --project "$LAB_PROJECT" \
|
||||
--image rustfs-inline-validation:local --revision "$LAB_REVISION" \
|
||||
--binary "$LAB_BUILD_DIR/rustfs-linux" --port 19080
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" deploy \
|
||||
--bucket inline-geometry-probe --shard-bytes 0
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" probe \
|
||||
--bucket inline-geometry-probe
|
||||
```
|
||||
|
||||
The probe verifies complete bytes, HEAD sizes, Range contents, EC12+4 and the actual persisted block. It records inline markers for 4 KiB, 1 MiB, 1 MiB + 1 byte, 1.25 MiB, 1,572,856 bytes, 1,572,857 bytes and 1.5 MiB objects in `probe.json`. The server-only image includes the existing filemeta decoder through a small geometry inspector. Console assets and the administration CLI are outside this lab image.
|
||||
|
||||
## ABBA deployment hook
|
||||
|
||||
Use `inline_docker_lab.py ... deploy` in the existing `run_hotpath_warp_abba.sh` external deploy hook. Export `RUSTFS_ERASURE_SHARD_BYTES` according to `HOTPATH_ABBA_PHASE`. The adapter uses the supplied leg, phase, bucket, binary and evidence-file context, resets only owned lab volumes, checks running configuration and binary hashes, creates the fresh bucket and writes the runner's deployment evidence.
|
||||
|
||||
Use the same frozen binary for both sides of a geometry experiment. Fix the inline budget and encoding mode, then compare zero with one candidate shard value. Include the CPU, memory and image identities from every node in each result. The state `config.json` supplies the generated test credentials for the load driver. Set `DOCKER_CONTEXT` to the selected context when the driver captures Docker resource snapshots.
|
||||
|
||||
For content compatibility across an image or geometry change, configure the next selected image and artifact using the same state, then deploy with `--keep-data` and an existing bucket. This retains owned volumes. Compare existing objects and version IDs with the manifest prepared before the change.
|
||||
|
||||
## Cleanup
|
||||
|
||||
```bash
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" down --purge
|
||||
```
|
||||
|
||||
Cleanup checks ownership for containers, volumes and networks. A configuration cannot adopt another project's resources. Raw measurements, state, Git bundles and build outputs stay in task artifact directories.
|
||||
|
||||
Unit checks run with `scripts/python_bin.sh scripts/test_inline_docker_lab.py` and are included in `make script-tests`.
|
||||
|
||||
## Linux content-probe client
|
||||
|
||||
The PUT after-probe requires a working `mc`. For a macOS client affected by cgo initialization failures, compile the same frozen public mc module for Linux with CGO disabled, build it with `Dockerfile.mc`, and use `scripts/inline_docker_mc.sh` as the process-local `mc` command. Record the module version, checksum and resulting image ID with the run.
|
||||
|
||||
The adapter rewrites only the lab loopback alias endpoint to the private `node1` endpoint. Alias configuration stays in the probe's own `MC_CONFIG_DIR`, and `cat` streams raw bytes. The existing HEAD, GET and hash checks continue to execute.
|
||||
|
||||
```bash
|
||||
export RUSTFS_INLINE_LAB_STATE="$LAB_STATE"
|
||||
export RUSTFS_INLINE_MC_IMAGE="$LAB_MC_IMAGE"
|
||||
mkdir -p "$LAB_TOOLS"
|
||||
ln -s "$PWD/scripts/inline_docker_mc.sh" "$LAB_TOOLS/mc"
|
||||
export PATH="$LAB_TOOLS:$PATH"
|
||||
```
|
||||
|
||||
## Compare transport and placement independently
|
||||
|
||||
`deploy --rpc-mode json-compat` explicitly clears both MessagePack-only flags on every node. `deploy --rpc-mode msgpack-only` enables both flags for the selected homogeneous lab fleet. Deployment evidence records the mode and checks both running flags. The default is `json-compat`. The ABBA hook may select the same mode through `RUSTFS_INLINE_LAB_RPC_MODE`.
|
||||
|
||||
First compare the base and bounded-multi-block binaries at `--shard-bytes 0` in JSON compatibility mode. Then repeat with MessagePack-only on both sides. A same-binary transport ABBA isolates the encoding effect. Keep the 128 KiB budget, CPU sets, strict durability, load and object set fixed. Check actual markers and block sizes at the boundary sizes before timing. Retain baseline-drift and candidate gates, error logs and raw request distributions.
|
||||
|
||||
For a rolling downgrade, redeploy the current binary with `--keep-data --rpc-mode json-compat` before introducing an older node. Check old and new reads, version IDs and batch deletes against the persisted manifest. A running MessagePack-only flag pair establishes the lab configuration. Compatibility with a particular old reader requires the explicit reader test.
|
||||
|
||||
## Preserve a compatibility manifest
|
||||
|
||||
Run the seed once in a fresh lab bucket. It records full byte oracles and the observed version inventory, including any additional version matching a known body, and leaves a 5 MiB multipart part ready for the next reader. Each bucket has its own private `compat-<bucket>.json` manifest. Keep the same state and volumes across image changes. A fresh deployment or cleanup that resets volumes moves existing compatibility manifests into `compat-history`. Retained-data deployments preserve the active manifests.
|
||||
|
||||
```bash
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" compat-seed --bucket inline-compat
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" compat-check --bucket inline-compat
|
||||
# Configure the next frozen image and binary, then retain the data.
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" deploy \
|
||||
--bucket inline-compat --shard-bytes 0 --rpc-mode json-compat --keep-data
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" compat-check --bucket inline-compat
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" compat-finish-mpu --bucket inline-compat
|
||||
```
|
||||
|
||||
The check compares every persisted version ID, HEAD length, complete SHA256, cross-block Range and per-version geometry from `xl.meta`. The seed includes zero-length objects, null versions, the 192 KiB versioned admission boundary and the accumulated 128 KiB shard boundary. Seed a second bucket on the new writer before the downgrade, then check both manifests using the selected old reader. Complete an old upload on the new binary and a new upload on the old binary. After returning to the current fleet, `compat-delete --bucket <bucket>` deletes the recorded versions in one S3 batch and checks the exact remaining inventory and bytes.
|
||||
|
||||
Successful multipart-completion and batch-delete receipts are saved before subsequent content and geometry checks. If either check fails, rerun the same command with the same state directory to resume verification from the saved completion state. After a seed check fails with a saved manifest, use `compat-check` to resume verification.
|
||||
|
||||
For mixed-version testing, retain JSON compatibility mode throughout the rolling replacement. Verify each node's immutable image and binary, then perform manifest checks and new writes through both coordinator versions using `--coordinator node1` and `--coordinator node4`. The manifest records the actual writer coordinator revision from the running image. Stop one owned node to check read availability at EC12+4, restore it and check the manifests again. Record the exact oldest tested source revision, toolchain and build profile with the evidence. The version boundary is established by these executed cells.
|
||||
|
||||
An owned lab can replace just its fourth node through a compose image override. Select the frozen old image first and verify its revision label. Keep `compose.json` in JSON compatibility mode for this cell.
|
||||
|
||||
```bash
|
||||
set -euo pipefail
|
||||
OLD_IMAGE_ID="$(docker --context "$LAB_CONTEXT" image inspect "$OLD_IMAGE" --format '{{.Id}}')"
|
||||
test "$(docker --context "$LAB_CONTEXT" image inspect "$OLD_IMAGE_ID" --format '{{index .Config.Labels "org.opencontainers.image.revision"}}')" = "$OLD_REVISION"
|
||||
INLINE_OLD_IMAGE_ID="$OLD_IMAGE_ID" scripts/python_bin.sh - <<'PYTHON' > "$LAB_STATE/mixed-image.json"
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
override = {"services": {"node4": {"image": os.environ["INLINE_OLD_IMAGE_ID"]}}}
|
||||
json.dump(override, sys.stdout)
|
||||
PYTHON
|
||||
docker --context "$LAB_CONTEXT" compose --project-name "$LAB_PROJECT" \
|
||||
--file "$LAB_STATE/compose.json" --file "$LAB_STATE/mixed-image.json" \
|
||||
up --detach --no-deps --no-build --pull never node4
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" compat-check \
|
||||
--bucket inline-compat --coordinator node4
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" compat-seed \
|
||||
--bucket inline-mixed-writer --coordinator node4
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" compat-check \
|
||||
--bucket inline-mixed-writer --coordinator node1
|
||||
```
|
||||
|
||||
Inspect all four running image IDs and binary hashes before recording the mixed-fleet result. Redeploy the configured current image with `--keep-data --rpc-mode json-compat` to restore a homogeneous fleet. The same manifest checks apply after stopping and restarting one owned node through the base compose file.
|
||||
@@ -0,0 +1,54 @@
|
||||
# Copyright 2026 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Both-stack leg
|
||||
description: Prove which S3 stack one e2e leg's servers selected and upload the leg for the summary (call with if always)
|
||||
|
||||
inputs:
|
||||
profile:
|
||||
description: The nextest profile the leg ran
|
||||
required: true
|
||||
stack:
|
||||
description: The stack the leg requested, legacy or gateway
|
||||
required: true
|
||||
logs:
|
||||
description: Directory or file holding the servers' output
|
||||
required: true
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Prove which stack the servers selected
|
||||
shell: bash
|
||||
env:
|
||||
PROFILE: ${{ inputs.profile }}
|
||||
STACK: ${{ inputs.stack }}
|
||||
LOGS: ${{ inputs.logs }}
|
||||
run: |
|
||||
# Outside the workspace: e2e_binary.py refuses a run whose source tree changed.
|
||||
mkdir -p "$RUNNER_TEMP/leg"
|
||||
junit="target/nextest/${PROFILE}/junit.xml"
|
||||
if [ -s "$junit" ]; then cp "$junit" "$RUNNER_TEMP/leg/junit.xml"; fi
|
||||
# A gateway leg stopped by its preflight has no output; the preflight's stack.json stays.
|
||||
if [ -e "$LOGS" ]; then python3 scripts/dual_stack.py stack --expect "$STACK" --out "$RUNNER_TEMP/leg/stack.json" "$LOGS"; fi
|
||||
|
||||
- name: Upload the leg
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: dual-stack-leg-${{ inputs.profile }}-${{ inputs.stack }}
|
||||
overwrite: true
|
||||
path: ${{ runner.temp }}/leg
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
@@ -0,0 +1,56 @@
|
||||
# Copyright 2026 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Both-stack summary
|
||||
description: Diff the legacy and gateway legs of this run (scripts/junit_diff.py) and upload one dated summary
|
||||
|
||||
inputs:
|
||||
profiles:
|
||||
description: Comma-separated nextest profiles whose two legs this run uploaded
|
||||
required: true
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Download the legs
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
pattern: dual-stack-leg-*
|
||||
path: legs
|
||||
|
||||
- name: Diff every profile and write the summary
|
||||
id: summary
|
||||
shell: bash
|
||||
env:
|
||||
PROFILES: ${{ inputs.profiles }}
|
||||
run: |
|
||||
# The shell runs with -e; the exit status is reported after the summary is written.
|
||||
set +e -u -o pipefail
|
||||
date="$(date -u +%F)"
|
||||
echo "date=${date}" >> "$GITHUB_OUTPUT"
|
||||
mkdir -p legs
|
||||
python3 scripts/dual_stack.py summary --profiles "$PROFILES" --legs legs --date "$date" --out-dir dual-stack
|
||||
status=$?
|
||||
if [ -f dual-stack/summary.md ]; then cat dual-stack/summary.md >> "$GITHUB_STEP_SUMMARY"; fi
|
||||
exit "$status"
|
||||
|
||||
- name: Upload the dated summary
|
||||
if: always() && steps.summary.outputs.date != ''
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: dual-stack-summary-${{ steps.summary.outputs.date }}
|
||||
overwrite: true
|
||||
path: dual-stack
|
||||
if-no-files-found: error
|
||||
retention-days: 30
|
||||
@@ -814,7 +814,7 @@
|
||||
"monitor": "workflow",
|
||||
"max_age_hours": 36,
|
||||
"result_owner": "dual-stack-nightly.yml",
|
||||
"disposition": "nightly run of the two e2e profiles only ci.yml runs, on the gateway stack, with the JUnit reports retained for 30 days",
|
||||
"disposition": "both-stack JUnit diff of the two profiles only ci.yml runs; the gateway legs are report-only until T2.10",
|
||||
"failure_owner": "watchdog",
|
||||
"origin": "added"
|
||||
},
|
||||
|
||||
+173
-1
@@ -92,6 +92,7 @@ jobs:
|
||||
connect: ${{ steps.scope.outputs.connect || 'true' }}
|
||||
packages: ${{ steps.scope.outputs.packages }}
|
||||
full_validation: ${{ steps.scope.outputs.full_validation || 'true' }}
|
||||
gateway_smoke: ${{ steps.scope.outputs.gateway_smoke || 'true' }}
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
@@ -118,7 +119,7 @@ jobs:
|
||||
echo "Base CI policy unavailable; running the full matrix."
|
||||
fi
|
||||
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=8$' "$GITHUB_OUTPUT"; then
|
||||
printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
|
||||
printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'gateway_smoke=true' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
|
||||
echo "Base policy does not cover this workflow contract; retaining the full matrix."
|
||||
fi
|
||||
- name: Retain verification plan
|
||||
@@ -675,6 +676,54 @@ jobs:
|
||||
retention-days: 3
|
||||
if-no-files-found: warn
|
||||
|
||||
test-and-lint-gateway-stack:
|
||||
name: Test and Lint (gateway-stack)
|
||||
# The gateway S3 stack (rustfs/backlog#2734) sits behind the rustfs
|
||||
# `gateway-stack` feature, which no other pull-request lane lints or tests.
|
||||
# Unlike the dormant feature lanes above, this one runs for every code
|
||||
# change: each operation slice puts its handler tests and its refusal-matrix
|
||||
# row behind the feature, and a lane that ran only on demand would let them
|
||||
# rot behind a green pull request.
|
||||
if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
# Self-hosted for build scale and cache ownership: a workspace-scale Cargo
|
||||
# build that reads the ci-dev cache written on this image.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: ci-dev-ea7eec74fe14
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
# A feature that does not compile fails here; nothing below is skipped
|
||||
# into a pass.
|
||||
- name: Run clippy with gateway-stack
|
||||
run: cargo clippy --locked -p rustfs --all-targets --features gateway-stack -- -D warnings
|
||||
|
||||
# The gateway stack's own tests: the assembly, the operation table and
|
||||
# its completeness, the shared handler sequence, the stack selection, and
|
||||
# the embedded server driven through the whole middleware stack. The
|
||||
# filter names modules, never operations.
|
||||
- name: Run tests with gateway-stack
|
||||
env:
|
||||
# The same bounded link concurrency as the other feature lanes.
|
||||
CARGO_BUILD_JOBS: "2"
|
||||
run: |
|
||||
cargo nextest run --profile ci --locked -p rustfs --features gateway-stack --no-tests fail \
|
||||
--status-level all --final-status-level all \
|
||||
-E 'test(/gateway_stack|gateway_handlers|server::stack::/) | binary(embedded_gateway_stack_test)'
|
||||
|
||||
build-rustfs-debug-binary:
|
||||
name: Build RustFS Debug Binary
|
||||
if: contains(fromJSON('["full", "e2e"]'), needs.classify-changes.outputs.mode) && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
@@ -937,6 +986,8 @@ jobs:
|
||||
# Retain the producer image and fixture paths until the runtime pilot passes.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
RUSTFS_S3_STACK: legacy
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -1057,6 +1108,125 @@ jobs:
|
||||
path: ${{ runner.temp }}/rustfs-e2e-*/rustfs.log
|
||||
retention-days: 3
|
||||
|
||||
# The same e2e-smoke suite against the same debug binary with the gateway
|
||||
# S3 stack selected (rustfs/backlog#2734). Report-only: it is outside the
|
||||
# Test and Lint aggregate and becomes required at T2.10. The test-binary
|
||||
# download, identity, membership, run and completeness steps must stay
|
||||
# identical to e2e-tests; scripts/ci_gate.py enforces that.
|
||||
#
|
||||
# When it runs: every push to main or gateway/integration, merge group,
|
||||
# nightly schedule and manual dispatch, and every pull request that changes a
|
||||
# gateway-stack input (GATEWAY_SMOKE_PATHS and the rustfs-gateway pins in
|
||||
# scripts/ci_gate.py, published by Select CI scope as gateway_smoke).
|
||||
# Why it is skipped on every other pull request: until T2.10 the gateway
|
||||
# stack does not serve the whole smoke suite, so the lane fails on any
|
||||
# revision. A pull request that touches none of its inputs can neither cause
|
||||
# nor fix that failure, and a red check there says nothing about the change.
|
||||
# Skipped is the honest state; a failure must never be rendered green, so no
|
||||
# step here may carry continue-on-error. A selection that is missing or
|
||||
# unreadable runs the lane. At T2.10, when the lane becomes required, it
|
||||
# must run on every pull request again: delete the selection, do not extend
|
||||
# it.
|
||||
e2e-tests-gateway:
|
||||
name: e2e-smoke (gateway, report-only until T2.10)
|
||||
needs: [ classify-changes, build-rustfs-debug-binary ]
|
||||
if: github.event_name != 'pull_request' || needs.classify-changes.outputs.gateway_smoke != 'false'
|
||||
runs-on: sm-standard-2
|
||||
timeout-minutes: 30
|
||||
continue-on-error: true
|
||||
env:
|
||||
RUSTFS_S3_STACK: gateway
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install verification runtime
|
||||
uses: ./.github/actions/test-runtime
|
||||
|
||||
- name: Download E2E test binaries
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: e2e-test-binaries
|
||||
path: ${{ runner.temp }}
|
||||
|
||||
- name: Verify E2E archive identity
|
||||
run: python3 scripts/ci_test_artifact.py verify --configuration e2e --archive "$RUNNER_TEMP/e2e-tests.tar.zst"
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install awscurl
|
||||
run: |
|
||||
python3 -m pip install --user --upgrade pip "awscurl==0.44"
|
||||
echo "AWSCURL_PATH=$HOME/.local/bin/awscurl" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Verify awscurl
|
||||
run: test -x "$AWSCURL_PATH"
|
||||
|
||||
# Download after the cache restore so the freshly built binary from the
|
||||
# build job always wins over anything restored into target/debug.
|
||||
- name: Download debug binary
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: rustfs-debug-binary
|
||||
path: target/debug
|
||||
|
||||
- name: Make binary executable
|
||||
run: chmod +x ./target/debug/rustfs
|
||||
|
||||
# Build the e2e test graph once. The archive is reused by the smoke
|
||||
# selection guard, security exact-count check, and run below, avoiding a
|
||||
# second compile of the same e2e_test target on cold runners (backlog#1645).
|
||||
- name: Verify e2e smoke test membership
|
||||
env:
|
||||
NEXTEST_ARCHIVE: ${{ runner.temp }}/e2e-tests.tar.zst
|
||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-smoke-list.json
|
||||
run: |
|
||||
cargo nextest list --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" --workspace-remap "$GITHUB_WORKSPACE" --extract-to "$GITHUB_WORKSPACE" --extract-overwrite --message-format json > "${NEXTEST_LISTING}"
|
||||
python3 ./scripts/check_test_wiring.py --check-profile e2e-smoke "${NEXTEST_LISTING}"
|
||||
./scripts/check_security_smoke_count.sh check "${NEXTEST_LISTING}"
|
||||
|
||||
# PR smoke subset of the in-repo e2e suite (backlog#1149 ci-4). The
|
||||
# profile.e2e-smoke default-filter in .config/nextest.toml is the single
|
||||
# wiring mechanism for e2e tests in CI — extend that filter instead of
|
||||
# adding new e2e jobs here. Each test spawns its own rustfs server on a
|
||||
# random port and reuses the downloaded debug binary above.
|
||||
- name: Run e2e smoke suite
|
||||
env:
|
||||
NEXTEST_ARCHIVE: ${{ runner.temp }}/e2e-tests.tar.zst
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-smoke-logs
|
||||
SAMPLER_INTERVAL_SECS: "15"
|
||||
run: |
|
||||
./scripts/ci/resource_sampler.sh start e2e-smoke
|
||||
trap './scripts/ci/resource_sampler.sh stop' EXIT
|
||||
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-smoke --cargo-metadata target/nextest/cargo-metadata.json --binaries-metadata target/nextest/binaries-metadata.json --workspace-remap "$GITHUB_WORKSPACE" --target-dir-remap "$GITHUB_WORKSPACE/target" \
|
||||
--status-level all --final-status-level all --failure-output final
|
||||
|
||||
- name: Verify e2e smoke execution completeness
|
||||
run: python3 scripts/ci_test_artifact.py evidence "$RUNNER_TEMP/rustfs-e2e-smoke-list.json" target/nextest/e2e-smoke/junit.xml
|
||||
|
||||
- name: Upload e2e smoke diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-smoke-gateway-diagnostics-${{ github.run_number }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
${{ runner.temp }}/rustfs-e2e-smoke-logs/
|
||||
${{ runner.temp }}/rustfs-e2e-smoke-list.json
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Upload e2e smoke JUnit report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-smoke-gateway-junit-${{ github.run_number }}-${{ github.run_attempt }}
|
||||
path: target/nextest/e2e-smoke/junit.xml
|
||||
if-no-files-found: warn
|
||||
|
||||
e2e-full:
|
||||
name: End-to-End Tests (full merge gate)
|
||||
# Merge gate only (backlog#1149 ci-5): the never-automated user-visible
|
||||
@@ -1442,6 +1612,7 @@ jobs:
|
||||
- fips-provider
|
||||
- test-and-lint-rio-v2
|
||||
- test-and-lint-protocols
|
||||
- test-and-lint-gateway-stack
|
||||
- build-rustfs-debug-binary
|
||||
- uring-integration
|
||||
- e2e-tests
|
||||
@@ -1477,6 +1648,7 @@ jobs:
|
||||
- core-contract
|
||||
- test-and-lint-rio-v2
|
||||
- test-and-lint-protocols
|
||||
- test-and-lint-gateway-stack
|
||||
- build-rustfs-debug-binary
|
||||
- build-rustfs-debug-binary-rio-v2
|
||||
- uring-integration
|
||||
|
||||
@@ -12,18 +12,21 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Nightly run of the two e2e profiles only ci.yml runs, e2e-smoke and e2e-full
|
||||
# (rustfs/backlog#2772, T3.1), on the gateway stack, which is the only S3 stack.
|
||||
# The workflow keeps its file and registry name from the period when it diffed two
|
||||
# stacks. Each leg uploads its JUnit report and server logs for 30 days; the one
|
||||
# comparison against the last legacy record (the `dual-stack-leg-<profile>-legacy`
|
||||
# artifacts of the final both-stack run) is made by hand with scripts/junit_diff.py.
|
||||
# Nightly both-stack diff of the two e2e profiles only ci.yml runs, e2e-smoke
|
||||
# and e2e-full (rustfs/backlog#2772, T3.1). The other profiles carry a `stack`
|
||||
# dimension in their own workflows, where the existing lane is the legacy leg.
|
||||
#
|
||||
# Left out: `dual_stack_differential::` (servers with an explicit
|
||||
# One build serves every leg. One preflight starts that binary with
|
||||
# RUSTFS_S3_STACK=gateway (scripts/dual_stack.py preflight); without a proven
|
||||
# gateway start the gateway legs do not run and the summary says `skipped:
|
||||
# gateway stack refuses startup: <why>`. Which stack a leg ran is read from the
|
||||
# servers' own start-up line. The gateway legs are report-only until T2.10.
|
||||
#
|
||||
# Left out of both stacks: `dual_stack_differential::` (servers with an explicit
|
||||
# stack), `fake_s3_target::`, the upgrade write-readiness cases and one deadline
|
||||
# case (no server), and `distributed_startup_regression_test::` (needs ci.yml's
|
||||
# hooks-binary manifest). e2e-full uses ci.yml's pinned mc and Vault binaries
|
||||
# and network fault-injection tools.
|
||||
# hooks-binary manifest). Both e2e-full stacks use ci.yml's pinned mc and
|
||||
# Vault binaries and network fault-injection tools.
|
||||
|
||||
name: dual-stack-nightly
|
||||
|
||||
@@ -47,11 +50,13 @@ defaults:
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Build once
|
||||
name: Build once and probe the gateway stack
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
# The runner and cache of ci.yml's build-rustfs-debug-binary.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 60
|
||||
outputs:
|
||||
stacks: ${{ steps.preflight.outputs.stacks }}
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
@@ -81,15 +86,47 @@ jobs:
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
# Without a proven gateway start there is no gateway leg to run.
|
||||
- name: Gateway preflight
|
||||
id: preflight
|
||||
run: |
|
||||
if python3 scripts/dual_stack.py preflight --binary target/debug/rustfs --out "$RUNNER_TEMP/preflight/stack.json"; then
|
||||
echo 'stacks=["legacy","gateway"]' >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo 'stacks=["legacy"]' >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# The evidence stands in for a gateway leg that is skipped; a leg that runs overwrites it.
|
||||
- name: Upload the preflight evidence for e2e-smoke
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: dual-stack-leg-e2e-smoke-gateway
|
||||
path: ${{ runner.temp }}/preflight
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
|
||||
- name: Upload the preflight evidence for e2e-full
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: dual-stack-leg-e2e-full-gateway
|
||||
path: ${{ runner.temp }}/preflight
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
|
||||
leg:
|
||||
name: ${{ matrix.profile }}
|
||||
name: ${{ matrix.profile }} (${{ matrix.stack }}${{ matrix.stack == 'gateway' && ', report-only until T2.10' || '' }})
|
||||
needs: [ build ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 120
|
||||
# Report-only until T2.10: the gateway stack cannot serve the suites yet.
|
||||
continue-on-error: ${{ matrix.stack == 'gateway' }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
profile: [ e2e-smoke, e2e-full ]
|
||||
stack: ${{ fromJSON(needs.build.outputs.stacks) }}
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
@@ -171,9 +208,10 @@ jobs:
|
||||
run: |
|
||||
"$RUSTFS_TEST_VAULT_BIN" version
|
||||
|
||||
- name: Run the profile
|
||||
- name: Run the profile on the ${{ matrix.stack }} stack
|
||||
env:
|
||||
PROFILE: ${{ matrix.profile }}
|
||||
RUSTFS_S3_STACK: ${{ matrix.stack }}
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/server-logs
|
||||
run: |
|
||||
chmod +x target/debug/rustfs
|
||||
@@ -182,14 +220,23 @@ jobs:
|
||||
filter="$filter & not test(=list_objects_v2_pagination_test::tests::test_list_objects_v2_deadline_rejects_unresponsive_peer)"
|
||||
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile "$PROFILE" -p e2e_test -E "$filter"
|
||||
|
||||
- name: Upload the leg
|
||||
- name: Prove the stack and upload the leg
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
uses: ./.github/actions/dual-stack-leg
|
||||
with: { profile: "${{ matrix.profile }}", stack: "${{ matrix.stack }}", logs: "${{ runner.temp }}/server-logs" }
|
||||
|
||||
summary:
|
||||
name: Both-stack diff summary
|
||||
needs: [ build, leg ]
|
||||
if: github.repository == 'rustfs/rustfs' && always()
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
name: e2e-nightly-${{ matrix.profile }}
|
||||
overwrite: true
|
||||
path: |
|
||||
target/nextest/${{ matrix.profile }}/junit.xml
|
||||
${{ runner.temp }}/server-logs
|
||||
if-no-files-found: warn
|
||||
retention-days: 30
|
||||
persist-credentials: false
|
||||
|
||||
- uses: ./.github/actions/dual-stack-summary
|
||||
with:
|
||||
profiles: e2e-smoke,e2e-full
|
||||
|
||||
@@ -53,11 +53,14 @@ concurrency:
|
||||
jobs:
|
||||
distributed:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Distributed 4-node 4-disk e2e
|
||||
name: Distributed 4-node 4-disk e2e (${{ matrix.stack }}${{ matrix.stack == 'gateway' && ', report-only until T2.10' || '' }})
|
||||
# GitHub-hosted VM: loop and tmpfs mounts work here. sm-standard-4 is an
|
||||
# ARC pod and rejects both (`mount -o loop` ENOENT, tmpfs "read-only").
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 180
|
||||
# The gateway leg is report-only until T2.10.
|
||||
continue-on-error: ${{ matrix.stack == 'gateway' }}
|
||||
strategy: { fail-fast: false, matrix: { stack: [legacy, gateway] } }
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
NO_PROXY: 127.0.0.1,localhost
|
||||
@@ -129,6 +132,11 @@ jobs:
|
||||
run: |
|
||||
python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks
|
||||
|
||||
# A gateway binary that refuses to start skips the suite; the summary says why.
|
||||
- name: Gateway preflight
|
||||
if: matrix.stack == 'gateway'
|
||||
run: python3 scripts/dual_stack.py preflight --binary target/debug/rustfs --out "$RUNNER_TEMP/leg/stack.json"
|
||||
|
||||
- name: Verify distributed e2e membership
|
||||
env:
|
||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-distributed-list.json
|
||||
@@ -140,6 +148,7 @@ jobs:
|
||||
env:
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-distributed-logs
|
||||
FILTER: ${{ inputs.filter }}
|
||||
RUSTFS_S3_STACK: ${{ matrix.stack }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
rm -f target/nextest/e2e-distributed/junit.xml
|
||||
@@ -150,11 +159,19 @@ jobs:
|
||||
fi
|
||||
test -s target/nextest/e2e-distributed/junit.xml
|
||||
|
||||
- name: Prove the stack and upload the leg
|
||||
if: always()
|
||||
uses: ./.github/actions/dual-stack-leg
|
||||
with:
|
||||
profile: e2e-distributed
|
||||
stack: ${{ matrix.stack }}
|
||||
logs: ${{ runner.temp }}/rustfs-e2e-distributed-logs
|
||||
|
||||
- name: Upload distributed e2e diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-distributed-${{ github.run_number }}
|
||||
name: e2e-distributed-${{ github.run_number }}-${{ matrix.stack }}
|
||||
path: |
|
||||
target/nextest/e2e-distributed/junit.xml
|
||||
${{ runner.temp }}/rustfs-e2e-distributed-list.json
|
||||
@@ -174,6 +191,20 @@ jobs:
|
||||
fi
|
||||
done
|
||||
|
||||
dual-stack-summary:
|
||||
name: Both-stack diff summary
|
||||
needs: [distributed]
|
||||
if: github.repository == 'rustfs/rustfs' && always()
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: ./.github/actions/dual-stack-summary
|
||||
with:
|
||||
profiles: e2e-distributed
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [distributed]
|
||||
|
||||
@@ -54,15 +54,18 @@ concurrency:
|
||||
jobs:
|
||||
repl-nightly:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Replication e2e (nightly)
|
||||
name: Replication e2e (nightly, ${{ matrix.stack }}${{ matrix.stack == 'gateway' && ', report-only until T2.10' || '' }})
|
||||
# dual-node tests spawn two full rustfs servers each; the extra headroom
|
||||
# over the per-PR sm-standard-2 e2e job keeps the parallel process fan-out
|
||||
# from starving the runner. Mirrors the ILM serial lane's runner class.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 45
|
||||
# The gateway leg is report-only until T2.10.
|
||||
continue-on-error: ${{ matrix.stack == 'gateway' }}
|
||||
strategy: { fail-fast: false, matrix: { stack: [legacy, gateway] } }
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
RUSTFS_BUILD_FEATURES: ""
|
||||
RUSTFS_BUILD_FEATURES: ${{ matrix.stack == 'gateway' && 'gateway-stack' || '' }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -96,6 +99,11 @@ jobs:
|
||||
run: |
|
||||
python3 scripts/e2e_binary.py build --bins --features "$RUSTFS_BUILD_FEATURES"
|
||||
|
||||
# A gateway binary that refuses to start skips the suite; the summary says why.
|
||||
- name: Gateway preflight
|
||||
if: matrix.stack == 'gateway'
|
||||
run: python3 scripts/dual_stack.py preflight --binary target/debug/rustfs --out "$RUNNER_TEMP/leg/stack.json"
|
||||
|
||||
- name: Verify replication e2e membership
|
||||
env:
|
||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-repl-nightly-list.json
|
||||
@@ -106,13 +114,22 @@ jobs:
|
||||
- name: Run replication e2e nightly suite
|
||||
env:
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-repl-nightly-logs
|
||||
RUSTFS_S3_STACK: ${{ matrix.stack }}
|
||||
run: python3 scripts/e2e_binary.py run --features "$RUSTFS_BUILD_FEATURES" -- cargo nextest run --profile e2e-repl-nightly -p e2e_test
|
||||
|
||||
- name: Prove the stack and upload the leg
|
||||
if: always()
|
||||
uses: ./.github/actions/dual-stack-leg
|
||||
with:
|
||||
profile: e2e-repl-nightly
|
||||
stack: ${{ matrix.stack }}
|
||||
logs: ${{ runner.temp }}/rustfs-e2e-repl-nightly-logs
|
||||
|
||||
- name: Upload nextest junit report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-replication-nightly-junit-${{ github.run_number }}
|
||||
name: e2e-replication-nightly-junit-${{ github.run_number }}-${{ matrix.stack }}
|
||||
path: |
|
||||
target/nextest/e2e-repl-nightly/junit.xml
|
||||
${{ runner.temp }}/rustfs-e2e-repl-nightly-list.json
|
||||
@@ -122,9 +139,12 @@ jobs:
|
||||
|
||||
cluster-nightly:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Cluster fault e2e (nightly)
|
||||
name: Cluster fault e2e (nightly, ${{ matrix.stack }}${{ matrix.stack == 'gateway' && ', report-only until T2.10' || '' }})
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
# The gateway leg is report-only until T2.10.
|
||||
continue-on-error: ${{ matrix.stack == 'gateway' }}
|
||||
strategy: { fail-fast: false, matrix: { stack: [legacy, gateway] } }
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
@@ -145,6 +165,11 @@ jobs:
|
||||
run: |
|
||||
python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks
|
||||
|
||||
# A gateway binary that refuses to start skips the suite; the summary says why.
|
||||
- name: Gateway preflight
|
||||
if: matrix.stack == 'gateway'
|
||||
run: python3 scripts/dual_stack.py preflight --binary target/debug/rustfs --out "$RUNNER_TEMP/leg/stack.json"
|
||||
|
||||
- name: Verify cluster fault e2e membership
|
||||
env:
|
||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-nightly-list.json
|
||||
@@ -156,13 +181,22 @@ jobs:
|
||||
env:
|
||||
CARGO_BIN_EXE_rustfs: ${{ github.workspace }}/target/debug/rustfs
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-nightly-logs
|
||||
RUSTFS_S3_STACK: ${{ matrix.stack }}
|
||||
run: python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-nightly -p e2e_test
|
||||
|
||||
- name: Prove the stack and upload the leg
|
||||
if: always()
|
||||
uses: ./.github/actions/dual-stack-leg
|
||||
with:
|
||||
profile: e2e-nightly
|
||||
stack: ${{ matrix.stack }}
|
||||
logs: ${{ runner.temp }}/rustfs-e2e-nightly-logs
|
||||
|
||||
- name: Upload cluster fault diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-cluster-nightly-${{ github.run_number }}
|
||||
name: e2e-cluster-nightly-${{ github.run_number }}-${{ matrix.stack }}
|
||||
path: |
|
||||
target/nextest/e2e-nightly/junit.xml
|
||||
${{ runner.temp }}/rustfs-e2e-nightly-list.json
|
||||
@@ -172,12 +206,15 @@ jobs:
|
||||
|
||||
protocols-nightly:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Protocol e2e (nightly)
|
||||
name: Protocol e2e (nightly, ${{ matrix.stack }}${{ matrix.stack == 'gateway' && ', report-only until T2.10' || '' }})
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
# The gateway leg is report-only until T2.10.
|
||||
continue-on-error: ${{ matrix.stack == 'gateway' }}
|
||||
strategy: { fail-fast: false, matrix: { stack: [legacy, gateway] } }
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
RUSTFS_BUILD_FEATURES: ftps,webdav,sftp
|
||||
RUSTFS_BUILD_FEATURES: ${{ matrix.stack == 'gateway' && 'ftps,webdav,sftp,gateway-stack' || 'ftps,webdav,sftp' }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -201,6 +238,11 @@ jobs:
|
||||
- name: Build protocol server
|
||||
run: python3 scripts/e2e_binary.py build --features "$RUSTFS_BUILD_FEATURES"
|
||||
|
||||
# A gateway binary that refuses to start skips the suite; the summary says why.
|
||||
- name: Gateway preflight
|
||||
if: matrix.stack == 'gateway'
|
||||
run: python3 scripts/dual_stack.py preflight --binary target/debug/rustfs --out "$RUNNER_TEMP/leg/stack.json"
|
||||
|
||||
# The suite owns fixed protocol ports and serializes its internal cases.
|
||||
- name: Verify protocol e2e membership
|
||||
env:
|
||||
@@ -212,16 +254,28 @@ jobs:
|
||||
- name: Run protocol e2e nightly suite
|
||||
env:
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-protocol-e2e-logs
|
||||
RUSTFS_S3_STACK: ${{ matrix.stack }}
|
||||
# These servers set no log level of their own; the stack proof reads their start-up line.
|
||||
RUST_LOG: rustfs=info
|
||||
# The servers inherit nextest's stdout (--no-capture): keep it where the stack proof reads it.
|
||||
run: |
|
||||
set -o pipefail
|
||||
mkdir -p "$RUSTFS_E2E_LOG_DIR"
|
||||
python3 scripts/e2e_binary.py run --features "$RUSTFS_BUILD_FEATURES" -- cargo nextest run -j 1 --profile e2e-protocols -p e2e_test --no-capture 2>&1 | tee "$RUSTFS_E2E_LOG_DIR/nextest.log"
|
||||
|
||||
- name: Prove the stack and upload the leg
|
||||
if: always()
|
||||
uses: ./.github/actions/dual-stack-leg
|
||||
with:
|
||||
profile: e2e-protocols
|
||||
stack: ${{ matrix.stack }}
|
||||
logs: ${{ runner.temp }}/rustfs-protocol-e2e-logs
|
||||
|
||||
- name: Upload protocol diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-protocol-nightly-${{ github.run_number }}
|
||||
name: e2e-protocol-nightly-${{ github.run_number }}-${{ matrix.stack }}
|
||||
path: |
|
||||
target/nextest/e2e-protocols/junit.xml
|
||||
${{ runner.temp }}/rustfs-e2e-protocols-list.json
|
||||
@@ -229,6 +283,20 @@ jobs:
|
||||
retention-days: 7
|
||||
if-no-files-found: warn
|
||||
|
||||
dual-stack-summary:
|
||||
name: Both-stack diff summary
|
||||
needs: [repl-nightly, cluster-nightly, protocols-nightly]
|
||||
if: github.repository == 'rustfs/rustfs' && always()
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: ./.github/actions/dual-stack-summary
|
||||
with:
|
||||
profiles: e2e-repl-nightly,e2e-nightly,e2e-protocols
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [repl-nightly, cluster-nightly, protocols-nightly]
|
||||
|
||||
@@ -15,20 +15,31 @@
|
||||
# Gateway black-box acceptance lanes (rustfs/backlog#2775 T3.4, #2776 T3.5).
|
||||
#
|
||||
# Runs two suites of the rustfs/gateway repository against the candidate binary,
|
||||
# one server:
|
||||
# once per S3 stack (RUSTFS_S3_STACK=legacy and =gateway), one server per stack:
|
||||
#
|
||||
# - the conformance corpus, `--profile rustfs`, judged by
|
||||
# scripts/check_conformance_rulings.py against .config/conformance-rulings.toml
|
||||
# (red: an unruled failure, a ruled case failing differently, a stale ruling,
|
||||
# a skip on the server's own answer);
|
||||
# - the client matrix, ci/compat/run_matrix.sh --external, judged by
|
||||
# - the conformance corpus, `--profile rustfs`. That profile describes the
|
||||
# gateway edge, so scripts/check_conformance_rulings.py judges the gateway
|
||||
# stack against .config/conformance-rulings.toml (red: an unruled failure, a
|
||||
# ruled case failing differently, a stale ruling, a skip on the server's own
|
||||
# answer) and publishes the legacy run as a reference that is never judged:
|
||||
# it only has to serve, stay up and complete;
|
||||
# - the client matrix, ci/compat/run_matrix.sh --external, judged per stack by
|
||||
# scripts/compat_matrix_report.py (no `fail`, no `sut-unregistered`, every
|
||||
# client ran; a client that could not be installed is reported as skipped).
|
||||
# The legacy matrix is the baseline: every cell that passes on legacy must
|
||||
# pass on gateway (`compare`).
|
||||
#
|
||||
# The suites are checked out at the rustfs/gateway rev Cargo.toml pins, so the
|
||||
# suite and the framework under test never drift apart. The verdict is computed
|
||||
# from what the run left on disk (startup record, runner exit code, liveness
|
||||
# after the corpus, report), never from the exit status of an earlier step.
|
||||
# suite and the framework under test never drift apart. The verdict of each
|
||||
# stack is computed from what the run left on disk (startup record, the stack
|
||||
# the server's own log names, runner exit code, liveness after the corpus,
|
||||
# report), never from the exit status of an earlier step. Needs
|
||||
# scripts/dual_stack.py and the `s3_stack` startup field of rustfs/rustfs#9367.
|
||||
#
|
||||
# The gateway stack is report-only until T2.10: its job may fail without failing
|
||||
# the workflow, and its summary says what it observed (did not start, or how many
|
||||
# cases ran, passed, failed and were answered 501). The stack comparison is
|
||||
# report-only for the same reason; its conformance half is information only.
|
||||
#
|
||||
# Not a pull-request gate. Scheduled failures are owned by the scheduled-validation
|
||||
# watchdog (.github/scheduled-validations.json).
|
||||
@@ -155,15 +166,21 @@ jobs:
|
||||
retention-days: 1
|
||||
|
||||
acceptance:
|
||||
name: Acceptance
|
||||
name: ${{ matrix.stack == 'gateway' && 'Acceptance (gateway, report-only until T2.10)' || 'Acceptance (legacy)' }}
|
||||
needs: [ plan, build-candidate, build-gateway-tools ]
|
||||
# GitHub-hosted like the s3-tests targets of reference-parity.yml: the job
|
||||
# only runs prebuilt binaries and the pinned clients, none of which needs Docker.
|
||||
runs-on: ubuntu-latest
|
||||
# About 3x the 15 minutes the job took in run 43325101234 (client install about
|
||||
# About 3x the 15 minutes the legacy job took in run 43325101234 (client install about
|
||||
# 8, matrix about 3, corpus about 3), so a hung server costs 45 runner minutes, not more.
|
||||
timeout-minutes: 45
|
||||
continue-on-error: ${{ matrix.stack == 'gateway' }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
stack: [ legacy, gateway ]
|
||||
env:
|
||||
STACK: ${{ matrix.stack }}
|
||||
ENDPOINT: http://127.0.0.1:9000
|
||||
# The conformance suite signs with these fixed fixture credentials (the AWS
|
||||
# documentation example pair), so the candidate is started with them and the
|
||||
@@ -196,8 +213,10 @@ jobs:
|
||||
name: gateway-acceptance-tools
|
||||
path: tools
|
||||
|
||||
# Records `ready`, or what the process said when it exited, in lane/startup.txt.
|
||||
- name: Start candidate
|
||||
# Records `ready`, or what the process said when it exited, in lane/startup.txt. `ready`
|
||||
# also needs the server's own `server_starting` log event to name the requested stack
|
||||
# (its `s3_stack` field comes from rustfs/rustfs#9367; scripts/dual_stack.py reads it).
|
||||
- name: Start candidate (${{ matrix.stack }} stack)
|
||||
run: |
|
||||
set -uo pipefail
|
||||
chmod +x target/debug/rustfs tools/*
|
||||
@@ -205,7 +224,7 @@ jobs:
|
||||
target/debug/rustfs --version | tee lane/candidate-version.txt
|
||||
# The four directories share the runner's one disk; the bypass is the CI case the
|
||||
# physical-disk guard sanctions (as in mint.yml).
|
||||
RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true RUST_LOG=info target/debug/rustfs --address 127.0.0.1:9000 \
|
||||
RUSTFS_S3_STACK="$STACK" RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true RUST_LOG=info target/debug/rustfs --address 127.0.0.1:9000 \
|
||||
--access-key "$ACCESS_KEY" --secret-key "$SECRET_KEY" "$RUNNER_TEMP"/data/rustfs{0..3} \
|
||||
> lane/rustfs.log 2>&1 &
|
||||
pid=$!
|
||||
@@ -220,8 +239,13 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
if curl -sf "$ENDPOINT/health/ready" >/dev/null; then
|
||||
echo ready > lane/startup.txt
|
||||
exit 0
|
||||
if python3 scripts/dual_stack.py stack --expect "$STACK" --out lane/stack.json lane/rustfs.log > /dev/null; then
|
||||
echo ready > lane/startup.txt
|
||||
exit 0
|
||||
fi
|
||||
echo "did not start: stack not observed" > lane/startup.txt
|
||||
cat lane/startup.txt
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
@@ -314,23 +338,58 @@ jobs:
|
||||
run: |
|
||||
set -uo pipefail
|
||||
status=0
|
||||
python3 scripts/check_conformance_rulings.py --run-dir lane | tee -a "$GITHUB_STEP_SUMMARY" || status=1
|
||||
python3 scripts/compat_matrix_report.py check --run-dir lane | tee -a "$GITHUB_STEP_SUMMARY" || status=1
|
||||
python3 scripts/check_conformance_rulings.py --stack "$STACK" --run-dir lane | tee -a "$GITHUB_STEP_SUMMARY" || status=1
|
||||
python3 scripts/compat_matrix_report.py check --stack "$STACK" --run-dir lane | tee -a "$GITHUB_STEP_SUMMARY" || status=1
|
||||
exit "$status"
|
||||
|
||||
- name: Upload lane results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: gateway-acceptance-lane
|
||||
name: gateway-acceptance-lane-${{ matrix.stack }}
|
||||
overwrite: true
|
||||
path: |
|
||||
lane/*.txt
|
||||
lane/*.log
|
||||
lane/report.json
|
||||
lane/stack.json
|
||||
lane/compat/matrix.json
|
||||
lane/compat/probe.jsonl
|
||||
lane/compat/sut.log
|
||||
lane/compat/results/**
|
||||
if-no-files-found: error
|
||||
retention-days: 14
|
||||
|
||||
compare:
|
||||
name: Compare stacks (report-only until T2.10)
|
||||
needs: [ acceptance ]
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download lane results
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
pattern: gateway-acceptance-lane-*
|
||||
path: lanes
|
||||
|
||||
- name: Diff legacy and gateway
|
||||
run: |
|
||||
set -uo pipefail
|
||||
status=0
|
||||
{
|
||||
python3 scripts/check_conformance_rulings.py \
|
||||
--diff lanes/gateway-acceptance-lane-legacy lanes/gateway-acceptance-lane-gateway || status=1
|
||||
echo
|
||||
python3 scripts/compat_matrix_report.py diff \
|
||||
--legacy lanes/gateway-acceptance-lane-legacy --gateway lanes/gateway-acceptance-lane-gateway || status=1
|
||||
} > diff.md
|
||||
cat diff.md
|
||||
cat diff.md >> "$GITHUB_STEP_SUMMARY"
|
||||
exit "$status"
|
||||
|
||||
@@ -44,6 +44,13 @@
|
||||
# well inside the 120-minute budget and buys deterministic infra. The
|
||||
# docker-capable self-hosted `dind-sm-standard-2` label was the alternative but
|
||||
# has fewer cores and reintroduces fleet-state risk for no reliability gain.
|
||||
#
|
||||
# Stack dimension (rustfs/backlog#2774, T3.3): one image, run with RUSTFS_S3_STACK=legacy and
|
||||
# =gateway. `compare` (scripts/mint_stack_diff.py) holds the gate that every case legacy passes
|
||||
# passes on gateway with no new failure. The gateway leg and `compare` are report-only until
|
||||
# T2.10. A gateway server that refuses to start stops its leg at "Wait for RustFS ready", before
|
||||
# mint runs; the summary says `skipped: gateway stack refuses startup: <why>`. The stack is read
|
||||
# from the server's own start-up line (scripts/dual_stack.py stack).
|
||||
|
||||
name: mint
|
||||
|
||||
@@ -138,8 +145,16 @@ jobs:
|
||||
retention-days: 1
|
||||
|
||||
mint:
|
||||
name: mint
|
||||
name: mint (${{ matrix.stack }}${{ matrix.stack == 'gateway' && ', report-only until T2.10' || '' }})
|
||||
needs: [image]
|
||||
# Report-only until T2.10: the gateway stack cannot serve the suites yet.
|
||||
continue-on-error: ${{ matrix.stack == 'gateway' }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
stack: [legacy, gateway]
|
||||
env:
|
||||
STACK: ${{ matrix.stack }}
|
||||
# GitHub-hosted: reliably provides a running Docker daemon + buildx +
|
||||
# python3. See the header note (ci-2) for why the self-hosted sm-standard-4
|
||||
# label was abandoned (no docker.sock on the pod its only run landed on).
|
||||
@@ -179,6 +194,7 @@ jobs:
|
||||
-e RUSTFS_SECRET_KEY="${S3_SECRET_KEY}" \
|
||||
-e RUSTFS_VOLUMES="/data/rustfs{0...3}" \
|
||||
-e RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true \
|
||||
-e RUSTFS_S3_STACK="${STACK}" \
|
||||
-e RUST_LOG=rustfs=info \
|
||||
-v /tmp/rustfs-mint:/data \
|
||||
rustfs-ci
|
||||
@@ -275,13 +291,47 @@ jobs:
|
||||
mkdir -p artifacts/rustfs-mint
|
||||
docker logs rustfs-mint > artifacts/rustfs-mint/rustfs.log 2>&1 || true
|
||||
|
||||
- name: Prove which stack the server selected
|
||||
if: always()
|
||||
run: python3 scripts/dual_stack.py stack --expect "${STACK}" --out artifacts/stack.json artifacts/rustfs-mint
|
||||
|
||||
- name: Upload artifacts
|
||||
if: always() && env.ACT != 'true'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: mint
|
||||
name: mint-${{ matrix.stack }}
|
||||
path: artifacts/**
|
||||
|
||||
compare:
|
||||
name: Compare stacks (report-only until T2.10)
|
||||
needs: [mint]
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download both legs
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
pattern: mint-{legacy,gateway}
|
||||
path: legs
|
||||
|
||||
# A leg that never uploaded leaves no directory: create both so that the
|
||||
# comparison reports "no result" for it instead of failing to find it.
|
||||
- name: Compare the stacks
|
||||
run: |
|
||||
# The shell runs with -e; the exit status is reported after the summary is written.
|
||||
set +e -u -o pipefail
|
||||
mkdir -p legs/mint-legacy legs/mint-gateway
|
||||
python3 scripts/mint_stack_diff.py legs/mint-legacy legs/mint-gateway | tee report.txt
|
||||
status="${PIPESTATUS[0]}"
|
||||
{ echo '## Mint: gateway versus legacy'; echo; echo '```'; cat report.txt; echo '```'; } >> "$GITHUB_STEP_SUMMARY"
|
||||
exit "$status"
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [image, mint]
|
||||
|
||||
@@ -68,11 +68,14 @@ env:
|
||||
|
||||
jobs:
|
||||
minio-source:
|
||||
name: MinIO source (read-through, list-through, backfill)
|
||||
name: MinIO source (read-through, list-through, backfill, ${{ matrix.stack }}${{ matrix.stack == 'gateway' && ', report-only until T2.10' || '' }})
|
||||
# Keep this scheduled interoperability lane scoped to the upstream repo.
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 90
|
||||
# The gateway leg is report-only until T2.10.
|
||||
continue-on-error: ${{ matrix.stack == 'gateway' }}
|
||||
strategy: { fail-fast: false, matrix: { stack: [legacy, gateway] } }
|
||||
env:
|
||||
NO_PROXY: 127.0.0.1,localhost
|
||||
# Fixed credentials of the container this job starts and throws away;
|
||||
@@ -137,6 +140,11 @@ jobs:
|
||||
- name: Build the RustFS binary under test
|
||||
run: python3 scripts/e2e_binary.py build --bins
|
||||
|
||||
# A gateway binary that refuses to start skips the suite; the summary says why.
|
||||
- name: Gateway preflight
|
||||
if: matrix.stack == 'gateway'
|
||||
run: python3 scripts/dual_stack.py preflight --binary target/debug/rustfs --out "$RUNNER_TEMP/leg/stack.json"
|
||||
|
||||
# The lane selects tests by module, so a rename would quietly shrink it.
|
||||
# The committed digest in .config/e2e-odm-interop-selection.txt fails
|
||||
# closed on that.
|
||||
@@ -147,9 +155,18 @@ jobs:
|
||||
|
||||
- name: Run the interop cases against MinIO
|
||||
env:
|
||||
RUSTFS_S3_STACK: ${{ matrix.stack }}
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-logs
|
||||
run: python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-odm-interop -p e2e_test --no-tests=fail
|
||||
|
||||
- name: Prove the stack and upload the leg
|
||||
if: always()
|
||||
uses: ./.github/actions/dual-stack-leg
|
||||
with:
|
||||
profile: e2e-odm-interop
|
||||
stack: ${{ matrix.stack }}
|
||||
logs: ${{ runner.temp }}/rustfs-e2e-logs
|
||||
|
||||
- name: Build the MinIO interop report
|
||||
if: always()
|
||||
uses: ./.github/actions/odm-interop-report
|
||||
@@ -173,7 +190,7 @@ jobs:
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: odm-interop-minio-${{ github.run_number }}-${{ github.run_attempt }}
|
||||
name: odm-interop-minio-${{ github.run_number }}-${{ github.run_attempt }}-${{ matrix.stack }}
|
||||
path: |
|
||||
artifacts/odm-interop/minio
|
||||
target/nextest/e2e-odm-interop/junit.xml
|
||||
@@ -299,6 +316,20 @@ jobs:
|
||||
target/nextest/e2e-odm-interop/junit.xml
|
||||
retention-days: 14
|
||||
|
||||
dual-stack-summary:
|
||||
name: Both-stack diff summary
|
||||
needs: [minio-source]
|
||||
if: github.repository == 'rustfs/rustfs' && always()
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: ./.github/actions/dual-stack-summary
|
||||
with:
|
||||
profiles: e2e-odm-interop
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [minio-source, cloud-source]
|
||||
|
||||
@@ -32,9 +32,9 @@
|
||||
#
|
||||
# The `differential` job adds the 73-operation dual-stack differential
|
||||
# (crates/e2e_test/src/dual_stack_differential, rustfs/backlog#2740) against the
|
||||
# same pinned reference: side A is the released binary (RUSTFS_E2E_REFERENCE_BINARY),
|
||||
# side B is the candidate built above.
|
||||
# It reuses the candidate build artifact, never builds the candidate
|
||||
# same pinned reference: side A is the released binary (RUSTFS_E2E_REFERENCE_BINARY,
|
||||
# started without RUSTFS_S3_STACK), side B is the candidate built above on its
|
||||
# legacy stack. It reuses the candidate build artifact, never builds the candidate
|
||||
# twice, and prints reference_side, version_a and version_b from the report the test
|
||||
# wrote; it fails when side A was not the reference or both sides report the same
|
||||
# version. Every (op, field, a, b) difference goes to the job summary. Report-only
|
||||
@@ -310,7 +310,7 @@ jobs:
|
||||
chmod +x target/debug/rustfs
|
||||
test -x "$REFERENCE_PATH"
|
||||
mkdir -p "$RUSTFS_E2E_LOG_DIR"
|
||||
# Side A: the released binary. Side B: the candidate.
|
||||
# Side A: the released binary. Side B: the candidate on its legacy stack.
|
||||
export RUSTFS_E2E_REFERENCE_BINARY="$REFERENCE_PATH"
|
||||
test_name='dual_stack_differential::tests::both_stacks_answer_every_operation_identically'
|
||||
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- \
|
||||
|
||||
@@ -13,8 +13,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Deployment
|
||||
|
||||
- **The S3 protocol stack is now rustfs/gateway**, and it is the only S3 stack. The legacy s3s-based stack is removed, and so are the `RUSTFS_S3_STACK` environment variable and the `gateway-stack` cargo feature. There is no on-disk format change. `RUSTFS_S3_STACK` no longer selects anything: a server started with it set, to any value, refuses to start and prints `RUSTFS_S3_STACK is no longer supported; the gateway stack is the only S3 stack since <version>`, where `<version>` is the release's version. Remove the variable from service units, container environments, Helm values and orchestration files before upgrading. To roll back, run the previous release binary on the same data.
|
||||
|
||||
- Before upgrading, verify that the configured root access key is at least 3 bytes and the root secret key is at least 8 bytes. RustFS now rejects shorter values at startup and identifies the setting to update. Version 1.0.5 accepted these settings at startup and returned HTTP 500 when root requested STS credentials. Coordinate root credential changes across all nodes and clients. See [root credential rotation considerations](docs/operations/two-factor-auth.md#root-credentials-cannot-be-changed-at-runtime).
|
||||
|
||||
- Existing volumes created by older root-running container images must be writable by the non-root runtime identity before upgrading. The default container UID/GID is `10001:10001`; stop the affected instances, verify host mount paths, and for ordinary bind mounts run `sudo chown -R 10001:10001 /actual/data /actual/logs` on each RustFS data/log mount. Account for ACLs, NFS, Kubernetes, and rootless UID mappings. Image-build ownership changes do not migrate mounted data. See the Docker Quick Start in [README.md](README.md).
|
||||
|
||||
Generated
+388
-451
File diff suppressed because it is too large
Load Diff
+32
-35
@@ -151,16 +151,14 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.5" }
|
||||
# engine crates move onto from s3s (rustfs/backlog#2745). The one pin of the
|
||||
# gateway in this workspace; no crate may enable a `compat-s3s*` feature except
|
||||
# the s3s-facing edge of the rustfs binary.
|
||||
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "d6840d54187382d31526317e61369734c759e2cd", version = "0.42.0", default-features = false }
|
||||
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "d6840d54187382d31526317e61369734c759e2cd", version = "0.2.0", default-features = false }
|
||||
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "2a6b8d1f7663db978d8784ab3ea331cade19237b", version = "0.40.0", default-features = false }
|
||||
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "2a6b8d1f7663db978d8784ab3ea331cade19237b", version = "0.2.0", default-features = false }
|
||||
# The gateway framework itself, consumed for its extension traits by the ring-2
|
||||
# adapter. `default-features = false` keeps the `server` feature, and with it the
|
||||
# server crate, out of the build.
|
||||
rustfs-gateway = { git = "https://github.com/rustfs/gateway.git", rev = "d6840d54187382d31526317e61369734c759e2cd", version = "0.66.0", default-features = false }
|
||||
rustfs-gateway = { git = "https://github.com/rustfs/gateway.git", rev = "2a6b8d1f7663db978d8784ab3ea331cade19237b", version = "0.63.0", default-features = false }
|
||||
# Test-only: builds the `EffectiveHost` a host-resolution test queries with.
|
||||
rustfs-gateway-http = { git = "https://github.com/rustfs/gateway.git", rev = "d6840d54187382d31526317e61369734c759e2cd", default-features = false }
|
||||
rustfs-gateway-core = { git = "https://github.com/rustfs/gateway.git", rev = "d6840d54187382d31526317e61369734c759e2cd", version = "0.63.0", default-features = false }
|
||||
rustfs-gateway-dialect-rustfs-admin = { git = "https://github.com/rustfs/gateway.git", rev = "d6840d54187382d31526317e61369734c759e2cd", version = "0.13.0", default-features = false }
|
||||
rustfs-gateway-http = { git = "https://github.com/rustfs/gateway.git", rev = "2a6b8d1f7663db978d8784ab3ea331cade19237b", default-features = false }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-channel = "2.5.0"
|
||||
@@ -168,13 +166,13 @@ async_zip = { default-features = false, version = "0.0.19" }
|
||||
mysql_async = { default-features = false, version = "0.37.1" }
|
||||
async-compression = { version = "0.4.52" }
|
||||
async-trait = "0.1.92"
|
||||
async-nats = { version = "0.53.0", default-features = false }
|
||||
async-nats = { version = "0.52.0", default-features = false }
|
||||
axum = "0.8.9"
|
||||
futures = "0.3.34"
|
||||
futures-core = "0.3.34"
|
||||
futures-lite = "2.6.1"
|
||||
futures-util = "0.3.34"
|
||||
pollster = "1.0.2"
|
||||
pollster = "1.0.1"
|
||||
pulsar = { default-features = false, version = "=6.9.0" }
|
||||
lapin = { default-features = false, version = "4.12.3" }
|
||||
hyper = { version = "1.12.0" }
|
||||
@@ -188,7 +186,6 @@ memchr = "2.8.3"
|
||||
minlz = "1.2.3"
|
||||
reqwest = "0.13.5"
|
||||
rustfs-kafka-async = { version = "1.4.1" }
|
||||
slab = { version = "0.4.12" }
|
||||
socket2 = { version = "0.6.5" }
|
||||
tokio = { version = "1.54.1" }
|
||||
tokio-rustls = { default-features = false, version = "0.26.6" }
|
||||
@@ -209,8 +206,8 @@ quinn = { version = "0.11.12", default-features = false, features = [
|
||||
] }
|
||||
|
||||
# Serialization and Data Formats
|
||||
apache-avro = { version = "0.23.0", features = ["snappy", "zstandard"] }
|
||||
bytes = { version = "1.12.2" }
|
||||
apache-avro = { version = "0.22.0", features = ["snappy", "zstandard"] }
|
||||
bytes = { version = "1.12.1" }
|
||||
csv-core = "0.1.13"
|
||||
bytesize = "2.7.0"
|
||||
byteorder = "1.5.0"
|
||||
@@ -242,7 +239,7 @@ jsonwebtoken = { version = "12.0.0" }
|
||||
openidconnect = { default-features = false, version = "4.1.0" }
|
||||
pbkdf2 = "0.13.0"
|
||||
p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
|
||||
rsa = { version = "0.10.0", features = ["pkcs1v15"] }
|
||||
rsa = { version = "=0.10.0-rc.18" }
|
||||
rustls = { default-features = false, version = "0.23.46" }
|
||||
rustls-native-certs = "0.8"
|
||||
rustls-pki-types = "1.16.0"
|
||||
@@ -250,13 +247,13 @@ x509-parser = "0.18.1"
|
||||
sha1 = "0.11.0"
|
||||
sha2 = "0.11.1"
|
||||
subtle = "2.6"
|
||||
zeroize = { version = "1.9.2" }
|
||||
zeroize = { version = "1.9.1" }
|
||||
proptest = "1"
|
||||
|
||||
# Time and Date
|
||||
chrono = { version = "0.4.45" }
|
||||
humantime = "2.4.0"
|
||||
jiff = { version = "1.0.1" }
|
||||
jiff = { version = "0.2.43" }
|
||||
time = { version = "0.3.55" }
|
||||
|
||||
# Database
|
||||
@@ -273,18 +270,18 @@ tar-codec = "0.0.16"
|
||||
tar-framing = "0.0.16"
|
||||
atoi = "3.1.1"
|
||||
atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.13.3" }
|
||||
aws-config = { version = "1.13.2" }
|
||||
aws-credential-types = { version = "1.3.1" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.133.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.160.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.127.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.131.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.159.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.126.0" }
|
||||
aws-smithy-async = { version = "1.3.1" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.5.3" }
|
||||
aws-smithy-runtime-api = { version = "1.22.0" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.5.2" }
|
||||
aws-smithy-runtime-api = { version = "1.21.0" }
|
||||
aws-smithy-types = { version = "1.9.4" }
|
||||
base64-simd = "0.8.0"
|
||||
brotli = "9.1.3"
|
||||
clap = { version = "4.6.8" }
|
||||
brotli = "9.1.1"
|
||||
clap = { version = "4.6.7" }
|
||||
const-str = { version = "1.1.0" }
|
||||
convert_case = "0.12.0"
|
||||
criterion = { version = "0.8" }
|
||||
@@ -292,7 +289,7 @@ crossbeam-queue = "0.3.14"
|
||||
crossbeam-channel = "0.5.17"
|
||||
crossbeam-deque = "0.8.8"
|
||||
crossbeam-utils = "0.8.23"
|
||||
datafusion = { default-features = false, version = "55.3.0" }
|
||||
datafusion = { default-features = false, version = "55.2.0" }
|
||||
derive_builder = "0.20.2"
|
||||
ed25519-dalek = "3.0.0"
|
||||
enumset = "1.1.15"
|
||||
@@ -311,14 +308,14 @@ hostname = "0.4.2"
|
||||
ipnetwork = { version = "0.21.1" }
|
||||
lazy_static = "1.5.1"
|
||||
libc = "0.2.191"
|
||||
cuobject = { version = "0.1.0", default-features = false }
|
||||
cuobject = "0.1.0"
|
||||
libsystemd = "0.7.2"
|
||||
local-ip-address = "0.6.13"
|
||||
log = "0.4"
|
||||
memmap2 = "0.9.12"
|
||||
nixl-sys = { git = "https://github.com/houseme/nixl", rev = "cc636c83f13d692e85b67e55aed092413d16fb41" }
|
||||
lz4 = "1.28.1"
|
||||
lz4_flex = { version = "0.14.2", default-features = false, features = ["std", "frame"] }
|
||||
lz4_flex = { version = "=0.14.0", default-features = false, features = ["std", "frame"] }
|
||||
matchit = "0.9.2"
|
||||
md-5 = "0.11.0"
|
||||
mime_guess = "2.0.5"
|
||||
@@ -337,14 +334,14 @@ percent-encoding = "2.3.2"
|
||||
qrcode-rs = { version = "2.2.0", default-features = false, features = ["std", "svg"] }
|
||||
pin-project-lite = "0.2.17"
|
||||
pretty_assertions = "1.4.1"
|
||||
rand = { version = "0.10.4" }
|
||||
rand = { version = "0.10.3" }
|
||||
ratelimit = "2.0.0"
|
||||
rayon = "1.12.0"
|
||||
rustfs-erasure-codec = { version = "9.1.0" }
|
||||
reed-solomon-simd = "3.1.0"
|
||||
regex = { version = "1.13.2" }
|
||||
regex = { version = "1.13.1" }
|
||||
rumqttc = { package = "rumqttc-next", version = "0.34.0" }
|
||||
redis = { version = "1.8.2" }
|
||||
redis = { version = "1.8.1" }
|
||||
rustify = { version = "0.7", default-features = false }
|
||||
rustix = { version = "1.1.6" }
|
||||
rust-embed = { version = "8.14.0" }
|
||||
@@ -364,7 +361,7 @@ snap = "1.1.2"
|
||||
starshard = { version = "2.4.1" }
|
||||
strum = { version = "0.28.0" }
|
||||
sync_wrapper = "1.0.2"
|
||||
sysinfo = "0.40.4"
|
||||
sysinfo = "0.40.3"
|
||||
temp-env = "0.3.6"
|
||||
tempfile = "3.27.0"
|
||||
test-case = "3.4.1"
|
||||
@@ -378,15 +375,15 @@ tracing-subscriber = { version = "0.3.23" }
|
||||
transform-stream = "0.3.1"
|
||||
url = "2.5.8"
|
||||
urlencoding = "2.1.3"
|
||||
uuid = { version = "1.29.3" }
|
||||
uuid = { version = "1.29.2" }
|
||||
vaultrs = { version = "0.8.0" }
|
||||
tar = "0.4.46"
|
||||
walkdir = "2.5.0"
|
||||
winapi-util = "0.1.11"
|
||||
windows = { version = "0.62.2" }
|
||||
windows-sys = "0.61.2"
|
||||
xxhash-rust = { version = "0.8.21" }
|
||||
zip = "9.7.0"
|
||||
xxhash-rust = { version = "0.8.19" }
|
||||
zip = "9.5.1"
|
||||
zstd = "0.14.4"
|
||||
|
||||
# Observability and Metrics
|
||||
@@ -405,9 +402,9 @@ pyroscope = { version = "2.1.5" }
|
||||
# FTP and SFTP
|
||||
libunftp = { version = "0.25.0" }
|
||||
unftp-core = "0.1.1"
|
||||
suppaftp = { version = "13.0.0" }
|
||||
suppaftp = { version = "12.2.1" }
|
||||
rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
russh = { version = "0.65.2" }
|
||||
russh = { version = "0.65.1" }
|
||||
russh-sftp = "3.0.1"
|
||||
|
||||
# WebDAV
|
||||
@@ -416,7 +413,7 @@ dav-server = "0.12.0"
|
||||
# Performance Analysis and Memory Profiling
|
||||
rustfs-mimalloc = { version = "0.7.4" }
|
||||
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
|
||||
hotpath = { version = "0.32.1", default-features = false }
|
||||
hotpath = { version = "0.31.2", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.49" }
|
||||
|
||||
|
||||
@@ -302,8 +302,6 @@ Drive timeout profile preset:
|
||||
|
||||
RustFS production guidance remains direct-attached local POSIX filesystems. Network-mounted filesystems (for example `nfs`, `cifs`, `smb2`, and `fuse.*`) are treated as unsupported by this startup guard. On Windows, endpoints on network drives (UNC shares such as `\\server\share` and mapped drive letters) are treated the same way, even though Windows reports the server's filesystem (usually `NTFS`) for them.
|
||||
|
||||
A local POSIX filesystem bind-mounted into a gVisor sandbox with directfs is a supported storage path. gVisor presents that bind mount as `9p`/`V9FS`, so the host filesystem type is not visible; the startup guard recognizes mountinfo options `trans=fd` and `directfs` and does not treat the mount as unsupported. Genuine 9p, virtio, and NFS shares remain unsupported.
|
||||
|
||||
## 📄 License
|
||||
|
||||
This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.
|
||||
|
||||
@@ -152,33 +152,6 @@ pub const DEFAULT_SHARD_INTEGRITY_WRITE: bool = false;
|
||||
pub const ENV_SHARD_INTEGRITY_FLEET_CONFIRMED: &str = "RUSTFS_SHARD_INTEGRITY_FLEET_CONFIRMED";
|
||||
pub const DEFAULT_SHARD_INTEGRITY_FLEET_CONFIRMED: bool = false;
|
||||
|
||||
/// Request parsing switch for the per-object erasure-code block-size hint.
|
||||
/// The default keeps the feature inert during rolling upgrades.
|
||||
pub const ENV_EC_BLOCK_SIZE_HINT_ENABLE: &str = "RUSTFS_EC_BLOCK_SIZE_HINT_ENABLE";
|
||||
pub const DEFAULT_EC_BLOCK_SIZE_HINT_ENABLE: bool = false;
|
||||
|
||||
/// Operator confirmation that every writer and reader in the fleet understands
|
||||
/// the persisted erasure block-size geometry.
|
||||
pub const ENV_EC_BLOCK_SIZE_HINT_FLEET_CONFIRMED: &str = "RUSTFS_EC_BLOCK_SIZE_HINT_FLEET_CONFIRMED";
|
||||
pub const DEFAULT_EC_BLOCK_SIZE_HINT_FLEET_CONFIRMED: bool = false;
|
||||
|
||||
/// Write new EC shards with a bounded bitrot frame size so Range GETs do not
|
||||
/// inherit the full persisted layout block as their physical read quantum.
|
||||
/// The marker is only emitted after the fleet confirmation gate is enabled;
|
||||
/// readers without this feature continue to consume legacy objects unchanged.
|
||||
pub const ENV_EC_READ_QUANTUM_ENABLE: &str = "RUSTFS_EC_READ_QUANTUM_ENABLE";
|
||||
pub const DEFAULT_EC_READ_QUANTUM_ENABLE: bool = false;
|
||||
|
||||
/// Operator confirmation that every reader and writer understands the
|
||||
/// persisted EC read-quantum marker and its per-frame bitrot layout.
|
||||
pub const ENV_EC_READ_QUANTUM_FLEET_CONFIRMED: &str = "RUSTFS_EC_READ_QUANTUM_FLEET_CONFIRMED";
|
||||
pub const DEFAULT_EC_READ_QUANTUM_FLEET_CONFIRMED: bool = false;
|
||||
|
||||
const _: () = assert!(!DEFAULT_EC_BLOCK_SIZE_HINT_ENABLE);
|
||||
const _: () = assert!(!DEFAULT_EC_BLOCK_SIZE_HINT_FLEET_CONFIRMED);
|
||||
const _: () = assert!(!DEFAULT_EC_READ_QUANTUM_ENABLE);
|
||||
const _: () = assert!(!DEFAULT_EC_READ_QUANTUM_FLEET_CONFIRMED);
|
||||
|
||||
const _: () = assert!(!DEFAULT_SHARD_INTEGRITY_WRITE);
|
||||
const _: () = assert!(!DEFAULT_SHARD_INTEGRITY_FLEET_CONFIRMED);
|
||||
|
||||
|
||||
@@ -48,7 +48,7 @@ pub const ENV_RUNTIME_DIAL9_S3_PREFIX: &str = "RUSTFS_RUNTIME_DIAL9_S3_PREFIX";
|
||||
pub const DEFAULT_WORKER_THREADS: usize = 16;
|
||||
pub const DEFAULT_MAX_BLOCKING_THREADS: usize = 1024;
|
||||
pub const DEFAULT_THREAD_PRINT_ENABLED: bool = false;
|
||||
pub const DEFAULT_THREAD_STACK_SIZE: usize = 2 * MI_B; // 2 MiB
|
||||
pub const DEFAULT_THREAD_STACK_SIZE: usize = MI_B; // 1 MiB
|
||||
pub const DEFAULT_THREAD_KEEP_ALIVE: u64 = 60; // seconds
|
||||
pub const DEFAULT_GLOBAL_QUEUE_INTERVAL: u32 = 31;
|
||||
pub const DEFAULT_THREAD_NAME: &str = "rustfs-worker";
|
||||
|
||||
@@ -36,7 +36,7 @@ jsonwebtoken = { workspace = true, features = ["aws_lc_rs"] }
|
||||
base64-simd = { workspace = true }
|
||||
pbkdf2 = { workspace = true, optional = true }
|
||||
rand = { workspace = true, features = ["serde"] }
|
||||
rsa = { workspace = true }
|
||||
rsa = { workspace = true, features = ["sha2"] }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
sha2 = { workspace = true, optional = true }
|
||||
thiserror.workspace = true
|
||||
|
||||
@@ -13,9 +13,9 @@
|
||||
// limitations under the License.
|
||||
|
||||
use rsa::{
|
||||
Pkcs1v15Encrypt, PrivateKey, PublicKey,
|
||||
Pkcs1v15Encrypt, RsaPrivateKey, RsaPublicKey,
|
||||
pkcs8::{DecodePrivateKey, DecodePublicKey},
|
||||
pss::{Signature, SigningKey, VerifyingKey},
|
||||
pss::{BlindedSigningKey, Signature, VerifyingKey},
|
||||
sha2::Sha256,
|
||||
signature::{RandomizedSigner, Verifier},
|
||||
traits::PublicKeyParts,
|
||||
@@ -37,7 +37,7 @@ pub struct Token {
|
||||
pub fn gencode(token: &Token, key: &str) -> Result<String> {
|
||||
let data = serde_json::to_vec(token)?;
|
||||
let mut rng = rand::rng();
|
||||
let public_key = PublicKey::from_public_key_pem(key).map_err(Error::other)?;
|
||||
let public_key = RsaPublicKey::from_public_key_pem(key).map_err(Error::other)?;
|
||||
let encrypted_data = public_key.encrypt(&mut rng, Pkcs1v15Encrypt, &data).map_err(Error::other)?;
|
||||
Ok(base64_simd::URL_SAFE_NO_PAD.encode_to_string(&encrypted_data))
|
||||
}
|
||||
@@ -51,7 +51,7 @@ pub fn parse(token: &str, key: &str) -> Result<Token> {
|
||||
let encrypted_data = base64_simd::URL_SAFE_NO_PAD
|
||||
.decode_to_vec(token.as_bytes())
|
||||
.map_err(Error::other)?;
|
||||
let private_key = PrivateKey::from_pkcs8_pem(key).map_err(Error::other)?;
|
||||
let private_key = RsaPrivateKey::from_pkcs8_pem(key).map_err(Error::other)?;
|
||||
let decrypted_data = private_key.decrypt(Pkcs1v15Encrypt, &encrypted_data).map_err(Error::other)?;
|
||||
serde_json::from_slice(&decrypted_data).map_err(Error::other)
|
||||
}
|
||||
@@ -63,8 +63,8 @@ pub fn parse(token: &str, key: &str) -> Result<Token> {
|
||||
pub fn sign_license_token(token: &Token, private_key_pem: &str) -> Result<String> {
|
||||
let payload = serde_json::to_vec(token)?;
|
||||
let mut rng = rand::rng();
|
||||
let private_key = PrivateKey::from_pkcs8_pem(private_key_pem).map_err(Error::other)?;
|
||||
let signing_key = SigningKey::<Sha256>::new(private_key);
|
||||
let private_key = RsaPrivateKey::from_pkcs8_pem(private_key_pem).map_err(Error::other)?;
|
||||
let signing_key = BlindedSigningKey::<Sha256>::new(private_key);
|
||||
let signature: Signature = signing_key.try_sign_with_rng(&mut rng, &payload).map_err(Error::other)?;
|
||||
let signature: Box<[u8]> = signature.into();
|
||||
|
||||
@@ -80,7 +80,7 @@ pub fn parse_signed_license_token(token: &str, public_key_pem: &str) -> Result<T
|
||||
let signed_payload = base64_simd::URL_SAFE_NO_PAD
|
||||
.decode_to_vec(token.as_bytes())
|
||||
.map_err(Error::other)?;
|
||||
let public_key = PublicKey::from_public_key_pem(public_key_pem).map_err(Error::other)?;
|
||||
let public_key = RsaPublicKey::from_public_key_pem(public_key_pem).map_err(Error::other)?;
|
||||
let signature_len = public_key.size();
|
||||
|
||||
if signed_payload.len() <= signature_len {
|
||||
@@ -103,7 +103,7 @@ pub fn parse_license_with_public_key(license: &str, public_key: &str) -> Result<
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rsa::{
|
||||
PrivateKey,
|
||||
RsaPrivateKey,
|
||||
pkcs8::{EncodePrivateKey, EncodePublicKey, LineEnding},
|
||||
};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
@@ -112,8 +112,8 @@ mod tests {
|
||||
fn test_sign_license_token_and_parse_signed_license_token() {
|
||||
let mut rng = rand::rng();
|
||||
let bits = 2048;
|
||||
let private_key = PrivateKey::new(&mut rng, bits).expect("Failed to generate private key");
|
||||
let public_key = PublicKey::from(&private_key);
|
||||
let private_key = RsaPrivateKey::new(&mut rng, bits).expect("Failed to generate private key");
|
||||
let public_key = RsaPublicKey::from(&private_key);
|
||||
|
||||
let private_key_pem = private_key
|
||||
.to_pkcs8_pem(LineEnding::LF)
|
||||
@@ -144,8 +144,8 @@ mod tests {
|
||||
fn test_legacy_gencode_and_parse_roundtrip() {
|
||||
let mut rng = rand::rng();
|
||||
let bits = 2048;
|
||||
let private_key = PrivateKey::new(&mut rng, bits).expect("Failed to generate private key");
|
||||
let public_key = PublicKey::from(&private_key);
|
||||
let private_key = RsaPrivateKey::new(&mut rng, bits).expect("Failed to generate private key");
|
||||
let public_key = RsaPublicKey::from(&private_key);
|
||||
|
||||
let private_key_pem = private_key
|
||||
.to_pkcs8_pem(LineEnding::LF)
|
||||
@@ -173,8 +173,8 @@ mod tests {
|
||||
#[test]
|
||||
fn test_parse_signed_license_token_rejects_tampered_payload() {
|
||||
let mut rng = rand::rng();
|
||||
let private_key = PrivateKey::new(&mut rng, 2048).expect("Failed to generate private key");
|
||||
let public_key = PublicKey::from(&private_key);
|
||||
let private_key = RsaPrivateKey::new(&mut rng, 2048).expect("Failed to generate private key");
|
||||
let public_key = RsaPublicKey::from(&private_key);
|
||||
let private_key_pem = private_key
|
||||
.to_pkcs8_pem(LineEnding::LF)
|
||||
.expect("failed to encode private key pem");
|
||||
@@ -206,8 +206,8 @@ mod tests {
|
||||
#[test]
|
||||
fn test_parse_signed_license_token_rejects_invalid_token() {
|
||||
let mut rng = rand::rng();
|
||||
let private_key = PrivateKey::new(&mut rng, 2048).expect("Failed to generate private key");
|
||||
let public_key = PublicKey::from(&private_key);
|
||||
let private_key = RsaPrivateKey::new(&mut rng, 2048).expect("Failed to generate private key");
|
||||
let public_key = RsaPublicKey::from(&private_key);
|
||||
let public_key_pem = public_key
|
||||
.to_public_key_pem(LineEnding::LF)
|
||||
.expect("failed to encode public key pem");
|
||||
|
||||
@@ -66,12 +66,6 @@ pub const LEGACY_DATA_USAGE_OBJECT_NAME: &str = ".usage.json";
|
||||
/// stale tier names from growing persisted per-tier maps without bound.
|
||||
pub const UNKNOWN_TIER: &str = "UNKNOWN_TIER";
|
||||
|
||||
/// Whether a remote tier name can be represented in scanner accounting.
|
||||
/// Local storage classes are checked separately by the configuration owner.
|
||||
pub fn is_scannable_tier_name(name: &str) -> bool {
|
||||
!name.is_empty() && name.len() <= 256 && !name.bytes().any(|byte| byte.is_ascii_control()) && name != UNKNOWN_TIER
|
||||
}
|
||||
|
||||
/// Returns true when `existing_last_update` is ahead of `now` by more than
|
||||
/// [`USAGE_LAST_UPDATE_FUTURE_TOLERANCE`], i.e. the persisted timestamp cannot be
|
||||
/// trusted for staleness comparisons and a fresh snapshot save must be allowed.
|
||||
|
||||
@@ -98,6 +98,7 @@ rustfs-madmin.workspace = true
|
||||
rustfs-filemeta.workspace = true
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
|
||||
aws-sdk-sts = { workspace = true, default-features = false, features = ["default-https-client", "rt-tokio"] }
|
||||
aws-smithy-http-client = { workspace = true, default-features = false, features = ["rustls-aws-lc"] }
|
||||
aws-smithy-types.workspace = true
|
||||
async-trait = { workspace = true }
|
||||
@@ -123,12 +124,11 @@ urlencoding.workspace = true
|
||||
walkdir.workspace = true
|
||||
base64-simd = { workspace = true }
|
||||
rand = { workspace = true, features = ["serde"] }
|
||||
jiff = { workspace = true, features = ["serde"] }
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
hex-simd = { workspace = true }
|
||||
md-5 = { workspace = true }
|
||||
prost.workspace = true
|
||||
sha2 = { workspace = true }
|
||||
rustfs-s3-types = { workspace = true }
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
zstd.workspace = true
|
||||
time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }
|
||||
@@ -141,7 +141,6 @@ clap = { workspace = true, features = ["derive", "env"] }
|
||||
[dev-dependencies]
|
||||
async-compression = { workspace = true, features = ["tokio", "bzip2", "lz4", "xz"] }
|
||||
aws-config = { workspace = true }
|
||||
aws-sdk-sts = { workspace = true, default-features = false, features = ["default-https-client", "rt-tokio"] }
|
||||
opentelemetry-proto = { workspace = true }
|
||||
russh = { workspace = true, features = ["serde"] }
|
||||
russh-sftp = { workspace = true }
|
||||
|
||||
@@ -29,7 +29,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
|
||||
| **distributed 4×4** | [`src/distributed/`](src/distributed) | Nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
|
||||
| **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
|
||||
| **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
|
||||
| **dual-stack differential** | `dual_stack_differential` | Two single-node servers answer one SDK script for all 73 S3 operations and every differing `(op, field, a, b)` is reported. Both sides run the binary under test on its only (gateway) stack unless `RUSTFS_E2E_REFERENCE_BINARY` names a released `rustfs` binary, which then runs side A. A missing, non-file or non-executable path fails the run; it never skips or falls back to the same-binary mode. `RUSTFS_E2E_STACK_B_REGION` starts side B in another region to inject a difference. The report in `<target>/dual-stack/<run>.json` records `reference_side` and each side's `--version` output |
|
||||
| **dual-stack differential** | `dual_stack_differential` | Two single-node servers answer one SDK script for all 73 S3 operations and every differing `(op, field, a, b)` is reported. Side A is the binary under test on `RUSTFS_S3_STACK=legacy`; side B runs `RUSTFS_E2E_STACK_B`. Set `RUSTFS_E2E_REFERENCE_BINARY` to a released `rustfs` binary to make side A that binary instead (started without `RUSTFS_S3_STACK`, which older releases do not know). A missing, non-file or non-executable path fails the run; it never skips or falls back to the same-binary mode. The report in `<target>/dual-stack/<run>.json` records `reference_side` and each side's `--version` output |
|
||||
|
||||
The external-tool `storage_metric_ownership_test` validates the OTLP/Collector/Prometheus path, including a rolling upgrade and node failures. See the [storage metrics guide](../../docs/operations/storage-metrics.md) for its required binaries and focused command.
|
||||
|
||||
|
||||
@@ -21,8 +21,8 @@ use crate::common::{
|
||||
};
|
||||
use http::header::{AUTHORIZATION, HOST};
|
||||
use reqwest::StatusCode;
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::{constants::UNSIGNED_PAYLOAD, sign_v4};
|
||||
use s3s::Body;
|
||||
use serde_json::Value;
|
||||
use std::{error::Error, time::Duration};
|
||||
|
||||
|
||||
@@ -146,11 +146,7 @@ async fn create_service_account_for(
|
||||
Some(serde_json::json!({ "targetUser": parent }).to_string()),
|
||||
)
|
||||
.await?;
|
||||
service_account_keys(&response)
|
||||
}
|
||||
|
||||
fn service_account_keys(response: &str) -> Result<(String, String), Box<dyn Error + Send + Sync>> {
|
||||
let response: serde_json::Value = serde_json::from_str(response)?;
|
||||
let response: serde_json::Value = serde_json::from_str(&response)?;
|
||||
let access_key = response["credentials"]["accessKey"]
|
||||
.as_str()
|
||||
.ok_or("service account response should contain credentials.accessKey")?
|
||||
@@ -348,134 +344,6 @@ async fn test_update_service_account_enforces_owner_and_parent_scope() -> TestRe
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_delegated_service_account_creation_through_console_sessions() -> TestResult {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_without_cleanup(vec![]).await?;
|
||||
let bucket = "delegated-backups";
|
||||
let target = "backup-user";
|
||||
let operator = "backup-operator";
|
||||
let operator_secret = "operatorSecret123";
|
||||
let wildcard_admin = "backup-admin";
|
||||
let wildcard_secret = "adminSecret123";
|
||||
let s3 = env.create_s3_client();
|
||||
s3.create_bucket().bucket(bucket).send().await?;
|
||||
for (name, secret) in [
|
||||
(target, "backupSecret123"),
|
||||
(operator, operator_secret),
|
||||
(wildcard_admin, wildcard_secret),
|
||||
("other-user", "otherUserSecret123"),
|
||||
] {
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::PUT,
|
||||
&format!("/rustfs/admin/v3/add-user?accessKey={name}"),
|
||||
Some(serde_json::json!({"secretKey":secret,"status":"enabled"}).to_string()),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
let delegation = serde_json::json!({
|
||||
"Version":"2012-10-17",
|
||||
"Statement":[
|
||||
{"Effect":"Allow","Action":"admin:CreateServiceAccount"},
|
||||
{"Effect":"Allow","Action":"sts:AssumeRole","Resource":"arn:aws:s3:::*"},
|
||||
{"Effect":"Allow","Action":"admin:CreateServiceAccountForUser","Condition":{"StringEquals":{"svc:TargetUser":target}}}
|
||||
]
|
||||
});
|
||||
let wildcard = serde_json::json!({
|
||||
"Version":"2012-10-17",
|
||||
"Statement":[
|
||||
{"Effect":"Allow","Action":"admin:*"},
|
||||
{"Effect":"Allow","Action":"sts:AssumeRole","Resource":"arn:aws:s3:::*"}
|
||||
]
|
||||
});
|
||||
for (user, name, policy) in [
|
||||
(wildcard_admin, "backup-admin-wildcard", wildcard.to_string()),
|
||||
(target, "backup-writer", bucket_rw_policy(bucket)),
|
||||
(operator, "backup-key-manager", delegation.to_string()),
|
||||
] {
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::PUT,
|
||||
&format!("/rustfs/admin/v3/add-canned-policy?name={name}"),
|
||||
Some(policy),
|
||||
)
|
||||
.await?;
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::POST,
|
||||
"/rustfs/admin/v3/idp/builtin/policy/attach",
|
||||
Some(serde_json::json!({"policies":[name],"user":user}).to_string()),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
for (name, secret) in [
|
||||
(operator, operator_secret),
|
||||
(wildcard_admin, wildcard_secret),
|
||||
(env.access_key.as_str(), env.secret_key.as_str()),
|
||||
] {
|
||||
let session = build_test_sts_client(&env.url, name, secret, None, "delegated-create")
|
||||
.assume_role()
|
||||
.role_arn("arn:aws:iam::123456789012:role/console")
|
||||
.role_session_name("console")
|
||||
.send()
|
||||
.await?;
|
||||
let caller = session.credentials().ok_or("missing console session")?;
|
||||
let create = |user: &str| {
|
||||
admin_request_with_session_token(
|
||||
&env.url,
|
||||
http::Method::PUT,
|
||||
"/rustfs/admin/v3/add-service-account",
|
||||
Some(serde_json::json!({"targetUser":user}).to_string()),
|
||||
caller.access_key_id(),
|
||||
caller.secret_access_key(),
|
||||
Some(caller.session_token()),
|
||||
)
|
||||
};
|
||||
let (status, response) = create(target).await?;
|
||||
assert_eq!(status, StatusCode::OK, "create for target as {name}");
|
||||
let (key, secret) = service_account_keys(&response)?;
|
||||
let client = build_s3_client(&env.url, &key, &secret);
|
||||
wait_for_s3_put(&client, bucket, "backup", Duration::from_secs(10)).await?;
|
||||
let denied = client
|
||||
.create_bucket()
|
||||
.bucket("outside-backup-policy")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("target cannot create unrelated buckets");
|
||||
assert_eq!(denied.as_service_error().and_then(|e| e.code()), Some("AccessDenied"));
|
||||
if name != env.access_key {
|
||||
let (status, _) = create(&env.access_key).await?;
|
||||
assert_eq!(status, StatusCode::FORBIDDEN, "delegated targets exclude root");
|
||||
}
|
||||
if name == operator {
|
||||
let (status, _) = create("other-user").await?;
|
||||
assert_eq!(status, StatusCode::FORBIDDEN, "outside delegated target");
|
||||
}
|
||||
}
|
||||
let restricted = build_test_sts_client(&env.url, &env.access_key, &env.secret_key, None, "restricted-root")
|
||||
.assume_role()
|
||||
.role_arn("arn:aws:iam::123456789012:role/console")
|
||||
.role_session_name("restricted-console")
|
||||
.policy(r#"{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"admin:CreateServiceAccount"}]}"#)
|
||||
.send()
|
||||
.await?;
|
||||
let caller = restricted.credentials().ok_or("missing restricted root session")?;
|
||||
let (status, _) = admin_request_with_session_token(
|
||||
&env.url,
|
||||
http::Method::PUT,
|
||||
"/rustfs/admin/v3/add-service-account",
|
||||
Some(serde_json::json!({"targetUser":target}).to_string()),
|
||||
caller.access_key_id(),
|
||||
caller.secret_access_key(),
|
||||
Some(caller.session_token()),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(status, StatusCode::FORBIDDEN, "restricted root session requires delegated permission");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Full user -> policy -> service-account lifecycle, proving each management
|
||||
/// call takes effect on the data plane, not just that the endpoint answers 200.
|
||||
#[tokio::test]
|
||||
|
||||
@@ -37,9 +37,9 @@ mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use hmac::{Hmac, KeyInit as _, Mac};
|
||||
use http::header::HOST;
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use sha1::Sha1;
|
||||
use std::error::Error;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
@@ -15,9 +15,9 @@
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use http::header::HOST;
|
||||
use reqwest::StatusCode;
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use serde::Deserialize;
|
||||
use std::error::Error;
|
||||
|
||||
|
||||
@@ -17,9 +17,9 @@ use aws_sdk_s3::primitives::ByteStream;
|
||||
use http::header::HOST;
|
||||
use reqwest::StatusCode;
|
||||
use rustfs_madmin::{ITEM_OFFLINE, InfoMessage, StorageInfo};
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use serde::Deserialize;
|
||||
use std::error::Error;
|
||||
use tokio::time::{Duration, sleep, timeout};
|
||||
|
||||
@@ -21,9 +21,9 @@ mod tests {
|
||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use reqwest::StatusCode;
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::{pre_sign_v4, sign_v4};
|
||||
use s3s::Body;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::error::Error;
|
||||
use std::io::{Cursor, Write};
|
||||
|
||||
@@ -37,9 +37,9 @@
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, local_http_client};
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::error::Error;
|
||||
@@ -170,15 +170,6 @@ impl DiskFaultHarness {
|
||||
/// Start (or restart after [`Self::kill_server`]) the RustFS server with
|
||||
/// all volume directories and the same address.
|
||||
pub async fn start_server(&mut self) -> ChaosResult<()> {
|
||||
self.start_server_with_readiness(false).await
|
||||
}
|
||||
|
||||
/// Start a process once its node read-readiness probe passes.
|
||||
pub async fn start_server_at_read_quorum(&mut self) -> ChaosResult<()> {
|
||||
self.start_server_with_readiness(true).await
|
||||
}
|
||||
|
||||
async fn start_server_with_readiness(&mut self, read_quorum_only: bool) -> ChaosResult<()> {
|
||||
// The common helper always appends `env.temp_dir` as the final storage
|
||||
// path: point it at the last disk for startup and pass the remaining
|
||||
// disks explicitly, then restore it to the root so Drop cleans up
|
||||
@@ -190,13 +181,7 @@ impl DiskFaultHarness {
|
||||
let arg_refs: Vec<&str> = disk_args.iter().map(String::as_str).collect();
|
||||
let env_refs: Vec<(&str, &str)> = self.extra_env.iter().map(|(k, v)| (k.as_str(), v.as_str())).collect();
|
||||
|
||||
let result = if read_quorum_only {
|
||||
self.env
|
||||
.start_rustfs_server_with_env_at_read_quorum(arg_refs, &env_refs)
|
||||
.await
|
||||
} else {
|
||||
self.env.start_rustfs_server_with_env(arg_refs, &env_refs).await
|
||||
};
|
||||
let result = self.env.start_rustfs_server_with_env(arg_refs, &env_refs).await;
|
||||
self.env.temp_dir = self.root.to_string_lossy().to_string();
|
||||
result
|
||||
}
|
||||
|
||||
@@ -29,9 +29,9 @@ use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use reqwest::Client as HttpClient;
|
||||
use reqwest::StatusCode;
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use serde_json;
|
||||
use std::fs as stdfs;
|
||||
use std::io::{self, ErrorKind};
|
||||
@@ -722,7 +722,7 @@ impl RustFSTestEnvironment {
|
||||
cleanup_existing: bool,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let binary_path = rustfs_binary_path();
|
||||
self.start_rustfs_server_inner_with_binary(&binary_path, extra_args, extra_env, cleanup_existing, false)
|
||||
self.start_rustfs_server_inner_with_binary(&binary_path, extra_args, extra_env, cleanup_existing)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -732,7 +732,6 @@ impl RustFSTestEnvironment {
|
||||
extra_args: Vec<&str>,
|
||||
extra_env: &[(&str, &str)],
|
||||
cleanup_existing: bool,
|
||||
read_quorum_only: bool,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
if cleanup_existing {
|
||||
self.cleanup_existing_processes().await?;
|
||||
@@ -757,7 +756,7 @@ impl RustFSTestEnvironment {
|
||||
self.process = Some(process);
|
||||
|
||||
// Wait for server to be ready
|
||||
self.wait_for_server_ready_with_probe(read_quorum_only).await?;
|
||||
self.wait_for_server_ready().await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -771,7 +770,7 @@ impl RustFSTestEnvironment {
|
||||
extra_args: Vec<&str>,
|
||||
extra_env: &[(&str, &str)],
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, true, false)
|
||||
self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, true)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -783,7 +782,7 @@ impl RustFSTestEnvironment {
|
||||
extra_args: Vec<&str>,
|
||||
extra_env: &[(&str, &str)],
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, false, false)
|
||||
self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, false)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -801,18 +800,6 @@ impl RustFSTestEnvironment {
|
||||
self.start_rustfs_server_inner(extra_args, extra_env, true).await
|
||||
}
|
||||
|
||||
/// Start a degraded read-serving process using the node readiness probe.
|
||||
/// ListBuckets retains its namespace write-quorum requirement.
|
||||
pub async fn start_rustfs_server_with_env_at_read_quorum(
|
||||
&mut self,
|
||||
extra_args: Vec<&str>,
|
||||
extra_env: &[(&str, &str)],
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let binary_path = rustfs_binary_path();
|
||||
self.start_rustfs_server_inner_with_binary(&binary_path, extra_args, extra_env, true, true)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Start RustFS server without cleaning up other running RustFS processes.
|
||||
///
|
||||
/// This is useful for tests that need multiple independent RustFS instances
|
||||
@@ -841,16 +828,8 @@ impl RustFSTestEnvironment {
|
||||
/// Fails immediately if the spawned server process has already exited
|
||||
/// (e.g. a port bind failure) instead of polling out the full timeout.
|
||||
pub async fn wait_for_server_ready(&mut self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
self.wait_for_server_ready_with_probe(false).await
|
||||
}
|
||||
|
||||
async fn wait_for_server_ready_with_probe(
|
||||
&mut self,
|
||||
read_quorum_only: bool,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
info!("Waiting for RustFS server to be ready on {}", self.address);
|
||||
let client = self.create_s3_client();
|
||||
let health_client = HttpClient::builder().no_proxy().timeout(Duration::from_secs(2)).build()?;
|
||||
|
||||
for i in 0..60 {
|
||||
if let Some(process) = self.process.as_mut()
|
||||
@@ -859,16 +838,7 @@ impl RustFSTestEnvironment {
|
||||
return Err(format!("RustFS server process exited before becoming ready: {status}").into());
|
||||
}
|
||||
|
||||
let ready = if read_quorum_only {
|
||||
health_client
|
||||
.get(format!("{}/minio/health/ready", self.url))
|
||||
.send()
|
||||
.await
|
||||
.is_ok_and(|response| response.status().is_success())
|
||||
} else {
|
||||
TcpStream::connect(&self.address).await.is_ok() && client.list_buckets().send().await.is_ok()
|
||||
};
|
||||
if ready {
|
||||
if TcpStream::connect(&self.address).await.is_ok() && client.list_buckets().send().await.is_ok() {
|
||||
info!("✅ RustFS server is ready after {} attempts", i + 1);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
@@ -13,8 +13,8 @@
|
||||
// limitations under the License.
|
||||
|
||||
//! Over-the-wire smoke net for the embedded console listener (backlog#1154
|
||||
//! peri-4). The console is a second HTTP listener for the control-plane and
|
||||
//! unauthenticated console routes under `/rustfs/console`;
|
||||
//! peri-4). The console is a second HTTP listener that serves the full S3/admin
|
||||
//! surface plus the unauthenticated console routes under `/rustfs/console`;
|
||||
//! before this test its behaviour was only covered by in-process router tests
|
||||
//! (`rustfs/src/admin/console.rs`), so a misconfiguration that exposed
|
||||
//! credentials through the public console endpoints or fail-opened the
|
||||
@@ -26,13 +26,13 @@
|
||||
//! * `/rustfs/console/license` exposes only the coarse `licensed` flag.
|
||||
//! * the console SPA prefix dispatches to the static handler, never to the
|
||||
//! S3 API (no S3 error XML), whether or not console assets are embedded.
|
||||
//! * unauthenticated admin API requests are denied with 403 AccessDenied.
|
||||
//! * S3 root/object requests on the console listener return an empty 404,
|
||||
//! including authenticated requests, before S3 dispatch.
|
||||
//! * unauthenticated requests to the admin API and the S3 root on the console
|
||||
//! listener are denied (403 AccessDenied) — the extra listener does not
|
||||
//! fail-open the protected surface.
|
||||
//! * a listener with the console disabled (the main S3 port here) does not
|
||||
//! serve the unauthenticated console endpoints at all.
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client, signed_request_with_client};
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use std::error::Error;
|
||||
use tokio::time::{Duration, sleep};
|
||||
|
||||
@@ -166,39 +166,13 @@ async fn test_console_over_the_wire_smoke() -> TestResult {
|
||||
let admin_body = admin_response.text().await?;
|
||||
assert!(admin_body.contains("AccessDenied"), "admin denial must be AccessDenied: {admin_body}");
|
||||
|
||||
let signed_data_root = signed_request_with_client(
|
||||
&client,
|
||||
http::Method::GET,
|
||||
&format!("{}/", env.url),
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
let s3_root_response = client.get(format!("{console_base}/")).send().await?;
|
||||
assert_eq!(
|
||||
signed_data_root.status(),
|
||||
reqwest::StatusCode::OK,
|
||||
"the same credentials must reach S3 on the data listener"
|
||||
s3_root_response.status(),
|
||||
reqwest::StatusCode::FORBIDDEN,
|
||||
"unauthenticated S3 root on the console listener must be denied"
|
||||
);
|
||||
|
||||
for path in ["/", "/console-object-bucket/key"] {
|
||||
let url = format!("{console_base}{path}");
|
||||
for signed in [false, true] {
|
||||
let response = if signed {
|
||||
signed_request_with_client(&client, http::Method::GET, &url, &env.access_key, &env.secret_key, None, None).await?
|
||||
} else {
|
||||
client.get(&url).send().await?
|
||||
};
|
||||
assert_eq!(
|
||||
response.status(),
|
||||
reqwest::StatusCode::NOT_FOUND,
|
||||
"S3 route {path}, signed={signed} must be isolated from the console listener"
|
||||
);
|
||||
assert!(response.bytes().await?.is_empty(), "isolated S3 routes must not reach a handler");
|
||||
}
|
||||
}
|
||||
|
||||
// --- console disabled on a listener means no console surface at all ------
|
||||
// The main S3 listener of this same process runs with the console disabled,
|
||||
// so its console paths must not answer with the unauthenticated console
|
||||
|
||||
@@ -44,9 +44,9 @@ mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use http::header::HOST;
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use std::error::Error;
|
||||
|
||||
/// Signed raw `PUT` copy request with an explicit copy-source conditional
|
||||
|
||||
@@ -21,9 +21,9 @@ mod tests {
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
||||
use http::header::HOST;
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use std::error::Error;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpStream;
|
||||
|
||||
@@ -26,7 +26,7 @@ use aws_sdk_s3::types::{
|
||||
DefaultRetention, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHoldStatus, ObjectLockRetentionMode,
|
||||
ObjectLockRule,
|
||||
};
|
||||
use jiff::{SignedDuration, Timestamp};
|
||||
use chrono::{Duration as ChronoDuration, Utc};
|
||||
|
||||
fn delete_denied(error: &SdkError<DeleteObjectError>, context: &str) -> TestResult {
|
||||
let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
|
||||
@@ -66,7 +66,7 @@ async fn four_node_four_drive_object_lock_worm_blocks_delete() -> TestResult {
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let retain_until = Timestamp::now() + SignedDuration::from_hours(24);
|
||||
let retain_until = Utc::now() + ChronoDuration::days(1);
|
||||
|
||||
let compliance_key = "compliance.bin";
|
||||
let compliance_version = put_object_with_retention(
|
||||
@@ -176,10 +176,7 @@ async fn four_node_default_retention_is_visible_and_non_lock_bucket_rejects_conf
|
||||
let retain_until = retention
|
||||
.retain_until_date()
|
||||
.ok_or("default retention omitted retain-until date")?;
|
||||
assert!(
|
||||
retain_until.secs() > Timestamp::now().as_second(),
|
||||
"default retention is not in the future"
|
||||
);
|
||||
assert!(retain_until.secs() > Utc::now().timestamp(), "default retention is not in the future");
|
||||
|
||||
let versioning = reader.get_bucket_versioning().bucket(&bucket).send().await?;
|
||||
assert_eq!(versioning.status().map(|status| status.as_str()), Some("Enabled"));
|
||||
|
||||
@@ -38,8 +38,8 @@ use aws_sdk_s3::types::{
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, ObjectLockLegalHoldStatus, ObjectLockRetentionMode,
|
||||
VersioningConfiguration,
|
||||
};
|
||||
use chrono::{DateTime, Utc};
|
||||
use http::{Method, StatusCode};
|
||||
use jiff::Timestamp;
|
||||
use serde_json::Value;
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::path::{Path, PathBuf};
|
||||
@@ -526,8 +526,8 @@ async fn put_multipart(client: &Client, bucket: &str, key: &str) -> TestResult<V
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
fn retention_deadline() -> TestResult<Timestamp> {
|
||||
Ok(Timestamp::now() + jiff::SignedDuration::from_hours(24))
|
||||
fn retention_deadline() -> TestResult<DateTime<Utc>> {
|
||||
DateTime::<Utc>::from_timestamp(Utc::now().timestamp() + 24 * 3600, 0).ok_or_else(|| "retention deadline out of range".into())
|
||||
}
|
||||
|
||||
/// Seed the deterministic inventory into the seed pool and record it.
|
||||
@@ -654,7 +654,7 @@ pub(crate) async fn seed_manifest(run: &TopologyRun) -> TestResult<TopologyManif
|
||||
let mut entry = ManifestEntry::body(&locked, &key, "compliance", Some(version_id.clone()), Some(version_id), &body);
|
||||
entry.retention = Some(RetentionRecord {
|
||||
mode: "COMPLIANCE".to_string(),
|
||||
retain_until_unix: retain_until.as_second(),
|
||||
retain_until_unix: retain_until.timestamp(),
|
||||
});
|
||||
entries.push(entry);
|
||||
}
|
||||
|
||||
@@ -12,15 +12,14 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Differential: two `rustfs` processes, one SDK script (rustfs/backlog#2740).
|
||||
//! Dual-stack differential: one `rustfs` binary, two processes, one SDK script (rustfs/backlog#2740).
|
||||
//!
|
||||
//! Side B is the binary under test. Side A is the binary under test too, unless
|
||||
//! `RUSTFS_E2E_REFERENCE_BINARY` names a released binary: then A is that binary, and a bad path
|
||||
//! fails the run instead of falling back (see `reference`). The report names which side is the
|
||||
//! reference and the `--version` each side ran. `RUSTFS_E2E_STACK_B_REGION`, when set, starts B
|
||||
//! in another region so that a difference can be injected by hand. Neither side is given the
|
||||
//! retired stack selector: the binary under test refuses to start with it, and the gateway stack
|
||||
//! is its only S3 stack. Both servers
|
||||
//! Stack A runs `RUSTFS_S3_STACK=legacy` of the binary under test, unless `RUSTFS_E2E_REFERENCE_BINARY`
|
||||
//! names a released binary: then A is that binary, started without `RUSTFS_S3_STACK` (older
|
||||
//! releases do not know it), and a bad path fails the run instead of falling back (see
|
||||
//! `reference`). The report names which side is the reference and the `--version` each side ran. Stack B runs the value of `RUSTFS_E2E_STACK_B`
|
||||
//! (default `legacy` until the gateway stack is assembled), and `RUSTFS_E2E_STACK_B_REGION`, when
|
||||
//! set, starts B in another region so that a difference can be injected by hand. Both servers
|
||||
//! answer the script in `scenarios`, which covers every operation of `impl S3 for FS`; `compare`
|
||||
//! reports each `(op, field, a, b)` on which the answers differ, and every run writes its report
|
||||
//! to `<target>/dual-stack/<run>.json`. Two processes, not two embedded servers: bucket metadata
|
||||
@@ -38,18 +37,20 @@ use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
type BoxError = Box<dyn std::error::Error + Send + Sync>;
|
||||
|
||||
/// Label of a side that runs the binary under test.
|
||||
const UNDER_TEST: &str = "under-test";
|
||||
const STACK_A: &str = "legacy";
|
||||
/// Label of side A in the report when it is the released reference binary.
|
||||
const STACK_A_REFERENCE: &str = "reference";
|
||||
const ENV_STACK_B: &str = "RUSTFS_E2E_STACK_B";
|
||||
const ENV_STACK_B_REGION: &str = "RUSTFS_E2E_STACK_B_REGION";
|
||||
/// Becomes `gateway` once rustfs/backlog#2734 T2.5 assembles the gateway stack.
|
||||
const DEFAULT_STACK_B: &str = "legacy";
|
||||
/// The region the negative control gives stack A explicitly, so an inherited `RUSTFS_REGION`
|
||||
/// cannot erase the injected difference. Not `us-east-1`: that region answers GetBucketLocation
|
||||
/// with an empty LocationConstraint and re-creating an owned bucket with 200, so it would not
|
||||
/// report its own name and would add CreateBucket differences to the control.
|
||||
const CONTROL_REGION_A: &str = "us-west-2";
|
||||
const CONTROL_REGION_B: &str = "eu-west-1";
|
||||
/// `normal` cases side A must answer with a 2xx. The comparison alone cannot tell a
|
||||
/// `normal` cases the legacy stack must answer with a 2xx. The comparison alone cannot tell a
|
||||
/// script that exercises real behavior from one in which both stacks fail the same way. Five of
|
||||
/// the 73 fail on legacy by design of this script: PutBucketReplication (no registered remote
|
||||
/// target), GetBucketReplication (so no configuration), PutBucketNotificationConfiguration (no
|
||||
@@ -59,6 +60,7 @@ const MIN_NORMAL_SUCCESSES: usize = 68;
|
||||
|
||||
struct PairConfig {
|
||||
side_a: SideA,
|
||||
stack_b: String,
|
||||
region_a: Option<String>,
|
||||
region_b: Option<String>,
|
||||
}
|
||||
@@ -67,6 +69,7 @@ impl PairConfig {
|
||||
fn from_env() -> Result<Self, BoxError> {
|
||||
Ok(Self {
|
||||
side_a: SideA::from_env()?,
|
||||
stack_b: std::env::var(ENV_STACK_B).unwrap_or_else(|_| DEFAULT_STACK_B.to_owned()),
|
||||
region_a: None,
|
||||
region_b: std::env::var(ENV_STACK_B_REGION).ok(),
|
||||
})
|
||||
@@ -74,16 +77,16 @@ impl PairConfig {
|
||||
|
||||
fn stack_a_label(&self) -> &'static str {
|
||||
match self.side_a {
|
||||
SideA::SameBinary => UNDER_TEST,
|
||||
SideA::SameBinary => STACK_A,
|
||||
SideA::Reference(_) => STACK_A_REFERENCE,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Starts one server of the binary under test, overriding its region only when asked.
|
||||
async fn start_under_test(region: Option<&str>) -> Result<RustFSTestEnvironment, BoxError> {
|
||||
/// Starts one server of the binary under test on `stack`, overriding its region only when asked.
|
||||
async fn start_stack(stack: &str, region: Option<&str>) -> Result<RustFSTestEnvironment, BoxError> {
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
let mut vars = Vec::new();
|
||||
let mut vars = vec![("RUSTFS_S3_STACK", stack)];
|
||||
if let Some(region) = region {
|
||||
vars.push(("RUSTFS_REGION", region));
|
||||
}
|
||||
@@ -91,7 +94,8 @@ async fn start_under_test(region: Option<&str>) -> Result<RustFSTestEnvironment,
|
||||
Ok(env)
|
||||
}
|
||||
|
||||
/// Starts one server of a released binary.
|
||||
/// Starts one server of a released binary. `RUSTFS_S3_STACK` is not passed: older releases do not
|
||||
/// know it and have exactly one stack.
|
||||
async fn start_reference(binary: &Path, region: Option<&str>) -> Result<RustFSTestEnvironment, BoxError> {
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
let mut vars = Vec::new();
|
||||
@@ -107,11 +111,11 @@ async fn start_reference(binary: &Path, region: Option<&str>) -> Result<RustFSTe
|
||||
async fn spawn_pair(config: &PairConfig) -> Result<(RustFSTestEnvironment, RustFSTestEnvironment), BoxError> {
|
||||
let side_a = async {
|
||||
match &config.side_a {
|
||||
SideA::SameBinary => start_under_test(config.region_a.as_deref()).await,
|
||||
SideA::SameBinary => start_stack(STACK_A, config.region_a.as_deref()).await,
|
||||
SideA::Reference(binary) => start_reference(binary, config.region_a.as_deref()).await,
|
||||
}
|
||||
};
|
||||
tokio::try_join!(side_a, start_under_test(config.region_b.as_deref()))
|
||||
tokio::try_join!(side_a, start_stack(&config.stack_b, config.region_b.as_deref()))
|
||||
}
|
||||
|
||||
fn report_directory() -> PathBuf {
|
||||
@@ -156,7 +160,7 @@ async fn run_differential(label: &str, config: &PairConfig) -> Run {
|
||||
schema: 2,
|
||||
run: format!("{label}-{millis}-{}", std::process::id()),
|
||||
stack_a: config.stack_a_label().to_owned(),
|
||||
stack_b: UNDER_TEST.to_owned(),
|
||||
stack_b: config.stack_b.clone(),
|
||||
reference_side: matches!(config.side_a, SideA::Reference(_)).then_some("a"),
|
||||
version_a,
|
||||
version_b,
|
||||
@@ -231,6 +235,7 @@ mod tests {
|
||||
init_logging();
|
||||
let config = PairConfig {
|
||||
side_a: SideA::SameBinary,
|
||||
stack_b: STACK_A.to_owned(),
|
||||
region_a: Some(CONTROL_REGION_A.to_owned()),
|
||||
region_b: Some(CONTROL_REGION_B.to_owned()),
|
||||
};
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
//! Which binary side A of the differential runs, and what it reports about itself.
|
||||
//!
|
||||
//! `RUSTFS_E2E_REFERENCE_BINARY` names a released `rustfs` binary to use as the reference. Unset,
|
||||
//! side A is the binary under test. Set, it must resolve to an
|
||||
//! side A is the binary under test on `RUSTFS_S3_STACK=legacy`. Set, it must resolve to an
|
||||
//! executable file or the run fails: a reference that was asked for and silently replaced by the
|
||||
//! binary under test would report an agreement with itself as agreement with the release.
|
||||
|
||||
@@ -28,9 +28,9 @@ pub(super) const ENV_REFERENCE_BINARY: &str = "RUSTFS_E2E_REFERENCE_BINARY";
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(super) enum SideA {
|
||||
/// Both sides are the binary under test.
|
||||
/// Both sides are the binary under test; A runs `RUSTFS_S3_STACK=legacy`.
|
||||
SameBinary,
|
||||
/// A is this released binary.
|
||||
/// A is this released binary, started without `RUSTFS_S3_STACK`.
|
||||
Reference(PathBuf),
|
||||
}
|
||||
|
||||
|
||||
@@ -29,11 +29,10 @@ use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::ServerSideEncryption;
|
||||
use base64_simd::STANDARD as BASE64;
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use jiff::Timestamp;
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use serde_json;
|
||||
use std::process::{Child, Command};
|
||||
use std::time::Duration;
|
||||
@@ -387,7 +386,7 @@ pub async fn create_key_with_specific_id(key_dir: &str, key_id: &str) -> Result<
|
||||
"usage": "EncryptDecrypt",
|
||||
"status": "Active",
|
||||
"metadata": HashMap::<String, String>::new(),
|
||||
"created_at": format!("{}[UTC]", Timestamp::now().strftime("%Y-%m-%dT%H:%M:%S%.fZ")),
|
||||
"created_at": format!("{}[UTC]", chrono::Utc::now().to_rfc3339()),
|
||||
"rotated_at": serde_json::Value::Null,
|
||||
"created_by": "e2e-test",
|
||||
"encrypted_key_material": BASE64.encode_to_string(key_data),
|
||||
|
||||
@@ -31,8 +31,8 @@ pub mod chaos;
|
||||
#[cfg(test)]
|
||||
pub mod fake_s3_target;
|
||||
|
||||
// Differential (rustfs/backlog#2740): two processes, the binary under test against
|
||||
// itself or a released reference binary, the same SDK script for every S3 operation,
|
||||
// Dual-stack differential (rustfs/backlog#2740): one binary, two processes on
|
||||
// different RUSTFS_S3_STACK values, the same SDK script for every S3 operation,
|
||||
// compared on status, error code, allowlisted headers and typed body fields.
|
||||
#[cfg(test)]
|
||||
mod dual_stack_differential;
|
||||
|
||||
@@ -24,9 +24,9 @@
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use http::header::HOST;
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use std::error::Error;
|
||||
|
||||
/// Sends a SigV4-signed `GET` where the signature is computed over `sign_path`
|
||||
|
||||
@@ -21,9 +21,9 @@ mod tests {
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
||||
use http::header::HOST;
|
||||
use reqwest::StatusCode;
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use std::error::Error;
|
||||
use tracing::info;
|
||||
|
||||
|
||||
@@ -20,9 +20,9 @@ mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use http::header::HOST;
|
||||
use reqwest::StatusCode;
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use std::error::Error;
|
||||
use tracing::info;
|
||||
|
||||
|
||||
@@ -23,17 +23,17 @@ use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
ServerSideEncryption, ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule,
|
||||
};
|
||||
use chrono::{Duration as ChronoDuration, Utc};
|
||||
use flate2::{
|
||||
Compression,
|
||||
write::{GzEncoder, ZlibEncoder},
|
||||
};
|
||||
use http::HeaderValue;
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use jiff::{SignedDuration, Timestamp};
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use std::collections::HashMap;
|
||||
use std::error::Error;
|
||||
use std::io::Cursor;
|
||||
@@ -42,8 +42,8 @@ use tokio::io::AsyncWriteExt;
|
||||
use uuid::Uuid;
|
||||
|
||||
fn encode_post_policy(conditions: Vec<serde_json::Value>) -> String {
|
||||
let expiration = (Timestamp::now() + SignedDuration::from_hours(1))
|
||||
.strftime("%Y-%m-%dT%H:%M:%S.000Z")
|
||||
let expiration = (Utc::now() + ChronoDuration::hours(1))
|
||||
.format("%Y-%m-%dT%H:%M:%S.000Z")
|
||||
.to_string();
|
||||
let policy = serde_json::json!({
|
||||
"expiration": expiration,
|
||||
|
||||
@@ -283,7 +283,6 @@ async fn test_cold_degraded_object_mutations_require_only_storage_quorum() -> Te
|
||||
#[tokio::test]
|
||||
async fn test_degraded_cluster_read_quorum_follows_erasure_layout() -> TestResult {
|
||||
crate::common::init_logging();
|
||||
assert_single_node_cold_start_with_missing_volume().await?;
|
||||
|
||||
for (node_count, parity) in [(3, 1), (4, 2), (6, 3), (6, 2), (4, 0)] {
|
||||
let read_quorum = node_count - parity;
|
||||
@@ -431,138 +430,6 @@ async fn test_degraded_cluster_read_quorum_follows_erasure_layout() -> TestResul
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_single_node_cold_start_with_missing_volume() -> TestResult {
|
||||
for drive_count in [2, 3, 4] {
|
||||
let mut server = crate::chaos::DiskFaultHarness::new(drive_count).await?;
|
||||
server.set_env("RUSTFS_STORAGE_CLASS_STANDARD", format!("EC:{}", drive_count / 2));
|
||||
server.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false");
|
||||
server.set_env("RUSTFS_OBS_METRICS_EXPORT_ENABLED", "false");
|
||||
server.set_env("RUSTFS_HEALTH_ENDPOINT_ENABLE", "true");
|
||||
server.set_env("RUST_LOG", "warn");
|
||||
server.start_server_at_read_quorum().await?;
|
||||
let client = server.env.create_s3_client();
|
||||
let client = Client::from_conf(
|
||||
client
|
||||
.config()
|
||||
.to_builder()
|
||||
.retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(1))
|
||||
.build(),
|
||||
);
|
||||
let bucket = format!("missing-volume-{drive_count}");
|
||||
client.create_bucket().bucket(&bucket).send().await?;
|
||||
let small = vec![17; 4096];
|
||||
let large = vec![73; 8 * 1024 * 1024];
|
||||
for (key, body) in [("small", &small), ("large", &large)] {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.body(Bytes::copy_from_slice(body).into())
|
||||
.send()
|
||||
.await?;
|
||||
for disk in 0..drive_count {
|
||||
let census = crate::chaos::census_object_version_on_disk(server.disk_path(disk), &bucket, key, None)?;
|
||||
assert!(census.is_complete(), "seed shard must exist before restart: {census:?}");
|
||||
}
|
||||
}
|
||||
server.kill_server();
|
||||
let missing = drive_count - 1;
|
||||
server.take_disk_offline(missing)?;
|
||||
server.start_server_at_read_quorum().await?;
|
||||
|
||||
for (key, body) in [("small", &small), ("large", &large)] {
|
||||
assert_quorum_object_body(&client, &bucket, key, body).await?;
|
||||
let head = client.head_object().bucket(&bucket).key(key).send().await?;
|
||||
assert_eq!(head.content_length(), Some(i64::try_from(body.len())?));
|
||||
}
|
||||
let listing = client.list_objects_v2().bucket(&bucket).send().await?;
|
||||
let keys: Vec<_> = listing.contents().iter().filter_map(|object| object.key()).collect();
|
||||
assert_eq!(keys, ["large", "small"]);
|
||||
|
||||
let put = client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key("degraded-write")
|
||||
.body(Bytes::from_static(b"degraded write").into())
|
||||
.send()
|
||||
.await;
|
||||
if drive_count == 2 {
|
||||
let error = put.expect_err("one of two drives cannot commit a new object");
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(503));
|
||||
let error = client
|
||||
.delete_object()
|
||||
.bucket(&bucket)
|
||||
.key("large")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("one of two drives cannot commit an object deletion");
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(503));
|
||||
assert_quorum_object_body(&client, &bucket, "large", &large).await?;
|
||||
} else {
|
||||
put?;
|
||||
assert_quorum_object_body(&client, &bucket, "degraded-write", b"degraded write").await?;
|
||||
client.delete_object().bucket(&bucket).key("degraded-write").send().await?;
|
||||
}
|
||||
assert!(
|
||||
!server.disk_path(missing).exists(),
|
||||
"degraded startup and S3 operations must leave the root absent"
|
||||
);
|
||||
|
||||
if drive_count == 2 {
|
||||
server.kill_server();
|
||||
std::fs::create_dir(server.disk_path(missing))?;
|
||||
server.start_server_at_read_quorum().await?;
|
||||
for (key, body) in [("small", &small), ("large", &large)] {
|
||||
assert_quorum_object_body(&client, &bucket, key, body).await?;
|
||||
client.head_object().bucket(&bucket).key(key).send().await?;
|
||||
}
|
||||
assert_eq!(client.list_objects_v2().bucket(&bucket).send().await?.contents().len(), 2);
|
||||
let error = client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key("empty-root-write")
|
||||
.body(Bytes::from_static(b"must remain unavailable").into())
|
||||
.send()
|
||||
.await
|
||||
.expect_err("an empty replacement cannot restore write quorum");
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(503));
|
||||
assert!(
|
||||
!server
|
||||
.disk_path(missing)
|
||||
.join(crate::storage_api::RUSTFS_META_BUCKET)
|
||||
.join("format.json")
|
||||
.exists(),
|
||||
"read admission must not format the empty replacement"
|
||||
);
|
||||
}
|
||||
server.kill_server();
|
||||
server.bring_disk_online(missing)?;
|
||||
server.start_server_at_read_quorum().await?;
|
||||
for (key, body) in [("small", &small), ("large", &large)] {
|
||||
assert_quorum_object_body(&client, &bucket, key, body).await?;
|
||||
}
|
||||
for key in ["degraded-write", "empty-root-write"] {
|
||||
let error = client
|
||||
.head_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("rejected or deleted degraded write must stay absent after recovery");
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(404));
|
||||
}
|
||||
client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key("recovery-write")
|
||||
.body(Bytes::from_static(b"recovered write").into())
|
||||
.send()
|
||||
.await?;
|
||||
assert_quorum_object_body(&client, &bucket, "recovery-write", b"recovered write").await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cold_start_read_quorum_follows_erasure_layout() -> TestResult {
|
||||
crate::common::init_logging();
|
||||
|
||||
@@ -17,9 +17,9 @@ use aws_sdk_s3::primitives::ByteStream;
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use reqwest::StatusCode;
|
||||
use rustfs_config::{ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, ENV_NOTIFY_ENABLE};
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::pre_sign_v4;
|
||||
use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS;
|
||||
use s3s::Body;
|
||||
use std::collections::HashMap;
|
||||
use std::error::Error;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
@@ -29,7 +29,7 @@ use aws_sdk_s3::types::{
|
||||
DefaultRetention, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHold, ObjectLockLegalHoldStatus, ObjectLockMode,
|
||||
ObjectLockRetention, ObjectLockRetentionMode, ObjectLockRule,
|
||||
};
|
||||
use jiff::{SignedDuration, Timestamp};
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use tracing::info;
|
||||
|
||||
use crate::common::RustFSTestEnvironment;
|
||||
@@ -126,10 +126,10 @@ pub async fn put_object_with_retention(
|
||||
key: &str,
|
||||
data: &[u8],
|
||||
mode: ObjectLockRetentionMode,
|
||||
retain_until: Timestamp,
|
||||
retain_until: DateTime<Utc>,
|
||||
) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
|
||||
// AWS SDK requires UTC time without timezone offset (e.g., "2026-01-24T11:20:14Z")
|
||||
let retain_until_str = retain_until.strftime("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||||
let retain_until_str = retain_until.format("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||||
let retain_until_datetime =
|
||||
aws_sdk_s3::primitives::DateTime::from_str(&retain_until_str, aws_sdk_s3::primitives::DateTimeFormat::DateTime)?;
|
||||
|
||||
@@ -181,10 +181,10 @@ pub async fn put_object_retention(
|
||||
key: &str,
|
||||
version_id: Option<&str>,
|
||||
mode: ObjectLockRetentionMode,
|
||||
retain_until: Timestamp,
|
||||
retain_until: DateTime<Utc>,
|
||||
bypass_governance: bool,
|
||||
) -> Result<(), Box<SdkError<PutObjectRetentionError>>> {
|
||||
let retain_until_datetime = aws_sdk_s3::primitives::DateTime::from_secs(retain_until.as_second());
|
||||
let retain_until_datetime = aws_sdk_s3::primitives::DateTime::from_secs(retain_until.timestamp());
|
||||
|
||||
let retention = ObjectLockRetention::builder()
|
||||
.mode(mode.clone())
|
||||
@@ -253,6 +253,6 @@ pub async fn delete_object_with_bypass(
|
||||
}
|
||||
|
||||
/// Calculate a future retain_until date
|
||||
pub fn future_retain_until(days: i64) -> Timestamp {
|
||||
Timestamp::now() + SignedDuration::from_hours(days * 24)
|
||||
pub fn future_retain_until(days: i64) -> DateTime<Utc> {
|
||||
Utc::now() + Duration::days(days)
|
||||
}
|
||||
|
||||
@@ -34,7 +34,7 @@ use aws_sdk_s3::types::{
|
||||
CompletedMultipartUpload, CompletedPart, Delete, MetadataDirective, ObjectIdentifier, ObjectLockLegalHoldStatus,
|
||||
ObjectLockMode, ObjectLockRetentionMode,
|
||||
};
|
||||
use jiff::{SignedDuration, Timestamp};
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use tracing::info;
|
||||
|
||||
/// Initialize test logging
|
||||
@@ -67,7 +67,7 @@ async fn put_bucket_deny_policy(
|
||||
}
|
||||
|
||||
fn retention_timestamp(days: i64) -> aws_sdk_s3::primitives::DateTime {
|
||||
let retain_until = future_retain_until(days).strftime("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||||
let retain_until = future_retain_until(days).format("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||||
aws_sdk_s3::primitives::DateTime::from_str(&retain_until, aws_sdk_s3::primitives::DateTimeFormat::DateTime)
|
||||
.expect("retention timestamp should parse")
|
||||
}
|
||||
@@ -118,14 +118,13 @@ where
|
||||
);
|
||||
}
|
||||
|
||||
fn parse_s3_datetime(value: &aws_sdk_s3::primitives::DateTime) -> Timestamp {
|
||||
fn parse_s3_datetime(value: &aws_sdk_s3::primitives::DateTime) -> DateTime<Utc> {
|
||||
let formatted = value
|
||||
.fmt(DateTimeFormat::DateTime)
|
||||
.expect("S3 timestamp should format as RFC3339");
|
||||
let normalized = formatted
|
||||
.strip_suffix('Z')
|
||||
.map_or(formatted.clone(), |prefix| format!("{prefix}+00:00"));
|
||||
Timestamp::strptime("%Y-%m-%dT%H:%M:%S%.f%:z", normalized).expect("S3 timestamp should parse as RFC3339")
|
||||
DateTime::parse_from_rfc3339(&formatted)
|
||||
.expect("S3 timestamp should parse as RFC3339")
|
||||
.with_timezone(&Utc)
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
@@ -471,7 +470,7 @@ async fn test_get_object_retention_returns_configured_values() {
|
||||
let bucket = "test-get-retention";
|
||||
let key = "retained-object";
|
||||
let retain_until = future_retain_until(30);
|
||||
let retain_until_expected = retain_until.strftime("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||||
let retain_until_expected = retain_until.format("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||||
|
||||
env.create_object_lock_bucket(bucket).await.unwrap();
|
||||
|
||||
@@ -1925,7 +1924,7 @@ async fn test_copy_object_retention_uses_destination_policy() {
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let copy_started = Timestamp::now();
|
||||
let copy_started = Utc::now();
|
||||
client
|
||||
.copy_object()
|
||||
.copy_source(format!("{src_bucket}/{src_key}"))
|
||||
@@ -1948,7 +1947,7 @@ async fn test_copy_object_retention_uses_destination_policy() {
|
||||
assert_eq!(retention.mode().map(|value| value.as_str()), Some("GOVERNANCE"));
|
||||
let retain_until = parse_s3_datetime(retention.retain_until_date().expect("retain-until date should be present"));
|
||||
assert!(
|
||||
retain_until < copy_started + SignedDuration::from_hours(3 * 24),
|
||||
retain_until < copy_started + Duration::days(3),
|
||||
"destination default retention should not inherit the source's longer retention"
|
||||
);
|
||||
|
||||
@@ -2009,7 +2008,7 @@ async fn test_copy_object_retention_uses_destination_policy() {
|
||||
.expect("explicit retain-until date should be present"),
|
||||
);
|
||||
assert!(
|
||||
explicit_retain_until > Timestamp::now() + SignedDuration::from_hours(20 * 24),
|
||||
explicit_retain_until > Utc::now() + Duration::days(20),
|
||||
"explicit retention should override the shorter bucket default"
|
||||
);
|
||||
|
||||
@@ -2089,7 +2088,7 @@ async fn test_multipart_default_retention_fixed_at_create() {
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let create_started = Timestamp::now();
|
||||
let create_started = Utc::now();
|
||||
let create_output = client.create_multipart_upload().bucket(bucket).key(key).send().await.unwrap();
|
||||
let upload_id = create_output.upload_id().unwrap();
|
||||
|
||||
@@ -2132,7 +2131,7 @@ async fn test_multipart_default_retention_fixed_at_create() {
|
||||
assert_eq!(retention.mode().map(|value| value.as_str()), Some("GOVERNANCE"));
|
||||
let retain_until = parse_s3_datetime(retention.retain_until_date().expect("retain-until date should be present"));
|
||||
assert!(
|
||||
retain_until < create_started + SignedDuration::from_hours(3 * 24),
|
||||
retain_until < create_started + Duration::days(3),
|
||||
"CompleteMultipartUpload should keep the default retention calculated at create time"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -693,7 +693,7 @@ impl OdmTestEnv {
|
||||
.uri(uri)
|
||||
.header(http::header::HOST, authority)
|
||||
.header("x-amz-content-sha256", rustfs_signer::constants::UNSIGNED_PAYLOAD)
|
||||
.body(rustfs_s3_types::Body::empty())?;
|
||||
.body(s3s::Body::empty())?;
|
||||
let signed = rustfs_signer::sign_v4(request, 0, &self.rustfs.access_key, &self.rustfs.secret_key, "", "us-east-1");
|
||||
|
||||
let mut builder =
|
||||
|
||||
@@ -20,9 +20,9 @@
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use http::header::{CONTENT_LENGTH, HOST, TRANSFER_ENCODING};
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::sign_v4;
|
||||
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
|
||||
use s3s::Body;
|
||||
use std::error::Error;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpStream;
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
|
||||
use async_trait::async_trait;
|
||||
use rustfs_lock::{
|
||||
CacheFenceStamp, LockClient, LockError, LockId, LockInfo, LockRequest, LockResponse, LockStats, LockStatus, LockType, Result,
|
||||
LockClient, LockError, LockId, LockInfo, LockRequest, LockResponse, LockStats, LockStatus, LockType, Result,
|
||||
types::{LockMetadata, LockPriority},
|
||||
};
|
||||
use rustfs_protos::proto_gen::node_service::{BatchGenerallyLockRequest, GenerallyLockRequest, PingRequest};
|
||||
@@ -64,7 +64,6 @@ impl GrpcLockClient {
|
||||
deadlock_detection: false,
|
||||
suppress_contention_logs: false,
|
||||
refresh_interval: None,
|
||||
cache_fence_stamp_requested: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -125,15 +124,10 @@ impl LockClient for GrpcLockClient {
|
||||
|
||||
// Check if the lock acquisition was successful
|
||||
if resp.success {
|
||||
let info = Self::build_lock_info(request, resp.lock_info);
|
||||
match resp
|
||||
.cache_fence_stamp
|
||||
.as_deref()
|
||||
.and_then(|encoded| CacheFenceStamp::decode(encoded).ok())
|
||||
{
|
||||
Some(stamp) => Ok(LockResponse::success_with_cache_fence_stamp(info, std::time::Duration::ZERO, stamp)),
|
||||
None => Ok(LockResponse::success(info, std::time::Duration::ZERO)),
|
||||
}
|
||||
Ok(LockResponse::success(
|
||||
Self::build_lock_info(request, resp.lock_info),
|
||||
std::time::Duration::ZERO,
|
||||
))
|
||||
} else {
|
||||
// Lock acquisition failed
|
||||
Ok(LockResponse::failure(
|
||||
@@ -166,15 +160,7 @@ impl LockClient for GrpcLockClient {
|
||||
.enumerate()
|
||||
.map(|(idx, request)| match resp.results.get(idx) {
|
||||
Some(result) if result.success => {
|
||||
let info = Self::build_lock_info(request, result.lock_info.clone());
|
||||
match result
|
||||
.cache_fence_stamp
|
||||
.as_deref()
|
||||
.and_then(|encoded| CacheFenceStamp::decode(encoded).ok())
|
||||
{
|
||||
Some(stamp) => LockResponse::success_with_cache_fence_stamp(info, std::time::Duration::ZERO, stamp),
|
||||
None => LockResponse::success(info, std::time::Duration::ZERO),
|
||||
}
|
||||
LockResponse::success(Self::build_lock_info(request, result.lock_info.clone()), std::time::Duration::ZERO)
|
||||
}
|
||||
Some(result) => LockResponse::failure(
|
||||
result
|
||||
|
||||
@@ -43,7 +43,6 @@ fn lock_result_from_response(response: rustfs_lock::LockResponse) -> GenerallyLo
|
||||
success: response.success,
|
||||
error_info: response.error,
|
||||
lock_info: response.lock_info.and_then(|info| serde_json::to_string(&info).ok()),
|
||||
cache_fence_stamp: response.cache_fence_stamp.map(|stamp| stamp.encode().to_vec().into()),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
@@ -181,7 +180,6 @@ impl NodeService for MinimalLockNodeService {
|
||||
success: result.success,
|
||||
error_info: result.error,
|
||||
lock_info: lock_info_json,
|
||||
cache_fence_stamp: result.cache_fence_stamp.map(|stamp| stamp.encode().to_vec().into()),
|
||||
..Default::default()
|
||||
}))
|
||||
}
|
||||
@@ -573,15 +571,6 @@ impl NodeService for MinimalLockNodeService {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
type RenameDataBatchStream = ResponseStream<rustfs_protos::proto_gen::node_service::RenameDataBatchResult>;
|
||||
|
||||
async fn rename_data_batch(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::RenameDataBatchRequest>,
|
||||
) -> Result<Response<Self::RenameDataBatchStream>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn rename_multipart_data(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::RenameDataRequest>,
|
||||
|
||||
@@ -27,9 +27,9 @@ use http::Version;
|
||||
use http::header::HOST;
|
||||
use rcgen::generate_simple_self_signed;
|
||||
use reqwest::{Certificate, Client, Response, StatusCode};
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use std::error::Error;
|
||||
use std::path::Path;
|
||||
use std::process::Command;
|
||||
|
||||
@@ -59,6 +59,7 @@ use rustfs_madmin::{
|
||||
AddServiceAccountReq, ListServiceAccountsResp, PeerInfo, PeerSite, ReplicateAddStatus, ReplicateEditStatus,
|
||||
ReplicateRemoveStatus, SRRemoveReq, SRResyncOpStatus, SRStatusInfo, SiteReplicationInfo, SyncStatus,
|
||||
};
|
||||
use s3s::header::{X_AMZ_REPLICATION_STATUS, X_AMZ_TAGGING};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::BTreeMap;
|
||||
use std::convert::Infallible;
|
||||
@@ -77,8 +78,6 @@ use tokio::task::JoinHandle;
|
||||
use tokio::task::JoinSet;
|
||||
use tokio::time::{Duration, sleep, timeout};
|
||||
|
||||
const X_AMZ_REPLICATION_STATUS: http::HeaderName = http::HeaderName::from_static("x-amz-replication-status");
|
||||
const X_AMZ_TAGGING: http::HeaderName = http::HeaderName::from_static("x-amz-tagging");
|
||||
type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
|
||||
type BacklogMetricPoints = Arc<Mutex<BTreeMap<String, BTreeMap<String, (u64, f64)>>>>;
|
||||
|
||||
|
||||
@@ -32,9 +32,9 @@ mod tests {
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use http::StatusCode;
|
||||
use http::header::HOST;
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use std::error::Error;
|
||||
use tracing::{debug, info};
|
||||
|
||||
|
||||
@@ -23,9 +23,9 @@ mod tests {
|
||||
use aws_sdk_s3::types::{ObjectAttributes, StorageClass};
|
||||
use http::header::HOST;
|
||||
use reqwest::StatusCode;
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use serde_json::Value;
|
||||
use std::error::Error;
|
||||
use std::path::Path;
|
||||
|
||||
@@ -126,7 +126,6 @@ test-util = []
|
||||
e2e-test-hooks = []
|
||||
|
||||
[dependencies]
|
||||
slab.workspace = true
|
||||
hashbrown.workspace = true
|
||||
hotpath.workspace = true
|
||||
rustfs-filemeta.workspace = true
|
||||
@@ -154,6 +153,7 @@ arc-swap.workspace = true
|
||||
async-trait.workspace = true
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
byteorder = { workspace = true }
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
jiff = { workspace = true, features = ["serde"] }
|
||||
thiserror.workspace = true
|
||||
flatbuffers.workspace = true
|
||||
|
||||
@@ -393,8 +393,7 @@ pub mod cluster {
|
||||
|
||||
pub mod compression {
|
||||
pub use crate::io_support::compress::{
|
||||
DISK_COMPRESSION_EXCLUDED_CONTENT_TYPES, MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible, is_disk_compression_enabled,
|
||||
is_multipart_disk_compression_enabled,
|
||||
MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible, is_disk_compression_enabled, is_multipart_disk_compression_enabled,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -615,21 +614,15 @@ pub mod object {
|
||||
};
|
||||
pub use crate::erasure::codec::shard_emit::ShardEmitter;
|
||||
pub use crate::object_api::{
|
||||
BLOCK_SIZE_V2, ClientEcBlockSizeHint, DEFAULT_EC_BLOCK_SIZE, EC_BLOCK_SIZE_CANDIDATES, EC_BLOCK_SIZE_HINT_INTERNAL_KEY,
|
||||
EC_BLOCK_SIZE_HINT_INTERNAL_SUFFIX, EC_BLOCK_SIZE_HINT_METADATA_KEY, EC_BLOCK_SIZE_HINT_REASON_HEADER,
|
||||
EC_BLOCK_SIZE_HINT_STATUS_HEADER, EC_BLOCK_SIZE_INFO_INTERNAL_KEY, EC_BLOCK_SIZE_INFO_INTERNAL_SUFFIX,
|
||||
EC_BLOCK_SIZE_RESPONSE_HEADER, EC_READ_QUANTUM_INFO_INTERNAL_KEY, EC_READ_QUANTUM_INFO_INTERNAL_SUFFIX,
|
||||
ERASURE_ALGORITHM, EcBlockSizeHintError, EcBlockSizeHintIgnoreReason, EcBlockSizeHintOutcome, EncryptionResolutionError,
|
||||
EncryptionResolutionErrorKind, GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup, GetObjectBodySource,
|
||||
GetObjectReadCondition, GetObjectReadMetadata, GetObjectReader, GetObjectSink, GetObjectSinkSegmentFactory,
|
||||
GetObjectSinkSegments, GetObjectSinkStream, GetObjectSinkWriter, NamespaceLockFence, ObjectEncryptionResolver,
|
||||
ObjectInfo, ObjectLockConfigSnapshot, ObjectMutationHook, ObjectOptions, PutObjReader, QuotaAdmission,
|
||||
RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest, ResolvedEcBlockSize,
|
||||
BLOCK_SIZE_V2, ERASURE_ALGORITHM, EncryptionResolutionError, EncryptionResolutionErrorKind, GetObjectBodyCacheHook,
|
||||
GetObjectBodyCacheHookLookup, GetObjectBodySource, GetObjectReadCondition, GetObjectReadMetadata, GetObjectReader,
|
||||
GetObjectSink, GetObjectSinkSegmentFactory, GetObjectSinkSegments, GetObjectSinkStream, GetObjectSinkWriter,
|
||||
NamespaceLockFence, ObjectEncryptionResolver, ObjectInfo, ObjectLockConfigSnapshot, ObjectMutationHook, ObjectOptions,
|
||||
PutObjReader, QuotaAdmission, RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest,
|
||||
SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, ScannerPublicationCommitScope, ScannerPublicationCommitStartError,
|
||||
ScannerPublicationCommitState, ShardIntegrityWriteMode, StreamConsumer, WriteCommitGuard, WriteCompletion,
|
||||
get_object_body_cache_plaintext_len, is_supported_ec_block_size, lookup_get_object_body_cache_hook,
|
||||
parse_ec_block_size_hint, register_get_object_body_cache_hook, register_object_mutation_hook, resolve_ec_block_size_hint,
|
||||
take_ec_block_size_hint, unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
|
||||
get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, register_get_object_body_cache_hook,
|
||||
register_object_mutation_hook, unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
|
||||
with_get_object_read_condition,
|
||||
};
|
||||
pub use crate::store::{
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -5591,7 +5591,7 @@ impl RestoreRequestOps for RestoreRequest {
|
||||
}
|
||||
|
||||
// For non-SELECT requests, days must be at least 1
|
||||
if !valid_restore_days(self) {
|
||||
if self.type_.is_none() && self.days.is_none_or(|d| d <= 0) {
|
||||
return Err(std::io::Error::other("restoration days should be at least 1"));
|
||||
}
|
||||
|
||||
@@ -5599,10 +5599,6 @@ impl RestoreRequestOps for RestoreRequest {
|
||||
}
|
||||
}
|
||||
|
||||
fn valid_restore_days(request: &RestoreRequest) -> bool {
|
||||
request.type_.as_deref() == Some("SELECT") || request.days.is_some_and(|days| days > 0)
|
||||
}
|
||||
|
||||
const _MAX_RESTORE_OBJECT_REQUEST_SIZE: i64 = 2 << 20;
|
||||
|
||||
/// Builds an exact-version lifecycle delete target with the concrete
|
||||
@@ -9377,18 +9373,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn standard_restore_days_are_positive_with_or_without_type() {
|
||||
let mut request = select_restore_request(None);
|
||||
for kind in [None, Some("STANDARD".to_owned())] {
|
||||
request.type_ = kind;
|
||||
for days in [None, Some(0), Some(-1), Some(1)] {
|
||||
request.days = days;
|
||||
assert_eq!(super::valid_restore_days(&request), days == Some(1));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn select_restore_s3_location_rejects_missing_s3_output_location() {
|
||||
let request = select_restore_request(Some(OutputLocation { s3: None }));
|
||||
|
||||
@@ -1136,14 +1136,6 @@ impl BucketMetadata {
|
||||
/// delivered late (backlog#2292). Only replication receivers should pass
|
||||
/// a foreign time; local edits keep [`Self::update_config`].
|
||||
pub fn update_config_at(&mut self, config_file: &str, data: Vec<u8>, updated: OffsetDateTime) -> Result<OffsetDateTime> {
|
||||
const MAX_BUCKET_CONFIG_BYTES: usize = 2 * 1024 * 1024;
|
||||
if data.len() > MAX_BUCKET_CONFIG_BYTES {
|
||||
return Err(Error::InvalidArgument(
|
||||
self.name.clone(),
|
||||
String::new(),
|
||||
"bucket configuration exceeds 2 MiB".to_owned(),
|
||||
));
|
||||
}
|
||||
match config_file {
|
||||
BUCKET_POLICY_CONFIG => {
|
||||
self.policy_config_json = data;
|
||||
@@ -1351,39 +1343,6 @@ impl BucketMetadata {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Parse only the configuration used for foreground write/delete admission.
|
||||
pub(super) fn parse_admission_configs(&mut self) {
|
||||
self.object_lock_config = None;
|
||||
if !self.object_lock_config_xml.is_empty()
|
||||
&& let Err(e) =
|
||||
deserialize::<ObjectLockConfiguration>(&self.object_lock_config_xml).map(|c| self.object_lock_config = Some(c))
|
||||
{
|
||||
tracing::warn!(
|
||||
event = "bucket_metadata_parse_failed",
|
||||
component = "ecstore",
|
||||
subsystem = "bucket_metadata",
|
||||
bucket = %self.name,
|
||||
config = "object_lock",
|
||||
error = %e,
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
}
|
||||
self.quota_config = None;
|
||||
if !self.quota_config_json.is_empty()
|
||||
&& let Err(e) = serde_json::from_slice(&self.quota_config_json).map(|c| self.quota_config = Some(c))
|
||||
{
|
||||
tracing::warn!(
|
||||
event = "bucket_metadata_parse_failed",
|
||||
component = "ecstore",
|
||||
subsystem = "bucket_metadata",
|
||||
bucket = %self.name,
|
||||
config = "quota",
|
||||
error = %e,
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode every stored sub-configuration into its typed field.
|
||||
///
|
||||
/// A decode failure never fails the whole load: this runs on every bucket
|
||||
@@ -1405,7 +1364,7 @@ impl BucketMetadata {
|
||||
/// | public access block | Fails closed in `get_public_access_block_config`: degrading grants the anonymous access the operator asked to block. |
|
||||
/// | quota | Fails closed in `get_quota_config`; the enforcement path in `quota::checker` already re-parses the raw JSON and refuses on error. |
|
||||
/// | lifecycle | `get_lifecycle_config` fails closed, so GetBucketLifecycle reports the fault instead of NoSuchLifecycleConfiguration. ILM, scanner and expiry-header consumers still degrade to "no rules": nothing is deleted or moved on the strength of an unreadable rule set, and the bucket keeps serving reads and writes. |
|
||||
/// | notification | `get_notification_config` fails closed, so GetBucketNotificationConfiguration reports the fault and startup skips that bucket's rule registration; metadata-bound server dispatch suppresses external delivery for the unreadable snapshot. Other buckets are unaffected. |
|
||||
/// | notification | `get_notification_config` fails closed, so GetBucketNotificationConfiguration reports the fault and startup leaves that one bucket's rules unchanged instead of clearing them; other buckets are unaffected. |
|
||||
/// | tagging, CORS, logging, website, accelerate, request payment | The getter fails closed so the matching GET API reports the fault instead of "not configured". Per-request consumers (CORS response headers) still degrade to "not configured", which is the restrictive direction. |
|
||||
/// | bucket ACL | Safe to degrade: it only shapes an optional response. |
|
||||
///
|
||||
@@ -1459,7 +1418,21 @@ impl BucketMetadata {
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
}
|
||||
self.parse_admission_configs();
|
||||
self.object_lock_config = None;
|
||||
if !self.object_lock_config_xml.is_empty()
|
||||
&& let Err(e) =
|
||||
deserialize::<ObjectLockConfiguration>(&self.object_lock_config_xml).map(|c| self.object_lock_config = Some(c))
|
||||
{
|
||||
tracing::warn!(
|
||||
event = "bucket_metadata_parse_failed",
|
||||
component = "ecstore",
|
||||
subsystem = "bucket_metadata",
|
||||
bucket = %self.name,
|
||||
config = "object_lock",
|
||||
error = %e,
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
}
|
||||
self.versioning_config = None;
|
||||
if !self.versioning_config_xml.is_empty()
|
||||
&& let Err(e) =
|
||||
@@ -1504,6 +1477,19 @@ impl BucketMetadata {
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
}
|
||||
if !self.quota_config_json.is_empty()
|
||||
&& let Err(e) = serde_json::from_slice(&self.quota_config_json).map(|c| self.quota_config = Some(c))
|
||||
{
|
||||
tracing::warn!(
|
||||
event = "bucket_metadata_parse_failed",
|
||||
component = "ecstore",
|
||||
subsystem = "bucket_metadata",
|
||||
bucket = %self.name,
|
||||
config = "quota",
|
||||
error = %e,
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
}
|
||||
self.replication_config = None;
|
||||
if !self.replication_config_xml.is_empty()
|
||||
&& let Err(e) = deserialize::<PersistedReplicationConfiguration>(&self.replication_config_xml)
|
||||
@@ -2089,33 +2075,6 @@ mod test {
|
||||
assert!(bm.object_locking());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bucket_config_size_limit_preserves_last_accepted_policy() {
|
||||
let mut metadata = BucketMetadata::new("policy-size-limit");
|
||||
let stamp = OffsetDateTime::now_utc();
|
||||
let mut policy = br#"{"Version":"2012-10-17","Statement":[]}"#.to_vec();
|
||||
policy.resize(2 * 1024 * 1024, b' ');
|
||||
metadata
|
||||
.update_config_at(BUCKET_POLICY_CONFIG, policy.clone(), stamp)
|
||||
.expect("accept a policy exactly at the size limit");
|
||||
let mut oversized = policy.clone();
|
||||
oversized.push(b' ');
|
||||
let result = metadata.update_config_at(BUCKET_POLICY_CONFIG, oversized, stamp + time::Duration::seconds(1));
|
||||
assert!(matches!(result, Err(Error::InvalidArgument(..))));
|
||||
assert!(metadata.policy_config_json == policy, "rejection must preserve the accepted policy");
|
||||
assert_eq!(metadata.policy_config_updated_at, stamp);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn admission_configs_leave_logging_unparsed() {
|
||||
let mut bm = BucketMetadata::new("bucket");
|
||||
bm.logging_config_xml = b"<BucketLoggingStatus/>".to_vec();
|
||||
bm.parse_admission_configs();
|
||||
assert!(bm.logging_config.is_none());
|
||||
bm.parse_all_configs().expect("parse logging config");
|
||||
assert!(bm.logging_config.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_all_configs_parses_stored_configs_without_store_dependency() {
|
||||
let mut bm = BucketMetadata::new("test-bucket");
|
||||
|
||||
@@ -1794,12 +1794,9 @@ async fn get_delete_and_quota_config_from_disk_in(
|
||||
if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() =>
|
||||
{
|
||||
if let Some(expected) = expiry_policy {
|
||||
let raw = &metadata.lifecycle_config_xml;
|
||||
let current = (!raw.is_empty())
|
||||
.then(|| super::utils::deserialize::<PersistedLifecycleConfiguration>(raw))
|
||||
.transpose()
|
||||
.map_err(|_| unreadable_config_error(bucket, BUCKET_LIFECYCLE_CONFIG, raw.len()))?;
|
||||
if !expected.matches_configuration(current.as_ref()) {
|
||||
let current = ConfigState::of(&metadata.lifecycle_config_xml, &metadata.lifecycle_config)
|
||||
.require(bucket, BUCKET_LIFECYCLE_CONFIG)?;
|
||||
if !expected.matches_configuration(current) {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
}
|
||||
@@ -1835,9 +1832,7 @@ pub(crate) async fn integrity_migration_metadata_in(
|
||||
BucketMetadataAuthority::Authoritative(metadata)
|
||||
if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() =>
|
||||
{
|
||||
let mut metadata = Arc::unwrap_or_clone(metadata);
|
||||
metadata.parse_all_configs()?;
|
||||
Ok(Arc::new(metadata))
|
||||
Ok(metadata)
|
||||
}
|
||||
BucketMetadataAuthority::MissingBucket => Err(Error::BucketNotFound(bucket.to_string())),
|
||||
_ => Err(Error::other("migration requires authoritative bucket metadata")),
|
||||
@@ -2612,10 +2607,6 @@ impl BucketMetadataSys {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn cached_metadata(&self) -> Vec<Arc<BucketMetadata>> {
|
||||
self.metadata_map.read().await.values().cloned().collect()
|
||||
}
|
||||
|
||||
pub async fn set(&self, bucket: String, bm: Arc<BucketMetadata>) {
|
||||
self.revoke_pending_bucket_creation(&bucket).await;
|
||||
bump_local_meta_mutation_gen();
|
||||
@@ -3517,7 +3508,7 @@ impl BucketMetadataSys {
|
||||
Some(guard),
|
||||
bucket,
|
||||
"bucket metadata authoritative snapshot",
|
||||
load_bucket_metadata_parse_with_presence(Arc::clone(&store), bucket, false),
|
||||
load_bucket_metadata_parse_with_presence(Arc::clone(&store), bucket, true),
|
||||
)),
|
||||
);
|
||||
match bucket_info_res {
|
||||
@@ -3526,12 +3517,11 @@ impl BucketMetadataSys {
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
|
||||
let (mut metadata, persisted) = metadata_res?;
|
||||
let (metadata, persisted) = metadata_res?;
|
||||
if persisted && metadata.lock_enabled && !metadata.bucket_creation_committed {
|
||||
return Err(Error::ErasureWriteQuorum);
|
||||
}
|
||||
if persisted {
|
||||
metadata.parse_admission_configs();
|
||||
let metadata = Arc::new(metadata);
|
||||
if can_use_local_authority_cache
|
||||
&& metadata.bucket_incarnation_sidecar
|
||||
@@ -5764,64 +5754,6 @@ mod tests {
|
||||
assert_eq!(tags.tag_set.len(), WRITERS, "every concurrent rewrite must survive: {tags:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn notification_snapshots_follow_periodic_refresh_after_lost_peer_signals() {
|
||||
let (dirs, store) = isolated_store_over_temp_disks().await;
|
||||
let writer = BucketMetadataSys::new(store.clone());
|
||||
let peer = Arc::new(RwLock::new(BucketMetadataSys::new(store.clone())));
|
||||
let bucket = "notification-periodic-snapshots";
|
||||
for dir in &dirs {
|
||||
std::fs::create_dir_all(dir.path().join(bucket)).expect("create empty fixture bucket");
|
||||
}
|
||||
let mut original = BucketMetadata::new(bucket);
|
||||
original.notification_config_xml = concat!(
|
||||
"<NotificationConfiguration><QueueConfiguration><Event>s3:ObjectCreated:Put</Event>",
|
||||
"<Queue>arn:rustfs:sqs:us-east-1:inert:webhook</Queue></QueueConfiguration></NotificationConfiguration>"
|
||||
)
|
||||
.as_bytes()
|
||||
.to_vec();
|
||||
original.parse_all_configs().expect("parse inert subscription");
|
||||
writer
|
||||
.persist_new_and_set(original)
|
||||
.await
|
||||
.expect("persist initial subscription");
|
||||
peer.read().await.reload_from_store(bucket).await.expect("prime peer cache");
|
||||
let old = peer.read().await.get(bucket).await.expect("cached original snapshot");
|
||||
|
||||
let mut removed = (*old).clone();
|
||||
removed.notification_config_xml = b"<NotificationConfiguration/>".to_vec();
|
||||
removed.parse_all_configs().expect("parse removed subscription");
|
||||
writer
|
||||
.persist_new_and_set(removed)
|
||||
.await
|
||||
.expect("persist removal without peer callback");
|
||||
assert!(Arc::ptr_eq(&old, &peer.read().await.get(bucket).await.expect("peer remains stale")));
|
||||
|
||||
// Exercise the timer's actual one-pass entry point, not the peer RPC.
|
||||
refresh_buckets_metadata_once(peer.clone()).await;
|
||||
let snapshots = peer.read().await.cached_metadata().await;
|
||||
assert_eq!(snapshots.len(), 1);
|
||||
assert!(!Arc::ptr_eq(&old, &snapshots[0]));
|
||||
assert_eq!(old.bucket_incarnation_id, snapshots[0].bucket_incarnation_id);
|
||||
assert_eq!(snapshots[0].notification_config_xml, b"<NotificationConfiguration/>");
|
||||
assert!(
|
||||
snapshots[0]
|
||||
.notification_config
|
||||
.as_ref()
|
||||
.expect("parsed removal")
|
||||
.behavior()
|
||||
.queue_routes
|
||||
.is_empty()
|
||||
);
|
||||
|
||||
for dir in &dirs {
|
||||
std::fs::remove_dir(dir.path().join(bucket)).expect("remove empty fixture bucket without peer callback");
|
||||
}
|
||||
refresh_buckets_metadata_once(peer.clone()).await;
|
||||
assert!(peer.read().await.cached_metadata().await.is_empty());
|
||||
assert!(peer.read().await.get(bucket).await.is_err());
|
||||
}
|
||||
|
||||
/// Pins the peer reload-notification contract (`reload_from_store`, the
|
||||
/// LoadBucketMetadata RPC path): only metadata actually read from
|
||||
/// persisted storage enters the cache. A load miss errors out and leaves
|
||||
|
||||
@@ -1607,7 +1607,6 @@ fn shard_object(ledger_object: &str, shard_index: u16, bucket_incarnation: Uuid,
|
||||
}
|
||||
|
||||
fn shard_index(object: &str) -> u16 {
|
||||
let object = rustfs_utils::path::encode_dir_object_ref(object);
|
||||
let digest = Sha256::digest(object.as_bytes());
|
||||
u16::from_be_bytes([digest[0], digest[1]]) % SHARDED_LEDGER_COUNT
|
||||
}
|
||||
@@ -2315,12 +2314,6 @@ fn log_deferred_settlement(data: &ReservationLedgerData, state: &'static str, er
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn quota_shard_agrees_with_directory_listing_name() {
|
||||
for listed in ["photos/", "reports/june/"] {
|
||||
assert_eq!(shard_index(listed), shard_index(&rustfs_utils::path::encode_dir_object(listed)));
|
||||
}
|
||||
}
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use serial_test::serial;
|
||||
|
||||
|
||||
@@ -256,64 +256,6 @@ pub(crate) fn remote_sdk_credentials(credentials: &RemoteCredentials, now: Syste
|
||||
Ok(builder.build())
|
||||
}
|
||||
|
||||
/// Bound the bytes Smithy can aggregate for a control document or service error.
|
||||
/// Successful GetObject responses are streamed directly to the caller.
|
||||
#[derive(Debug)]
|
||||
struct ControlResponseLimit(usize);
|
||||
|
||||
#[derive(Debug)]
|
||||
struct OversizedResponseRetryClassifier;
|
||||
|
||||
impl aws_smithy_runtime_api::client::retries::classifiers::ClassifyRetry for OversizedResponseRetryClassifier {
|
||||
fn classify_retry(
|
||||
&self,
|
||||
ctx: &aws_smithy_runtime_api::client::interceptors::context::InterceptorContext,
|
||||
) -> aws_smithy_runtime_api::client::retries::classifiers::RetryAction {
|
||||
use aws_smithy_runtime_api::client::retries::classifiers::RetryAction;
|
||||
if let Some(Err(error)) = ctx.output_or_error() {
|
||||
let mut cause = std::error::Error::source(error);
|
||||
while let Some(error) = cause {
|
||||
if error.is::<http_body_util::LengthLimitError>() {
|
||||
return RetryAction::RetryForbidden;
|
||||
}
|
||||
cause = error.source();
|
||||
}
|
||||
}
|
||||
RetryAction::NoActionIndicated
|
||||
}
|
||||
|
||||
fn name(&self) -> &'static str {
|
||||
"RustfsOversizedResponseRetryClassifier"
|
||||
}
|
||||
}
|
||||
|
||||
impl Intercept for ControlResponseLimit {
|
||||
fn name(&self) -> &'static str {
|
||||
"RustfsControlResponseLimit"
|
||||
}
|
||||
|
||||
fn modify_before_deserialization(
|
||||
&self,
|
||||
context: &mut BeforeDeserializationInterceptorContextMut<'_>,
|
||||
_runtime_components: &RuntimeComponents,
|
||||
cfg: &mut ConfigBag,
|
||||
) -> Result<(), BoxError> {
|
||||
let operation = cfg
|
||||
.load::<aws_smithy_runtime_api::client::orchestrator::Metadata>()
|
||||
.map(|m| m.name());
|
||||
bound_control_response(context.response_mut(), operation, self.0);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn bound_control_response(response: &mut HttpResponse, operation: Option<&str>, limit: usize) {
|
||||
if operation == Some("GetObject") && response.status().is_success() {
|
||||
return;
|
||||
}
|
||||
let body = std::mem::replace(response.body_mut(), SdkBody::taken());
|
||||
*response.body_mut() = SdkBody::from_body_1_x(http_body_util::Limited::new(body, limit));
|
||||
}
|
||||
|
||||
/// Only installed on non-streaming upload operations. The budget begins at
|
||||
/// response headers, independently of the potentially long request upload.
|
||||
#[derive(Debug)]
|
||||
@@ -433,8 +375,6 @@ pub async fn build_remote_s3_config(spec: &RemoteS3EndpointSpec) -> Result<aws_s
|
||||
.region(SdkRegion::new(spec.region.clone()))
|
||||
.behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
|
||||
.request_checksum_calculation(replication_request_checksum_calculation())
|
||||
.interceptor(ControlResponseLimit(16 * 1024 * 1024))
|
||||
.retry_classifier(OversizedResponseRetryClassifier)
|
||||
.retry_config(spec.retry.retry_config());
|
||||
|
||||
if spec.path_style.force_path_style() {
|
||||
@@ -845,33 +785,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remote_control_response_limit_preserves_object_streams() {
|
||||
use http_body_util::BodyExt;
|
||||
for (operation, status, bytes, allowed) in [
|
||||
(Some("ListObjectsV2"), 200, 4, true),
|
||||
(Some("ListObjectsV2"), 200, 5, false),
|
||||
(Some("GetObject"), 206, 5, true),
|
||||
(Some("GetObject"), 404, 5, false),
|
||||
(None, 200, 5, false),
|
||||
] {
|
||||
let mut response = HttpResponse::try_from(
|
||||
http::Response::builder()
|
||||
.status(status)
|
||||
.body(SdkBody::from(vec![1; bytes]))
|
||||
.expect("fixture response"),
|
||||
)
|
||||
.expect("Smithy response");
|
||||
bound_control_response(&mut response, operation, 4);
|
||||
let result = response.into_body().collect().await;
|
||||
if allowed {
|
||||
assert_eq!(result.expect("body is permitted").to_bytes().len(), bytes);
|
||||
} else {
|
||||
assert!(result.is_err(), "oversized control response must fail");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn build_remote_s3_client_rejects_loopback_and_metadata_endpoints() {
|
||||
// Default (no loopback opt-in): loopback in IPv4, IPv6 and hostname
|
||||
@@ -973,59 +886,6 @@ mod tests {
|
||||
wire_requests.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn sdk_remote_control_limit_stops_chunked_errors_and_preserves_downloads() {
|
||||
#[derive(Debug)]
|
||||
struct ChunkConnector(u16, Arc<AtomicUsize>);
|
||||
impl SmithyHttpConnector for ChunkConnector {
|
||||
fn call(&self, _: HttpRequest) -> HttpConnectorFuture {
|
||||
self.1.fetch_add(1, Ordering::SeqCst);
|
||||
let frames =
|
||||
[b"abc", b"def"].map(|s| Ok::<_, std::io::Error>(http_body::Frame::data(bytes::Bytes::from_static(s))));
|
||||
let body = http_body_util::StreamBody::new(futures::stream::iter(frames));
|
||||
HttpConnectorFuture::ready(Ok(HttpResponse::new(
|
||||
SmithyStatusCode::try_from(self.0).expect("fixture status"),
|
||||
SdkBody::from_body_1_x(body),
|
||||
)))
|
||||
}
|
||||
}
|
||||
for status in [200, 503] {
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let connector = SharedHttpConnector::new(ChunkConnector(status, calls.clone()));
|
||||
let mut target = spec("s3.example.com", true);
|
||||
target.retry = REPLICATION_TARGET_RETRY_POLICY;
|
||||
let config = build_remote_s3_config(&target)
|
||||
.await
|
||||
.expect("build remote config")
|
||||
.http_client(http_client_fn(move |_, _| connector.clone()))
|
||||
.interceptor(ControlResponseLimit(4))
|
||||
.build();
|
||||
let result = S3Client::from_conf(config)
|
||||
.get_object()
|
||||
.bucket("bucket")
|
||||
.key("key")
|
||||
.send()
|
||||
.await;
|
||||
if status == 200 {
|
||||
let bytes = result
|
||||
.expect("successful object stream")
|
||||
.body
|
||||
.collect()
|
||||
.await
|
||||
.expect("collect fixture object")
|
||||
.into_bytes();
|
||||
assert_eq!(&bytes[..], b"abcdef");
|
||||
} else {
|
||||
assert!(result.is_err(), "oversized error response must fail");
|
||||
}
|
||||
assert_eq!(
|
||||
calls.load(Ordering::SeqCst),
|
||||
1,
|
||||
"oversized control responses must not trigger SDK retries"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn retry_policy_decides_how_many_wire_requests_one_call_costs() {
|
||||
assert_eq!(
|
||||
@@ -1070,8 +930,8 @@ mod tests {
|
||||
.unwrap_or_else(|| panic!("signed request must contain {name}"))
|
||||
}
|
||||
|
||||
fn signing_time(headers: &[(String, String)]) -> jiff::Timestamp {
|
||||
jiff::Timestamp::strptime("%Y%m%dT%H%M%SZ", recorded_header(headers, "x-amz-date"))
|
||||
fn signing_time(headers: &[(String, String)]) -> chrono::NaiveDateTime {
|
||||
chrono::NaiveDateTime::parse_from_str(recorded_header(headers, "x-amz-date"), "%Y%m%dT%H%M%SZ")
|
||||
.expect("SDK signing timestamp must use the SigV4 format")
|
||||
}
|
||||
|
||||
@@ -1086,8 +946,8 @@ mod tests {
|
||||
.map(|(key, value)| (key.to_string(), value.to_string()))
|
||||
.collect(),
|
||||
);
|
||||
let server_time = jiff::Timestamp::try_from(self.clock.now()).expect("clock timestamp")
|
||||
+ jiff::SignedDuration::from_secs(self.skew_seconds);
|
||||
let server_time = chrono::DateTime::<chrono::Utc>::from(self.clock.now()).naive_utc()
|
||||
+ chrono::Duration::seconds(self.skew_seconds);
|
||||
let (status, body) = if headers.len() == 1 {
|
||||
(
|
||||
403,
|
||||
@@ -1098,7 +958,7 @@ mod tests {
|
||||
};
|
||||
let response = http::Response::builder()
|
||||
.status(status)
|
||||
.header("date", server_time.strftime("%a, %d %b %Y %H:%M:%S GMT").to_string())
|
||||
.header("date", server_time.format("%a, %d %b %Y %H:%M:%S GMT").to_string())
|
||||
.header("content-type", "application/xml")
|
||||
.header("content-length", body.len())
|
||||
.body(SdkBody::from(body))
|
||||
@@ -1136,7 +996,7 @@ mod tests {
|
||||
for error_code in ["RequestTimeTooSkewed", "SignatureDoesNotMatch"] {
|
||||
for skew_seconds in [-600, 600] {
|
||||
let (client, headers, clock) = clock_skew_client(error_code, skew_seconds, REPLICATION_TARGET_RETRY_POLICY).await;
|
||||
let initial = jiff::Timestamp::try_from(clock.now()).expect("clock timestamp");
|
||||
let initial = chrono::DateTime::<chrono::Utc>::from(clock.now()).naive_utc();
|
||||
client
|
||||
.get_object()
|
||||
.bucket("bucket")
|
||||
@@ -1163,12 +1023,12 @@ mod tests {
|
||||
assert_eq!(signing_time(&headers[0]), initial, "the first attempt must use the injected clock");
|
||||
assert_eq!(
|
||||
signing_time(&headers[1]),
|
||||
initial + jiff::SignedDuration::from_secs(skew_seconds),
|
||||
initial + chrono::Duration::seconds(skew_seconds),
|
||||
"{error_code}: retry must apply the measured offset exactly"
|
||||
);
|
||||
assert_eq!(
|
||||
signing_time(&headers[2]),
|
||||
initial + jiff::SignedDuration::from_secs(skew_seconds + 17),
|
||||
initial + chrono::Duration::seconds(skew_seconds + 17),
|
||||
"{error_code}: the next operation must apply cached skew to the advanced signing clock"
|
||||
);
|
||||
let signature = |index: usize| {
|
||||
|
||||
@@ -251,10 +251,10 @@ const METRIC_VERSION_IDENTITY_DRIFT_TOTAL: &str = "rustfs_replication_version_id
|
||||
/// the condition discoverable in any recent window.
|
||||
const VERSION_IDENTITY_DRIFT_LOG_INTERVAL: TokioDuration = TokioDuration::from_secs(600);
|
||||
|
||||
/// When each source bucket and ARN last reported version-identity drift. Throttling is
|
||||
/// When each target last reported version-identity drift, by ARN. Throttling is
|
||||
/// advisory only — the metric still counts every drifting PUT.
|
||||
type SourceTargetDriftTimes = HashMap<String, HashMap<String, Instant>>;
|
||||
static VERSION_IDENTITY_WARNED_ARNS: LazyLock<StdMutex<SourceTargetDriftTimes>> = LazyLock::new(|| StdMutex::new(HashMap::new()));
|
||||
static VERSION_IDENTITY_WARNED_ARNS: LazyLock<StdMutex<HashMap<String, Instant>>> =
|
||||
LazyLock::new(|| StdMutex::new(HashMap::new()));
|
||||
|
||||
/// Version purges a target refused for a reason a retry cannot change on its
|
||||
/// own. Two shapes today:
|
||||
@@ -395,13 +395,13 @@ fn audit_target_version_identity(tgt_client: &TargetClient, source_version_id: &
|
||||
// below (`replica_head_fallback`) knows whether a 404 on a
|
||||
// version-addressed HEAD can mean "replica missing" on this target.
|
||||
if let Some(capability) = version_identity_capability_from_put(source_version_id, assigned_version_id) {
|
||||
tgt_client.record_version_identity_capability(capability);
|
||||
ReplicationTargetStore::record_version_identity_capability(&tgt_client.arn, capability);
|
||||
}
|
||||
if !version_identity_drifted(source_version_id, assigned_version_id) {
|
||||
return;
|
||||
}
|
||||
counter!(METRIC_VERSION_IDENTITY_DRIFT_TOTAL).increment(1);
|
||||
if !version_identity_drift_log_due(&tgt_client.source_bucket, &tgt_client.arn, Instant::now()) {
|
||||
if !version_identity_drift_log_due(&tgt_client.arn, Instant::now()) {
|
||||
return;
|
||||
}
|
||||
// `error`, not `warn`: the target silently refuses the addressing scheme
|
||||
@@ -424,14 +424,14 @@ fn audit_target_version_identity(tgt_client: &TargetClient, source_version_id: &
|
||||
/// Whether this ARN's version-identity drift is due to be logged again at
|
||||
/// `now`, re-arming the throttle when it is. Split out from the audit so the
|
||||
/// interval policy is testable without a target client.
|
||||
fn version_identity_drift_log_due(bucket: &str, arn: &str, now: Instant) -> bool {
|
||||
fn version_identity_drift_log_due(arn: &str, now: Instant) -> bool {
|
||||
let mut warned = VERSION_IDENTITY_WARNED_ARNS
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
match warned.get(bucket).and_then(|entries| entries.get(arn)) {
|
||||
match warned.get(arn) {
|
||||
Some(last) if now.duration_since(*last) < VERSION_IDENTITY_DRIFT_LOG_INTERVAL => false,
|
||||
_ => {
|
||||
warned.entry(bucket.to_owned()).or_default().insert(arn.to_owned(), now);
|
||||
warned.insert(arn.to_string(), now);
|
||||
true
|
||||
}
|
||||
}
|
||||
@@ -697,7 +697,7 @@ async fn replica_head_fallback(
|
||||
if let Some(ledger_version_id) = ledger_version_id {
|
||||
// A ledger entry is direct evidence of the identity contract; it
|
||||
// survives a restart while the runtime verdict cache does not.
|
||||
tgt_client.record_version_identity_capability(VersionIdentityCapability::MintsOwn);
|
||||
ReplicationTargetStore::record_version_identity_capability(&tgt_client.arn, VersionIdentityCapability::MintsOwn);
|
||||
match head_object_for_worker(tgt_client, &tgt_client.bucket, object, Some(ledger_version_id.to_string())).await {
|
||||
Ok(head) => {
|
||||
return Some(Ok(Some(LocatedReplica {
|
||||
@@ -717,7 +717,7 @@ async fn replica_head_fallback(
|
||||
if !(encrypted && content_identity.is_some()) {
|
||||
return Some(head_object_fallback(tgt_client, object).await);
|
||||
}
|
||||
} else if !tgt_client.version_identity_capability().version_addressing_unreliable() {
|
||||
} else if !ReplicationTargetStore::version_identity_capability(&tgt_client.arn).version_addressing_unreliable() {
|
||||
return None;
|
||||
}
|
||||
let siblings = content_identity?;
|
||||
@@ -989,14 +989,14 @@ async fn sync_drifted_replica_metadata(
|
||||
/// (no PUT is ever sent — the object stays on the normal MRF retry channel
|
||||
/// and re-audits once the verdict's TTL expires or replication-check
|
||||
/// re-probes the target) and returns `None`.
|
||||
fn resolve_ssec_passthrough_gate(
|
||||
async fn resolve_ssec_passthrough_gate(
|
||||
ssec: bool,
|
||||
tgt_client: &TargetClient,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
rinfo: &mut ReplicatedTargetInfo,
|
||||
) -> Option<bool> {
|
||||
let (capability, expired) = tgt_client.ssec_passthrough_capability();
|
||||
let (capability, expired) = ReplicationTargetStore::ssec_passthrough_capability(&tgt_client.arn).await;
|
||||
match ssec_passthrough_gate(ssec, capability, expired) {
|
||||
SsecPassthroughGate::Proceed => Some(false),
|
||||
SsecPassthroughGate::ProceedWithAudit => Some(true),
|
||||
@@ -1023,7 +1023,7 @@ fn resolve_ssec_passthrough_gate(
|
||||
/// capability verdict for the target. Returns true when the SSE-C material
|
||||
/// provably survived; otherwise records `Unsupported` and settles `rinfo` as
|
||||
/// FAILED so the attempt never reports a silently unreadable COMPLETED.
|
||||
fn settle_ssec_passthrough_evidence(
|
||||
async fn settle_ssec_passthrough_evidence(
|
||||
head: &HeadObjectOutput,
|
||||
tgt_client: &TargetClient,
|
||||
bucket: &str,
|
||||
@@ -1031,10 +1031,10 @@ fn settle_ssec_passthrough_evidence(
|
||||
rinfo: &mut ReplicatedTargetInfo,
|
||||
) -> bool {
|
||||
if ssec_passthrough_evidence_present(head) {
|
||||
tgt_client.record_ssec_passthrough_capability(SsecPassthroughCapability::Supported);
|
||||
ReplicationTargetStore::record_ssec_passthrough_capability(&tgt_client.arn, SsecPassthroughCapability::Supported).await;
|
||||
return true;
|
||||
}
|
||||
tgt_client.record_ssec_passthrough_capability(SsecPassthroughCapability::Unsupported);
|
||||
ReplicationTargetStore::record_ssec_passthrough_capability(&tgt_client.arn, SsecPassthroughCapability::Unsupported).await;
|
||||
rinfo.replication_status = ReplicationStatusType::Failed;
|
||||
rinfo.error = Some(ERR_REPLICATION_SSEC_PASSTHROUGH_UNSUPPORTED.to_string());
|
||||
warn!(
|
||||
@@ -1070,7 +1070,7 @@ async fn audit_ssec_passthrough_replica(
|
||||
// version, so the audit HEAD does not 4xx-loop on those objects.
|
||||
let version_id = resolve_read_api_version_id(version_id);
|
||||
match head_object_for_worker(tgt_client.as_ref(), &tgt_client.bucket, object, version_id).await {
|
||||
Ok(head) => settle_ssec_passthrough_evidence(&head, tgt_client, bucket, object, rinfo),
|
||||
Ok(head) => settle_ssec_passthrough_evidence(&head, tgt_client, bucket, object, rinfo).await,
|
||||
Err(e) => {
|
||||
rinfo.replication_status = ReplicationStatusType::Failed;
|
||||
rinfo.error = Some(format!("SSE-C passthrough audit HEAD failed: {e}"));
|
||||
@@ -1967,7 +1967,8 @@ async fn verify_resync_head_result(
|
||||
// source marker id; verify the marker on the latest version.
|
||||
DeleteMarkerHeadOutcome::Ambiguous => {
|
||||
let mints_own = rejects_source_version_id(&err)
|
||||
|| target_client.version_identity_capability().version_addressing_unreliable();
|
||||
|| ReplicationTargetStore::version_identity_capability(&target_client.arn)
|
||||
.version_addressing_unreliable();
|
||||
!(mints_own && latest_target_version_is_delete_marker(target_client, &roi.name).await)
|
||||
}
|
||||
DeleteMarkerHeadOutcome::Exists | DeleteMarkerHeadOutcome::Missing => false,
|
||||
@@ -3664,10 +3665,10 @@ async fn resolve_purge_replica(
|
||||
return PurgeReplicaResolution::LedgerCorrupt;
|
||||
}
|
||||
if let Some(assigned) = source.ledger.get(&tgt_client.arn) {
|
||||
tgt_client.record_version_identity_capability(VersionIdentityCapability::MintsOwn);
|
||||
ReplicationTargetStore::record_version_identity_capability(&tgt_client.arn, VersionIdentityCapability::MintsOwn);
|
||||
return PurgeReplicaResolution::Resolved(assigned.clone());
|
||||
}
|
||||
if !force_lookup && !tgt_client.version_identity_capability().version_addressing_unreliable() {
|
||||
if !force_lookup && !ReplicationTargetStore::version_identity_capability(&tgt_client.arn).version_addressing_unreliable() {
|
||||
return PurgeReplicaResolution::SourceVersion;
|
||||
}
|
||||
let etag = source.etag.as_deref().filter(|etag| !etag.trim().is_empty());
|
||||
@@ -3827,7 +3828,8 @@ async fn replicate_delete_to_target(
|
||||
// A target that mints its own version ids never holds the source marker
|
||||
// id, so a version-addressed pre-check cannot succeed there; the marker
|
||||
// is created by the versionless DELETE below.
|
||||
let version_addressable = !tgt_client.version_identity_capability().version_addressing_unreliable();
|
||||
let version_addressable =
|
||||
!ReplicationTargetStore::version_identity_capability(&tgt_client.arn).version_addressing_unreliable();
|
||||
if version_addressable
|
||||
&& !is_version_purge
|
||||
&& dobj.delete_object.delete_marker
|
||||
@@ -3844,7 +3846,7 @@ async fn replicate_delete_to_target(
|
||||
// The verdict is not known yet (e.g. no PUT since restart) and the
|
||||
// target rejected the source id: learn it as the PUT path does.
|
||||
Err(e) if rejects_source_version_id(e.as_ref()) => {
|
||||
tgt_client.record_version_identity_capability(VersionIdentityCapability::MintsOwn);
|
||||
ReplicationTargetStore::record_version_identity_capability(&tgt_client.arn, VersionIdentityCapability::MintsOwn);
|
||||
}
|
||||
Ok(head) if head.delete_marker == Some(true) => {
|
||||
// Some targets expose an existing marker as a successful HEAD
|
||||
@@ -3920,7 +3922,7 @@ async fn replicate_delete_to_target(
|
||||
let object = &dobj.delete_object.object_name;
|
||||
match resolve_purge_replica(&tgt_client, object, source, true).await {
|
||||
PurgeReplicaResolution::Resolved(assigned) => {
|
||||
tgt_client.record_version_identity_capability(VersionIdentityCapability::MintsOwn);
|
||||
ReplicationTargetStore::record_version_identity_capability(&tgt_client.arn, VersionIdentityCapability::MintsOwn);
|
||||
counter!(METRIC_VERSION_PURGE_REPLICA_TOTAL, "resolution" => "resolved").increment(1);
|
||||
version_id = Some(assigned);
|
||||
removed = tgt_client
|
||||
@@ -4624,7 +4626,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
// N2 fail-closed: never PUT SSE-C ciphertext at a target known to drop
|
||||
// the passthrough transport headers, and never trust a convergence HEAD
|
||||
// against such a target — a previous broken replica matches by ETag.
|
||||
let Some(ssec_audit_required) = resolve_ssec_passthrough_gate(self.ssec, &tgt_client, &bucket, &object, &mut rinfo)
|
||||
let Some(ssec_audit_required) = resolve_ssec_passthrough_gate(self.ssec, &tgt_client, &bucket, &object, &mut rinfo).await
|
||||
else {
|
||||
send_local_event(EventArgs {
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
@@ -4745,7 +4747,9 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
// An SSE-C replica only counts as converged when the same
|
||||
// HEAD proves its decryption material survived; a broken
|
||||
// ciphertext copy from an earlier attempt matches by ETag.
|
||||
if ssec_audit_required && !settle_ssec_passthrough_evidence(&oi, &tgt_client, &bucket, &object, &mut rinfo) {
|
||||
if ssec_audit_required
|
||||
&& !settle_ssec_passthrough_evidence(&oi, &tgt_client, &bucket, &object, &mut rinfo).await
|
||||
{
|
||||
return rinfo;
|
||||
}
|
||||
rinfo.replication_status = ReplicationStatusType::Completed;
|
||||
@@ -4780,6 +4784,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
Ok(Some(located)) if located_replica_matches(identity.as_ref(), &located) => {
|
||||
if ssec_audit_required
|
||||
&& !settle_ssec_passthrough_evidence(&located.head, &tgt_client, &bucket, &object, &mut rinfo)
|
||||
.await
|
||||
{
|
||||
return rinfo;
|
||||
}
|
||||
@@ -4979,7 +4984,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
|
||||
// N2 fail-closed: see the gate in `replicate_object` — the same policy
|
||||
// applies to the metadata/existing-object transport.
|
||||
let Some(ssec_audit_required) = resolve_ssec_passthrough_gate(self.ssec, &tgt_client, &bucket, &object, &mut rinfo)
|
||||
let Some(ssec_audit_required) = resolve_ssec_passthrough_gate(self.ssec, &tgt_client, &bucket, &object, &mut rinfo).await
|
||||
else {
|
||||
send_local_event(EventArgs {
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
@@ -5352,7 +5357,7 @@ async fn resolve_replicate_all_action(
|
||||
// An SSE-C replica only counts as converged when the same HEAD
|
||||
// proves its decryption material survived; a broken ciphertext
|
||||
// copy from an earlier attempt matches by ETag.
|
||||
if ssec_audit_required && !settle_ssec_passthrough_evidence(&oi, tgt_client, bucket, object, rinfo) {
|
||||
if ssec_audit_required && !settle_ssec_passthrough_evidence(&oi, tgt_client, bucket, object, rinfo).await {
|
||||
rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs();
|
||||
return None;
|
||||
}
|
||||
@@ -5427,7 +5432,7 @@ async fn resolve_replicate_all_action(
|
||||
}
|
||||
if replica_matches
|
||||
&& ssec_audit_required
|
||||
&& !settle_ssec_passthrough_evidence(oi, tgt_client, bucket, object, rinfo)
|
||||
&& !settle_ssec_passthrough_evidence(oi, tgt_client, bucket, object, rinfo).await
|
||||
{
|
||||
rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs();
|
||||
return None;
|
||||
@@ -6701,8 +6706,6 @@ mod tests {
|
||||
.build();
|
||||
|
||||
Arc::new(TargetClient {
|
||||
source_bucket: "source".to_owned(),
|
||||
capabilities: Default::default(),
|
||||
endpoint,
|
||||
credentials: None,
|
||||
bucket: "target-bucket".to_string(),
|
||||
@@ -6907,7 +6910,7 @@ mod tests {
|
||||
async fn heal_redrive_locates_replica_by_etag_on_target_that_mints_own_version_ids() {
|
||||
let (endpoint, server) = spawn_drifted_target_server(3);
|
||||
let target = test_target_client(endpoint);
|
||||
target.record_version_identity_capability(VersionIdentityCapability::MintsOwn);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let (roi, object_info) = drifted_roi_and_object();
|
||||
let mut rinfo = replicate_all_target_info(&roi, &target);
|
||||
|
||||
@@ -6976,7 +6979,7 @@ mod tests {
|
||||
async fn resync_verification_counts_drifted_replica_as_replicated() {
|
||||
let (endpoint, server) = spawn_drifted_target_server(3);
|
||||
let target = test_target_client(endpoint);
|
||||
target.record_version_identity_capability(VersionIdentityCapability::MintsOwn);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let (roi, _) = drifted_roi_and_object();
|
||||
let mut st = TargetReplicationResyncStatus::default();
|
||||
|
||||
@@ -7090,7 +7093,7 @@ mod tests {
|
||||
static VERSIONS: &[(&str, i64)] = &[(DRIFTED_ASSIGNED_VERSION_ID, 4)];
|
||||
let (endpoint, server) = spawn_computed_etag_target(3, VERSIONS);
|
||||
let target = test_target_client(endpoint);
|
||||
target.record_version_identity_capability(VersionIdentityCapability::MintsOwn);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let (roi, object_info) = encrypted_roi_and_object(replica_written_secs() - 1);
|
||||
let mut rinfo = replicate_all_target_info(&roi, &target);
|
||||
|
||||
@@ -7112,7 +7115,7 @@ mod tests {
|
||||
static OTHER_SIZE: &[(&str, i64)] = &[(DRIFTED_ASSIGNED_VERSION_ID, 36)];
|
||||
let (endpoint, server) = spawn_computed_etag_target(2, OTHER_SIZE);
|
||||
let target = test_target_client(endpoint);
|
||||
target.record_version_identity_capability(VersionIdentityCapability::MintsOwn);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let (roi, object_info) = encrypted_roi_and_object(replica_written_secs() - 1);
|
||||
let mut rinfo = replicate_all_target_info(&roi, &target);
|
||||
let action = resolve_heal_action(&target, &roi, object_info, &mut rinfo).await;
|
||||
@@ -7123,7 +7126,7 @@ mod tests {
|
||||
static SAME_SIZE: &[(&str, i64)] = &[(DRIFTED_ASSIGNED_VERSION_ID, 4)];
|
||||
let (endpoint, server) = spawn_computed_etag_target(2, SAME_SIZE);
|
||||
let target = test_target_client(endpoint);
|
||||
target.record_version_identity_capability(VersionIdentityCapability::MintsOwn);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let (roi, object_info) = encrypted_roi_and_object(replica_written_secs() + 1);
|
||||
let mut rinfo = replicate_all_target_info(&roi, &target);
|
||||
let action = resolve_heal_action(&target, &roi, object_info, &mut rinfo).await;
|
||||
@@ -7136,7 +7139,7 @@ mod tests {
|
||||
static VERSIONS: &[(&str, i64)] = &[(DRIFTED_ASSIGNED_VERSION_ID, 4), ("another-minted-id", 4)];
|
||||
let (endpoint, server) = spawn_computed_etag_target(2, VERSIONS);
|
||||
let target = test_target_client(endpoint);
|
||||
target.record_version_identity_capability(VersionIdentityCapability::MintsOwn);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let (roi, object_info) = encrypted_roi_and_object(replica_written_secs() - 1);
|
||||
let mut rinfo = replicate_all_target_info(&roi, &target);
|
||||
|
||||
@@ -7158,7 +7161,7 @@ mod tests {
|
||||
static VERSIONS: &[(&str, i64)] = &[(DRIFTED_ASSIGNED_VERSION_ID, 4)];
|
||||
let (endpoint, server) = spawn_computed_etag_target(2, VERSIONS);
|
||||
let target = test_target_client(endpoint);
|
||||
target.record_version_identity_capability(VersionIdentityCapability::MintsOwn);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let (roi, mut object_info) = drifted_roi_and_object();
|
||||
object_info.mod_time = Some(OffsetDateTime::from_unix_timestamp(replica_written_secs() - 1).expect("valid timestamp"));
|
||||
let mut rinfo = replicate_all_target_info(&roi, &target);
|
||||
@@ -7176,7 +7179,7 @@ mod tests {
|
||||
static VERSIONS: &[(&str, i64)] = &[("null", 4)];
|
||||
let (endpoint, server) = spawn_computed_etag_target(2, VERSIONS);
|
||||
let target = test_target_client(endpoint);
|
||||
target.record_version_identity_capability(VersionIdentityCapability::MintsOwn);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let (roi, object_info) = encrypted_roi_and_object(replica_written_secs() - 1);
|
||||
let mut rinfo = replicate_all_target_info(&roi, &target);
|
||||
|
||||
@@ -7261,7 +7264,7 @@ mod tests {
|
||||
static VERSIONS: &[(&str, i64)] = &[(DRIFTED_ASSIGNED_VERSION_ID, 4)];
|
||||
let (endpoint, server) = spawn_computed_etag_target(3, VERSIONS);
|
||||
let target = test_target_client(endpoint);
|
||||
target.record_version_identity_capability(VersionIdentityCapability::MintsOwn);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let (roi, object_info) = encrypted_roi_and_object(replica_written_secs() - 1);
|
||||
let mut st = TargetReplicationResyncStatus::default();
|
||||
|
||||
@@ -7301,7 +7304,7 @@ mod tests {
|
||||
});
|
||||
let target = test_target_client(endpoint);
|
||||
register_test_target(&target).await;
|
||||
target.record_version_identity_capability(VersionIdentityCapability::MintsOwn);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let dobj = version_purge_dobj(&target.arn);
|
||||
|
||||
let rinfo = replicate_delete_to_target(&dobj, target.clone(), Some(encrypted_purge_source())).await;
|
||||
@@ -7327,7 +7330,7 @@ mod tests {
|
||||
});
|
||||
let target = test_target_client(endpoint);
|
||||
register_test_target(&target).await;
|
||||
target.record_version_identity_capability(VersionIdentityCapability::MintsOwn);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let dobj = version_purge_dobj(&target.arn);
|
||||
|
||||
let rinfo = replicate_delete_to_target(&dobj, target.clone(), Some(encrypted_purge_source())).await;
|
||||
@@ -7474,7 +7477,10 @@ mod tests {
|
||||
let rinfo = replicate_delete_to_target(&dobj, target.clone(), None).await;
|
||||
|
||||
assert_eq!(rinfo.replication_status, ReplicationStatusType::Completed, "{rinfo:?}");
|
||||
assert_eq!(target.version_identity_capability(), VersionIdentityCapability::MintsOwn);
|
||||
assert_eq!(
|
||||
ReplicationTargetStore::version_identity_capability(&target.arn),
|
||||
VersionIdentityCapability::MintsOwn
|
||||
);
|
||||
let seen = server.join().expect("test HTTP server should finish");
|
||||
assert_eq!(seen.len(), 2, "{seen:?}");
|
||||
assert!(seen[0].starts_with("HEAD "), "{seen:?}");
|
||||
@@ -7503,7 +7509,10 @@ mod tests {
|
||||
let rinfo = replicate_delete_to_target(&dobj, target.clone(), None).await;
|
||||
|
||||
assert_eq!(rinfo.replication_status, ReplicationStatusType::Failed, "{rinfo:?}");
|
||||
assert_ne!(target.version_identity_capability(), VersionIdentityCapability::MintsOwn);
|
||||
assert_ne!(
|
||||
ReplicationTargetStore::version_identity_capability(&target.arn),
|
||||
VersionIdentityCapability::MintsOwn
|
||||
);
|
||||
let seen = server.join().expect("test HTTP server should finish");
|
||||
assert_eq!(seen.len(), 1, "a denied HEAD must not trigger DELETE: {seen:?}");
|
||||
assert!(seen[0].starts_with("HEAD "), "{seen:?}");
|
||||
@@ -7522,7 +7531,7 @@ mod tests {
|
||||
});
|
||||
let target = test_target_client(endpoint);
|
||||
register_test_target(&target).await;
|
||||
target.record_version_identity_capability(VersionIdentityCapability::MintsOwn);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let dobj = marker_creation_dobj(&target.arn);
|
||||
|
||||
let rinfo = replicate_delete_to_target(&dobj, target.clone(), None).await;
|
||||
@@ -7884,7 +7893,7 @@ mod tests {
|
||||
seen[0]
|
||||
);
|
||||
assert!(
|
||||
target.version_identity_capability().version_addressing_unreliable(),
|
||||
ReplicationTargetStore::version_identity_capability(&target.arn).version_addressing_unreliable(),
|
||||
"a ledger entry proves the target mints its own ids"
|
||||
);
|
||||
}
|
||||
@@ -7902,7 +7911,7 @@ mod tests {
|
||||
});
|
||||
let target = test_target_client(endpoint);
|
||||
register_test_target(&target).await;
|
||||
target.record_version_identity_capability(VersionIdentityCapability::MintsOwn);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let dobj = version_purge_dobj(&target.arn);
|
||||
|
||||
let rinfo = replicate_delete_to_target(&dobj, target.clone(), Some(purge_source(&[]))).await;
|
||||
@@ -7929,7 +7938,7 @@ mod tests {
|
||||
});
|
||||
let target = test_target_client(endpoint);
|
||||
register_test_target(&target).await;
|
||||
target.record_version_identity_capability(VersionIdentityCapability::MintsOwn);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let dobj = version_purge_dobj(&target.arn);
|
||||
assert!(!purge_backoff_active(&dobj));
|
||||
|
||||
@@ -7957,7 +7966,7 @@ mod tests {
|
||||
});
|
||||
let target = test_target_client(endpoint);
|
||||
register_test_target(&target).await;
|
||||
target.record_version_identity_capability(VersionIdentityCapability::MintsOwn);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let dobj = version_purge_dobj(&target.arn);
|
||||
|
||||
let rinfo = replicate_delete_to_target(&dobj, target.clone(), Some(purge_source(&[]))).await;
|
||||
@@ -8029,7 +8038,7 @@ mod tests {
|
||||
assert!(seen[1].starts_with("GET "), "{}", seen[1]);
|
||||
assert!(seen[2].contains(DRIFTED_ASSIGNED_VERSION_ID), "{}", seen[2]);
|
||||
assert!(
|
||||
target.version_identity_capability().version_addressing_unreliable(),
|
||||
ReplicationTargetStore::version_identity_capability(&target.arn).version_addressing_unreliable(),
|
||||
"the located replica proves the target mints its own ids"
|
||||
);
|
||||
}
|
||||
@@ -8049,7 +8058,7 @@ mod tests {
|
||||
});
|
||||
let target = test_target_client(endpoint);
|
||||
register_test_target(&target).await;
|
||||
target.record_version_identity_capability(VersionIdentityCapability::MintsOwn);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let dobj = version_purge_dobj(&target.arn);
|
||||
let source = Arc::new(VersionPurgeSource {
|
||||
etag: Some(DRIFTED_ETAG.to_string()),
|
||||
@@ -8090,7 +8099,7 @@ mod tests {
|
||||
}
|
||||
});
|
||||
let target = test_target_client(endpoint);
|
||||
target.record_version_identity_capability(VersionIdentityCapability::MintsOwn);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let (roi, object_info) = drifted_roi_and_object();
|
||||
let mut rinfo = replicate_all_target_info(&roi, &target);
|
||||
|
||||
@@ -8241,12 +8250,21 @@ mod tests {
|
||||
let target = test_target_client("http://127.0.0.1:1".to_string());
|
||||
let source = Uuid::new_v4().to_string();
|
||||
audit_target_version_identity(&target, &source, Some(DRIFTED_ASSIGNED_VERSION_ID));
|
||||
assert_eq!(target.version_identity_capability(), VersionIdentityCapability::MintsOwn);
|
||||
assert_eq!(
|
||||
ReplicationTargetStore::version_identity_capability(&target.arn),
|
||||
VersionIdentityCapability::MintsOwn
|
||||
);
|
||||
audit_target_version_identity(&target, &source, Some(&source));
|
||||
assert_eq!(target.version_identity_capability(), VersionIdentityCapability::Adopts);
|
||||
assert_eq!(
|
||||
ReplicationTargetStore::version_identity_capability(&target.arn),
|
||||
VersionIdentityCapability::Adopts
|
||||
);
|
||||
// An unversioned write carries no contract and must not overwrite it.
|
||||
audit_target_version_identity(&target, "null", None);
|
||||
assert_eq!(target.version_identity_capability(), VersionIdentityCapability::Adopts);
|
||||
assert_eq!(
|
||||
ReplicationTargetStore::version_identity_capability(&target.arn),
|
||||
VersionIdentityCapability::Adopts
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -9432,13 +9450,13 @@ mod tests {
|
||||
let arn = "arn:replication::drift-throttle-test";
|
||||
let start = Instant::now();
|
||||
|
||||
assert!(version_identity_drift_log_due("source", arn, start), "first drift must be reported");
|
||||
assert!(version_identity_drift_log_due(arn, start), "first drift must be reported");
|
||||
assert!(
|
||||
!version_identity_drift_log_due("source", arn, start + VERSION_IDENTITY_DRIFT_LOG_INTERVAL / 2),
|
||||
!version_identity_drift_log_due(arn, start + VERSION_IDENTITY_DRIFT_LOG_INTERVAL / 2),
|
||||
"a second drift inside the interval must stay throttled"
|
||||
);
|
||||
assert!(
|
||||
version_identity_drift_log_due("source", arn, start + VERSION_IDENTITY_DRIFT_LOG_INTERVAL),
|
||||
version_identity_drift_log_due(arn, start + VERSION_IDENTITY_DRIFT_LOG_INTERVAL),
|
||||
"drift must become visible again once the interval elapses, instead of \
|
||||
going silent for the rest of the process lifetime"
|
||||
);
|
||||
@@ -9448,10 +9466,9 @@ mod tests {
|
||||
fn version_identity_drift_throttles_each_target_independently() {
|
||||
let now = Instant::now();
|
||||
|
||||
assert!(version_identity_drift_log_due("source", "arn:replication::drift-a", now));
|
||||
assert!(version_identity_drift_log_due("other-source", "arn:replication::drift-a", now));
|
||||
assert!(version_identity_drift_log_due("arn:replication::drift-a", now));
|
||||
assert!(
|
||||
version_identity_drift_log_due("source", "arn:replication::drift-b", now),
|
||||
version_identity_drift_log_due("arn:replication::drift-b", now),
|
||||
"one target's report must not silence another's"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -197,9 +197,31 @@ impl ReplicationTargetStore {
|
||||
BucketTargetSys::get().mark_target_offline(target_client).await
|
||||
}
|
||||
|
||||
/// Returns the cached verdict and whether it has outlived its TTL.
|
||||
pub(crate) async fn ssec_passthrough_capability(arn: &str) -> (SsecPassthroughCapability, bool) {
|
||||
BucketTargetSys::get().ssec_passthrough_capability(arn).await
|
||||
}
|
||||
|
||||
pub(crate) async fn record_ssec_passthrough_capability(arn: &str, capability: SsecPassthroughCapability) {
|
||||
BucketTargetSys::get()
|
||||
.record_ssec_passthrough_capability(arn, capability)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) fn version_identity_capability(arn: &str) -> VersionIdentityCapability {
|
||||
BucketTargetSys::get().version_identity_capability(arn)
|
||||
}
|
||||
|
||||
pub(crate) fn record_version_identity_capability(arn: &str, capability: VersionIdentityCapability) {
|
||||
BucketTargetSys::get().record_version_identity_capability(arn, capability)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn register_test_target(target_client: &Arc<TargetClient>) {
|
||||
BucketTargetSys::get().register_test_client(target_client).await;
|
||||
BucketTargetSys::get().arn_remotes_map.write().await.insert(
|
||||
target_client.arn.clone(),
|
||||
crate::bucket::bucket_target_sys::ArnTarget::with_client(target_client.clone()),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -28,7 +28,7 @@ const REDACTED_CREDENTIAL: &str = "<redacted>";
|
||||
const GO_YEAR_ONE_START_UNIX_SECONDS: i64 = -62_135_596_800;
|
||||
const GO_YEAR_TWO_START_UNIX_SECONDS: i64 = -62_104_060_800;
|
||||
|
||||
#[derive(Deserialize, Serialize, Default, Clone, PartialEq, Eq)]
|
||||
#[derive(Deserialize, Serialize, Default, Clone)]
|
||||
pub struct Credentials {
|
||||
#[serde(rename = "accessKey")]
|
||||
pub access_key: String,
|
||||
|
||||
@@ -73,11 +73,10 @@ use rustfs_protos::proto_gen::node_service::{
|
||||
PreparePartTransactionRequest, ReadAllRequest, ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse,
|
||||
ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RecoverDeleteTransactionRequest,
|
||||
RecoverMultipartSourceRequest as RecoverMultipartSourceRpcRequest,
|
||||
RecoverMultipartSourceResponse as RecoverMultipartSourceRpcResponse, RenameDataBatchItem, RenameDataBatchRequest,
|
||||
RenameDataBatchResult, RenameDataRequest, RenameFileRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest,
|
||||
SnapshotLeaseRenewRequest, SnapshotLeaseRequest, SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest,
|
||||
VerifyFileRequest, WriteAllRequest, WriteAllResponse, WriteFormatControlRequest, WriteMetadataRequest,
|
||||
node_service_client::NodeServiceClient,
|
||||
RecoverMultipartSourceResponse as RecoverMultipartSourceRpcResponse, RenameDataRequest, RenameFileRequest,
|
||||
SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest,
|
||||
SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteAllResponse,
|
||||
WriteFormatControlRequest, WriteMetadataRequest, node_service_client::NodeServiceClient,
|
||||
};
|
||||
use rustfs_protos::{
|
||||
ChannelClass, ChannelRebuildReason, ConnectionEvictionLogLevel, evict_failed_connection_with_reason, prewarm_channel_class,
|
||||
@@ -105,7 +104,7 @@ use tokio::{
|
||||
time::timeout,
|
||||
};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tonic::{Code, Request, Streaming, service::interceptor::InterceptedService};
|
||||
use tonic::{Code, Request, service::interceptor::InterceptedService};
|
||||
use tracing::{debug, trace, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -1046,7 +1045,6 @@ pub struct RemoteDisk {
|
||||
/// Optional per-peer gate held through complete bulk-unary decoding and validation.
|
||||
bulk_body_admission: Option<Arc<Semaphore>>,
|
||||
bulk_body_wait_timeout: Duration,
|
||||
rename_admission: Option<Arc<Semaphore>>,
|
||||
}
|
||||
|
||||
/// Shortest gap between two failure warnings of one kind from one remote disk.
|
||||
@@ -1062,251 +1060,6 @@ struct BulkBodyAdmissionRegistry {
|
||||
static BULK_BODY_ADMISSION_REGISTRY: std::sync::LazyLock<StdMutex<BulkBodyAdmissionRegistry>> =
|
||||
std::sync::LazyLock::new(|| StdMutex::new(BulkBodyAdmissionRegistry::default()));
|
||||
|
||||
const ENV_RUSTFS_EXPERIMENTAL_RENAME_STREAM_BATCH: &str = "RUSTFS_EXPERIMENTAL_RENAME_STREAM_BATCH";
|
||||
const RENAME_ADMISSION_REGISTRY_CAPACITY: usize = 512;
|
||||
|
||||
#[derive(Default)]
|
||||
struct RenameAdmissionRegistry {
|
||||
peers: HashMap<String, Weak<Semaphore>>,
|
||||
overflow: Option<Arc<Semaphore>>,
|
||||
}
|
||||
|
||||
static RENAME_ADMISSION_REGISTRY: std::sync::LazyLock<StdMutex<RenameAdmissionRegistry>> =
|
||||
std::sync::LazyLock::new(|| StdMutex::new(RenameAdmissionRegistry::default()));
|
||||
|
||||
fn peer_rename_admission(addr: &str, enabled: bool) -> Option<Arc<Semaphore>> {
|
||||
if !enabled {
|
||||
return None;
|
||||
}
|
||||
let mut registry = RENAME_ADMISSION_REGISTRY
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
if let Some(semaphore) = registry.peers.get(addr).and_then(Weak::upgrade) {
|
||||
return Some(semaphore);
|
||||
}
|
||||
if registry.peers.len() >= RENAME_ADMISSION_REGISTRY_CAPACITY {
|
||||
registry.peers.retain(|_, semaphore| semaphore.strong_count() > 0);
|
||||
}
|
||||
if registry.peers.len() >= RENAME_ADMISSION_REGISTRY_CAPACITY {
|
||||
return Some(Arc::clone(registry.overflow.get_or_insert_with(|| {
|
||||
Arc::new(Semaphore::new(rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT))
|
||||
})));
|
||||
}
|
||||
let semaphore = Arc::new(Semaphore::new(rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT));
|
||||
registry.peers.insert(addr.to_owned(), Arc::downgrade(&semaphore));
|
||||
Some(semaphore)
|
||||
}
|
||||
|
||||
struct RenameAdmissionGrant {
|
||||
permit: Option<OwnedSemaphorePermit>,
|
||||
batch: bool,
|
||||
}
|
||||
|
||||
impl RenameAdmissionGrant {
|
||||
fn release_one(&mut self) -> bool {
|
||||
self.permit.as_mut().is_some_and(|permit| permit.split(1).is_some())
|
||||
}
|
||||
|
||||
fn release_all(mut self) {
|
||||
drop(self.permit.take());
|
||||
}
|
||||
|
||||
fn quarantine_until(mut self, deadline: tokio::time::Instant) {
|
||||
let Some(permit) = self.permit.take() else {
|
||||
return;
|
||||
};
|
||||
let items = permit.num_permits();
|
||||
if items == 0 {
|
||||
return;
|
||||
}
|
||||
let hold = deadline.saturating_duration_since(tokio::time::Instant::now());
|
||||
rustfs_io_metrics::record_ecstore_rename_quarantine(self.batch, items, hold);
|
||||
let batch = self.batch;
|
||||
if let Ok(handle) = tokio::runtime::Handle::try_current() {
|
||||
handle.spawn(async move {
|
||||
tokio::time::sleep_until(deadline).await;
|
||||
drop(permit);
|
||||
rustfs_io_metrics::record_ecstore_rename_quarantine_released(batch, items);
|
||||
});
|
||||
} else {
|
||||
// Without a runtime there is no safe deadline task. Leaking until
|
||||
// process exit is safer than admitting work over a still-running mutation.
|
||||
rustfs_io_metrics::record_ecstore_rename_quarantine_no_runtime(batch, items);
|
||||
std::mem::forget(permit);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct RenameBatchCallGuard {
|
||||
admission: Option<RenameAdmissionGrant>,
|
||||
deadline: tokio::time::Instant,
|
||||
dispatch_state: RenameAdmissionDispatchState,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
enum RenameAdmissionDispatchState {
|
||||
NotDispatched,
|
||||
MayHavePublished,
|
||||
}
|
||||
|
||||
impl RenameBatchCallGuard {
|
||||
fn mark_dispatched(&mut self) {
|
||||
self.dispatch_state = RenameAdmissionDispatchState::MayHavePublished;
|
||||
}
|
||||
|
||||
fn complete(mut self) {
|
||||
debug_assert_eq!(self.dispatch_state, RenameAdmissionDispatchState::MayHavePublished);
|
||||
if let Some(admission) = self.admission.take() {
|
||||
admission.release_all();
|
||||
}
|
||||
}
|
||||
|
||||
fn release_without_quarantine(mut self) {
|
||||
if let Some(admission) = self.admission.take() {
|
||||
admission.release_all();
|
||||
}
|
||||
}
|
||||
|
||||
fn into_stream(mut self, inner: Streaming<RenameDataBatchResult>, expected_mask: u16) -> RemoteRenameBatchStream {
|
||||
debug_assert_eq!(self.dispatch_state, RenameAdmissionDispatchState::MayHavePublished);
|
||||
RemoteRenameBatchStream {
|
||||
inner,
|
||||
expected_mask,
|
||||
seen_mask: 0,
|
||||
admission: self.admission.take(),
|
||||
deadline: self.deadline,
|
||||
terminal: false,
|
||||
stream_started_at: rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now),
|
||||
first_item_at: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RenameBatchCallGuard {
|
||||
fn drop(&mut self) {
|
||||
if self.dispatch_state == RenameAdmissionDispatchState::MayHavePublished
|
||||
&& let Some(admission) = self.admission.take()
|
||||
{
|
||||
admission.quarantine_until(self.deadline);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct RenameBatchClientItem<'a> {
|
||||
pub slot: usize,
|
||||
pub disk: String,
|
||||
pub file_info: &'a FileInfo,
|
||||
}
|
||||
|
||||
pub(crate) enum RenameBatchStartError {
|
||||
NotDispatched(Error),
|
||||
UnsupportedBeforeStream,
|
||||
MayHavePublished(Error),
|
||||
}
|
||||
|
||||
pub(crate) struct RemoteRenameBatchOutcome {
|
||||
pub slot: usize,
|
||||
pub result: Result<RenameDataResp>,
|
||||
}
|
||||
|
||||
pub(crate) struct RemoteRenameBatchStream {
|
||||
inner: Streaming<RenameDataBatchResult>,
|
||||
expected_mask: u16,
|
||||
seen_mask: u16,
|
||||
admission: Option<RenameAdmissionGrant>,
|
||||
deadline: tokio::time::Instant,
|
||||
terminal: bool,
|
||||
stream_started_at: Option<Instant>,
|
||||
first_item_at: Option<Instant>,
|
||||
}
|
||||
|
||||
impl RemoteRenameBatchStream {
|
||||
fn terminate_ambiguous(&mut self, error: Error) -> Error {
|
||||
if !self.terminal {
|
||||
let missing = (self.expected_mask & !self.seen_mask).count_ones() as usize;
|
||||
rustfs_io_metrics::record_ecstore_rename_batch_terminal_missing(missing);
|
||||
}
|
||||
self.terminal = true;
|
||||
if let Some(admission) = self.admission.take() {
|
||||
admission.quarantine_until(self.deadline);
|
||||
}
|
||||
error
|
||||
}
|
||||
|
||||
pub(crate) async fn next_outcome(&mut self) -> Result<Option<RemoteRenameBatchOutcome>> {
|
||||
if self.terminal {
|
||||
return Ok(None);
|
||||
}
|
||||
let response = match tokio::time::timeout_at(self.deadline, self.inner.message()).await {
|
||||
Err(_) => return Err(self.terminate_ambiguous(DiskError::Timeout)),
|
||||
Ok(Err(status)) => return Err(self.terminate_ambiguous(status.into())),
|
||||
Ok(Ok(None)) if self.seen_mask == self.expected_mask => {
|
||||
self.terminal = true;
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Ok(None)) => return Err(self.terminate_ambiguous(DiskError::Unexpected)),
|
||||
Ok(Ok(Some(response))) => response,
|
||||
};
|
||||
let Ok(slot) = usize::try_from(response.item_index) else {
|
||||
return Err(self.terminate_ambiguous(DiskError::Unexpected));
|
||||
};
|
||||
if slot >= 16 {
|
||||
return Err(self.terminate_ambiguous(DiskError::Unexpected));
|
||||
}
|
||||
let bit = 1_u16 << slot;
|
||||
if self.expected_mask & bit == 0 || self.seen_mask & bit != 0 {
|
||||
return Err(self.terminate_ambiguous(DiskError::Unexpected));
|
||||
}
|
||||
self.seen_mask |= bit;
|
||||
if self.admission.as_mut().is_some_and(|admission| !admission.release_one()) {
|
||||
return Err(self.terminate_ambiguous(DiskError::Unexpected));
|
||||
}
|
||||
if let Some(stream_started_at) = self.stream_started_at {
|
||||
let completed_at = Instant::now();
|
||||
match self.first_item_at {
|
||||
Some(first_item_at) if self.seen_mask == self.expected_mask => {
|
||||
rustfs_io_metrics::record_ecstore_rename_batch_client_first_to_last(
|
||||
completed_at.saturating_duration_since(first_item_at),
|
||||
);
|
||||
}
|
||||
Some(_) => {}
|
||||
None => {
|
||||
rustfs_io_metrics::record_ecstore_rename_batch_client_stream_to_first(
|
||||
completed_at.saturating_duration_since(stream_started_at),
|
||||
);
|
||||
self.first_item_at = Some(completed_at);
|
||||
if self.seen_mask == self.expected_mask {
|
||||
rustfs_io_metrics::record_ecstore_rename_batch_client_first_to_last(Duration::ZERO);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
rustfs_io_metrics::record_ecstore_rename_batch_item_completed();
|
||||
let result = if response.success {
|
||||
if response.error.is_some() {
|
||||
Err(DiskError::FileCorrupt)
|
||||
} else {
|
||||
decode_msgpack_strict(&response.rename_data_resp_msgpack)
|
||||
}
|
||||
} else {
|
||||
Err(response.error.unwrap_or_default().into())
|
||||
};
|
||||
Ok(Some(RemoteRenameBatchOutcome { slot, result }))
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RemoteRenameBatchStream {
|
||||
fn drop(&mut self) {
|
||||
if !self.terminal {
|
||||
let missing = (self.expected_mask & !self.seen_mask).count_ones() as usize;
|
||||
rustfs_io_metrics::record_ecstore_rename_batch_terminal_missing(missing);
|
||||
}
|
||||
if let Some(admission) = self.admission.take() {
|
||||
admission.quarantine_until(self.deadline);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct BulkBodyPermit {
|
||||
_metrics: rustfs_io_metrics::internode_metrics::InternodeBulkBodyAdmissionGuard,
|
||||
_permit: OwnedSemaphorePermit,
|
||||
@@ -1473,10 +1226,6 @@ fn spawn_control_channel_prewarm(addr: String) {
|
||||
}
|
||||
|
||||
impl RemoteDisk {
|
||||
pub(crate) fn rename_stream_batch_enabled(&self) -> bool {
|
||||
self.rename_admission.is_some()
|
||||
}
|
||||
|
||||
async fn guarded_delete_versions_rpc(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -1802,8 +1551,6 @@ impl RemoteDisk {
|
||||
rustfs_utils::get_env_bool(ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING);
|
||||
let bulk_body_admission = peer_bulk_body_admission(&addr, bulk_body_concurrency_limit()?);
|
||||
let bulk_body_wait_timeout = bulk_body_wait_timeout();
|
||||
let rename_admission =
|
||||
peer_rename_admission(&addr, rustfs_utils::get_env_bool(ENV_RUSTFS_EXPERIMENTAL_RENAME_STREAM_BATCH, false));
|
||||
|
||||
let disk = Self {
|
||||
handle_id: Uuid::new_v4(),
|
||||
@@ -1826,7 +1573,6 @@ impl RemoteDisk {
|
||||
network_error_logs: rustfs_utils::LogThrottle::new(REMOTE_DISK_RPC_FAILURE_LOG_INTERVAL_MS),
|
||||
bulk_body_admission,
|
||||
bulk_body_wait_timeout,
|
||||
rename_admission,
|
||||
};
|
||||
record_drive_runtime_state(ep, RuntimeDriveHealthState::Online);
|
||||
|
||||
@@ -2562,54 +2308,6 @@ impl RemoteDisk {
|
||||
}
|
||||
}
|
||||
|
||||
async fn execute_peer_batch_with_timeout<T, F, Fut>(
|
||||
&self,
|
||||
op: &'static str,
|
||||
operation: F,
|
||||
timeout_duration: Duration,
|
||||
) -> Result<T>
|
||||
where
|
||||
F: FnOnce() -> Fut,
|
||||
Fut: std::future::Future<Output = Result<T>>,
|
||||
{
|
||||
if timeout_duration == Duration::ZERO {
|
||||
self.record_timeout(op, timeout_duration);
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
match time::timeout(timeout_duration, operation()).await {
|
||||
Ok(operation_result) => {
|
||||
if operation_result.is_ok() {
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
|
||||
}
|
||||
let evict = operation_result
|
||||
.as_ref()
|
||||
.err()
|
||||
.is_some_and(|error| !error.is_metacache_output_stream_closed() && is_network_like_disk_error(error));
|
||||
self.handle_network_like_error(op, timeout_duration, &operation_result, FailureHealthAction::IgnoreFailure)
|
||||
.await;
|
||||
if evict {
|
||||
evict_failed_connection_with_reason(
|
||||
&self.addr,
|
||||
ChannelRebuildReason::RpcFailure,
|
||||
ConnectionEvictionLogLevel::Debug,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
operation_result
|
||||
}
|
||||
Err(_) => {
|
||||
self.record_timeout(op, timeout_duration);
|
||||
evict_failed_connection_with_reason(
|
||||
&self.addr,
|
||||
ChannelRebuildReason::OperationTimeout,
|
||||
ConnectionEvictionLogLevel::Debug,
|
||||
)
|
||||
.await;
|
||||
Err(DiskError::Timeout)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn record_timeout(&self, op: &'static str, timeout_duration: Duration) {
|
||||
counter!(
|
||||
"rustfs_drive_op_timeout_total",
|
||||
@@ -2796,38 +2494,6 @@ impl RemoteDisk {
|
||||
}))
|
||||
}
|
||||
|
||||
async fn acquire_rename_admission_until(
|
||||
&self,
|
||||
item_count: usize,
|
||||
deadline: tokio::time::Instant,
|
||||
) -> Result<Option<RenameAdmissionGrant>> {
|
||||
let Some(semaphore) = &self.rename_admission else {
|
||||
return Ok(None);
|
||||
};
|
||||
if item_count == 0 || item_count > rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT {
|
||||
return Err(Error::other("invalid internode rename admission weight"));
|
||||
}
|
||||
if let Some(err) = self.offline_bypass_error() {
|
||||
return Err(err);
|
||||
}
|
||||
let weight = u32::try_from(item_count).map_err(Error::other)?;
|
||||
let started = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now);
|
||||
let (result, timed_out) = match tokio::time::timeout_at(deadline, Arc::clone(semaphore).acquire_many_owned(weight)).await
|
||||
{
|
||||
Ok(Ok(permit)) => (Ok(permit), false),
|
||||
Ok(Err(_)) => (Err(Error::RemoteClientUnavailable("internode rename admission closed".to_owned())), false),
|
||||
Err(_) => (Err(DiskError::DiskOngoingReq), true),
|
||||
};
|
||||
if let Some(started) = started {
|
||||
rustfs_io_metrics::record_ecstore_rename_admission(item_count > 1, item_count, started.elapsed(), timed_out);
|
||||
}
|
||||
let permit = result?;
|
||||
Ok(Some(RenameAdmissionGrant {
|
||||
permit: Some(permit),
|
||||
batch: item_count > 1,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn get_bulk_client(&self) -> Result<NodeRpcClient> {
|
||||
if let Some(err) = self.offline_bypass_error() {
|
||||
return Err(err);
|
||||
@@ -2893,44 +2559,9 @@ fn encode_file_info_versions_msgpack(value: &FileInfoVersions) -> Result<Vec<u8>
|
||||
encode_msgpack_with_capacity(value, capacity)
|
||||
}
|
||||
|
||||
fn encode_delete_versions_request(
|
||||
disk: String,
|
||||
volume: &str,
|
||||
versions: &[FileInfoVersions],
|
||||
opts: &DeleteOptions,
|
||||
) -> Result<DeleteVersionsRequest> {
|
||||
// Binary-only batches require fleet-confirmed receivers that traverse the
|
||||
// binary list. Capture that policy for the complete batch and its options.
|
||||
let msgpack_only = rustfs_protos::internode_rpc_msgpack_only();
|
||||
let opts_json = if msgpack_only {
|
||||
String::new()
|
||||
} else {
|
||||
serde_json::to_string(opts)?
|
||||
};
|
||||
let mut versions_json = if msgpack_only {
|
||||
Vec::new()
|
||||
} else {
|
||||
Vec::with_capacity(versions.len())
|
||||
};
|
||||
let mut versions_bin = Vec::with_capacity(versions.len());
|
||||
for version in versions {
|
||||
if !msgpack_only {
|
||||
versions_json.push(serde_json::to_string(version)?);
|
||||
}
|
||||
versions_bin.push(encode_file_info_versions_msgpack(version)?.into());
|
||||
}
|
||||
Ok(DeleteVersionsRequest {
|
||||
disk,
|
||||
volume: volume.to_owned(),
|
||||
versions: versions_json,
|
||||
opts: opts_json,
|
||||
versions_bin,
|
||||
opts_bin: encode_delete_options_msgpack(opts)?.into(),
|
||||
})
|
||||
}
|
||||
|
||||
/// JSON compatibility string for a dual-encoded request field. The captured
|
||||
/// send policy selects the encoding, and its generation guard binds dispatch.
|
||||
/// JSON compatibility string for a dual-encoded (`_bin` + text) request field. Returns an empty
|
||||
/// string only when msgpack-only mode and its explicit fleet confirmation guard are both enabled;
|
||||
/// otherwise the legacy JSON encoding is retained for old peers.
|
||||
fn compat_json<T: Serialize>(value: &T, policy: &MsgpackSendPolicy) -> Result<String> {
|
||||
if policy.omit_eligible_json() {
|
||||
return Ok(String::new());
|
||||
@@ -2968,18 +2599,6 @@ fn decode_msgpack_or_json<T: DeserializeOwned>(binary: &[u8], json: &str, value_
|
||||
}
|
||||
}
|
||||
|
||||
fn decode_msgpack_strict<T: DeserializeOwned>(binary: &[u8]) -> Result<T> {
|
||||
if binary.is_empty() {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
let mut cursor = Cursor::new(binary);
|
||||
let value = rmp_serde::from_read(&mut cursor).map_err(|_| DiskError::FileCorrupt)?;
|
||||
if cursor.position() != u64::try_from(binary.len()).map_err(Error::other)? {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
Ok(value)
|
||||
}
|
||||
|
||||
fn decode_multipart_publication_response(binary: &[u8]) -> Result<RenameDataResp> {
|
||||
let result = if binary.len() > MultipartSourceOperation::max_wire_payload_bytes() {
|
||||
Err(DiskError::FileCorrupt)
|
||||
@@ -3177,97 +2796,6 @@ impl RemoteDisk {
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub(crate) async fn start_rename_data_batch(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
items: Vec<RenameBatchClientItem<'_>>,
|
||||
) -> std::result::Result<RemoteRenameBatchStream, RenameBatchStartError> {
|
||||
if self.rename_admission.is_none() || items.is_empty() || items.len() > 16 {
|
||||
return Err(RenameBatchStartError::NotDispatched(DiskError::UnsupportedDisk));
|
||||
}
|
||||
let deadline = tokio::time::Instant::now() + get_max_timeout_duration();
|
||||
let mut expected_mask = 0_u16;
|
||||
let mut request_items = Vec::with_capacity(items.len());
|
||||
for item in items {
|
||||
if item.slot >= 16 {
|
||||
return Err(RenameBatchStartError::NotDispatched(DiskError::Unexpected));
|
||||
}
|
||||
let bit = 1_u16 << item.slot;
|
||||
if expected_mask & bit != 0 {
|
||||
return Err(RenameBatchStartError::NotDispatched(DiskError::Unexpected));
|
||||
}
|
||||
expected_mask |= bit;
|
||||
let item_index = u32::try_from(item.slot)
|
||||
.map_err(Error::other)
|
||||
.map_err(RenameBatchStartError::NotDispatched)?;
|
||||
let file_info_msgpack = encode_file_info_msgpack(item.file_info).map_err(RenameBatchStartError::NotDispatched)?;
|
||||
request_items.push(RenameDataBatchItem {
|
||||
item_index,
|
||||
disk: item.disk,
|
||||
file_info_msgpack: file_info_msgpack.into(),
|
||||
});
|
||||
}
|
||||
let mut request = Request::new(RenameDataBatchRequest {
|
||||
src_volume: src_volume.to_owned(),
|
||||
src_path: src_path.to_owned(),
|
||||
dst_volume: dst_volume.to_owned(),
|
||||
dst_path: dst_path.to_owned(),
|
||||
items: request_items,
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_rename_data_batch_request_body(request.get_ref())
|
||||
.map_err(Error::other)
|
||||
.map_err(RenameBatchStartError::NotDispatched)?;
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical_body)
|
||||
.map_err(Error::other)
|
||||
.map_err(RenameBatchStartError::NotDispatched)?;
|
||||
let mut client = match tokio::time::timeout_at(deadline, self.get_client()).await {
|
||||
Ok(result) => result.map_err(RenameBatchStartError::NotDispatched)?,
|
||||
Err(_) => return Err(RenameBatchStartError::NotDispatched(DiskError::Timeout)),
|
||||
};
|
||||
let admission = self
|
||||
.acquire_rename_admission_until(request.get_ref().items.len(), deadline)
|
||||
.await
|
||||
.map_err(RenameBatchStartError::NotDispatched)?;
|
||||
let guard = RenameBatchCallGuard {
|
||||
admission,
|
||||
deadline,
|
||||
dispatch_state: RenameAdmissionDispatchState::NotDispatched,
|
||||
};
|
||||
let call_timeout = deadline.saturating_duration_since(tokio::time::Instant::now());
|
||||
if call_timeout == Duration::ZERO {
|
||||
guard.release_without_quarantine();
|
||||
return Err(RenameBatchStartError::NotDispatched(DiskError::Timeout));
|
||||
}
|
||||
let mut guard = guard;
|
||||
guard.mark_dispatched();
|
||||
let response = match self
|
||||
.execute_peer_batch_with_timeout(
|
||||
"rename_data_batch",
|
||||
|| async {
|
||||
match client.rename_data_batch(request).await {
|
||||
Ok(response) => Ok(response),
|
||||
Err(status) if status.code() == Code::Unimplemented => Err(DiskError::UnsupportedDisk),
|
||||
Err(status) => Err(status.into()),
|
||||
}
|
||||
},
|
||||
call_timeout,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Err(DiskError::UnsupportedDisk) => {
|
||||
guard.release_without_quarantine();
|
||||
return Err(RenameBatchStartError::UnsupportedBeforeStream);
|
||||
}
|
||||
Err(error) => return Err(RenameBatchStartError::MayHavePublished(error)),
|
||||
Ok(response) => response,
|
||||
};
|
||||
Ok(guard.into_stream(response.into_inner(), expected_mask))
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub(crate) async fn rename_data_borrowed_with_fence(
|
||||
&self,
|
||||
@@ -3339,27 +2867,6 @@ impl RemoteDisk {
|
||||
let scanner_publication_lease = scanner_publication_lease_token
|
||||
.map(|token| Bytes::copy_from_slice(token.as_bytes()))
|
||||
.unwrap_or_default();
|
||||
let rename_deadline = tokio::time::Instant::now() + get_max_timeout_duration();
|
||||
let rename_admission = self.acquire_rename_admission_until(1, rename_deadline).await?;
|
||||
let mut rename_admission_guard = RenameBatchCallGuard {
|
||||
admission: rename_admission,
|
||||
deadline: rename_deadline,
|
||||
dispatch_state: RenameAdmissionDispatchState::NotDispatched,
|
||||
};
|
||||
let rpc_kind = if multipart_rename {
|
||||
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_MULTIPART
|
||||
} else if pool_meta_cas {
|
||||
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_POOL_META_CAS
|
||||
} else if quota_fenced {
|
||||
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_QUOTA_FENCE
|
||||
} else if incarnation_bound {
|
||||
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_INCARNATION
|
||||
} else if scanner_publication_lease_token.is_some() {
|
||||
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_SCANNER_LEASE
|
||||
} else {
|
||||
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_ORDINARY
|
||||
};
|
||||
rustfs_io_metrics::record_ecstore_rename_remote_rpc(rpc_kind);
|
||||
let rpc_started = rustfs_io_metrics::put_stage_timer();
|
||||
let mut retried_dual_write = false;
|
||||
let response = loop {
|
||||
@@ -3416,7 +2923,6 @@ impl RemoteDisk {
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_PREPARE,
|
||||
attempt_prepare_started,
|
||||
);
|
||||
rename_admission_guard.mark_dispatched();
|
||||
let result = rustfs_io_metrics::observe_put_stage_future(
|
||||
async {
|
||||
if multipart_rename {
|
||||
@@ -3447,7 +2953,6 @@ impl RemoteDisk {
|
||||
Err(status) => return Err(status.into()),
|
||||
}
|
||||
};
|
||||
rename_admission_guard.complete();
|
||||
let response_process_started = rustfs_io_metrics::put_stage_timer();
|
||||
let response_headers_started = rustfs_io_metrics::put_stage_timer();
|
||||
let _ = response.metadata().len();
|
||||
@@ -4042,10 +3547,52 @@ impl DiskAPI for RemoteDisk {
|
||||
return vec![Some(DiskError::FaultyDisk); versions.len()];
|
||||
}
|
||||
|
||||
let request = match encode_delete_versions_request(self.endpoint.to_string(), volume, &versions, &opts) {
|
||||
Ok(request) => request,
|
||||
Err(err) => return vec![Some(Error::other(err.to_string())); versions.len()],
|
||||
// `_bin` support for DeleteVersions is new (grpc-optimization P2); always dual-write JSON +
|
||||
// msgpack until its fallback counter has read zero across a release window.
|
||||
let opts_bin = match encode_delete_options_msgpack(&opts) {
|
||||
Ok(opts_bin) => opts_bin,
|
||||
Err(err) => {
|
||||
let mut errors = Vec::with_capacity(versions.len());
|
||||
for _ in 0..versions.len() {
|
||||
errors.push(Some(Error::other(err.to_string())));
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
};
|
||||
let opts = match serde_json::to_string(&opts) {
|
||||
Ok(opts) => opts,
|
||||
Err(err) => {
|
||||
let mut errors = Vec::with_capacity(versions.len());
|
||||
for _ in 0..versions.len() {
|
||||
errors.push(Some(Error::other(err.to_string())));
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
};
|
||||
let mut versions_str = Vec::with_capacity(versions.len());
|
||||
let mut versions_bin = Vec::with_capacity(versions.len());
|
||||
for file_info_versions in versions.iter() {
|
||||
versions_str.push(match serde_json::to_string(file_info_versions) {
|
||||
Ok(versions_str) => versions_str,
|
||||
Err(err) => {
|
||||
let mut errors = Vec::with_capacity(versions.len());
|
||||
for _ in 0..versions.len() {
|
||||
errors.push(Some(Error::other(err.to_string())));
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
});
|
||||
versions_bin.push(match encode_file_info_versions_msgpack(file_info_versions) {
|
||||
Ok(versions_bin) => Bytes::from(versions_bin),
|
||||
Err(err) => {
|
||||
let mut errors = Vec::with_capacity(versions.len());
|
||||
for _ in 0..versions.len() {
|
||||
errors.push(Some(Error::other(err.to_string())));
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
});
|
||||
}
|
||||
let mut client = match self.get_client().await {
|
||||
Ok(client) => client,
|
||||
Err(err) => {
|
||||
@@ -4057,7 +3604,14 @@ impl DiskAPI for RemoteDisk {
|
||||
}
|
||||
};
|
||||
|
||||
let mut request = Request::new(request);
|
||||
let mut request = Request::new(DeleteVersionsRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
volume: volume.to_string(),
|
||||
versions: versions_str,
|
||||
opts,
|
||||
versions_bin,
|
||||
opts_bin: opts_bin.into(),
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_delete_versions_request_body(request.get_ref());
|
||||
if let Err(err) = attach_mutation_body_digest(&mut request, canonical_body, "delete_versions") {
|
||||
let mut errors = Vec::with_capacity(versions.len());
|
||||
@@ -6783,94 +6337,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_versions_request_encoding_preserves_batches_and_fleet_guards() {
|
||||
use tonic_prost::prost::Message;
|
||||
let groups: Vec<_> = ["first", "second"]
|
||||
.into_iter()
|
||||
.map(|name| FileInfoVersions {
|
||||
volume: "bucket".to_owned(),
|
||||
name: name.to_owned(),
|
||||
versions: vec![sample_rename_data_file_info()],
|
||||
..Default::default()
|
||||
})
|
||||
.collect();
|
||||
let opts = DeleteOptions {
|
||||
undo_write: true,
|
||||
old_data_dir: Some(Uuid::from_u128(7)),
|
||||
..Default::default()
|
||||
};
|
||||
let mut binary_wire_size = 0;
|
||||
let mut compat_wire_size = 0;
|
||||
for (requested, confirmed, peer_capability, binary_only) in [
|
||||
(None, None, None, false),
|
||||
(Some("true"), None, None, false),
|
||||
(Some("false"), Some("true"), None, false),
|
||||
(Some("true"), Some("false"), None, false),
|
||||
(Some("true"), None, Some("true"), false),
|
||||
(Some("true"), Some("false"), Some("true"), false),
|
||||
(Some("true"), Some("true"), None, true),
|
||||
(Some("true"), Some("true"), Some("true"), true),
|
||||
] {
|
||||
with_internode_msgpack_env(
|
||||
[
|
||||
(rustfs_config::ENV_INTERNODE_RPC_MSGPACK_ONLY, requested),
|
||||
(rustfs_config::ENV_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED, confirmed),
|
||||
(rustfs_config::ENV_INTERNODE_RPC_MSGPACK_PEER_CAPABILITY, peer_capability),
|
||||
],
|
||||
|| {
|
||||
for batch in [&groups[..], &[]] {
|
||||
let disk = "disk".to_owned();
|
||||
let disk_buffer = disk.as_ptr();
|
||||
let request = encode_delete_versions_request(disk, "bucket", batch, &opts).expect("encode batch");
|
||||
assert_eq!(request.disk, "disk");
|
||||
assert_eq!(request.disk.as_ptr(), disk_buffer, "move the prepared endpoint into the request");
|
||||
assert_eq!(request.volume, "bucket");
|
||||
assert_eq!(request.versions_bin.len(), batch.len());
|
||||
assert_eq!(request.opts.is_empty(), binary_only);
|
||||
let decoded_opts: DeleteOptions = rmp_serde::from_slice(&request.opts_bin).expect("binary options");
|
||||
assert!(decoded_opts.undo_write);
|
||||
assert_eq!(decoded_opts.old_data_dir, opts.old_data_dir);
|
||||
for (encoded, expected) in request.versions_bin.iter().zip(batch) {
|
||||
let decoded: FileInfoVersions = rmp_serde::from_slice(encoded).expect("binary batch item");
|
||||
assert_eq!(decoded.name, expected.name);
|
||||
assert_eq!(decoded.versions, expected.versions);
|
||||
}
|
||||
if binary_only {
|
||||
assert!(request.versions.is_empty());
|
||||
} else {
|
||||
assert_eq!(request.opts, serde_json::to_string(&opts).expect("legacy options"));
|
||||
assert_eq!(request.versions.len(), batch.len());
|
||||
for (encoded, expected) in request.versions.iter().zip(batch) {
|
||||
let decoded: FileInfoVersions = serde_json::from_str(encoded).expect("legacy batch item");
|
||||
assert_eq!(decoded.name, expected.name);
|
||||
assert_eq!(decoded.versions, expected.versions);
|
||||
}
|
||||
}
|
||||
let canonical = rustfs_protos::canonical_delete_versions_request_body(&request).expect("bound batch");
|
||||
let mut modified = request.clone();
|
||||
modified.opts_bin = encode_delete_options_msgpack(&DeleteOptions::default())
|
||||
.expect("other options")
|
||||
.into();
|
||||
assert_ne!(
|
||||
canonical,
|
||||
rustfs_protos::canonical_delete_versions_request_body(&modified).expect("other bound batch")
|
||||
);
|
||||
if !batch.is_empty() {
|
||||
if binary_only {
|
||||
binary_wire_size = request.encoded_len();
|
||||
} else {
|
||||
compat_wire_size = request.encoded_len();
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
assert!(binary_wire_size > 0 && binary_wire_size < compat_wire_size);
|
||||
println!("delete batch encoded bytes compat={compat_wire_size} msgpack_only={binary_wire_size}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn msgpack_eligible_requests_bind_one_policy_and_bounded_local_retry() {
|
||||
let source = rustfs_protos::compat_manifest::production_source(include_str!("remote_disk.rs"), "remote_disk.rs");
|
||||
@@ -7821,10 +7287,7 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
fn with_internode_msgpack_env<R, const N: usize>(
|
||||
vars: [(&'static str, Option<&'static str>); N],
|
||||
f: impl FnOnce() -> R,
|
||||
) -> R {
|
||||
fn with_internode_msgpack_env<R>(vars: [(&'static str, Option<&'static str>); 2], f: impl FnOnce() -> R) -> R {
|
||||
temp_env::with_vars(vars, || {
|
||||
rustfs_protos::reset_internode_rpc_msgpack_only_cache();
|
||||
let result = f();
|
||||
@@ -13423,127 +12886,6 @@ mod tests {
|
||||
.expect("shared gate remains open");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rename_admission_is_weighted_shared_and_released_per_item() {
|
||||
use std::future::Future;
|
||||
use std::task::{Context, Poll, Waker};
|
||||
|
||||
let first = peer_rename_admission("http://rename-peer-a", true).expect("enabled rename gate");
|
||||
let same = peer_rename_admission("http://rename-peer-a", true).expect("same peer rename gate");
|
||||
let other = peer_rename_admission("http://rename-peer-b", true).expect("other peer rename gate");
|
||||
assert!(Arc::ptr_eq(&first, &same));
|
||||
assert!(!Arc::ptr_eq(&first, &other));
|
||||
assert!(peer_rename_admission("http://rename-disabled", false).is_none());
|
||||
assert_eq!(first.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
|
||||
let permit = Arc::clone(&first)
|
||||
.acquire_many_owned(4)
|
||||
.await
|
||||
.expect("weighted batch admission");
|
||||
let mut grant = RenameAdmissionGrant {
|
||||
permit: Some(permit),
|
||||
batch: true,
|
||||
};
|
||||
assert_eq!(first.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 4);
|
||||
assert!(grant.release_one());
|
||||
assert_eq!(first.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 3);
|
||||
drop(grant);
|
||||
assert_eq!(first.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
|
||||
let held = Arc::clone(&first)
|
||||
.acquire_many_owned(u32::try_from(rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 3).expect("small limit"))
|
||||
.await
|
||||
.expect("reserve all but three permits");
|
||||
let mut waiting = Box::pin(Arc::clone(&same).acquire_many_owned(4));
|
||||
assert!(matches!(waiting.as_mut().poll(&mut Context::from_waker(Waker::noop())), Poll::Pending));
|
||||
assert_eq!(first.available_permits(), 0, "the FIFO waiter reserves currently available permits");
|
||||
drop(waiting);
|
||||
assert_eq!(
|
||||
first.available_permits(),
|
||||
3,
|
||||
"cancelling a queued weighted acquire must return its reservation"
|
||||
);
|
||||
let waiting = Box::pin(Arc::clone(&same).acquire_many_owned(4));
|
||||
drop(held);
|
||||
let acquired = tokio::time::timeout(Duration::from_secs(1), waiting)
|
||||
.await
|
||||
.expect("weighted waiter wakes")
|
||||
.expect("rename gate remains open");
|
||||
drop(acquired);
|
||||
assert_eq!(first.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn rename_admission_quarantine_holds_until_absolute_deadline() {
|
||||
let gate = peer_rename_admission("http://rename-quarantine-peer", true).expect("enabled rename gate");
|
||||
let permit = Arc::clone(&gate)
|
||||
.acquire_many_owned(4)
|
||||
.await
|
||||
.expect("weighted batch admission");
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(1);
|
||||
RenameAdmissionGrant {
|
||||
permit: Some(permit),
|
||||
batch: true,
|
||||
}
|
||||
.quarantine_until(deadline);
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 4);
|
||||
|
||||
tokio::time::advance(Duration::from_millis(999)).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 4);
|
||||
tokio::time::advance(Duration::from_millis(1)).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn rename_call_guard_quarantines_only_after_dispatch() {
|
||||
let gate = peer_rename_admission("http://rename-call-guard-peer", true).expect("enabled rename gate");
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(1);
|
||||
let permit = Arc::clone(&gate).acquire_many_owned(4).await.expect("pre-dispatch permits");
|
||||
drop(RenameBatchCallGuard {
|
||||
admission: Some(RenameAdmissionGrant {
|
||||
permit: Some(permit),
|
||||
batch: true,
|
||||
}),
|
||||
deadline,
|
||||
dispatch_state: RenameAdmissionDispatchState::NotDispatched,
|
||||
});
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
|
||||
let permit = Arc::clone(&gate).acquire_many_owned(4).await.expect("dispatched permits");
|
||||
let mut guard = RenameBatchCallGuard {
|
||||
admission: Some(RenameAdmissionGrant {
|
||||
permit: Some(permit),
|
||||
batch: true,
|
||||
}),
|
||||
deadline,
|
||||
dispatch_state: RenameAdmissionDispatchState::NotDispatched,
|
||||
};
|
||||
guard.mark_dispatched();
|
||||
drop(guard);
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 4);
|
||||
tokio::time::advance(Duration::from_secs(1)).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
|
||||
let permit = Arc::clone(&gate)
|
||||
.acquire_many_owned(4)
|
||||
.await
|
||||
.expect("proven pre-mutation permits");
|
||||
let mut guard = RenameBatchCallGuard {
|
||||
admission: Some(RenameAdmissionGrant {
|
||||
permit: Some(permit),
|
||||
batch: true,
|
||||
}),
|
||||
deadline,
|
||||
dispatch_state: RenameAdmissionDispatchState::NotDispatched,
|
||||
};
|
||||
guard.mark_dispatched();
|
||||
guard.release_without_quarantine();
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remote_disk_builds_optional_complete_body_gate_from_configuration() {
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT, Some("3"))], async {
|
||||
|
||||
@@ -20,8 +20,8 @@ use async_trait::async_trait;
|
||||
use bytes::Bytes;
|
||||
use futures::{FutureExt, future::BoxFuture};
|
||||
use rustfs_lock::{
|
||||
BatchAcquireOutcome, CacheFenceStamp, LockCleanupOutcome, LockClient, LockError, LockInfo, LockRequest, LockResponse,
|
||||
LockStats, LockStatus, LockType, Result,
|
||||
BatchAcquireOutcome, LockCleanupOutcome, LockClient, LockError, LockInfo, LockRequest, LockResponse, LockStats, LockStatus,
|
||||
LockType, Result,
|
||||
types::{LockId, LockMetadata, LockPriority},
|
||||
};
|
||||
use rustfs_protos::proto_gen::node_service::{
|
||||
@@ -52,10 +52,6 @@ fn attach_lock_mutation_body_digest<T: rustfs_protos::CanonicalMutationBody>(req
|
||||
set_tonic_rolling_mutation_body_digest(request)
|
||||
}
|
||||
|
||||
fn decode_cache_fence_stamp(encoded: Option<&Bytes>) -> Option<CacheFenceStamp> {
|
||||
encoded.and_then(|encoded| CacheFenceStamp::decode(encoded).ok())
|
||||
}
|
||||
|
||||
/// Work to run if an RPC completes after its caller stops waiting.
|
||||
type LateCompletion<T> = Option<Box<dyn FnOnce(T) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send>>;
|
||||
type LateFailure = LateCompletion<()>;
|
||||
@@ -328,7 +324,6 @@ fn batch_acquire_outcome(lock_id: &LockId, lock_type: LockType, result: Option<&
|
||||
return BatchAcquireOutcome::Granted {
|
||||
lock_info,
|
||||
wait_time: Duration::ZERO,
|
||||
cache_fence_stamp: decode_cache_fence_stamp(result.cache_fence_stamp.as_ref()),
|
||||
};
|
||||
}
|
||||
if result.lock_info.is_some() {
|
||||
@@ -457,7 +452,6 @@ impl RemoteClient {
|
||||
deadlock_detection: false,
|
||||
suppress_contention_logs: false,
|
||||
refresh_interval: None,
|
||||
cache_fence_stamp_requested: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1579,11 +1573,10 @@ impl LockClient for RemoteClient {
|
||||
|
||||
// Check if the lock acquisition was successful
|
||||
if resp.success {
|
||||
let lock_info = Self::build_lock_info(request, resp.lock_info);
|
||||
match decode_cache_fence_stamp(resp.cache_fence_stamp.as_ref()) {
|
||||
Some(stamp) => Ok(LockResponse::success_with_cache_fence_stamp(lock_info, std::time::Duration::ZERO, stamp)),
|
||||
None => Ok(LockResponse::success(lock_info, std::time::Duration::ZERO)),
|
||||
}
|
||||
Ok(LockResponse::success(
|
||||
Self::build_lock_info(request, resp.lock_info),
|
||||
std::time::Duration::ZERO,
|
||||
))
|
||||
} else {
|
||||
// Lock acquisition failed
|
||||
Ok(LockResponse::failure(
|
||||
@@ -1601,19 +1594,12 @@ impl LockClient for RemoteClient {
|
||||
.iter()
|
||||
.zip(outcomes)
|
||||
.map(|(request, outcome)| match outcome {
|
||||
BatchAcquireOutcome::Granted {
|
||||
lock_info,
|
||||
wait_time,
|
||||
cache_fence_stamp,
|
||||
} => {
|
||||
let lock_info = lock_info
|
||||
BatchAcquireOutcome::Granted { lock_info, wait_time } => LockResponse::success(
|
||||
lock_info
|
||||
.map(|info| *info)
|
||||
.unwrap_or_else(|| Self::build_lock_info(request, None));
|
||||
match cache_fence_stamp {
|
||||
Some(stamp) => LockResponse::success_with_cache_fence_stamp(lock_info, wait_time, stamp),
|
||||
None => LockResponse::success(lock_info, wait_time),
|
||||
}
|
||||
}
|
||||
.unwrap_or_else(|| Self::build_lock_info(request, None)),
|
||||
wait_time,
|
||||
),
|
||||
BatchAcquireOutcome::Ambiguous {
|
||||
error: error @ LockError::Timeout { .. },
|
||||
..
|
||||
@@ -2381,38 +2367,6 @@ mod tests {
|
||||
assert!(matches!(classify(None), BatchAcquireOutcome::Ambiguous { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cache_fence_stamp_decoder_accepts_current_wire_shape_and_rejects_malformed_values() {
|
||||
let mut encoded = Vec::with_capacity(27);
|
||||
encoded.push(1);
|
||||
encoded.extend_from_slice(uuid::Uuid::from_u128(7).as_bytes());
|
||||
encoded.extend_from_slice(&17_u16.to_le_bytes());
|
||||
encoded.extend_from_slice(&42_u64.to_le_bytes());
|
||||
let encoded = Bytes::from(encoded);
|
||||
|
||||
let stamp = decode_cache_fence_stamp(Some(&encoded)).expect("current stamp wire shape");
|
||||
assert_eq!(stamp.voter_boot_id(), uuid::Uuid::from_u128(7));
|
||||
assert_eq!(stamp.shard_index(), 17);
|
||||
assert_eq!(stamp.generation(), 42);
|
||||
assert!(decode_cache_fence_stamp(None).is_none(), "legacy responses remain valid without proof");
|
||||
assert!(decode_cache_fence_stamp(Some(&Bytes::from_static(b"malformed"))).is_none());
|
||||
|
||||
let request = test_lock_request(Duration::from_secs(30));
|
||||
let result = GenerallyLockResult {
|
||||
success: true,
|
||||
cache_fence_stamp: Some(encoded),
|
||||
..Default::default()
|
||||
};
|
||||
let BatchAcquireOutcome::Granted {
|
||||
cache_fence_stamp: Some(batch_stamp),
|
||||
..
|
||||
} = batch_acquire_outcome(&request.lock_id, request.lock_type, Some(&result))
|
||||
else {
|
||||
panic!("valid batch grant must retain its cache-fence stamp");
|
||||
};
|
||||
assert_eq!(batch_stamp, stamp);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn batch_success_crossing_deadline_in_one_poll_runs_late_cleanup_once() {
|
||||
use crate::cluster::rpc::client::NoOpInterceptor;
|
||||
|
||||
@@ -101,6 +101,7 @@ const DEFAULT_RRS_STORAGE_CLASS: &str = "EC:1";
|
||||
const ZERO_SET_DRIVE_COUNT_ERROR: &str = "set drive count must be greater than zero";
|
||||
|
||||
pub static DEFAULT_INLINE_BLOCK: usize = 128 * 1024;
|
||||
const DEFAULT_INLINE_OBJECT_BUDGET: usize = 2 * DEFAULT_INLINE_BLOCK;
|
||||
|
||||
pub static DEFAULT_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
let kvs = vec![
|
||||
@@ -151,7 +152,6 @@ pub struct Config {
|
||||
inline_block: usize,
|
||||
initialized: bool,
|
||||
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
|
||||
// Preserve the explicit-limit marker when configurations round-trip through older readers.
|
||||
inline_block_explicit: bool,
|
||||
#[serde(skip)]
|
||||
standard_parities: Vec<PoolParity>,
|
||||
@@ -245,10 +245,8 @@ impl Config {
|
||||
return false;
|
||||
}
|
||||
|
||||
let Ok(shard_size) = usize::try_from(shard_size) else {
|
||||
return false;
|
||||
};
|
||||
let inline_block = self.inline_block();
|
||||
let shard_size = shard_size as usize;
|
||||
let inline_block = self.effective_inline_block(data_shards);
|
||||
|
||||
if versioned {
|
||||
shard_size <= inline_block / 8
|
||||
@@ -257,7 +255,27 @@ impl Config {
|
||||
}
|
||||
}
|
||||
|
||||
/// Per-shard inline data budget, defaulting to 128 KiB across EC layouts.
|
||||
/// Returns the per-shard inline budget used by both write admission and
|
||||
/// legacy read fallback.
|
||||
///
|
||||
/// The default budget is scaled by the number of data shards so a wider EC
|
||||
/// layout does not silently increase the maximum inline object size. An
|
||||
/// explicitly configured `inline_block` remains a fixed per-shard limit for
|
||||
/// compatibility with deployments that opted into the historical policy.
|
||||
pub(crate) fn effective_inline_block(&self, data_shards: usize) -> usize {
|
||||
if data_shards == 0 {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if self.initialized && self.inline_block_explicit {
|
||||
self.inline_block
|
||||
} else {
|
||||
// Keep the historical two-data-shard object budget while preventing
|
||||
// wider EC layouts from multiplying the maximum inline object size.
|
||||
DEFAULT_INLINE_OBJECT_BUDGET.div_ceil(data_shards).min(DEFAULT_INLINE_BLOCK)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn inline_block(&self) -> usize {
|
||||
if !self.initialized {
|
||||
DEFAULT_INLINE_BLOCK
|
||||
@@ -549,7 +567,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn should_inline_uses_fixed_default_threshold_for_every_ec_layout() {
|
||||
fn should_inline_scales_default_threshold_by_data_shards() {
|
||||
let config = lookup_config_for_pools_with_env(&KVS::new(), &[3, 12], no_env_overrides())
|
||||
.expect("default inline policy should resolve for EC2+1 and EC8+4");
|
||||
|
||||
@@ -558,14 +576,10 @@ mod tests {
|
||||
("EC2+1 unversioned above", 128 * 1024 + 1, 2, false, false),
|
||||
("EC2+1 versioned exact", 16 * 1024, 2, true, true),
|
||||
("EC2+1 versioned above", 16 * 1024 + 1, 2, true, false),
|
||||
("EC8+4 unversioned exact", 128 * 1024, 8, false, true),
|
||||
("EC8+4 unversioned above", 128 * 1024 + 1, 8, false, false),
|
||||
("EC8+4 versioned exact", 16 * 1024, 8, true, true),
|
||||
("EC8+4 versioned above", 16 * 1024 + 1, 8, true, false),
|
||||
("EC12+4 unversioned exact", 128 * 1024, 12, false, true),
|
||||
("EC12+4 unversioned above", 128 * 1024 + 1, 12, false, false),
|
||||
("EC12+4 versioned exact", 16 * 1024, 12, true, true),
|
||||
("EC12+4 versioned above", 16 * 1024 + 1, 12, true, false),
|
||||
("EC8+4 unversioned exact", 32 * 1024, 8, false, true),
|
||||
("EC8+4 unversioned above", 32 * 1024 + 1, 8, false, false),
|
||||
("EC8+4 versioned exact", 4 * 1024, 8, true, true),
|
||||
("EC8+4 versioned above", 4 * 1024 + 1, 8, true, false),
|
||||
("negative", -1, 2, false, false),
|
||||
("zero data shards", 0, 0, false, false),
|
||||
] {
|
||||
@@ -612,35 +626,37 @@ mod tests {
|
||||
let shard_size = erasure.shard_file_size(object_size);
|
||||
let inline = config.should_inline(shard_size, data_shards, false);
|
||||
|
||||
// The per-shard budget stays monotonic across object sizes.
|
||||
// PUT separately bounds inline body materialization.
|
||||
// The effective policy is monotonic across object sizes. This
|
||||
// table covers the boundaries that previously exposed the
|
||||
// fixed-shard read-ahead mismatch, including the 1 MiB case.
|
||||
assert!(!inline || previous, "inline decision must not re-enable at {object_size} bytes");
|
||||
previous = inline;
|
||||
}
|
||||
|
||||
assert!(
|
||||
config.should_inline(erasure.shard_file_size(1024 * 1024), data_shards, false),
|
||||
"1 MiB fits the per-shard budget for EC{data_shards}+{parity_shards}"
|
||||
!config.should_inline(erasure.shard_file_size(1024 * 1024), data_shards, false),
|
||||
"1 MiB must use the non-inline path for EC{data_shards}+{parity_shards}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inline_block_preserves_fixed_default_and_explicit_limits() {
|
||||
fn effective_inline_block_scales_default_budget_and_preserves_explicit_limit() {
|
||||
let config = Config::default();
|
||||
assert_eq!(config.inline_block(), 128 * 1024);
|
||||
assert!(!config.should_inline(0, 0, false));
|
||||
assert_eq!(config.effective_inline_block(8), 32 * 1024);
|
||||
assert_eq!(config.effective_inline_block(12), 21_846);
|
||||
assert_eq!(config.effective_inline_block(0), 0);
|
||||
|
||||
let explicit = lookup_config_for_pools_with_env(
|
||||
&KVS::new(),
|
||||
&[12],
|
||||
StorageClassEnvOverrides {
|
||||
inline_block: Some("32KiB".to_string()),
|
||||
inline_block: Some("128KiB".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.expect("explicit inline block should resolve");
|
||||
assert_eq!(explicit.inline_block(), 32 * 1024);
|
||||
assert_eq!(explicit.effective_inline_block(12), 128 * 1024);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -889,21 +905,6 @@ mod tests {
|
||||
assert!(decoded.should_inline(100 * 1024, 8, false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_scalar_inline_config_preserves_limit_without_explicit_marker() {
|
||||
let cfg = lookup_config_for_pools_with_env(&KVS::new(), &[12], no_env_overrides())
|
||||
.expect("default storage class should resolve");
|
||||
let mut legacy = serde_json::to_value(cfg).expect("serialize scalar config");
|
||||
legacy["inline_block"] = serde_json::json!(64 * 1024);
|
||||
legacy.as_object_mut().expect("config object").remove("inline_block_explicit");
|
||||
let decoded: Config = serde_json::from_value(legacy).expect("read historical scalar config");
|
||||
|
||||
assert!(decoded.should_inline(64 * 1024, 8, false));
|
||||
assert!(!decoded.should_inline(64 * 1024 + 1, 8, false));
|
||||
assert!(decoded.should_inline(8 * 1024, 8, true));
|
||||
assert!(!decoded.should_inline(8 * 1024 + 1, 8, true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lookup_config_reads_rrs_from_class_rrs_key() {
|
||||
// Regression: kvs.get(RRS) used RRS="REDUCED_REDUNDANCY" instead of
|
||||
|
||||
@@ -12378,7 +12378,6 @@ struct DecommissionCapacityLockOrderBarrierState {
|
||||
external_heal_target_lock_attempted: tokio::sync::Notify,
|
||||
target_gate_retry_entered: tokio::sync::Notify,
|
||||
target_gate_retry_entries: AtomicUsize,
|
||||
target_gate_retry_pause: std::sync::OnceLock<tokio::sync::Semaphore>,
|
||||
target_gate_inline_retry_entered: tokio::sync::Notify,
|
||||
target_gate_inline_retry_entries: AtomicUsize,
|
||||
target_gate_exact_reloads: AtomicUsize,
|
||||
@@ -12426,7 +12425,6 @@ impl DecommissionCapacityLockOrderBarrier {
|
||||
external_heal_target_lock_attempted: tokio::sync::Notify::new(),
|
||||
target_gate_retry_entered: tokio::sync::Notify::new(),
|
||||
target_gate_retry_entries: AtomicUsize::new(0),
|
||||
target_gate_retry_pause: std::sync::OnceLock::new(),
|
||||
target_gate_inline_retry_entered: tokio::sync::Notify::new(),
|
||||
target_gate_inline_retry_entries: AtomicUsize::new(0),
|
||||
target_gate_exact_reloads: AtomicUsize::new(0),
|
||||
@@ -12522,13 +12520,6 @@ impl DecommissionCapacityLockOrderBarrier {
|
||||
.expect("decommission entries should observe target gate contention");
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
pub(crate) fn pause_target_gate_retries(&self) -> &tokio::sync::Semaphore {
|
||||
self.state
|
||||
.target_gate_retry_pause
|
||||
.get_or_init(|| tokio::sync::Semaphore::new(0))
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
pub(crate) fn target_gate_inline_retries(&self) -> usize {
|
||||
self.state.target_gate_inline_retry_entries.load(Ordering::Acquire)
|
||||
@@ -12639,9 +12630,6 @@ impl DecommissionCapacityLockOrderBarrier {
|
||||
#[cfg(test)]
|
||||
impl Drop for DecommissionCapacityLockOrderBarrier {
|
||||
fn drop(&mut self) {
|
||||
if let Some(pause) = self.state.target_gate_retry_pause.get() {
|
||||
pause.close();
|
||||
}
|
||||
self.state.owner_release.notify_one();
|
||||
self.state.external_object_capacity_probe_release.notify_one();
|
||||
self.state.external_object_commit_phase_release.notify_one();
|
||||
@@ -12770,7 +12758,7 @@ async fn pause_decommission_cancel_target_timeout(store_id: uuid::Uuid) {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn observe_decommission_target_gate_retry(store_id: uuid::Uuid) {
|
||||
fn notify_decommission_target_gate_retry(store_id: uuid::Uuid) {
|
||||
let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
@@ -12781,11 +12769,6 @@ async fn observe_decommission_target_gate_retry(store_id: uuid::Uuid) {
|
||||
if let Some(barrier) = barrier {
|
||||
barrier.target_gate_retry_entries.fetch_add(1, Ordering::AcqRel);
|
||||
barrier.target_gate_retry_entered.notify_one();
|
||||
if let Some(pause) = barrier.target_gate_retry_pause.get()
|
||||
&& let Ok(permit) = pause.acquire().await
|
||||
{
|
||||
permit.forget();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17655,7 +17638,7 @@ impl ECStore {
|
||||
target_busy_attempt: usize,
|
||||
) -> Result<Option<DecommissionCapacityTargetPermit>> {
|
||||
#[cfg(test)]
|
||||
observe_decommission_target_gate_retry(self.id).await;
|
||||
notify_decommission_target_gate_retry(self.id);
|
||||
let mut wait_attempt = target_busy_attempt;
|
||||
let target_guard = loop {
|
||||
let retry_delay =
|
||||
|
||||
@@ -8360,16 +8360,13 @@ mod decommission_lock_order_tests {
|
||||
.await
|
||||
.expect("hold the target gate before starting both waiters");
|
||||
let retry_observer = DecommissionCapacityLockOrderBarrier::install(other_store.id, other_store.id);
|
||||
// Both workers must observe the original gate as busy, but slow disk I/O
|
||||
// in the holder or first waiter must not consume the second waiter's retry budget.
|
||||
let retry_pause = retry_observer.pause_target_gate_retries();
|
||||
|
||||
let mut workers = tokio::task::JoinSet::new();
|
||||
let mut workers = Vec::new();
|
||||
for object in objects {
|
||||
let worker_store = Arc::clone(&other_store);
|
||||
let worker_bucket = bucket.clone();
|
||||
let source_set = other_store.pools[0].get_disks_by_key(object);
|
||||
workers.spawn(async move {
|
||||
workers.push(tokio::spawn(async move {
|
||||
worker_store
|
||||
.decommission_entry_for_test(
|
||||
0,
|
||||
@@ -8381,7 +8378,7 @@ mod decommission_lock_order_tests {
|
||||
source_set,
|
||||
)
|
||||
.await
|
||||
});
|
||||
}));
|
||||
}
|
||||
retry_observer.wait_until_target_gate_retries(2).await;
|
||||
assert_eq!(
|
||||
@@ -8428,7 +8425,6 @@ mod decommission_lock_order_tests {
|
||||
);
|
||||
|
||||
drop(target_guard);
|
||||
retry_pause.add_permits(1);
|
||||
retry_observer.wait_until_owner_paused().await;
|
||||
assert_eq!(
|
||||
retry_observer.target_gate_exact_reloads(),
|
||||
@@ -8447,13 +8443,6 @@ mod decommission_lock_order_tests {
|
||||
.expect("a pinned waiter must not hold the alternate target gate");
|
||||
drop(alternate_probe);
|
||||
retry_observer.release_owner();
|
||||
tokio::time::timeout(Duration::from_secs(30), workers.join_next())
|
||||
.await
|
||||
.expect("the pinned waiter should finish before the other retries")
|
||||
.expect("the pinned waiter must exist")
|
||||
.expect("the pinned waiter task should not panic")
|
||||
.expect("the pinned waiter migration should succeed");
|
||||
retry_pause.add_permits(1);
|
||||
retry_observer.wait_until_owner_paused().await;
|
||||
assert_eq!(
|
||||
retry_observer.target_gate_exact_reloads(),
|
||||
@@ -8473,12 +8462,13 @@ mod decommission_lock_order_tests {
|
||||
drop(original_probe);
|
||||
retry_observer.release_owner();
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(30), workers.join_next())
|
||||
.await
|
||||
.expect("the reselected waiter should finish after permit transfer")
|
||||
.expect("the reselected waiter must exist")
|
||||
.expect("the reselected waiter task should not panic")
|
||||
.expect("the reselected waiter migration should succeed");
|
||||
for worker in workers {
|
||||
tokio::time::timeout(Duration::from_secs(30), worker)
|
||||
.await
|
||||
.expect("same-target waiter should finish after permit transfer")
|
||||
.expect("same-target waiter task should not panic")
|
||||
.expect("same-target waiter migration should succeed");
|
||||
}
|
||||
assert_eq!(
|
||||
retry_observer.target_gate_exact_reloads(),
|
||||
2,
|
||||
|
||||
@@ -842,7 +842,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for Sets {
|
||||
user_defined: dst_opts.user_defined.clone(),
|
||||
shard_integrity_write_mode: Some(src_info.shard_integrity_write_mode()),
|
||||
versioned: dst_opts.versioned,
|
||||
version_suspended: dst_opts.version_suspended,
|
||||
version_id: dst_opts.version_id.clone(),
|
||||
mod_time: dst_opts.mod_time,
|
||||
http_preconditions: dst_opts.http_preconditions.clone(),
|
||||
|
||||
@@ -180,9 +180,6 @@ pub(crate) const GET_METADATA_CACHE_DECISION_SKIP: &str = "skip";
|
||||
pub(crate) const GET_METADATA_CACHE_REASON_DATA_MOVEMENT: &str = "data_movement";
|
||||
pub(crate) const GET_METADATA_CACHE_REASON_DELETE_MARKER: &str = "delete_marker";
|
||||
pub(crate) const GET_METADATA_CACHE_REASON_DIST_ERASURE: &str = "dist_erasure";
|
||||
pub(crate) const GET_METADATA_CACHE_REASON_FENCE_LOCK_LOST: &str = "fence_lock_lost";
|
||||
pub(crate) const GET_METADATA_CACHE_REASON_FENCE_PROOF_MISMATCH: &str = "fence_proof_mismatch";
|
||||
pub(crate) const GET_METADATA_CACHE_REASON_FENCE_PROOF_UNAVAILABLE: &str = "fence_proof_unavailable";
|
||||
pub(crate) const GET_METADATA_CACHE_REASON_INCL_FREE_VERSIONS: &str = "incl_free_versions";
|
||||
pub(crate) const GET_METADATA_CACHE_REASON_INSUFFICIENT_CACHED_QUORUM: &str = "insufficient_cached_quorum";
|
||||
pub(crate) const GET_METADATA_CACHE_REASON_BODY_LAYOUT_REQUIRED: &str = "body_layout_required";
|
||||
|
||||
@@ -13,9 +13,11 @@
|
||||
// limitations under the License.
|
||||
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
fs::Metadata,
|
||||
path::{Path, PathBuf},
|
||||
time::Duration,
|
||||
sync::Mutex,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use tokio::{fs::File, io};
|
||||
|
||||
@@ -237,21 +239,72 @@ pub fn rename_std(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()
|
||||
std::fs::rename(from, to)
|
||||
}
|
||||
|
||||
// Only successful probes are cached. A missed bucket can be created at any time.
|
||||
static BUCKET_EXISTENCE_CACHE: std::sync::LazyLock<moka::sync::Cache<PathBuf, ()>> = std::sync::LazyLock::new(|| {
|
||||
moka::sync::Cache::builder()
|
||||
.max_capacity(4096)
|
||||
.time_to_live(Duration::from_secs(60))
|
||||
.build()
|
||||
});
|
||||
// Bucket existence cache - reduces statx syscalls for repeated bucket checks
|
||||
|
||||
/// Cache for bucket directory existence checks.
|
||||
struct BucketExistenceCache {
|
||||
cache: Mutex<HashMap<PathBuf, (Instant, bool)>>,
|
||||
ttl: Duration,
|
||||
}
|
||||
|
||||
impl BucketExistenceCache {
|
||||
fn new(ttl: Duration) -> Self {
|
||||
Self {
|
||||
cache: Mutex::new(HashMap::new()),
|
||||
ttl,
|
||||
}
|
||||
}
|
||||
|
||||
fn check_exists(&self, path: &Path) -> Option<bool> {
|
||||
let mut cache = self.cache.lock().ok()?;
|
||||
if let Some((timestamp, exists)) = cache.get(path) {
|
||||
if timestamp.elapsed() < self.ttl {
|
||||
return Some(*exists);
|
||||
}
|
||||
cache.remove(path);
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn record(&self, path: PathBuf, exists: bool) {
|
||||
if let Ok(mut cache) = self.cache.lock() {
|
||||
cache.insert(path, (Instant::now(), exists));
|
||||
}
|
||||
}
|
||||
|
||||
fn invalidate(&self, path: &Path) {
|
||||
if let Ok(mut cache) = self.cache.lock() {
|
||||
cache.remove(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static BUCKET_EXISTENCE_CACHE: std::sync::LazyLock<BucketExistenceCache> =
|
||||
std::sync::LazyLock::new(|| BucketExistenceCache::new(Duration::from_secs(60)));
|
||||
|
||||
/// Cached access check - reduces statx syscalls
|
||||
pub async fn cached_access(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
let path = path.as_ref();
|
||||
if BUCKET_EXISTENCE_CACHE.get(path).is_none() {
|
||||
access(path).await?;
|
||||
BUCKET_EXISTENCE_CACHE.insert(path.to_owned(), ());
|
||||
|
||||
if let Some(exists) = BUCKET_EXISTENCE_CACHE.check_exists(path) {
|
||||
if exists {
|
||||
return Ok(());
|
||||
}
|
||||
return Err(io::Error::new(io::ErrorKind::NotFound, "bucket not found (cached)"));
|
||||
}
|
||||
|
||||
let path_buf = path.to_path_buf();
|
||||
let result = access(&path_buf).await;
|
||||
|
||||
match &result {
|
||||
Ok(_) => BUCKET_EXISTENCE_CACHE.record(path_buf, true),
|
||||
Err(e) if e.kind() == io::ErrorKind::NotFound => {
|
||||
BUCKET_EXISTENCE_CACHE.record(path_buf, false);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
result?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -266,15 +319,6 @@ mod tests {
|
||||
use tempfile::TempDir;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
#[tokio::test]
|
||||
async fn bucket_creation_is_visible_after_a_cache_miss() {
|
||||
let root = TempDir::new().expect("bucket root");
|
||||
let path = root.path().join("bucket");
|
||||
assert_eq!(cached_access(&path).await.expect_err("not created yet").kind(), io::ErrorKind::NotFound);
|
||||
tokio::fs::create_dir(&path).await.expect("create bucket");
|
||||
cached_access(&path).await.expect("new bucket visible");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancelled_filesystem_jobs_retain_physical_resources() {
|
||||
use crate::disk::io_admission::{test_resource_units, with_optional_resource_unit};
|
||||
|
||||
+88
-1148
File diff suppressed because it is too large
Load Diff
@@ -231,26 +231,14 @@ fn rollback_metadata_std(destination_metadata: &Path, rollback_data_dir: Option<
|
||||
let Some(dst_parent) = destination_metadata.parent() else {
|
||||
return Err(std::io::Error::new(ErrorKind::InvalidInput, "missing object metadata parent"));
|
||||
};
|
||||
let rollback_parent = dst_parent.join(old_data_dir.to_string());
|
||||
let backup_path = rollback_parent.join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
std::fs::rename(&backup_path, destination_metadata)?;
|
||||
let _ = remove_file_if_exists(&backup_path);
|
||||
let _ = std::fs::remove_dir(rollback_parent);
|
||||
let backup_path = dst_parent.join(old_data_dir.to_string()).join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
std::fs::rename(backup_path, destination_metadata)?;
|
||||
} else {
|
||||
remove_file_if_exists(destination_metadata)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
pub(super) fn discard_staged_inline_rollback_backup_std(
|
||||
old_data_dir: Uuid,
|
||||
has_old_data_dir: Option<Uuid>,
|
||||
commit_guard: &os::RenameCommitGuard,
|
||||
) {
|
||||
os::discard_inline_rollback_backup_with_commit_guard_std(&old_data_dir.to_string(), has_old_data_dir.is_none(), commit_guard);
|
||||
}
|
||||
|
||||
fn rollback_inline_metadata_std(
|
||||
destination_metadata: &Path,
|
||||
rollback_data_dir: Option<Uuid>,
|
||||
@@ -389,77 +377,34 @@ pub(super) async fn lock_rename_commit_directories(
|
||||
result.map_err(to_file_error).map_err(DiskError::from)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[derive(Clone, Copy)]
|
||||
enum SourcePreparationMode {
|
||||
CreateParent,
|
||||
ExistingParent,
|
||||
FlatInlineTmp { defer_dst_preflight: bool },
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
enum PrepareCommitAndDestinationError {
|
||||
PreflightAccess(std::io::Error),
|
||||
Io(std::io::Error),
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn prepare_rename_commit_and_destination_metadata_std(
|
||||
mode: SourcePreparationMode,
|
||||
create_source_parent: bool,
|
||||
source_parent: &Path,
|
||||
destination_parent: &Path,
|
||||
destination_metadata: &Path,
|
||||
base_dir: &Path,
|
||||
publication_root: &os::PublicationRoot,
|
||||
) -> core::result::Result<(os::RenameCommitGuard, Option<os::ReadDestinationFile>), PrepareCommitAndDestinationError> {
|
||||
if matches!(mode, SourcePreparationMode::CreateParent) {
|
||||
std::fs::create_dir_all(source_parent).map_err(PrepareCommitAndDestinationError::Io)?;
|
||||
) -> std::io::Result<(os::RenameCommitGuard, Option<Bytes>)> {
|
||||
if create_source_parent {
|
||||
std::fs::create_dir_all(source_parent)?;
|
||||
}
|
||||
let defer_dst_preflight = matches!(
|
||||
mode,
|
||||
SourcePreparationMode::FlatInlineTmp {
|
||||
defer_dst_preflight: true
|
||||
}
|
||||
);
|
||||
let guard_res = if matches!(mode, SourcePreparationMode::FlatInlineTmp { .. }) {
|
||||
os::prepare_rename_commit_guard_flat_inline(source_parent, destination_parent, base_dir, publication_root)
|
||||
} else {
|
||||
os::prepare_rename_commit_guard(source_parent, destination_parent, base_dir, publication_root)
|
||||
};
|
||||
let guard = match guard_res {
|
||||
let guard = match os::prepare_rename_commit_guard(source_parent, destination_parent, base_dir, publication_root) {
|
||||
Ok(guard) => guard,
|
||||
Err(err) => {
|
||||
if err
|
||||
.get_ref()
|
||||
.and_then(|inner| inner.downcast_ref::<DiskError>())
|
||||
.is_some_and(|disk_err| matches!(disk_err, DiskError::InvalidPath))
|
||||
{
|
||||
return Err(PrepareCommitAndDestinationError::Io(err));
|
||||
}
|
||||
if defer_dst_preflight && let Err(access_err) = super::super::fs::access_std(base_dir) {
|
||||
return Err(PrepareCommitAndDestinationError::PreflightAccess(access_err));
|
||||
}
|
||||
return Err(PrepareCommitAndDestinationError::Io(match std::fs::symlink_metadata(base_dir) {
|
||||
return Err(match std::fs::symlink_metadata(base_dir) {
|
||||
Err(base_err) if base_err.kind() == ErrorKind::NotFound => base_err,
|
||||
_ => err,
|
||||
}));
|
||||
});
|
||||
}
|
||||
};
|
||||
if defer_dst_preflight
|
||||
&& !guard.has_prepared_destination()
|
||||
&& let Err(access_err) = super::super::fs::access_std(base_dir)
|
||||
{
|
||||
return Err(PrepareCommitAndDestinationError::PreflightAccess(access_err));
|
||||
}
|
||||
let data = os::read_destination_file_handle_with_commit_guard(destination_metadata, &guard)
|
||||
.map_err(PrepareCommitAndDestinationError::Io)?;
|
||||
Ok((guard, data))
|
||||
let data = os::read_destination_file_with_commit_guard(destination_metadata, &guard)?;
|
||||
Ok((guard, data.map(Bytes::from)))
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
enum FusedCommitAttempt {
|
||||
Committed(Box<RenameDataResp>),
|
||||
PreflightAccessErr(std::io::Error),
|
||||
CommitErr(DiskError),
|
||||
RestoreDataSource(os::RenameCommitGuard, DiskError),
|
||||
Fallback(os::RenameCommitGuard, Option<Bytes>, Box<FileInfo>),
|
||||
@@ -467,14 +412,6 @@ enum FusedCommitAttempt {
|
||||
ResumePanic(Box<dyn std::any::Any + Send + 'static>),
|
||||
}
|
||||
|
||||
/// File fdatasync grouping only owns staged shard files. Inline commits have
|
||||
/// no shard directory, so the switch must not evict them from the flat/fused
|
||||
/// metadata path where grouping can never run.
|
||||
#[cfg(any(target_os = "linux", test))]
|
||||
pub(super) fn file_fdatasync_grouping_requires_async_shard_sync(no_inline: bool) -> bool {
|
||||
no_inline && os::file_fdatasync_group_commit_enabled()
|
||||
}
|
||||
|
||||
async fn prepare_rename_commit_and_destination_metadata(
|
||||
create_source_parent: bool,
|
||||
source_parent: &Path,
|
||||
@@ -630,20 +567,7 @@ impl LocalDisk {
|
||||
// object commit is publishing into it. The peer's empty scan remains
|
||||
// optimistic; this lease establishes the local commit/delete order and
|
||||
// remains owned by any blocking syscall that outlives async cancellation.
|
||||
let dst_dir_group_commit = os::dst_dir_fsync_group_commit_enabled();
|
||||
let flat_inline_tmp = cfg!(target_os = "linux")
|
||||
&& (fi.data.is_some() || fi.size <= 0)
|
||||
&& src_volume == super::super::RUSTFS_META_TMP_BUCKET
|
||||
&& pool_meta_condition.is_none()
|
||||
&& !fi.is_healing()
|
||||
&& !dst_dir_group_commit
|
||||
&& !has_rename_data_test_failpoints(dst_path);
|
||||
let defer_dst_preflight = flat_inline_tmp && !skip_access_checks(dst_volume);
|
||||
let (dst_volume_dir, destination_object_path) = if defer_dst_preflight {
|
||||
self.io_get_bucket_and_object_open_path(dst_volume, dst_path)?
|
||||
} else {
|
||||
self.io_get_bucket_and_object_path(dst_volume, dst_path)?
|
||||
};
|
||||
let (dst_volume_dir, destination_object_path) = self.io_get_bucket_and_object_path(dst_volume, dst_path)?;
|
||||
let quota_fence_token =
|
||||
match rustfs_utils::http::metadata_compat::get_consistent_str(&fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX) {
|
||||
Some(value) => {
|
||||
@@ -792,8 +716,7 @@ impl LocalDisk {
|
||||
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
|
||||
}
|
||||
|
||||
if !defer_dst_preflight
|
||||
&& !skip_access_checks(dst_volume)
|
||||
if !skip_access_checks(dst_volume)
|
||||
&& let Err(e) = super::super::fs::access_std(&dst_volume_dir)
|
||||
{
|
||||
info!(
|
||||
@@ -819,6 +742,16 @@ impl LocalDisk {
|
||||
} else {
|
||||
STORAGE_FORMAT_FILE
|
||||
};
|
||||
let flat_inline_tmp = cfg!(target_os = "linux")
|
||||
&& !no_inline
|
||||
&& src_volume == super::super::RUSTFS_META_TMP_BUCKET
|
||||
&& !preserve_upload_metadata
|
||||
&& multipart_publication.is_none()
|
||||
&& pool_meta_condition.is_none()
|
||||
&& !fi_healing
|
||||
&& !os::dst_dir_fsync_group_commit_enabled()
|
||||
&& !os::file_fdatasync_group_commit_enabled()
|
||||
&& !has_rename_data_test_failpoints(dst_path);
|
||||
let src_file_path = match multipart_publication.as_ref() {
|
||||
Some(publication) => publication.canonical_head(),
|
||||
None if flat_inline_tmp => {
|
||||
@@ -900,13 +833,12 @@ impl LocalDisk {
|
||||
.ok_or_else(|| DiskError::other("missing object metadata parent"))?;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
let (rename_commit_guard, has_dst_buf) = if flat_inline_tmp
|
||||
|| (!preserve_upload_metadata
|
||||
&& multipart_publication.is_none()
|
||||
&& pool_meta_condition.is_none()
|
||||
&& !fi_healing
|
||||
&& !dst_dir_group_commit
|
||||
&& !file_fdatasync_grouping_requires_async_shard_sync(no_inline))
|
||||
let (rename_commit_guard, has_dst_buf) = if !preserve_upload_metadata
|
||||
&& multipart_publication.is_none()
|
||||
&& pool_meta_condition.is_none()
|
||||
&& !fi_healing
|
||||
&& !os::dst_dir_fsync_group_commit_enabled()
|
||||
&& !os::file_fdatasync_group_commit_enabled()
|
||||
{
|
||||
#[cfg(test)]
|
||||
if !no_inline {
|
||||
@@ -941,45 +873,25 @@ impl LocalDisk {
|
||||
let root_owned = self.root.clone();
|
||||
|
||||
let fused_attempt = move |is_cancelled: &dyn Fn() -> bool| -> std::io::Result<FusedCommitAttempt> {
|
||||
let source_mode = if flat_inline_tmp {
|
||||
SourcePreparationMode::FlatInlineTmp { defer_dst_preflight }
|
||||
} else if !no_inline {
|
||||
SourcePreparationMode::CreateParent
|
||||
} else {
|
||||
SourcePreparationMode::ExistingParent
|
||||
};
|
||||
let (guard, old_destination_file) = match prepare_rename_commit_and_destination_metadata_std(
|
||||
source_mode,
|
||||
let (guard, has_dst_buf) = prepare_rename_commit_and_destination_metadata_std(
|
||||
!no_inline && !flat_inline_tmp,
|
||||
&src_parent_owned,
|
||||
&dst_parent_owned,
|
||||
&dst_file_path_owned,
|
||||
&dst_volume_dir_owned,
|
||||
&publication_root_owned,
|
||||
) {
|
||||
Ok(res) => res,
|
||||
Err(PrepareCommitAndDestinationError::PreflightAccess(err)) => {
|
||||
return Ok(FusedCommitAttempt::PreflightAccessErr(err));
|
||||
}
|
||||
Err(PrepareCommitAndDestinationError::Io(err)) => return Err(err),
|
||||
};
|
||||
)?;
|
||||
// An existing xl.meta that replaces an existing version slot
|
||||
// requires a rollback directory; appending a new version keeps
|
||||
// an inline rollback backup for post-rename directory sync errors.
|
||||
if is_cancelled() {
|
||||
if flat_inline_tmp {
|
||||
guard.ensure_source_pinned()?;
|
||||
}
|
||||
return Ok(FusedCommitAttempt::Fallback(
|
||||
guard,
|
||||
old_destination_file.map(|read| read.bytes),
|
||||
Box::new(fi_moved),
|
||||
));
|
||||
return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, Box::new(fi_moved)));
|
||||
}
|
||||
let mut xlmeta = FileMeta::new();
|
||||
let mut dst_meta_unparsable = false;
|
||||
if let Some(ref old_file) = old_destination_file {
|
||||
if FileMeta::is_xl2_v1_format(&old_file.bytes)
|
||||
&& let Ok(nmeta) = FileMeta::load(&old_file.bytes)
|
||||
if let Some(ref dst_buf) = has_dst_buf {
|
||||
if FileMeta::is_xl2_v1_format(dst_buf)
|
||||
&& let Ok(nmeta) = FileMeta::load(dst_buf)
|
||||
{
|
||||
xlmeta = nmeta;
|
||||
} else {
|
||||
@@ -989,21 +901,14 @@ impl LocalDisk {
|
||||
let version_id = fi_moved.version_id.unwrap_or_default();
|
||||
let has_old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id));
|
||||
let rollback_data_dir = has_old_data_dir.or_else(|| {
|
||||
if old_destination_file.is_some() && xlmeta.find_version(Some(version_id)).is_ok() {
|
||||
if has_dst_buf.is_some() && xlmeta.find_version(Some(version_id)).is_ok() {
|
||||
Some(inline_metadata_rollback_dir(version_id, &xlmeta))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
});
|
||||
if rollback_data_dir.is_some() && has_rename_data_test_failpoints(&dst_path_str) {
|
||||
if flat_inline_tmp {
|
||||
guard.ensure_source_pinned()?;
|
||||
}
|
||||
return Ok(FusedCommitAttempt::Fallback(
|
||||
guard,
|
||||
old_destination_file.map(|read| read.bytes),
|
||||
Box::new(fi_moved),
|
||||
));
|
||||
return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, Box::new(fi_moved)));
|
||||
}
|
||||
#[cfg(test)]
|
||||
let probe_active = os::file_sync_probe::is_active();
|
||||
@@ -1015,17 +920,10 @@ impl LocalDisk {
|
||||
&& (probe_active || !super::is_presynced_tmp_data_dir(src_data_path))
|
||||
&& os::regular_files(src_data_path).is_ok_and(|files| files.len() >= os::PARALLEL_FILE_SYNC_THRESHOLD)
|
||||
{
|
||||
if flat_inline_tmp {
|
||||
guard.ensure_source_pinned()?;
|
||||
}
|
||||
return Ok(FusedCommitAttempt::Fallback(
|
||||
guard,
|
||||
old_destination_file.map(|read| read.bytes),
|
||||
Box::new(fi_moved),
|
||||
));
|
||||
return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, Box::new(fi_moved)));
|
||||
}
|
||||
let (old_current_size, old_current_source_checked, old_current_source) = observe_old_current(
|
||||
old_destination_file.is_some(),
|
||||
has_dst_buf.is_some(),
|
||||
dst_meta_unparsable,
|
||||
&xlmeta,
|
||||
&dst_volume_str,
|
||||
@@ -1038,9 +936,9 @@ impl LocalDisk {
|
||||
xlmeta.add_version(fi_moved)?;
|
||||
let version_signature = rename_data_versions_signature(&xlmeta);
|
||||
let new_dst_buf = xlmeta.marshal_msg().map_err(std::io::Error::other)?;
|
||||
let destination_was_absent = old_destination_file.is_none();
|
||||
let destination_was_absent = has_dst_buf.is_none();
|
||||
let mut skip_parent = dst_volume_dir_owned.clone();
|
||||
if old_destination_file.is_some() {
|
||||
if has_dst_buf.is_some() {
|
||||
skip_parent = dst_parent_owned.clone();
|
||||
}
|
||||
|
||||
@@ -1050,7 +948,7 @@ impl LocalDisk {
|
||||
os::create_prepared_rename_source_with_commit_guard(&src_file_path_owned, &dst_file_path_owned, &guard)?;
|
||||
let cleanup_flat_tmp = || {
|
||||
if flat_inline_tmp {
|
||||
os::unlink_flat_tmp_source_std(&src_file_path_owned, &src_parent_owned, &publication_root_owned);
|
||||
let _ = remove_file_if_exists(&src_file_path_owned);
|
||||
}
|
||||
};
|
||||
if let Err(err) = prepared_metadata_source.write_all(&new_dst_buf, false) {
|
||||
@@ -1063,14 +961,14 @@ impl LocalDisk {
|
||||
}
|
||||
|
||||
let mut staged_rollback_path = None;
|
||||
if rollback_data_dir.is_none()
|
||||
&& let Some(ref old_file) = old_destination_file
|
||||
&& cfg!(test)
|
||||
if (!no_inline || rollback_data_dir.is_none())
|
||||
&& let Some(ref old_metadata) = has_dst_buf
|
||||
&& (rollback_data_dir.is_some() || cfg!(test))
|
||||
{
|
||||
let backup_path = match create_local_inline_rollback_backup(
|
||||
&dst_file_path_owned,
|
||||
&src_file_path_owned,
|
||||
&old_file.bytes,
|
||||
old_metadata,
|
||||
sync_recovery_protocol,
|
||||
) {
|
||||
Ok(path) => path,
|
||||
@@ -1190,22 +1088,28 @@ impl LocalDisk {
|
||||
}
|
||||
return Ok(FusedCommitAttempt::RestoreDataSource(guard, DiskError::Unexpected));
|
||||
}
|
||||
if let (Some(old_data_dir), Some(old_file)) = (rollback_data_dir, old_destination_file.as_ref()) {
|
||||
if let (Some(old_data_dir), Some(dst_buf)) = (rollback_data_dir, has_dst_buf.as_ref()) {
|
||||
let backup_parent = dst_parent_owned.join(old_data_dir.to_string());
|
||||
if let Err(err) =
|
||||
os::mkdir_all_below_existing_base_std(&backup_parent, &skip_parent, &publication_root_owned)
|
||||
{
|
||||
return Ok(FusedCommitAttempt::RestoreDataSource(guard, to_file_error(err).into()));
|
||||
}
|
||||
let backup_path_guard = match guard.create_destination_directory_for_path_access(&backup_parent) {
|
||||
Ok(g) => g,
|
||||
Err(err) => return Ok(FusedCommitAttempt::RestoreDataSource(guard, to_file_error(err).into())),
|
||||
};
|
||||
let backup_path = backup_parent.join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
if let Err(err) = check_path_length(backup_path.to_string_lossy().as_ref()) {
|
||||
return Ok(FusedCommitAttempt::RestoreDataSource(guard, err));
|
||||
}
|
||||
#[cfg(test)]
|
||||
run_owned_file_before_open(&backup_path);
|
||||
if let Err(err) = os::stage_inline_rollback_backup_with_commit_guard_std(
|
||||
&dst_file_path_owned,
|
||||
&backup_parent,
|
||||
if let Err(err) = backup_path_guard.write_file_for_path_access(
|
||||
&backup_path,
|
||||
old_file,
|
||||
dst_buf.as_ref(),
|
||||
sync_recovery_protocol,
|
||||
sync_recovery_protocol,
|
||||
!should_fail_local_inline_rollback_hardlink(&dst_file_path_owned),
|
||||
&guard,
|
||||
) {
|
||||
return Ok(FusedCommitAttempt::RestoreDataSource(guard, to_file_error(err).into()));
|
||||
}
|
||||
@@ -1213,32 +1117,25 @@ impl LocalDisk {
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterBackupBeforeMetaCommit, &dst_path_str) {
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
|
||||
}
|
||||
} else if let (Some(old_data_dir), Some(old_file)) = (rollback_data_dir, old_destination_file.as_ref()) {
|
||||
} else if let (Some(old_data_dir), Some(staged_backup)) =
|
||||
(rollback_data_dir, staged_rollback_path.as_ref().map(StagedRollbackBackup::path))
|
||||
{
|
||||
let backup_parent = dst_parent_owned.join(old_data_dir.to_string());
|
||||
let backup_path = backup_parent.join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
if let Err(err) = check_path_length(backup_path.to_string_lossy().as_ref()) {
|
||||
drop(prepared_metadata_source);
|
||||
cleanup_flat_tmp();
|
||||
return Ok(FusedCommitAttempt::CommitErr(err));
|
||||
}
|
||||
#[cfg(test)]
|
||||
run_owned_file_before_open(&backup_path);
|
||||
if let Err(err) = os::stage_inline_rollback_backup_with_commit_guard_std(
|
||||
&dst_file_path_owned,
|
||||
&backup_parent,
|
||||
if let Err(err) = os::rename_all_with_precondition_and_parent_sync_std(
|
||||
staged_backup,
|
||||
&backup_path,
|
||||
old_file,
|
||||
&skip_parent,
|
||||
&publication_root_owned,
|
||||
None,
|
||||
sync_recovery_protocol,
|
||||
!should_fail_local_inline_rollback_hardlink(&dst_file_path_owned),
|
||||
&guard,
|
||||
) {
|
||||
drop(prepared_metadata_source);
|
||||
cleanup_flat_tmp();
|
||||
let _ = remove_file_if_exists(staged_backup);
|
||||
return Ok(FusedCommitAttempt::CommitErr(to_file_error(err).into()));
|
||||
}
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterBackupBeforeMetaCommit, &dst_path_str) {
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
|
||||
}
|
||||
staged_rollback_path = None;
|
||||
}
|
||||
|
||||
let rename_res = if should_fail_commit_rename(&dst_path_str) {
|
||||
@@ -1259,9 +1156,6 @@ impl LocalDisk {
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
if let Some(old_data_dir) = rollback_data_dir {
|
||||
discard_staged_inline_rollback_backup_std(old_data_dir, has_old_data_dir, &guard);
|
||||
}
|
||||
if no_inline {
|
||||
return Ok(FusedCommitAttempt::RestoreDataSource(guard, err));
|
||||
}
|
||||
@@ -1287,10 +1181,10 @@ impl LocalDisk {
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
|
||||
}
|
||||
|
||||
if durability.syncs_metadata_commit_directory(rollback_data_dir.is_some()) {
|
||||
if sync_commit_directories {
|
||||
if staged_rollback_path.is_none()
|
||||
&& rollback_data_dir.is_none()
|
||||
&& let Some(ref old_file) = old_destination_file
|
||||
&& let Some(ref old_metadata) = has_dst_buf
|
||||
&& (!no_inline || rollback_data_dir.is_none())
|
||||
{
|
||||
let fsync_res = (|| -> std::io::Result<()> {
|
||||
@@ -1301,7 +1195,7 @@ impl LocalDisk {
|
||||
})();
|
||||
if let Err(err) = fsync_res {
|
||||
let rollback_ok =
|
||||
write_local_inline_rollback_backup_from_bytes(&src_file_path_owned, &old_file.bytes, true)
|
||||
write_local_inline_rollback_backup_from_bytes(&src_file_path_owned, old_metadata, true)
|
||||
.and_then(|backup_path| {
|
||||
let backup = StagedRollbackBackup(backup_path);
|
||||
let res =
|
||||
@@ -1376,20 +1270,6 @@ impl LocalDisk {
|
||||
}
|
||||
return Ok(*resp);
|
||||
}
|
||||
Ok(FusedCommitAttempt::PreflightAccessErr(e)) => {
|
||||
info!(
|
||||
target: "rustfs_ecstore::disk::local",
|
||||
event = EVENT_DISK_LOCAL_ACCESS_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
|
||||
path = ?dst_volume_dir,
|
||||
operation = "rename_data_dst_access",
|
||||
error = %e,
|
||||
"Disk local access check failed"
|
||||
);
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
|
||||
}
|
||||
Ok(FusedCommitAttempt::RestoreDataSource(directory_guard, err)) => {
|
||||
let commit = RenameCommitContext {
|
||||
directory_guard,
|
||||
@@ -1426,7 +1306,7 @@ impl LocalDisk {
|
||||
&& !preserve_upload_metadata
|
||||
&& !publication.legacy
|
||||
&& !fi_healing
|
||||
&& !dst_dir_group_commit
|
||||
&& !os::dst_dir_fsync_group_commit_enabled()
|
||||
{
|
||||
enum MultipartFusedCommitOutcome {
|
||||
Committed(Box<RenameDataResp>),
|
||||
@@ -1456,19 +1336,15 @@ impl LocalDisk {
|
||||
true,
|
||||
move || -> std::io::Result<MultipartFusedCommitOutcome> {
|
||||
let (directory_guard_owned, has_dst_buf_owned) = match prepare_rename_commit_and_destination_metadata_std(
|
||||
SourcePreparationMode::ExistingParent,
|
||||
false,
|
||||
&src_parent_owned,
|
||||
&dst_parent_owned,
|
||||
&dst_file_path_owned,
|
||||
&dst_volume_dir_owned,
|
||||
&publication_root_owned,
|
||||
) {
|
||||
Ok((guard, old_file)) => (guard, old_file.map(|read| read.bytes)),
|
||||
Err(
|
||||
PrepareCommitAndDestinationError::PreflightAccess(err) | PrepareCommitAndDestinationError::Io(err),
|
||||
) => {
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into()));
|
||||
}
|
||||
Ok(res) => res,
|
||||
Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into())),
|
||||
};
|
||||
let mut xlmeta = FileMeta::new();
|
||||
let mut dst_meta_unparsable = false;
|
||||
@@ -2006,10 +1882,9 @@ impl LocalDisk {
|
||||
// durable. It is part of the metadata commit machinery, so relaxed
|
||||
// tiers leave it to the page cache like the xl.meta it mirrors.
|
||||
let backup_sync = durability.recovery_sync_mode();
|
||||
let recovery_backup = rollback_data_dir
|
||||
.zip(has_dst_buf.as_ref())
|
||||
.filter(|_| multipart_publication.as_ref().is_none_or(|publication| publication.legacy));
|
||||
if let Some((old_data_dir, dst_buf)) = recovery_backup {
|
||||
if let (Some(old_data_dir), Some(dst_buf)) = (rollback_data_dir, has_dst_buf.as_ref())
|
||||
&& multipart_publication.as_ref().is_none_or(|publication| publication.legacy)
|
||||
{
|
||||
let backup_parent = dst_file_parent.join(old_data_dir.to_string());
|
||||
#[cfg(not(windows))]
|
||||
if let Err(err) = os::make_dir_all(&backup_parent, &skip_parent).await {
|
||||
@@ -2123,7 +1998,7 @@ impl LocalDisk {
|
||||
// Persist the directory entries for both the data dir and xl.meta renames;
|
||||
// without this the commit itself can vanish on power loss. Relaxed tiers
|
||||
// accept that window (documented in docs/operations/durability-modes.md).
|
||||
if durability.syncs_metadata_commit_directory(recovery_backup.is_some())
|
||||
if durability.syncs_commit_directories()
|
||||
&& let Some(parent) = dst_file_path.parent()
|
||||
{
|
||||
let fsync_started = rustfs_io_metrics::put_stage_timer();
|
||||
@@ -2272,15 +2147,13 @@ impl LocalDisk {
|
||||
let dst_path_for_source = dst_path.to_string();
|
||||
let dst_volume_dir_for_commit = dst_volume_dir.clone();
|
||||
let publication_root_for_commit = self.publication_root.clone();
|
||||
let cleanup_src_parent = !flat_inline_tmp && multipart_publication.is_none() && !preserve_upload_metadata;
|
||||
let cleanup_src_parent = multipart_publication.is_none() && !preserve_upload_metadata;
|
||||
#[cfg(windows)]
|
||||
let source_parent = src_file_parent.to_path_buf();
|
||||
let rename_commit_guard_for_preparation = commit.directory_guard.clone();
|
||||
let sync_metadata_files = durability.syncs_metadata_files();
|
||||
let sync_commit_directories = durability.syncs_commit_directories();
|
||||
let sync_recovery_protocol = durability.syncs_recovery_protocol_state();
|
||||
let sync = durability.syncs_recovery_protocol_state();
|
||||
#[cfg(unix)]
|
||||
let receipt_identity = if sync_recovery_protocol
|
||||
let receipt_identity = if sync
|
||||
&& pool_meta_condition
|
||||
.as_ref()
|
||||
.is_some_and(|condition| condition.parent_raw_sha256.is_none())
|
||||
@@ -2308,7 +2181,7 @@ impl LocalDisk {
|
||||
}
|
||||
#[cfg(test)]
|
||||
run_inline_before_file_sync_admission(dst_path);
|
||||
let mut file_sync_admission = if sync_metadata_files || sync_commit_directories || sync_recovery_protocol {
|
||||
let mut file_sync_admission = if sync {
|
||||
let _admission = os::StageTimer::new(native_timing, "pool_meta_native_file_sync_admission");
|
||||
Some(
|
||||
os::acquire_file_sync_admission(self.file_sync_permits.clone())
|
||||
@@ -2376,10 +2249,14 @@ impl LocalDisk {
|
||||
xlmeta.add_version(fi)?;
|
||||
let version_signature = rename_data_versions_signature(&xlmeta);
|
||||
let new_buf = xlmeta.marshal_msg()?;
|
||||
// Inline payload shares xl.meta's file synchronization policy.
|
||||
// Write the staged xl.meta. Inline objects carry their data inside
|
||||
// xl.meta, so this is the durable preparation for the metadata commit:
|
||||
// relaxed tiers do no per-object fsync here at all (aligned
|
||||
// with MinIO's default), trading a documented power-loss
|
||||
// window for latency.
|
||||
{
|
||||
let _new_file = os::StageTimer::new(native_timing, "pool_meta_native_staged_write_sync");
|
||||
prepared_metadata_source.write_all(&new_buf, sync_metadata_files)?;
|
||||
prepared_metadata_source.write_all(&new_buf, sync)?;
|
||||
}
|
||||
run_inline_preparation_before_backup(&dst_path_for_failpoint);
|
||||
#[cfg(unix)]
|
||||
@@ -2410,10 +2287,10 @@ impl LocalDisk {
|
||||
if !backup.is_linked() {
|
||||
receipt_candidate = None;
|
||||
}
|
||||
if sync_recovery_protocol && !reuses_durability {
|
||||
if sync && !reuses_durability {
|
||||
let _backup_sync = os::StageTimer::new(native_timing, "pool_meta_native_backup_file_sync");
|
||||
backup.sync_data()?;
|
||||
} else if sync_recovery_protocol {
|
||||
} else if sync {
|
||||
rustfs_io_metrics::record_put_object_stage_count("pool_meta_native_backup_sync_reused", 1);
|
||||
}
|
||||
Some(backup)
|
||||
@@ -2427,7 +2304,7 @@ impl LocalDisk {
|
||||
let owned_backup = None;
|
||||
if let Some(ref old_metadata) = has_dst_buf
|
||||
&& owned_backup.is_none()
|
||||
&& (rollback_data_dir.is_some() || fi_healing || sync_recovery_protocol || cfg!(test))
|
||||
&& (rollback_data_dir.is_some() || fi_healing || sync || cfg!(test))
|
||||
{
|
||||
#[cfg(windows)]
|
||||
let backup_path = {
|
||||
@@ -2436,18 +2313,13 @@ impl LocalDisk {
|
||||
.parent()
|
||||
.ok_or_else(|| std::io::Error::new(ErrorKind::InvalidInput, "missing staging metadata parent"))?
|
||||
.join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
source_metadata_guard.write_file_for_path_access(
|
||||
&backup_path,
|
||||
old_metadata,
|
||||
sync_recovery_protocol,
|
||||
false,
|
||||
)?;
|
||||
source_metadata_guard.write_file_for_path_access(&backup_path, old_metadata, sync, false)?;
|
||||
backup_path
|
||||
};
|
||||
#[cfg(not(windows))]
|
||||
let backup_path = {
|
||||
let _backup = os::StageTimer::new(native_timing, "pool_meta_native_backup_create");
|
||||
create_local_inline_rollback_backup(&dst, &src, old_metadata, sync_recovery_protocol)?
|
||||
create_local_inline_rollback_backup(&dst, &src, old_metadata, sync)?
|
||||
};
|
||||
staged_rollback_path = Some(StagedRollbackBackup(backup_path));
|
||||
}
|
||||
@@ -2455,7 +2327,7 @@ impl LocalDisk {
|
||||
let destination_was_absent = has_dst_buf.is_none();
|
||||
if !is_cancelled()
|
||||
&& rollback_data_dir.is_none()
|
||||
&& (!sync_commit_directories || !os::dst_dir_fsync_group_commit_enabled())
|
||||
&& !os::dst_dir_fsync_group_commit_enabled()
|
||||
&& !has_inline_commit_test_failpoints(&dst_path_for_failpoint)
|
||||
{
|
||||
let rename_res = if should_fail_commit_rename(&dst_path_for_failpoint) {
|
||||
@@ -2472,10 +2344,6 @@ impl LocalDisk {
|
||||
)
|
||||
};
|
||||
if let Err(err) = rename_res {
|
||||
#[cfg(target_os = "linux")]
|
||||
if flat_inline_tmp {
|
||||
os::unlink_prepared_rename_source_with_commit_guard_std(&src, &rename_commit_guard_for_preparation);
|
||||
}
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
@@ -2489,7 +2357,7 @@ impl LocalDisk {
|
||||
return Ok(InlinePreparationOutcome::CommitErr(DiskError::from(err)));
|
||||
}
|
||||
|
||||
if sync_commit_directories
|
||||
if sync
|
||||
&& let Err(err) = fsync_inline_commit_std(
|
||||
&dst,
|
||||
&dst_volume_dir_for_commit,
|
||||
@@ -2591,10 +2459,6 @@ impl LocalDisk {
|
||||
}
|
||||
Ok(InlinePreparationOutcome::CommitErr(err)) | Err(err) => {
|
||||
drop(file_sync_admission.take());
|
||||
#[cfg(target_os = "linux")]
|
||||
if flat_inline_tmp {
|
||||
os::unlink_prepared_rename_source_with_commit_guard_std(&src_file_path, &commit.directory_guard);
|
||||
}
|
||||
for part_path in &invalidate_part_paths {
|
||||
self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
|
||||
}
|
||||
@@ -2688,7 +2552,7 @@ impl LocalDisk {
|
||||
}
|
||||
#[cfg(test)]
|
||||
run_rename_data_after_first_publication(&self.root, dst_volume, dst_path);
|
||||
if sync_commit_directories || sync_recovery_protocol {
|
||||
if sync {
|
||||
let _admission = os::StageTimer::new(native_timing, "pool_meta_native_file_sync_readmission");
|
||||
file_sync_admission = Some(
|
||||
os::acquire_file_sync_admission(self.file_sync_permits.clone())
|
||||
@@ -2737,10 +2601,6 @@ impl LocalDisk {
|
||||
};
|
||||
drop(head_publish);
|
||||
if let Err(err) = commit_result {
|
||||
#[cfg(target_os = "linux")]
|
||||
if flat_inline_tmp {
|
||||
os::unlink_prepared_rename_source_with_commit_guard_std(&src_file_path, &commit.directory_guard);
|
||||
}
|
||||
if let Some(backup_path) = local_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
@@ -2756,10 +2616,8 @@ impl LocalDisk {
|
||||
return Err(std::io::Error::other("test fail after metadata commit"));
|
||||
}
|
||||
|
||||
// The object directory also publishes the recovery backup's
|
||||
// directory entry, which file-sync must keep durable.
|
||||
if durability.syncs_metadata_commit_directory(rollback_data_dir.is_some())
|
||||
&& let Some(admission) = file_sync_admission.as_ref()
|
||||
// Persist the commit rename's directory entry across power loss.
|
||||
if let Some(admission) = file_sync_admission.as_ref()
|
||||
&& let Some(dst_parent) = dst_file_path.parent()
|
||||
{
|
||||
let _head_dir = os::StageTimer::new(native_timing, "pool_meta_native_head_dir_sync");
|
||||
@@ -2792,8 +2650,7 @@ impl LocalDisk {
|
||||
// not its own entry, so for a new inline object fsync the ancestor
|
||||
// chain up to and including the bucket. Overwrites already have a
|
||||
// durable object dir; the starts_with guard bounds the walk.
|
||||
if sync_commit_directories
|
||||
&& let Some(admission) = file_sync_admission.as_ref()
|
||||
if let Some(admission) = file_sync_admission.as_ref()
|
||||
&& destination_was_absent
|
||||
{
|
||||
let mut ancestor = dst_file_path.parent().and_then(|object_dir| object_dir.parent());
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,597 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Bounded emergency headroom for durable deletion. Healthy operations access
|
||||
//! only the shared owner's atomic state. Only allocation failures consume the
|
||||
//! reserve, without holding its mutex across transaction publication or quorum.
|
||||
|
||||
use std::{io, path::Path, sync::Arc};
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(super) const RESERVE_BYTES: u64 = 8 * 1024 * 1024;
|
||||
|
||||
pub(super) struct DeleteReserve {
|
||||
#[cfg(unix)]
|
||||
inner: unix::State,
|
||||
}
|
||||
|
||||
impl DeleteReserve {
|
||||
pub(super) fn acquire(root: &Path) -> io::Result<Arc<Self>> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
unix::acquire(root)
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
let _ = root;
|
||||
Ok(Arc::new(Self {}))
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn is_armed(&self) -> bool {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
self.inner.is_armed()
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
/// Populate the reserve on the disk's dedicated fsync lane.
|
||||
pub(super) fn initialize(&self) -> io::Result<()> {
|
||||
self.replenish()
|
||||
}
|
||||
|
||||
/// Refill only when sufficient free space remains for recovery. An armed
|
||||
/// owner returns immediately without opening files or probing capacity.
|
||||
pub(super) fn replenish(&self) -> io::Result<()> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
self.inner.replenish()
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Probe capacity during startup so a persisted reserve can make the disk
|
||||
/// writable before its normal writeability check.
|
||||
pub(super) fn release_if_low(&self) -> io::Result<()> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
self.inner.release_if_low()
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Retry an allocation-safe scratch operation once after ENOSPC. The first
|
||||
/// waiter releases the reserve; concurrent waiters serialize behind it and
|
||||
/// may use the same released capacity. The caller must ensure that a failed
|
||||
/// attempt leaves only disposable scratch or an idempotent primitive under
|
||||
/// the existing namespace owner. Never replay a transaction or a
|
||||
/// publication-plus-sync sequence with this helper.
|
||||
///
|
||||
/// The owner mutex covers release and the retry, excluding replenishment
|
||||
/// during this bounded operation. Later transaction phases and unrelated
|
||||
/// allocators can still compete for the released space.
|
||||
pub(super) fn retry_after_enospc<T>(&self, mut operation: impl FnMut() -> io::Result<T>) -> io::Result<T> {
|
||||
match operation() {
|
||||
#[cfg(unix)]
|
||||
Err(error) if error.kind() == io::ErrorKind::StorageFull => self.inner.retry_after_enospc(operation, error),
|
||||
result => result,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
mod unix {
|
||||
use super::*;
|
||||
use parking_lot::Mutex;
|
||||
use rustix::fs::{Mode, OFlags, mkdirat, open, openat};
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
fs::File,
|
||||
io::{Seek, SeekFrom, Write},
|
||||
os::unix::fs::MetadataExt,
|
||||
sync::{
|
||||
OnceLock, Weak,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
},
|
||||
};
|
||||
|
||||
const METADATA_DIR: &str = ".rustfs.sys";
|
||||
const RESERVE_NAME: &str = "delete-reserve";
|
||||
type RootIdentity = (u64, u64);
|
||||
type OwnerRegistry = Mutex<HashMap<RootIdentity, Weak<DeleteReserve>>>;
|
||||
static OWNERS: OnceLock<OwnerRegistry> = OnceLock::new();
|
||||
|
||||
pub(super) struct State {
|
||||
root: File,
|
||||
mutation: Mutex<()>,
|
||||
armed: AtomicBool,
|
||||
}
|
||||
|
||||
pub(super) fn acquire(path: &Path) -> io::Result<Arc<DeleteReserve>> {
|
||||
let root = File::from(open(
|
||||
path,
|
||||
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
|
||||
Mode::empty(),
|
||||
)?);
|
||||
let metadata = root.metadata()?;
|
||||
let identity = (metadata.dev(), metadata.ino());
|
||||
let mut registry = OWNERS.get_or_init(Mutex::default).lock();
|
||||
if let Some(owner) = registry.get(&identity).and_then(Weak::upgrade) {
|
||||
return Ok(owner);
|
||||
}
|
||||
registry.retain(|_, owner| owner.strong_count() != 0);
|
||||
let owner = Arc::new(DeleteReserve {
|
||||
inner: State {
|
||||
root,
|
||||
mutation: Mutex::new(()),
|
||||
armed: AtomicBool::new(false),
|
||||
},
|
||||
});
|
||||
registry.insert(identity, Arc::downgrade(&owner));
|
||||
Ok(owner)
|
||||
}
|
||||
|
||||
fn metadata_directory(root: &File, create: bool) -> io::Result<File> {
|
||||
if create {
|
||||
match mkdirat(root, METADATA_DIR, Mode::from_raw_mode(0o700)) {
|
||||
Ok(()) => root.sync_all()?,
|
||||
Err(rustix::io::Errno::EXIST) => {}
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
}
|
||||
Ok(File::from(openat(
|
||||
root,
|
||||
METADATA_DIR,
|
||||
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
|
||||
Mode::empty(),
|
||||
)?))
|
||||
}
|
||||
|
||||
fn reserve_file(parent: &File, create: bool) -> io::Result<File> {
|
||||
let flags = OFlags::RDWR | OFlags::NOFOLLOW | OFlags::NONBLOCK | OFlags::CLOEXEC;
|
||||
let file = File::from(openat(
|
||||
parent,
|
||||
RESERVE_NAME,
|
||||
if create { flags | OFlags::CREATE } else { flags },
|
||||
Mode::from_raw_mode(0o600),
|
||||
)?);
|
||||
let metadata = file.metadata()?;
|
||||
if !metadata.is_file() || metadata.nlink() != 1 || metadata.len() > RESERVE_BYTES {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid delete reserve file"));
|
||||
}
|
||||
Ok(file)
|
||||
}
|
||||
|
||||
fn absent_to_none<T>(result: io::Result<T>) -> io::Result<Option<T>> {
|
||||
match result {
|
||||
Ok(value) => Ok(Some(value)),
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None),
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
|
||||
impl State {
|
||||
pub(super) fn is_armed(&self) -> bool {
|
||||
self.armed.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub(super) fn replenish(&self) -> io::Result<()> {
|
||||
if self.is_armed() {
|
||||
return Ok(());
|
||||
}
|
||||
let _guard = self.mutation.lock();
|
||||
if self.is_armed() {
|
||||
return Ok(());
|
||||
}
|
||||
let existing = match absent_to_none(metadata_directory(&self.root, false))? {
|
||||
Some(parent) => absent_to_none(reserve_file(&parent, false))?.map(|file| (parent, file)),
|
||||
None => None,
|
||||
};
|
||||
if let Some((_, file)) = &existing {
|
||||
let metadata = file.metadata()?;
|
||||
if metadata.len() == RESERVE_BYTES && metadata.blocks().saturating_mul(512) >= RESERVE_BYTES {
|
||||
self.armed.store(true, Ordering::Release);
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
// A depleted reserve must not take newly released recovery space
|
||||
// away from a disk that is still nearly full.
|
||||
if available_bytes(&self.root)? < RESERVE_BYTES * 2 {
|
||||
return Ok(());
|
||||
}
|
||||
let (parent, mut file) = match existing {
|
||||
Some(existing) => existing,
|
||||
None => {
|
||||
let parent = metadata_directory(&self.root, true)?;
|
||||
let file = reserve_file(&parent, true)?;
|
||||
(parent, file)
|
||||
}
|
||||
};
|
||||
let metadata = file.metadata()?;
|
||||
// A length alone does not reserve blocks. Repair sparse files left
|
||||
// by interrupted allocation or older installations.
|
||||
if metadata.blocks().saturating_mul(512) < metadata.len() {
|
||||
file.set_len(0)?;
|
||||
}
|
||||
let length = file.seek(SeekFrom::End(0))?;
|
||||
let mut remaining = RESERVE_BYTES - length;
|
||||
let mut bytes = [0_u8; 64 * 1024];
|
||||
while remaining != 0 {
|
||||
let count = usize::try_from(remaining.min(64 * 1024)).map_err(io::Error::other)?;
|
||||
// Zero blocks can become holes on transparent-compression filesystems.
|
||||
rand::fill(&mut bytes[..count]);
|
||||
file.write_all(&bytes[..count])?;
|
||||
remaining -= u64::try_from(count).map_err(io::Error::other)?;
|
||||
}
|
||||
file.sync_all()?;
|
||||
if file.metadata()?.blocks().saturating_mul(512) < RESERVE_BYTES {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::Unsupported,
|
||||
"filesystem did not allocate delete reserve blocks",
|
||||
));
|
||||
}
|
||||
parent.sync_all()?;
|
||||
self.armed.store(true, Ordering::Release);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn release_locked(&self) -> io::Result<bool> {
|
||||
let Some(parent) = absent_to_none(metadata_directory(&self.root, false))? else {
|
||||
self.armed.store(false, Ordering::Release);
|
||||
return Ok(false);
|
||||
};
|
||||
let Some(file) = absent_to_none(reserve_file(&parent, false))? else {
|
||||
self.armed.store(false, Ordering::Release);
|
||||
return Ok(false);
|
||||
};
|
||||
let metadata = file.metadata()?;
|
||||
if metadata.len() == 0 {
|
||||
self.armed.store(false, Ordering::Release);
|
||||
return Ok(false);
|
||||
}
|
||||
file.set_len(0)?;
|
||||
self.armed.store(false, Ordering::Release);
|
||||
// The reserve contains no recovery authority. Truncation releases
|
||||
// blocks immediately and needs no durability barrier before retry.
|
||||
Ok(metadata.blocks() != 0)
|
||||
}
|
||||
|
||||
pub(super) fn release_if_low(&self) -> io::Result<()> {
|
||||
if available_bytes(&self.root)? < RESERVE_BYTES / 2 {
|
||||
let _guard = self.mutation.lock();
|
||||
self.release_locked()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn retry_after_enospc<T>(
|
||||
&self,
|
||||
mut operation: impl FnMut() -> io::Result<T>,
|
||||
original: io::Error,
|
||||
) -> io::Result<T> {
|
||||
let _guard = self.mutation.lock();
|
||||
match self.release_locked() {
|
||||
// A peer may have consumed the shared reserve while this
|
||||
// operation was failing. Waiting for the mutation lock and
|
||||
// replaying once lets it use the capacity that peer released.
|
||||
// The public contract only accepts disposable scratch or an
|
||||
// idempotent primitive, so the retry remains safe even when no
|
||||
// reserve was present. Invalid reserve state still fails closed.
|
||||
Ok(_) => operation(),
|
||||
Err(_) => Err(original),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn available_bytes(root: &File) -> io::Result<u64> {
|
||||
let stat = rustix::fs::fstatvfs(root)?;
|
||||
let block_size = if stat.f_frsize == 0 { stat.f_bsize } else { stat.f_frsize };
|
||||
let available = u128::from(stat.f_bavail) * u128::from(block_size);
|
||||
Ok(u64::try_from(available).unwrap_or(u64::MAX))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn reserve_path(root: &Path) -> std::path::PathBuf {
|
||||
root.join(METADATA_DIR).join(RESERVE_NAME)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_reserve_allocates_retries_once_and_replenishes_real_blocks() {
|
||||
let root = tempfile::tempdir().expect("create disk root");
|
||||
let owner = DeleteReserve::acquire(root.path()).expect("pin disk root");
|
||||
owner.initialize().expect("allocate reserve");
|
||||
let path = reserve_path(root.path());
|
||||
let initial = std::fs::metadata(&path).expect("reserve metadata");
|
||||
assert_eq!(initial.len(), RESERVE_BYTES);
|
||||
assert!(initial.blocks() * 512 >= RESERVE_BYTES);
|
||||
assert!(owner.is_armed());
|
||||
let mut calls = 0;
|
||||
let result = owner
|
||||
.retry_after_enospc(|| {
|
||||
calls += 1;
|
||||
if calls == 1 {
|
||||
Err(io::Error::new(io::ErrorKind::StorageFull, "scratch allocation"))
|
||||
} else {
|
||||
Ok(42)
|
||||
}
|
||||
})
|
||||
.expect("retry scratch allocation");
|
||||
assert_eq!(result, 42);
|
||||
assert_eq!(calls, 2);
|
||||
assert!(!owner.is_armed());
|
||||
assert_eq!(std::fs::metadata(&path).expect("released reserve").len(), 0);
|
||||
owner.replenish().expect("replenish reserve");
|
||||
assert!(owner.is_armed());
|
||||
assert_eq!(std::fs::metadata(&path).expect("replenished reserve").len(), RESERVE_BYTES);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_reserve_retries_once_without_local_reserve_capacity() {
|
||||
let root = tempfile::tempdir().expect("create disk root");
|
||||
let owner = DeleteReserve::acquire(root.path()).expect("pin disk root");
|
||||
let mut calls = 0;
|
||||
let error = owner
|
||||
.retry_after_enospc::<()>(|| {
|
||||
calls += 1;
|
||||
Err(io::Error::new(io::ErrorKind::StorageFull, format!("allocation {calls}")))
|
||||
})
|
||||
.expect_err("retry without a reserve still has no capacity");
|
||||
assert_eq!(calls, 2);
|
||||
assert_eq!(error.to_string(), "allocation 2");
|
||||
owner.initialize().expect("allocate reserve");
|
||||
let parent = metadata_directory(&owner.inner.root, false).expect("metadata directory");
|
||||
let reserve = reserve_file(&parent, false).expect("reserve file");
|
||||
reserve.set_len(0).expect("deplete reserve");
|
||||
calls = 0;
|
||||
assert!(
|
||||
owner
|
||||
.retry_after_enospc::<()>(|| {
|
||||
calls += 1;
|
||||
Err(io::Error::new(io::ErrorKind::StorageFull, "empty reserve"))
|
||||
})
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(calls, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_reserve_retry_ignores_other_errors_and_stays_bounded() {
|
||||
let root = tempfile::tempdir().expect("create disk root");
|
||||
let owner = DeleteReserve::acquire(root.path()).expect("pin disk root");
|
||||
owner.initialize().expect("allocate reserve");
|
||||
let mut calls = 0;
|
||||
let error = owner
|
||||
.retry_after_enospc::<()>(|| {
|
||||
calls += 1;
|
||||
Err(io::Error::new(io::ErrorKind::PermissionDenied, "permission failure"))
|
||||
})
|
||||
.expect_err("permission failures must propagate");
|
||||
assert_eq!(error.kind(), io::ErrorKind::PermissionDenied);
|
||||
assert_eq!(calls, 1);
|
||||
assert!(owner.is_armed());
|
||||
calls = 0;
|
||||
let error = owner
|
||||
.retry_after_enospc::<()>(|| {
|
||||
calls += 1;
|
||||
Err(io::Error::new(io::ErrorKind::StorageFull, format!("allocation {calls}")))
|
||||
})
|
||||
.expect_err("second allocation failure propagates");
|
||||
assert_eq!(calls, 2);
|
||||
assert_eq!(error.to_string(), "allocation 2");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_reserve_rejects_symlinks_and_hardlinks_without_hiding_operation_errors() {
|
||||
use std::os::unix::fs::symlink;
|
||||
let root = tempfile::tempdir().expect("create disk root");
|
||||
let owner = DeleteReserve::acquire(root.path()).expect("pin disk root");
|
||||
let external = tempfile::tempdir().expect("create external directory");
|
||||
let victim = external.path().join("victim");
|
||||
std::fs::write(&victim, b"preserve").expect("write victim");
|
||||
let metadata = root.path().join(METADATA_DIR);
|
||||
symlink(external.path(), &metadata).expect("symlink metadata directory");
|
||||
assert!(owner.replenish().is_err());
|
||||
std::fs::remove_file(&metadata).expect("remove directory link");
|
||||
std::fs::create_dir(&metadata).expect("create metadata directory");
|
||||
let reserve = reserve_path(root.path());
|
||||
symlink(&victim, &reserve).expect("symlink reserve");
|
||||
assert!(owner.replenish().is_err());
|
||||
let mut calls = 0;
|
||||
let error = owner
|
||||
.retry_after_enospc::<()>(|| {
|
||||
calls += 1;
|
||||
Err(io::Error::new(io::ErrorKind::StorageFull, "original error"))
|
||||
})
|
||||
.expect_err("invalid reserve cannot mask allocation failure");
|
||||
assert_eq!(calls, 1);
|
||||
assert_eq!(error.to_string(), "original error");
|
||||
std::fs::remove_file(&reserve).expect("remove reserve link");
|
||||
std::fs::hard_link(&victim, &reserve).expect("hardlink reserve");
|
||||
assert!(owner.replenish().is_err());
|
||||
assert!(owner.inner.release_locked().is_err());
|
||||
assert_eq!(std::fs::read(&victim).expect("read victim"), b"preserve");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_reserve_repairs_partial_and_sparse_files() {
|
||||
let root = tempfile::tempdir().expect("create disk root");
|
||||
let owner = DeleteReserve::acquire(root.path()).expect("pin disk root");
|
||||
let parent = metadata_directory(&owner.inner.root, true).expect("create metadata directory");
|
||||
let mut file = reserve_file(&parent, true).expect("create reserve file");
|
||||
file.write_all(b"partial allocation").expect("write partial reserve");
|
||||
owner.replenish().expect("finish partial allocation");
|
||||
let metadata = file.metadata().expect("partial reserve metadata");
|
||||
assert_eq!(metadata.len(), RESERVE_BYTES);
|
||||
assert!(metadata.blocks() * 512 >= RESERVE_BYTES);
|
||||
file.set_len(0).expect("truncate reserve");
|
||||
file.set_len(RESERVE_BYTES).expect("create sparse reserve");
|
||||
owner.inner.armed.store(false, Ordering::Release);
|
||||
assert!(file.metadata().expect("sparse reserve metadata").blocks() * 512 < RESERVE_BYTES);
|
||||
owner.replenish().expect("repair sparse allocation");
|
||||
let metadata = file.metadata().expect("repaired reserve metadata");
|
||||
assert_eq!(metadata.len(), RESERVE_BYTES);
|
||||
assert!(metadata.blocks() * 512 >= RESERVE_BYTES);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_reserve_shares_reopened_owner_and_serializes_recovery_with_replenishment() {
|
||||
let root = tempfile::tempdir().expect("create disk root");
|
||||
let first = DeleteReserve::acquire(root.path()).expect("open first root");
|
||||
let second = DeleteReserve::acquire(root.path()).expect("reopen root");
|
||||
assert!(Arc::ptr_eq(&first, &second));
|
||||
first.initialize().expect("allocate reserve");
|
||||
let (retry_entered_tx, retry_entered_rx) = std::sync::mpsc::channel();
|
||||
let (allow_retry_tx, allow_retry_rx) = std::sync::mpsc::channel();
|
||||
let (replenished_tx, replenished_rx) = std::sync::mpsc::channel();
|
||||
let recovery_owner = first.clone();
|
||||
let root_path = root.path().to_path_buf();
|
||||
std::thread::scope(|scope| {
|
||||
let recovery = scope.spawn(move || {
|
||||
let mut calls = 0;
|
||||
recovery_owner
|
||||
.retry_after_enospc(|| {
|
||||
calls += 1;
|
||||
if calls == 1 {
|
||||
return Err(io::Error::new(io::ErrorKind::StorageFull, "scratch allocation"));
|
||||
}
|
||||
assert!(recovery_owner.inner.mutation.try_lock().is_none(), "retry must retain the reserve mutex");
|
||||
retry_entered_tx.send(()).expect("notify retry");
|
||||
allow_retry_rx
|
||||
.recv_timeout(std::time::Duration::from_secs(10))
|
||||
.expect("allow retry completion");
|
||||
assert_eq!(std::fs::metadata(reserve_path(&root_path)).expect("depleted reserve").len(), 0);
|
||||
Ok(())
|
||||
})
|
||||
.expect("retry allocation");
|
||||
});
|
||||
retry_entered_rx
|
||||
.recv_timeout(std::time::Duration::from_secs(10))
|
||||
.expect("retry entered");
|
||||
let replenish = scope.spawn(move || {
|
||||
second.replenish().expect("concurrent replenishment");
|
||||
replenished_tx.send(()).expect("notify replenishment");
|
||||
});
|
||||
assert!(matches!(replenished_rx.try_recv(), Err(std::sync::mpsc::TryRecvError::Empty)));
|
||||
allow_retry_tx.send(()).expect("finish retry");
|
||||
recovery.join().expect("recovery worker");
|
||||
replenish.join().expect("replenishment worker");
|
||||
});
|
||||
assert!(first.is_armed());
|
||||
assert_eq!(std::fs::metadata(reserve_path(root.path())).expect("final reserve").len(), RESERVE_BYTES);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_reserve_gives_concurrent_enospc_waiters_one_retry() {
|
||||
let root = tempfile::tempdir().expect("create disk root");
|
||||
let first = DeleteReserve::acquire(root.path()).expect("open first owner");
|
||||
let second = DeleteReserve::acquire(root.path()).expect("open second owner");
|
||||
first.initialize().expect("allocate reserve");
|
||||
|
||||
let (first_retry_tx, first_retry_rx) = std::sync::mpsc::channel();
|
||||
let (allow_first_tx, allow_first_rx) = std::sync::mpsc::channel();
|
||||
let (second_failed_tx, second_failed_rx) = std::sync::mpsc::channel();
|
||||
let (second_done_tx, second_done_rx) = std::sync::mpsc::channel();
|
||||
|
||||
std::thread::scope(|scope| {
|
||||
let first_waiter = scope.spawn(move || {
|
||||
let mut calls = 0;
|
||||
first
|
||||
.retry_after_enospc(|| {
|
||||
calls += 1;
|
||||
if calls == 1 {
|
||||
return Err(io::Error::new(io::ErrorKind::StorageFull, "first ENOSPC"));
|
||||
}
|
||||
first_retry_tx.send(()).expect("notify first retry");
|
||||
allow_first_rx
|
||||
.recv_timeout(std::time::Duration::from_secs(10))
|
||||
.expect("allow first retry to finish");
|
||||
Ok(())
|
||||
})
|
||||
.expect("first waiter retries");
|
||||
assert_eq!(calls, 2);
|
||||
});
|
||||
|
||||
first_retry_rx
|
||||
.recv_timeout(std::time::Duration::from_secs(10))
|
||||
.expect("first waiter holds recovery lock");
|
||||
let second_waiter = scope.spawn(move || {
|
||||
let mut calls = 0;
|
||||
second
|
||||
.retry_after_enospc(|| {
|
||||
calls += 1;
|
||||
if calls == 1 {
|
||||
second_failed_tx.send(()).expect("notify second ENOSPC");
|
||||
return Err(io::Error::new(io::ErrorKind::StorageFull, "second ENOSPC"));
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.expect("second waiter retries after shared release");
|
||||
assert_eq!(calls, 2);
|
||||
second_done_tx.send(()).expect("notify second completion");
|
||||
});
|
||||
|
||||
second_failed_rx
|
||||
.recv_timeout(std::time::Duration::from_secs(10))
|
||||
.expect("second waiter observed ENOSPC");
|
||||
assert!(matches!(second_done_rx.try_recv(), Err(std::sync::mpsc::TryRecvError::Empty)));
|
||||
allow_first_tx.send(()).expect("finish first retry");
|
||||
first_waiter.join().expect("first waiter");
|
||||
second_done_rx
|
||||
.recv_timeout(std::time::Duration::from_secs(10))
|
||||
.expect("second waiter completed its bounded retry");
|
||||
second_waiter.join().expect("second waiter");
|
||||
});
|
||||
|
||||
assert_eq!(std::fs::metadata(reserve_path(root.path())).expect("depleted reserve").len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_reserve_rejects_oversize_files_and_fifos() {
|
||||
let root = tempfile::tempdir().expect("create disk root");
|
||||
let owner = DeleteReserve::acquire(root.path()).expect("pin disk root");
|
||||
let parent = metadata_directory(&owner.inner.root, true).expect("create metadata directory");
|
||||
let file = reserve_file(&parent, true).expect("create reserve file");
|
||||
file.set_len(RESERVE_BYTES + 1).expect("create oversized reserve");
|
||||
assert!(owner.replenish().is_err());
|
||||
assert!(owner.inner.release_locked().is_err());
|
||||
assert_eq!(file.metadata().expect("oversize metadata").len(), RESERVE_BYTES + 1);
|
||||
let path = reserve_path(root.path());
|
||||
std::fs::remove_file(&path).expect("remove oversized file");
|
||||
assert!(
|
||||
std::process::Command::new("mkfifo")
|
||||
.arg(&path)
|
||||
.status()
|
||||
.expect("create FIFO fixture")
|
||||
.success()
|
||||
);
|
||||
assert!(owner.replenish().is_err());
|
||||
assert!(owner.inner.release_locked().is_err());
|
||||
use std::os::unix::fs::FileTypeExt;
|
||||
assert!(std::fs::symlink_metadata(path).expect("FIFO metadata").file_type().is_fifo());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -17,27 +17,6 @@ use std::{
|
||||
};
|
||||
use uuid::Uuid;
|
||||
|
||||
fn verify_synced_target(
|
||||
dir: &Path,
|
||||
base: &Path,
|
||||
root: &os::PublicationRoot,
|
||||
length: u64,
|
||||
digest: [u8; 32],
|
||||
) -> std::io::Result<()> {
|
||||
// Target version histories are bounded by the prepared receipt, not the source-control record limit.
|
||||
let observed = os::observe_multipart_source_metadata(&dir.join(STORAGE_FORMAT_FILE), base, root, length).map_err(|err| {
|
||||
if err.kind() == std::io::ErrorKind::NotFound {
|
||||
IoError::other(DiskError::ConditionalDeleteConflict)
|
||||
} else {
|
||||
err
|
||||
}
|
||||
})?;
|
||||
if observed != (length, digest) {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
const STATE: &str = ".mpu-source-state";
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
@@ -2491,7 +2470,30 @@ impl NativeMultipartPublication {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
if target_already_synced {
|
||||
verify_synced_target(&target_dir, base, root, expected_length, expected_sha256)?;
|
||||
let pinned = os::pin_multipart_source_directory(&target_dir, base, root).map_err(|error| {
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
IoError::other(DiskError::ConditionalDeleteConflict)
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
let target = os::read_multipart_source_metadata_in_pinned(
|
||||
&pinned,
|
||||
&target_dir,
|
||||
std::ffi::OsStr::new(STORAGE_FORMAT_FILE),
|
||||
base,
|
||||
root,
|
||||
)
|
||||
.map_err(|error| {
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
IoError::other(DiskError::ConditionalDeleteConflict)
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
if target.len() as u64 != expected_length || raw_sha(&target) != expected_sha256 {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
} else {
|
||||
os::read_and_sync_multipart_target_metadata(
|
||||
&target_dir,
|
||||
@@ -2777,29 +2779,3 @@ impl LocalDisk {
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod target_receipt_tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn synced_target_uses_prepared_receipt_for_large_history() {
|
||||
let tmp = tempfile::tempdir().expect("temporary publication root");
|
||||
let base = tmp.path().canonicalize().expect("absolute root");
|
||||
let dir = base.join("target");
|
||||
std::fs::create_dir(&dir).expect("target directory");
|
||||
let root = os::PublicationRoot::new(&base).expect("publication root");
|
||||
let block = [0u8; 64 * 1024];
|
||||
let length = 16 * 1024 * 1024 + 1;
|
||||
std::fs::File::create(dir.join(STORAGE_FORMAT_FILE))
|
||||
.expect("target head")
|
||||
.set_len(length)
|
||||
.expect("sparse head");
|
||||
let mut hash = Sha256::new();
|
||||
for _ in 0..256 {
|
||||
hash.update(block);
|
||||
}
|
||||
hash.update([0]);
|
||||
verify_synced_target(&dir, &base, &root, length, hash.finalize().into()).expect("matching receipt");
|
||||
assert!(verify_synced_target(&dir, &base, &root, length - 1, [0; 32]).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
+152
-1179
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -53,8 +53,6 @@ pub(crate) struct DecodeOutcome {
|
||||
pub(crate) error: Option<std::io::Error>,
|
||||
pub(crate) exact_quorum: bool,
|
||||
pub(crate) repair: ShardReadRepair,
|
||||
pub(crate) frame_count: usize,
|
||||
pub(crate) read_dispatch_count: usize,
|
||||
}
|
||||
|
||||
type ShardIndexes = SmallVec<[usize; INLINE_SHARD_SLOTS]>;
|
||||
@@ -837,58 +835,6 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a reconstruction-verifying reader for one q-aligned slice of a
|
||||
/// logical EC block. The underlying erasure geometry still determines the
|
||||
/// number of data and parity readers, while `read_shard_size` describes the
|
||||
/// smaller byte vector that Reed-Solomon should reconstruct for this range.
|
||||
///
|
||||
/// The q-range caller opens every reader at the same logical shard offset,
|
||||
/// so this reader only needs one local stripe and no global shard cursor.
|
||||
fn new_for_quantized_decode(
|
||||
readers: Vec<Option<BitrotReader<R>>>,
|
||||
e: Erasure,
|
||||
read_shard_size: usize,
|
||||
metrics_path: Option<&'static str>,
|
||||
read_costs: Option<Vec<ShardReadCost>>,
|
||||
) -> Self {
|
||||
let mut reader = if let Some(read_costs) = read_costs {
|
||||
Self::new_with_metrics_path_read_costs_timeout_and_reconstruction_verification(
|
||||
readers,
|
||||
e,
|
||||
0,
|
||||
0,
|
||||
metrics_path,
|
||||
read_costs,
|
||||
get_object_disk_read_timeout(),
|
||||
true,
|
||||
)
|
||||
} else {
|
||||
Self::new_with_metrics_path_read_timeout_and_reconstruction_verification(
|
||||
readers,
|
||||
e,
|
||||
0,
|
||||
0,
|
||||
metrics_path,
|
||||
get_object_disk_read_timeout(),
|
||||
true,
|
||||
)
|
||||
};
|
||||
reader.offset = 0;
|
||||
reader.shard_size = read_shard_size;
|
||||
reader.shard_file_size = read_shard_size;
|
||||
// A q-range is already bounded to the exact shard frames needed by the
|
||||
// client request. Keep healthy reads demand-bound as well: read the
|
||||
// data quorum and retain parity as stripe-aligned reserves for a failed
|
||||
// data reader. The ordinary GET rollout remains controlled by its
|
||||
// deployment gate, while this bounded path avoids paying parity I/O for
|
||||
// every small Range request.
|
||||
reader.demand_bound_lockstep = true;
|
||||
for (index, engaged) in reader.engaged.iter_mut().enumerate() {
|
||||
*engaged = index < reader.data_shards;
|
||||
}
|
||||
reader
|
||||
}
|
||||
|
||||
pub(crate) fn with_repair_evidence(mut self, evidence: Arc<ShardReadRepair>) -> Self {
|
||||
self.repair_evidence = Some(evidence);
|
||||
self
|
||||
@@ -1053,15 +999,6 @@ where
|
||||
state.into_parts()
|
||||
}
|
||||
|
||||
pub(crate) fn read_shape(&self) -> (usize, usize) {
|
||||
self.readers.iter().flatten().fold((0, 0), |(frames, dispatches), reader| {
|
||||
(
|
||||
frames.saturating_add(reader.last_frame_count()),
|
||||
dispatches.saturating_add(reader.last_read_dispatch_count()),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
async fn read_into_state(&mut self, state: &mut StripeReadState) {
|
||||
// On the reconstruction-verifying GET path, read every live shard reader
|
||||
// in lockstep so all readers advance one block per stripe and stay
|
||||
@@ -1897,9 +1834,6 @@ where
|
||||
// fail a stripe that later reaches a real quorum.
|
||||
self.engaged[i] = false;
|
||||
} else {
|
||||
if let Some(evidence) = &self.repair_evidence {
|
||||
evidence.observe(&[Some(e.clone())]);
|
||||
}
|
||||
errs[i] = Some(e);
|
||||
// Lockstep cannot safely reuse a reader after any
|
||||
// error, even when the low-level classifier called it
|
||||
@@ -2509,301 +2443,6 @@ impl Erasure {
|
||||
.await
|
||||
}
|
||||
|
||||
/// Decode a range that was opened as one q-aligned slice of every data
|
||||
/// shard. The caller supplies the output offset within the concatenated
|
||||
/// q-slices (`shard_index * read_shard_size + offset_in_shard`). Keeping
|
||||
/// this path separate from `decode_inner` preserves the normal full-shard
|
||||
/// reader geometry for legacy and full-object GETs.
|
||||
/// `allow_direct_data_shard` requires the caller to have established its
|
||||
/// metadata/open quorum and enabled frame authentication; policies requiring
|
||||
/// reconstruction evidence must leave it false.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) async fn decode_quantized_range_with_stripe_handles_and_reopeners<W, R>(
|
||||
&self,
|
||||
writer: &mut W,
|
||||
mut readers: Vec<Option<BitrotReader<R>>>,
|
||||
output_offset: usize,
|
||||
length: usize,
|
||||
read_shard_size: usize,
|
||||
read_costs: Option<Vec<ShardReadCost>>,
|
||||
mut deferred_handles: Vec<Option<DeferredReaderStripeHandle>>,
|
||||
mut deferred_reopeners: Vec<Option<DeferredReaderReopener<R>>>,
|
||||
allow_direct_data_shard: bool,
|
||||
) -> DecodeOutcome
|
||||
where
|
||||
W: AsyncWrite + Send + Sync + Unpin,
|
||||
R: crate::erasure::coding::ShardSource,
|
||||
{
|
||||
if readers.len() != self.data_shards + self.parity_shards {
|
||||
record_get_object_pipeline_failure(GET_STAGE_RANGE, GetObjectFailureReason::RangeOrLengthInvalid);
|
||||
return DecodeOutcome {
|
||||
error: Some(io::Error::new(ErrorKind::InvalidInput, "Invalid number of readers")),
|
||||
..Default::default()
|
||||
};
|
||||
}
|
||||
if read_shard_size == 0 {
|
||||
record_get_object_pipeline_failure(GET_STAGE_RANGE, GetObjectFailureReason::RangeOrLengthInvalid);
|
||||
return DecodeOutcome {
|
||||
error: Some(io::Error::new(ErrorKind::InvalidInput, "quantized shard size must be non-zero")),
|
||||
..Default::default()
|
||||
};
|
||||
}
|
||||
let Some(required_len) = output_offset.checked_add(length) else {
|
||||
record_get_object_pipeline_failure(GET_STAGE_RANGE, GetObjectFailureReason::RangeOrLengthInvalid);
|
||||
return DecodeOutcome {
|
||||
error: Some(io::Error::new(ErrorKind::InvalidInput, "quantized output range overflows")),
|
||||
..Default::default()
|
||||
};
|
||||
};
|
||||
let Some(max_len) = self.data_shards.checked_mul(read_shard_size) else {
|
||||
record_get_object_pipeline_failure(GET_STAGE_RANGE, GetObjectFailureReason::RangeOrLengthInvalid);
|
||||
return DecodeOutcome {
|
||||
error: Some(io::Error::new(ErrorKind::InvalidInput, "quantized shard geometry overflows")),
|
||||
..Default::default()
|
||||
};
|
||||
};
|
||||
if required_len > max_len {
|
||||
record_get_object_pipeline_failure(GET_STAGE_RANGE, GetObjectFailureReason::RangeOrLengthInvalid);
|
||||
return DecodeOutcome {
|
||||
error: Some(io::Error::new(ErrorKind::InvalidInput, "quantized output range exceeds shard geometry")),
|
||||
..Default::default()
|
||||
};
|
||||
}
|
||||
if length == 0 {
|
||||
return DecodeOutcome::default();
|
||||
}
|
||||
|
||||
let metrics_path = Some(GET_OBJECT_PATH_LEGACY_DUPLEX);
|
||||
let target_shard = output_offset / read_shard_size;
|
||||
let within_shard = output_offset % read_shard_size;
|
||||
// The caller has already established the normal metadata and reader-
|
||||
// open quorum. A healthy systematic shard can serve a single-shard
|
||||
// range after authenticating every frame in its opened q-slice. Other
|
||||
// data shards contain no requested bytes; read them only if this shard
|
||||
// fails and reconstruction becomes necessary. Protected layouts and
|
||||
// policies requiring reconstruction evidence keep the quorum decoder.
|
||||
if allow_direct_data_shard
|
||||
&& !self.uses_legacy_codec()
|
||||
&& length <= read_shard_size - within_shard
|
||||
&& readers.iter().flatten().all(|reader| reader.integrity_proof().is_none())
|
||||
&& let Some(mut direct_reader) = readers[target_shard].take()
|
||||
{
|
||||
let stripe_read_stage_start = get_stage_timer_if_enabled(rustfs_io_metrics::get_stage_metrics_enabled());
|
||||
let read_cost = read_costs
|
||||
.as_ref()
|
||||
.and_then(|costs| costs.get(target_shard))
|
||||
.copied()
|
||||
.unwrap_or(ShardReadCost::Unknown);
|
||||
let read_timeout = get_object_disk_read_timeout();
|
||||
let mut direct_read = Box::pin(read_shard_result(
|
||||
target_shard,
|
||||
read_cost,
|
||||
&mut direct_reader,
|
||||
None,
|
||||
read_shard_size,
|
||||
self.data_shards,
|
||||
read_timeout,
|
||||
metrics_path,
|
||||
));
|
||||
let initial_result = if let Some(delay) = shard_read_hedge_delay(read_timeout) {
|
||||
tokio::select! {
|
||||
biased;
|
||||
result = &mut direct_read => Some(result.0),
|
||||
_ = tokio::time::sleep(delay) => None,
|
||||
}
|
||||
} else {
|
||||
Some((&mut direct_read).await.0)
|
||||
};
|
||||
let repair = Arc::new(ShardReadRepair::default());
|
||||
let mut outcome = DecodeOutcome::default();
|
||||
let selected_data;
|
||||
match initial_result {
|
||||
Some(Ok(data)) => selected_data = Some((data, within_shard)),
|
||||
initial_result => {
|
||||
// Keep a pending target alive through its real deadline.
|
||||
// The hedge only admits competing reconstruction; it is
|
||||
// not evidence that this healthy source is missing.
|
||||
if let Some(handle) = deferred_handles.get_mut(target_shard) {
|
||||
*handle = None;
|
||||
}
|
||||
if let Some(reopener) = deferred_reopeners.get_mut(target_shard) {
|
||||
*reopener = None;
|
||||
}
|
||||
if let Some(Err(error)) = &initial_result {
|
||||
repair.observe(&[Some(error.clone())]);
|
||||
}
|
||||
// Buffer at most one requested data-shard slice. Neither
|
||||
// contender writes to the downstream until authenticated
|
||||
// direct bytes or a complete verified quorum has won.
|
||||
let mut fallback_data = Vec::with_capacity(length);
|
||||
let mut fallback = Box::pin(self.decode_quantized_range_quorum(
|
||||
&mut fallback_data,
|
||||
readers,
|
||||
output_offset,
|
||||
length,
|
||||
read_shard_size,
|
||||
read_costs,
|
||||
deferred_handles,
|
||||
deferred_reopeners,
|
||||
Some(target_shard),
|
||||
Arc::clone(&repair),
|
||||
));
|
||||
if initial_result.is_some() {
|
||||
outcome = (&mut fallback).await;
|
||||
drop(fallback);
|
||||
selected_data = (outcome.written == length).then_some((fallback_data.into(), 0));
|
||||
} else {
|
||||
let race = tokio::select! {
|
||||
biased;
|
||||
result = &mut direct_read => Ok(result.0),
|
||||
outcome = &mut fallback => Err(outcome),
|
||||
};
|
||||
match race {
|
||||
Ok(Ok(data)) => {
|
||||
drop(fallback);
|
||||
selected_data = Some((data, within_shard));
|
||||
}
|
||||
Ok(Err(error)) => {
|
||||
repair.observe(&[Some(error)]);
|
||||
outcome = (&mut fallback).await;
|
||||
drop(fallback);
|
||||
selected_data = (outcome.written == length).then_some((fallback_data.into(), 0));
|
||||
}
|
||||
Err(fallback_outcome) => {
|
||||
drop(fallback);
|
||||
if fallback_outcome.written == length {
|
||||
outcome = fallback_outcome;
|
||||
selected_data = Some((fallback_data.into(), 0));
|
||||
} else {
|
||||
// A failed alternative quorum must not vote
|
||||
// away a still-pending healthy target.
|
||||
match (&mut direct_read).await.0 {
|
||||
Ok(data) => selected_data = Some((data, within_shard)),
|
||||
Err(error) => {
|
||||
repair.observe(&[Some(error)]);
|
||||
outcome = fallback_outcome;
|
||||
selected_data = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
drop(direct_read);
|
||||
outcome.frame_count = outcome.frame_count.saturating_add(direct_reader.last_frame_count());
|
||||
outcome.read_dispatch_count = outcome
|
||||
.read_dispatch_count
|
||||
.saturating_add(direct_reader.last_read_dispatch_count());
|
||||
outcome.repair = repair.snapshot();
|
||||
record_get_stage_duration_if_enabled(GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_STRIPE_READ, stripe_read_stage_start);
|
||||
if let Some((data, start)) = selected_data {
|
||||
let emit_stage_start = get_stage_timer_if_enabled(rustfs_io_metrics::get_stage_metrics_enabled());
|
||||
let result = writer.write_all(&data[start..start + length]).await;
|
||||
record_get_stage_duration_if_enabled(GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_EMIT, emit_stage_start);
|
||||
if let Err(error) = result {
|
||||
outcome.written = 0;
|
||||
outcome.error = Some(error);
|
||||
} else {
|
||||
outcome.written = length;
|
||||
}
|
||||
}
|
||||
return outcome;
|
||||
}
|
||||
self.decode_quantized_range_quorum(
|
||||
writer,
|
||||
readers,
|
||||
output_offset,
|
||||
length,
|
||||
read_shard_size,
|
||||
read_costs,
|
||||
deferred_handles,
|
||||
deferred_reopeners,
|
||||
None,
|
||||
Arc::new(ShardReadRepair::default()),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn decode_quantized_range_quorum<W, R>(
|
||||
&self,
|
||||
writer: &mut W,
|
||||
readers: Vec<Option<BitrotReader<R>>>,
|
||||
output_offset: usize,
|
||||
length: usize,
|
||||
read_shard_size: usize,
|
||||
read_costs: Option<Vec<ShardReadCost>>,
|
||||
deferred_handles: Vec<Option<DeferredReaderStripeHandle>>,
|
||||
deferred_reopeners: Vec<Option<DeferredReaderReopener<R>>>,
|
||||
pending_target: Option<usize>,
|
||||
repair: Arc<ShardReadRepair>,
|
||||
) -> DecodeOutcome
|
||||
where
|
||||
W: AsyncWrite + Send + Sync + Unpin,
|
||||
R: crate::erasure::coding::ShardSource,
|
||||
{
|
||||
let metrics_path = Some(GET_OBJECT_PATH_LEGACY_DUPLEX);
|
||||
let mut reader =
|
||||
ParallelReader::new_for_quantized_decode(readers, self.clone(), read_shard_size, metrics_path, read_costs)
|
||||
.with_deferred_parity_handles(deferred_handles)
|
||||
.with_deferred_parity_reopeners(deferred_reopeners)
|
||||
.with_repair_evidence(Arc::clone(&repair));
|
||||
let integrity = reader.integrity.clone();
|
||||
// The q-range reader still verifies each bitrot frame. Requiring an
|
||||
// additional reconstruction source for a healthy range would defeat
|
||||
// demand-bound parity and turn every request back into an all-shard
|
||||
// read; parity is admitted by the lockstep reader only when data is
|
||||
// missing or fails verification.
|
||||
let require_surplus_source = false;
|
||||
|
||||
let stripe_read_stage_start = get_stage_timer_if_enabled(rustfs_io_metrics::get_stage_metrics_enabled());
|
||||
let (mut shards, mut errs) = reader.read().await;
|
||||
if let Some(target_shard) = pending_target {
|
||||
// This slot is owned by the direct contender, rather than missing
|
||||
// from storage. Clear only its synthetic error; actual successful
|
||||
// shards still determine the reconstruction quorum below.
|
||||
errs[target_shard] = None;
|
||||
}
|
||||
let (frame_count, read_dispatch_count) = reader.read_shape();
|
||||
record_get_stage_duration_if_enabled(GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_STRIPE_READ, stripe_read_stage_start);
|
||||
let exact_quorum = shards.iter().filter(|shard| shard.is_some()).count() == self.data_shards;
|
||||
let mut written = 0;
|
||||
let mut ret_err = None;
|
||||
let flow = self
|
||||
.emit_decoded_stripe(
|
||||
writer,
|
||||
&mut shards,
|
||||
&errs,
|
||||
output_offset,
|
||||
length,
|
||||
&mut written,
|
||||
&mut ret_err,
|
||||
&repair,
|
||||
rustfs_io_metrics::get_stage_metrics_enabled(),
|
||||
require_surplus_source,
|
||||
integrity.as_ref(),
|
||||
0,
|
||||
)
|
||||
.await;
|
||||
if matches!(flow, StripeFlow::Continue) {
|
||||
reader.recycle_shards(&mut shards);
|
||||
}
|
||||
if ret_err.is_none() && written < length {
|
||||
ret_err = Some(Error::LessData.into());
|
||||
}
|
||||
|
||||
DecodeOutcome {
|
||||
written,
|
||||
error: ret_err,
|
||||
exact_quorum,
|
||||
repair: repair.snapshot(),
|
||||
frame_count,
|
||||
read_dispatch_count,
|
||||
}
|
||||
}
|
||||
|
||||
/// Reconstruct and emit one already-read stripe.
|
||||
///
|
||||
/// This is the shared per-stripe body used by both the serial legacy loop
|
||||
@@ -3254,8 +2893,6 @@ impl Erasure {
|
||||
error: ret_err,
|
||||
exact_quorum,
|
||||
repair,
|
||||
frame_count: 0,
|
||||
read_dispatch_count: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3409,372 +3046,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
async fn quantized_counting_readers(
|
||||
erasure: &Erasure,
|
||||
data: &[u8],
|
||||
frame_size: usize,
|
||||
slice_offset: usize,
|
||||
slice_length: usize,
|
||||
corrupt_shard: Option<usize>,
|
||||
) -> (Vec<Option<BitrotReader<CountingShardReader>>>, Vec<Arc<AtomicUsize>>) {
|
||||
let shards = erasure.encode_data(data).expect("quantized fixture must encode");
|
||||
let mut readers = Vec::with_capacity(shards.len());
|
||||
let mut counters = Vec::with_capacity(shards.len());
|
||||
for shard in shards {
|
||||
let mut writer = BitrotWriter::new(Vec::new(), frame_size, HashAlgorithm::HighwayHash256S);
|
||||
for frame in shard[slice_offset..slice_offset + slice_length].chunks(frame_size) {
|
||||
writer.write(frame).await.expect("quantized fixture frame must write");
|
||||
}
|
||||
let mut encoded = writer.into_inner();
|
||||
if corrupt_shard == Some(readers.len()) {
|
||||
*encoded.last_mut().expect("fixture has a frame payload") ^= 0x80;
|
||||
}
|
||||
let bytes_read = Arc::new(AtomicUsize::new(0));
|
||||
readers.push(Some(BitrotReader::new_with_frame_size(
|
||||
CountingShardReader {
|
||||
inner: Cursor::new(encoded),
|
||||
bytes_read: Arc::clone(&bytes_read),
|
||||
},
|
||||
erasure.shard_size(),
|
||||
frame_size,
|
||||
HashAlgorithm::HighwayHash256S,
|
||||
false,
|
||||
)));
|
||||
counters.push(bytes_read);
|
||||
}
|
||||
(readers, counters)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn quantized_direct_reads_only_the_requested_authenticated_data_shard() {
|
||||
let data = b"abcdefghijklmnop";
|
||||
let erasure = Erasure::new(2, 2, data.len());
|
||||
let (readers, counters) = quantized_counting_readers(&erasure, data, 4, 0, 4, None).await;
|
||||
let mut output = Vec::new();
|
||||
let outcome = erasure
|
||||
.decode_quantized_range_with_stripe_handles_and_reopeners(
|
||||
&mut output,
|
||||
readers,
|
||||
5,
|
||||
2,
|
||||
4,
|
||||
None,
|
||||
Vec::new(),
|
||||
Vec::new(),
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(outcome.error.is_none(), "authenticated direct range must succeed: {:?}", outcome.error);
|
||||
assert_eq!(outcome.written, 2);
|
||||
assert_eq!(output, &data[9..11]);
|
||||
assert_eq!(outcome.frame_count, 1);
|
||||
assert_eq!(outcome.read_dispatch_count, 1);
|
||||
for (index, counter) in counters.iter().enumerate() {
|
||||
assert_eq!(
|
||||
counter.load(Ordering::SeqCst),
|
||||
if index == 1 {
|
||||
HashAlgorithm::HighwayHash256S.size() + 4
|
||||
} else {
|
||||
0
|
||||
},
|
||||
"healthy direct range must not read unrequested shard {index}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn quantized_direct_can_be_disabled_without_changing_range_bytes() {
|
||||
let data = b"abcdefghijklmnop";
|
||||
let erasure = Erasure::new(2, 2, data.len());
|
||||
let (readers, counters) = quantized_counting_readers(&erasure, data, 4, 0, 4, None).await;
|
||||
let mut output = Vec::new();
|
||||
let outcome = erasure
|
||||
.decode_quantized_range_with_stripe_handles_and_reopeners(
|
||||
&mut output,
|
||||
readers,
|
||||
5,
|
||||
2,
|
||||
4,
|
||||
None,
|
||||
Vec::new(),
|
||||
Vec::new(),
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(outcome.error.is_none(), "quorum decoder range must succeed: {:?}", outcome.error);
|
||||
assert_eq!(output, &data[9..11]);
|
||||
assert!(counters[0].load(Ordering::SeqCst) > 0, "disabled direct path must read the data quorum");
|
||||
assert!(
|
||||
counters[1].load(Ordering::SeqCst) > 0,
|
||||
"disabled direct path must read the requested data shard"
|
||||
);
|
||||
assert_eq!(counters[2].load(Ordering::SeqCst), 0);
|
||||
assert_eq!(counters[3].load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn quantized_direct_verifies_every_frame_before_emitting_and_recovers_corruption() {
|
||||
let data = b"abcdefghijklmnop";
|
||||
let erasure = Erasure::new(2, 2, data.len());
|
||||
for corrupt in [false, true] {
|
||||
let (readers, counters) = quantized_counting_readers(&erasure, data, 4, 0, 8, corrupt.then_some(0)).await;
|
||||
let mut output = Vec::new();
|
||||
let outcome = erasure
|
||||
.decode_quantized_range_with_stripe_handles_and_reopeners(
|
||||
&mut output,
|
||||
readers,
|
||||
2,
|
||||
6,
|
||||
8,
|
||||
None,
|
||||
Vec::new(),
|
||||
Vec::new(),
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(
|
||||
outcome.written, 6,
|
||||
"corrupt={corrupt}: authenticated or reconstructed range must complete"
|
||||
);
|
||||
assert_eq!(output, &data[2..8], "corrupt={corrupt}: no corrupt prefix may reach the writer");
|
||||
assert_eq!(
|
||||
outcome.repair.needed(false),
|
||||
corrupt,
|
||||
"observed corruption must survive successful reconstruction"
|
||||
);
|
||||
assert_eq!(
|
||||
counters[0].load(Ordering::SeqCst),
|
||||
2 * (HashAlgorithm::HighwayHash256S.size() + 4),
|
||||
"target must authenticate both frames exactly once, including the failing second frame"
|
||||
);
|
||||
if !corrupt {
|
||||
assert_eq!(outcome.frame_count, 2);
|
||||
assert_eq!(counters[1].load(Ordering::SeqCst), 0);
|
||||
} else {
|
||||
assert!(
|
||||
counters[1].load(Ordering::SeqCst) > 0,
|
||||
"reconstruction must read the remaining data shard"
|
||||
);
|
||||
assert!(counters[2].load(Ordering::SeqCst) + counters[3].load(Ordering::SeqCst) > 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn quantized_direct_missing_target_reconstructs_the_same_slice() {
|
||||
let data = b"abcdefghijklmnop";
|
||||
let erasure = Erasure::new(2, 2, data.len());
|
||||
let (mut readers, counters) = quantized_counting_readers(&erasure, data, 4, 4, 4, None).await;
|
||||
readers[1] = None;
|
||||
let mut output = Vec::new();
|
||||
let outcome = erasure
|
||||
.decode_quantized_range_with_stripe_handles_and_reopeners(
|
||||
&mut output,
|
||||
readers,
|
||||
5,
|
||||
2,
|
||||
4,
|
||||
None,
|
||||
Vec::new(),
|
||||
Vec::new(),
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(outcome.written, 2, "missing target must reconstruct from a read quorum");
|
||||
assert_eq!(output, &data[13..15], "fallback must remain aligned to the nonzero q slice");
|
||||
assert_eq!(counters[1].load(Ordering::SeqCst), 0);
|
||||
assert!(counters[0].load(Ordering::SeqCst) > 0);
|
||||
assert!(counters[2].load(Ordering::SeqCst) + counters[3].load(Ordering::SeqCst) > 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn quantized_direct_unrecoverable_corruption_emits_no_bytes() {
|
||||
let data = b"abcdefghijklmnop";
|
||||
let erasure = Erasure::new(2, 2, data.len());
|
||||
let (mut readers, _) = quantized_counting_readers(&erasure, data, 4, 0, 4, Some(0)).await;
|
||||
readers[1] = None;
|
||||
readers[3] = None;
|
||||
let mut output = Vec::new();
|
||||
let outcome = erasure
|
||||
.decode_quantized_range_with_stripe_handles_and_reopeners(
|
||||
&mut output,
|
||||
readers,
|
||||
1,
|
||||
2,
|
||||
4,
|
||||
None,
|
||||
Vec::new(),
|
||||
Vec::new(),
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(outcome.error.is_some(), "insufficient reconstruction quorum must fail");
|
||||
assert_eq!(outcome.written, 0);
|
||||
assert!(output.is_empty(), "unverified data must never be emitted");
|
||||
assert!(outcome.repair.needed(false), "failed direct read must retain corruption evidence");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn quantized_direct_preserves_writer_failure() {
|
||||
let data = b"abcdefghijklmnop";
|
||||
let erasure = Erasure::new(2, 2, data.len());
|
||||
let (readers, _) = quantized_counting_readers(&erasure, data, 4, 0, 4, None).await;
|
||||
let outcome = erasure
|
||||
.decode_quantized_range_with_stripe_handles_and_reopeners(
|
||||
&mut FailingEmitWriter,
|
||||
readers,
|
||||
1,
|
||||
2,
|
||||
4,
|
||||
None,
|
||||
Vec::new(),
|
||||
Vec::new(),
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(outcome.error.expect("writer failure must escape").kind(), ErrorKind::BrokenPipe);
|
||||
assert_eq!(outcome.written, 0);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial_test::serial]
|
||||
async fn quantized_direct_stalled_target_recovers_without_waiting_for_disk_timeout() {
|
||||
let data = b"abcdefghijklmnop";
|
||||
let erasure = Erasure::new(2, 2, data.len());
|
||||
let (readers, _) = quantized_counting_readers(&erasure, data, 4, 4, 4, None).await;
|
||||
let slow_until = TokioInstant::now() + Duration::from_secs(60);
|
||||
let readers = readers
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(index, reader)| {
|
||||
let bytes = reader.expect("fixture reader").inner_ref().inner.get_ref().clone();
|
||||
let source = if index == 1 {
|
||||
TestShardReader::ReadyAt {
|
||||
cursor: Cursor::new(bytes),
|
||||
ready_at: slow_until,
|
||||
sleep: None,
|
||||
}
|
||||
} else {
|
||||
TestShardReader::Ready(Cursor::new(bytes))
|
||||
};
|
||||
Some(BitrotReader::new_with_frame_size(
|
||||
source,
|
||||
erasure.shard_size(),
|
||||
4,
|
||||
HashAlgorithm::HighwayHash256S,
|
||||
false,
|
||||
))
|
||||
})
|
||||
.collect();
|
||||
let mut output = Vec::new();
|
||||
let started = TokioInstant::now();
|
||||
let outcome = temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_DISK_READ_TIMEOUT, Some("30"))], async {
|
||||
erasure
|
||||
.decode_quantized_range_with_stripe_handles_and_reopeners(
|
||||
&mut output,
|
||||
readers,
|
||||
5,
|
||||
2,
|
||||
4,
|
||||
None,
|
||||
Vec::new(),
|
||||
Vec::new(),
|
||||
true,
|
||||
)
|
||||
.await
|
||||
})
|
||||
.await;
|
||||
|
||||
assert_eq!(outcome.written, 2, "fast parity must recover the stalled q slice");
|
||||
assert_eq!(output, &data[13..15], "timeout recovery must retain the original q offset");
|
||||
assert!(!outcome.repair.needed(true), "a hedged healthy target is not a missing storage shard");
|
||||
assert!(
|
||||
started.elapsed() <= Duration::from_millis(150),
|
||||
"direct attempt must admit reconstruction by the existing 100 ms hedge, elapsed={:?}",
|
||||
started.elapsed()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial_test::serial]
|
||||
async fn quantized_direct_pending_healthy_target_survives_missing_or_corrupt_parity() {
|
||||
let data = b"abcdefghijklmnop";
|
||||
let erasure = Erasure::new(2, 2, data.len());
|
||||
for corrupt_parity in [false, true] {
|
||||
let (readers, _) = quantized_counting_readers(&erasure, data, 4, 4, 4, None).await;
|
||||
let started = TokioInstant::now();
|
||||
let ready_at = started + Duration::from_millis(200);
|
||||
let readers = readers
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(index, reader)| {
|
||||
let mut bytes = reader.expect("fixture reader").inner_ref().inner.get_ref().clone();
|
||||
if index >= erasure.data_shards {
|
||||
if !corrupt_parity {
|
||||
return None;
|
||||
}
|
||||
*bytes.last_mut().expect("parity payload") ^= 0x80;
|
||||
}
|
||||
let source = if index == 1 {
|
||||
TestShardReader::ReadyAt {
|
||||
cursor: Cursor::new(bytes),
|
||||
ready_at,
|
||||
sleep: None,
|
||||
}
|
||||
} else {
|
||||
TestShardReader::Ready(Cursor::new(bytes))
|
||||
};
|
||||
Some(BitrotReader::new_with_frame_size(
|
||||
source,
|
||||
erasure.shard_size(),
|
||||
4,
|
||||
HashAlgorithm::HighwayHash256S,
|
||||
false,
|
||||
))
|
||||
})
|
||||
.collect();
|
||||
let mut output = Vec::new();
|
||||
let outcome = temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_DISK_READ_TIMEOUT, Some("30"))], async {
|
||||
erasure
|
||||
.decode_quantized_range_with_stripe_handles_and_reopeners(
|
||||
&mut output,
|
||||
readers,
|
||||
5,
|
||||
2,
|
||||
4,
|
||||
None,
|
||||
Vec::new(),
|
||||
Vec::new(),
|
||||
true,
|
||||
)
|
||||
.await
|
||||
})
|
||||
.await;
|
||||
|
||||
assert!(
|
||||
outcome.error.is_none(),
|
||||
"corrupt_parity={corrupt_parity}: healthy target must remain eligible"
|
||||
);
|
||||
assert_eq!(outcome.written, 2);
|
||||
assert_eq!(output, &data[13..15], "a failed competing quorum must not emit or duplicate range bytes");
|
||||
assert!(
|
||||
started.elapsed() >= Duration::from_millis(200) && started.elapsed() <= Duration::from_millis(250),
|
||||
"corrupt_parity={corrupt_parity}: wait for the healthy target, not the 30 s disk deadline"
|
||||
);
|
||||
assert_eq!(
|
||||
outcome.repair.needed(false),
|
||||
corrupt_parity,
|
||||
"actual observed parity corruption must survive a successful direct winner"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a production-shaped reader set: data shards as opened stream
|
||||
/// readers, parity shards as *unopened* deferred readers carrying stripe
|
||||
/// handles (mirroring `fill_deferred_bitrot_readers`). `truncate` shortens
|
||||
@@ -4034,12 +3305,6 @@ mod tests {
|
||||
|
||||
assert_eq!(defaulted.read_costs, vec![ShardReadCost::Unknown; 3]);
|
||||
assert!(defaulted.verify_reconstruction);
|
||||
|
||||
let quantized: ParallelReader<Cursor<Vec<u8>>> =
|
||||
ParallelReader::new_for_quantized_decode(vec![None, None, None], Erasure::new(2, 1, 64), 16, None, None);
|
||||
assert!(quantized.verify_reconstruction);
|
||||
assert!(quantized.demand_bound_lockstep);
|
||||
assert_eq!(quantized.engaged.as_slice(), &[true, true, false]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -39,7 +39,7 @@ use std::pin::Pin;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::task::{Context, Poll};
|
||||
use std::time::Instant;
|
||||
use tokio::io::{AsyncRead, AsyncReadExt, ReadBuf};
|
||||
use tokio::io::{AsyncRead, ReadBuf};
|
||||
use tracing::debug;
|
||||
|
||||
#[cfg(all(test, feature = "hotpath"))]
|
||||
@@ -142,7 +142,6 @@ struct BitrotReaderSource {
|
||||
path: String,
|
||||
offset: usize,
|
||||
length: usize,
|
||||
small_shard: bool,
|
||||
use_mmap_read: bool,
|
||||
stage_metrics: Option<BitrotReaderStageMetrics>,
|
||||
}
|
||||
@@ -156,7 +155,6 @@ impl BitrotReaderSource {
|
||||
&self.path,
|
||||
self.offset,
|
||||
self.length,
|
||||
self.small_shard,
|
||||
self.use_mmap_read,
|
||||
self.stage_metrics.map(|metrics| metrics.path),
|
||||
)
|
||||
@@ -172,7 +170,6 @@ async fn open_reader_source(
|
||||
path: &str,
|
||||
offset: usize,
|
||||
length: usize,
|
||||
small_shard: bool,
|
||||
use_mmap_read: bool,
|
||||
metrics_path: Option<&'static str>,
|
||||
) -> disk::error::Result<Option<BoxedObjectReader>> {
|
||||
@@ -185,45 +182,14 @@ async fn open_reader_source(
|
||||
} else if let Some(disk) = disk {
|
||||
#[cfg(test)]
|
||||
crate::set_disk::disk_call_counters::record_shard_open(path, false);
|
||||
let buffer_stream = small_shard && length >= MIN_BUFFERED_STREAM_BYTES;
|
||||
let reader = open_disk_reader(disk, bucket, path, offset, length, use_mmap_read, metrics_path).await?;
|
||||
let reader = if buffer_stream && matches!(reader, ShardReader::Stream(_)) && disk.is_local() {
|
||||
buffer_local_shard_stream(reader, length)?
|
||||
} else {
|
||||
reader
|
||||
};
|
||||
Ok(Some(reader))
|
||||
open_disk_reader(disk, bucket, path, offset, length, use_mmap_read, metrics_path)
|
||||
.await
|
||||
.map(Some)
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
const MIN_BUFFERED_STREAM_BYTES: usize = 1024 * 1024;
|
||||
|
||||
fn buffer_local_shard_stream(reader: ShardReader, length: usize) -> disk::error::Result<ShardReader> {
|
||||
if length < MIN_BUFFERED_STREAM_BYTES {
|
||||
return Ok(reader);
|
||||
}
|
||||
let ShardReader::Stream(reader) = reader else {
|
||||
return Ok(reader);
|
||||
};
|
||||
let limit = u64::try_from(length).map_err(|_| DiskError::FileCorrupt)?;
|
||||
if disk::has_resource_io_scope() {
|
||||
// ResourceFileReader reserves its buffer on demand. Prefetching here would
|
||||
// enlarge that reservation and allocate a second, unaccounted buffer.
|
||||
return Ok(ShardReader::Stream(Box::new(reader.take(limit))));
|
||||
}
|
||||
// Small layout stripes otherwise dispatch a blocking file read per frame.
|
||||
// Short ranges keep direct reads: the caller's conservative stream span
|
||||
// can include an extra stripe beyond the final requested block. Long
|
||||
// streams stay bounded to that span; prefetched frames still authenticate
|
||||
// individually when consumed by the bitrot reader.
|
||||
Ok(ShardReader::Stream(Box::new(tokio::io::BufReader::with_capacity(
|
||||
256 * 1024,
|
||||
reader.take(limit),
|
||||
))))
|
||||
}
|
||||
|
||||
struct FirstReadMetricsReader {
|
||||
inner: FileReader,
|
||||
metrics_path: &'static str,
|
||||
@@ -589,29 +555,6 @@ fn bitrot_encoded_range(offset: usize, length: usize, shard_size: usize, checksu
|
||||
adjust_shard_read_params(offset, length, shard_size, &checksum_algo)
|
||||
}
|
||||
|
||||
pub(crate) fn bitrot_encoded_range_with_frame_size(
|
||||
offset: usize,
|
||||
length: usize,
|
||||
shard_size: usize,
|
||||
frame_size: usize,
|
||||
checksum_algo: HashAlgorithm,
|
||||
) -> (usize, usize) {
|
||||
if frame_size == shard_size {
|
||||
return bitrot_encoded_range(offset, length, shard_size, checksum_algo);
|
||||
}
|
||||
|
||||
// A frame-aligned range may begin inside a logical shard. Skip the hashes
|
||||
// in preceding logical shards as well as those before this frame.
|
||||
let frames_per_shard = shard_size.div_ceil(frame_size);
|
||||
let hash_size = checksum_algo.size();
|
||||
let logical_shards = offset / shard_size;
|
||||
let within_shard = offset % shard_size;
|
||||
let frame_index = within_shard / frame_size;
|
||||
let physical_offset = logical_shards * (shard_size + frames_per_shard * hash_size) + within_shard + frame_index * hash_size;
|
||||
let physical_length = length + length.div_ceil(frame_size) * hash_size;
|
||||
(physical_offset, physical_length)
|
||||
}
|
||||
|
||||
/// Adjusts a raw (offset, length) pair to account for per-shard checksum overhead.
|
||||
/// Returns (adjusted_offset, adjusted_length).
|
||||
pub(crate) fn adjust_shard_read_params(
|
||||
@@ -743,44 +686,12 @@ pub(crate) async fn create_bitrot_reader_from_bytes_with_stage_metrics(
|
||||
skip_verify: bool,
|
||||
use_mmap_read: bool,
|
||||
stage_metrics: Option<BitrotReaderStageMetrics>,
|
||||
) -> disk::error::Result<Option<BitrotReader<ShardReader>>> {
|
||||
create_bitrot_reader_from_bytes_with_frame_size_and_stage_metrics(
|
||||
inline_data,
|
||||
disk,
|
||||
bucket,
|
||||
path,
|
||||
offset,
|
||||
length,
|
||||
shard_size,
|
||||
shard_size,
|
||||
checksum_algo,
|
||||
skip_verify,
|
||||
use_mmap_read,
|
||||
stage_metrics,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) async fn create_bitrot_reader_from_bytes_with_frame_size_and_stage_metrics(
|
||||
inline_data: Option<Bytes>,
|
||||
disk: Option<&DiskStore>,
|
||||
bucket: &str,
|
||||
path: &str,
|
||||
offset: usize,
|
||||
length: usize,
|
||||
shard_size: usize,
|
||||
frame_size: usize,
|
||||
checksum_algo: HashAlgorithm,
|
||||
skip_verify: bool,
|
||||
use_mmap_read: bool,
|
||||
stage_metrics: Option<BitrotReaderStageMetrics>,
|
||||
) -> disk::error::Result<Option<BitrotReader<ShardReader>>> {
|
||||
let stage_metrics = stage_metrics.filter(|_| rustfs_io_metrics::get_stage_metrics_enabled());
|
||||
let stage_metrics_enabled = stage_metrics.is_some();
|
||||
|
||||
let reader_construction_start = stage_metrics_enabled.then(Instant::now);
|
||||
let (offset, length) = bitrot_encoded_range_with_frame_size(offset, length, shard_size, frame_size, checksum_algo.clone());
|
||||
let (offset, length) = bitrot_encoded_range(offset, length, shard_size, checksum_algo.clone());
|
||||
if let Some(metrics) = stage_metrics {
|
||||
record_get_stage_duration_if_enabled(metrics.path, metrics.reader_construction_stage, reader_construction_start);
|
||||
}
|
||||
@@ -793,7 +704,6 @@ pub(crate) async fn create_bitrot_reader_from_bytes_with_frame_size_and_stage_me
|
||||
path,
|
||||
offset,
|
||||
length,
|
||||
shard_size != 0 && shard_size <= 128 * 1024,
|
||||
use_mmap_read,
|
||||
stage_metrics.map(|metrics| metrics.path),
|
||||
)
|
||||
@@ -803,8 +713,7 @@ pub(crate) async fn create_bitrot_reader_from_bytes_with_frame_size_and_stage_me
|
||||
}
|
||||
|
||||
let bitrot_reader_init_start = stage_metrics_enabled.then(Instant::now);
|
||||
let reader =
|
||||
reader.map(|reader| BitrotReader::new_with_frame_size(reader, shard_size, frame_size, checksum_algo, skip_verify));
|
||||
let reader = reader.map(|reader| BitrotReader::new(reader, shard_size, checksum_algo, skip_verify));
|
||||
if let Some(metrics) = stage_metrics {
|
||||
record_get_stage_duration_if_enabled(metrics.path, metrics.bitrot_reader_init_stage, bitrot_reader_init_start);
|
||||
}
|
||||
@@ -857,38 +766,8 @@ pub(crate) fn create_deferred_bitrot_reader_with_stripe_handle(
|
||||
skip_verify: bool,
|
||||
use_mmap_read: bool,
|
||||
) -> (BitrotReader<ShardReader>, DeferredReaderStripeHandle) {
|
||||
create_deferred_bitrot_reader_with_stripe_handle_and_frame_size(
|
||||
inline_data,
|
||||
disk,
|
||||
bucket,
|
||||
path,
|
||||
offset,
|
||||
length,
|
||||
shard_size,
|
||||
shard_size,
|
||||
checksum_algo,
|
||||
skip_verify,
|
||||
use_mmap_read,
|
||||
)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) fn create_deferred_bitrot_reader_with_stripe_handle_and_frame_size(
|
||||
inline_data: Option<Bytes>,
|
||||
disk: Option<DiskStore>,
|
||||
bucket: &str,
|
||||
path: &str,
|
||||
offset: usize,
|
||||
length: usize,
|
||||
shard_size: usize,
|
||||
frame_size: usize,
|
||||
checksum_algo: HashAlgorithm,
|
||||
skip_verify: bool,
|
||||
use_mmap_read: bool,
|
||||
) -> (BitrotReader<ShardReader>, DeferredReaderStripeHandle) {
|
||||
let frames_per_shard = shard_size.div_ceil(frame_size);
|
||||
let stripe_stride = shard_size + frames_per_shard * checksum_algo.size();
|
||||
let (offset, length) = bitrot_encoded_range_with_frame_size(offset, length, shard_size, frame_size, checksum_algo.clone());
|
||||
let stripe_stride = shard_size + checksum_algo.size();
|
||||
let (offset, length) = bitrot_encoded_range(offset, length, shard_size, checksum_algo.clone());
|
||||
let inline_source = inline_data.is_some();
|
||||
let source = BitrotReaderSource {
|
||||
inline_data,
|
||||
@@ -897,7 +776,6 @@ pub(crate) fn create_deferred_bitrot_reader_with_stripe_handle_and_frame_size(
|
||||
path: if inline_source { String::new() } else { path.to_string() },
|
||||
offset,
|
||||
length,
|
||||
small_shard: shard_size != 0 && shard_size <= 128 * 1024,
|
||||
use_mmap_read,
|
||||
stage_metrics: None,
|
||||
};
|
||||
@@ -907,13 +785,7 @@ pub(crate) fn create_deferred_bitrot_reader_with_stripe_handle_and_frame_size(
|
||||
// The deferred parity reader opens its source lazily, so it cannot hand out an
|
||||
// in-memory block up front; it stays on the streaming path. Parity shards are
|
||||
// only read when a data shard fails, so the fast path is not needed here.
|
||||
let reader = BitrotReader::new_with_frame_size(
|
||||
ShardReader::Stream(Box::new(deferred)),
|
||||
shard_size,
|
||||
frame_size,
|
||||
checksum_algo,
|
||||
skip_verify,
|
||||
);
|
||||
let reader = BitrotReader::new(ShardReader::Stream(Box::new(deferred)), shard_size, checksum_algo, skip_verify);
|
||||
(reader, handle)
|
||||
}
|
||||
|
||||
@@ -940,20 +812,12 @@ pub(crate) fn create_deferred_bitrot_reader_with_stripe_handle_and_frame_size(
|
||||
/// a fixed body length when locating the authenticated trailer. Clamping it
|
||||
/// to `0` would claim an empty body and misframe the stream. `0` stays `0`
|
||||
/// because a genuinely empty object still means an empty body.
|
||||
fn bitrot_create_file_size(length: i64, shard_size: usize, checksum_algo: &HashAlgorithm) -> disk::error::Result<i64> {
|
||||
fn bitrot_create_file_size(length: i64, shard_size: usize, checksum_algo: &HashAlgorithm) -> i64 {
|
||||
if length <= 0 {
|
||||
return Ok(length);
|
||||
return length;
|
||||
}
|
||||
if shard_size == 0 {
|
||||
return Err(DiskError::other("bitrot shard size is zero"));
|
||||
}
|
||||
let payload = usize::try_from(length).map_err(|_| DiskError::other("bitrot shard length exceeds usize"))?;
|
||||
let framed = payload
|
||||
.div_ceil(shard_size)
|
||||
.checked_mul(checksum_algo.size())
|
||||
.and_then(|overhead| payload.checked_add(overhead))
|
||||
.ok_or_else(|| DiskError::other("bitrot framed length overflows"))?;
|
||||
i64::try_from(framed).map_err(|_| DiskError::other("bitrot framed length exceeds i64"))
|
||||
let length = length as usize;
|
||||
(length.div_ceil(shard_size) * checksum_algo.size() + length) as i64
|
||||
}
|
||||
|
||||
pub async fn create_bitrot_writer(
|
||||
@@ -979,16 +843,12 @@ pub async fn create_bitrot_writer_with_orig_volume(
|
||||
shard_size: usize,
|
||||
checksum_algo: HashAlgorithm,
|
||||
) -> disk::error::Result<BitrotWriterWrapper> {
|
||||
let framed_length = bitrot_create_file_size(length, shard_size, &checksum_algo)?;
|
||||
let writer = if is_inline_buffer {
|
||||
if framed_length < 0 {
|
||||
CustomWriter::new_inline_buffer()
|
||||
} else {
|
||||
let limit = usize::try_from(framed_length).map_err(|_| DiskError::other("inline buffer limit exceeds usize"))?;
|
||||
CustomWriter::new_bounded_inline_buffer(limit)
|
||||
}
|
||||
CustomWriter::new_inline_buffer()
|
||||
} else if let Some(disk) = disk {
|
||||
let file = disk.create_file(origvolume, volume, path, framed_length).await?;
|
||||
let length = bitrot_create_file_size(length, shard_size, &checksum_algo);
|
||||
|
||||
let file = disk.create_file(origvolume, volume, path, length).await?;
|
||||
#[cfg(feature = "hotpath")]
|
||||
let file = instrument_raw_shard_writer(file, disk.is_local());
|
||||
CustomWriter::from_file_writer(file)
|
||||
@@ -1004,222 +864,26 @@ mod tests {
|
||||
use super::*;
|
||||
use rustfs_rio::ChunkReader;
|
||||
use std::collections::VecDeque;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
struct CountingWindowReader {
|
||||
source: Cursor<Vec<u8>>,
|
||||
reads: Arc<AtomicUsize>,
|
||||
bytes: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl AsyncRead for CountingWindowReader {
|
||||
fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, out: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
|
||||
let before = out.filled().len();
|
||||
let result = Pin::new(&mut self.source).poll_read(cx, out);
|
||||
self.reads.fetch_add(1, Ordering::Relaxed);
|
||||
self.bytes.fetch_add(out.filled().len() - before, Ordering::Relaxed);
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn buffered_local_shard_stream_coalesces_reads_without_crossing_window() {
|
||||
const SHARD: usize = 32 * 1024;
|
||||
const TAIL: usize = 12_001;
|
||||
let algo = HashAlgorithm::HighwayHash256S;
|
||||
let data = vec![37; 37 * SHARD + TAIL];
|
||||
let mut encoded = Vec::new();
|
||||
{
|
||||
let mut writer = crate::erasure::coding::BitrotWriter::new(&mut encoded, SHARD, algo.clone());
|
||||
for chunk in data.chunks(SHARD) {
|
||||
writer.write(chunk).await.expect("encode full shards and short tail");
|
||||
}
|
||||
}
|
||||
let window = encoded.len();
|
||||
encoded.extend_from_slice(&vec![99; 100_000]);
|
||||
let reads = Arc::new(AtomicUsize::new(0));
|
||||
let bytes = Arc::new(AtomicUsize::new(0));
|
||||
let source = ShardReader::Stream(Box::new(CountingWindowReader {
|
||||
source: Cursor::new(encoded),
|
||||
reads: Arc::clone(&reads),
|
||||
bytes: Arc::clone(&bytes),
|
||||
}));
|
||||
let source = buffer_local_shard_stream(source, window).expect("buffer the exact local window");
|
||||
let mut reader = BitrotReader::new(source, SHARD, algo, false);
|
||||
let mut actual = Vec::new();
|
||||
for chunk in data.chunks(SHARD) {
|
||||
reader
|
||||
.read_appending(&mut actual, chunk.len())
|
||||
.await
|
||||
.expect("verify each buffered shard");
|
||||
}
|
||||
assert_eq!(actual, data);
|
||||
assert_eq!(
|
||||
bytes.load(Ordering::Relaxed),
|
||||
window,
|
||||
"prefetch must not consume the sentinel beyond the window"
|
||||
);
|
||||
assert_eq!(
|
||||
reads.load(Ordering::Relaxed),
|
||||
5,
|
||||
"thirty-eight logical reads should share five bounded source reads"
|
||||
);
|
||||
let mut extra = vec![0; 1];
|
||||
assert_eq!(
|
||||
reader.read(&mut extra).await.expect_err("window ended").kind(),
|
||||
io::ErrorKind::UnexpectedEof
|
||||
);
|
||||
assert_eq!(bytes.load(Ordering::Relaxed), window);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn buffered_local_shard_stream_still_rejects_corrupt_prefetched_frame() {
|
||||
const SHARD: usize = 32 * 1024;
|
||||
let algo = HashAlgorithm::HighwayHash256S;
|
||||
let payload = vec![73; SHARD];
|
||||
let mut encoded = Vec::new();
|
||||
{
|
||||
let mut writer = crate::erasure::coding::BitrotWriter::new(&mut encoded, SHARD, algo.clone());
|
||||
for _ in 0..33 {
|
||||
writer.write(&payload).await.expect("encode fixture shards");
|
||||
}
|
||||
}
|
||||
encoded[SHARD + algo.size() + algo.size() + 7] ^= 1;
|
||||
let window = encoded.len();
|
||||
let reads = Arc::new(AtomicUsize::new(0));
|
||||
let bytes = Arc::new(AtomicUsize::new(0));
|
||||
let source = buffer_local_shard_stream(
|
||||
ShardReader::Stream(Box::new(CountingWindowReader {
|
||||
source: Cursor::new(encoded),
|
||||
reads: Arc::clone(&reads),
|
||||
bytes: Arc::clone(&bytes),
|
||||
})),
|
||||
window,
|
||||
)
|
||||
.expect("buffer corruption fixture");
|
||||
let mut reader = BitrotReader::new(source, SHARD, algo, false);
|
||||
let mut output = vec![0xa5; SHARD];
|
||||
reader.read(&mut output).await.expect("first frame is valid");
|
||||
assert_eq!(output, payload);
|
||||
assert_eq!(reads.load(Ordering::Relaxed), 1);
|
||||
assert_eq!(
|
||||
bytes.load(Ordering::Relaxed),
|
||||
256 * 1024,
|
||||
"long stream must prefetch the corrupt second frame"
|
||||
);
|
||||
output.fill(0xa5);
|
||||
assert_eq!(
|
||||
reader
|
||||
.read(&mut output)
|
||||
.await
|
||||
.expect_err("corrupt second frame must fail")
|
||||
.kind(),
|
||||
io::ErrorKind::InvalidData
|
||||
);
|
||||
assert!(
|
||||
output.iter().all(|byte| *byte == 0xa5),
|
||||
"corrupt prefetched data must not escape verification"
|
||||
);
|
||||
assert_eq!(
|
||||
reads.load(Ordering::Relaxed),
|
||||
1,
|
||||
"corrupt prefetched frame must fail without another source read"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn buffered_local_shard_stream_does_not_prefetch_conservative_short_range_tail() {
|
||||
const BLOCK: usize = 256 * 1024;
|
||||
const SHARD: usize = BLOCK / 2;
|
||||
let algo = HashAlgorithm::HighwayHash256S;
|
||||
let erasure = crate::erasure::coding::Erasure::new(2, 2, BLOCK);
|
||||
// An aligned 1 MiB Range consumes four data-shard frames, while the
|
||||
// existing reader planner conservatively opens five logical shards.
|
||||
let logical_window = erasure.shard_file_offset(0, 4 * BLOCK, 6 * BLOCK);
|
||||
assert_eq!(logical_window, 5 * SHARD);
|
||||
let (_, wire_window) = adjust_shard_read_params(0, logical_window, SHARD, &algo);
|
||||
let payload = vec![73; SHARD];
|
||||
let mut encoded = Vec::new();
|
||||
{
|
||||
let mut writer = crate::erasure::coding::BitrotWriter::new(&mut encoded, SHARD, algo.clone());
|
||||
for _ in 0..4 {
|
||||
writer.write(&payload).await.expect("encode requested range frames");
|
||||
}
|
||||
writer
|
||||
.write(&vec![99; SHARD])
|
||||
.await
|
||||
.expect("encode unrequested conservative tail frame");
|
||||
}
|
||||
assert_eq!(encoded.len(), wire_window);
|
||||
let reads = Arc::new(AtomicUsize::new(0));
|
||||
let bytes = Arc::new(AtomicUsize::new(0));
|
||||
let source = buffer_local_shard_stream(
|
||||
ShardReader::Stream(Box::new(CountingWindowReader {
|
||||
source: Cursor::new(encoded),
|
||||
reads: Arc::clone(&reads),
|
||||
bytes: Arc::clone(&bytes),
|
||||
})),
|
||||
wire_window,
|
||||
)
|
||||
.expect("open conservative short range stream");
|
||||
let mut reader = BitrotReader::new(source, SHARD, algo.clone(), false);
|
||||
let mut output = Vec::new();
|
||||
for _ in 0..4 {
|
||||
reader
|
||||
.read_appending(&mut output, SHARD)
|
||||
.await
|
||||
.expect("read and authenticate requested frame");
|
||||
}
|
||||
assert_eq!(output, payload.repeat(4));
|
||||
assert_eq!(
|
||||
bytes.load(Ordering::Relaxed),
|
||||
4 * (SHARD + algo.size()),
|
||||
"four decoded frames must not prefetch the unrequested fifth frame"
|
||||
);
|
||||
assert_eq!(
|
||||
reads.load(Ordering::Relaxed),
|
||||
4,
|
||||
"short range should retain direct per-frame source reads"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bitrot_create_file_size_grows_known_length_by_checksums() {
|
||||
// 10 bytes over 4-byte shards = 3 frames with a 32-byte prefix each.
|
||||
assert_eq!(
|
||||
bitrot_create_file_size(10, 4, &HashAlgorithm::HighwayHash256).expect("known frame size"),
|
||||
10 + 3 * 32
|
||||
);
|
||||
assert_eq!(bitrot_create_file_size(10, 4, &HashAlgorithm::None).expect("unhashed frame size"), 10);
|
||||
// 10 bytes over 4-byte shards = 3 shards, each followed by a 32-byte hash.
|
||||
assert_eq!(bitrot_create_file_size(10, 4, &HashAlgorithm::HighwayHash256), 10 + 3 * 32);
|
||||
assert_eq!(bitrot_create_file_size(10, 4, &HashAlgorithm::None), 10);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bitrot_create_file_size_keeps_empty_and_unknown_distinct() {
|
||||
assert_eq!(
|
||||
bitrot_create_file_size(0, 4, &HashAlgorithm::HighwayHash256).expect("empty frame size"),
|
||||
0
|
||||
);
|
||||
assert_eq!(bitrot_create_file_size(0, 4, &HashAlgorithm::HighwayHash256), 0);
|
||||
// SSE/compression streams advertise SIZE_PRESERVE_LAYER (-1); the remote
|
||||
// put_file_stream receiver relies on a non-positive size to parse the auth
|
||||
// trailer from the stream tail, so the sentinel must survive untouched.
|
||||
assert_eq!(
|
||||
bitrot_create_file_size(rustfs_rio::HashReader::SIZE_PRESERVE_LAYER, 4, &HashAlgorithm::HighwayHash256)
|
||||
.expect("unknown length sentinel"),
|
||||
bitrot_create_file_size(rustfs_rio::HashReader::SIZE_PRESERVE_LAYER, 4, &HashAlgorithm::HighwayHash256),
|
||||
rustfs_rio::HashReader::SIZE_PRESERVE_LAYER
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bitrot_create_file_size_rejects_invalid_geometry_and_overflow() {
|
||||
assert!(bitrot_create_file_size(1, 0, &HashAlgorithm::HighwayHash256S).is_err());
|
||||
assert!(bitrot_create_file_size(i64::MAX, 1, &HashAlgorithm::HighwayHash256S).is_err());
|
||||
assert_eq!(
|
||||
bitrot_create_file_size(-7, 0, &HashAlgorithm::HighwayHash256S).expect("negative sentinel"),
|
||||
-7
|
||||
);
|
||||
assert_eq!(bitrot_create_file_size(0, 0, &HashAlgorithm::HighwayHash256S).expect("empty body"), 0);
|
||||
}
|
||||
|
||||
struct TestChunkReader {
|
||||
chunks: VecDeque<Bytes>,
|
||||
}
|
||||
@@ -1374,96 +1038,6 @@ mod tests {
|
||||
(disk, dir)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn resource_scoped_local_shard_reads_on_demand_within_its_window() {
|
||||
use crate::disk::{BackgroundMemoryBudget, test_resource_units, with_optional_resource_unit};
|
||||
|
||||
let (disk, _dir) = local_test_disk().await;
|
||||
let window = MIN_BUFFERED_STREAM_BYTES;
|
||||
disk.make_volume("test-bucket").await.expect("create volume");
|
||||
disk.write_all("test-bucket", "obj/shard", Bytes::from(vec![37; window + 1]))
|
||||
.await
|
||||
.expect("write shard with a byte beyond the window");
|
||||
let budget = BackgroundMemoryBudget::new();
|
||||
budget.set_sampled_limit(128 * 1024, std::time::Duration::from_secs(60));
|
||||
let (resources, _) = test_resource_units(1);
|
||||
let mut reader = with_optional_resource_unit(
|
||||
Some(resources.with_memory_budget(budget.clone())),
|
||||
open_reader_source(None, Some(&disk), "test-bucket", "obj/shard", 0, window, true, false, None),
|
||||
)
|
||||
.await
|
||||
.expect("open resource-scoped shard")
|
||||
.expect("local reader");
|
||||
let mut chunk = [0u8; 8192];
|
||||
let mut total = 0;
|
||||
loop {
|
||||
let n = reader.read(&mut chunk).await.expect("read without a 256 KiB prefetch grant");
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
assert!(chunk[..n].iter().all(|&byte| byte == 37));
|
||||
total += n;
|
||||
}
|
||||
assert_eq!(total, window);
|
||||
drop(reader);
|
||||
assert_eq!(budget.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn deferred_local_opener_applies_buffer_gate_to_current_window() {
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
const SHARD: usize = 128 * 1024;
|
||||
const WINDOW: usize = 1024 * 1024;
|
||||
let (disk, _dir) = local_test_disk().await;
|
||||
let payload: Vec<_> = (0u8..=250).cycle().take(2 * WINDOW + 1).collect();
|
||||
disk.make_volume("test-bucket").await.expect("create test volume");
|
||||
disk.write_all("test-bucket", "obj/buffer-gate", Bytes::from(payload.clone()))
|
||||
.await
|
||||
.expect("write source with bytes beyond every requested window");
|
||||
for (shard_size, length, advance, expected_end) in [
|
||||
(SHARD, WINDOW, 0, WINDOW),
|
||||
(SHARD + 1, WINDOW, 0, payload.len()),
|
||||
(SHARD, WINDOW + SHARD, 1, WINDOW + SHARD),
|
||||
(SHARD, WINDOW, 1, payload.len()),
|
||||
] {
|
||||
let (_reader, handle) = create_deferred_bitrot_reader_with_stripe_handle(
|
||||
None,
|
||||
Some(disk.clone()),
|
||||
"test-bucket",
|
||||
"obj/buffer-gate",
|
||||
0,
|
||||
length,
|
||||
shard_size,
|
||||
HashAlgorithm::None,
|
||||
false,
|
||||
false,
|
||||
);
|
||||
assert!(handle.advance_stripes(advance), "advance the unopened source");
|
||||
let source = {
|
||||
let mut state = handle.state.lock().expect("lock pending reader state");
|
||||
let DeferredObjectReaderState::Pending(source) = &mut *state else {
|
||||
panic!("advancing stripes must leave the source unopened");
|
||||
};
|
||||
source.take().expect("pending source must be available")
|
||||
};
|
||||
let mut source = source
|
||||
.open()
|
||||
.await
|
||||
.expect("open advanced local source")
|
||||
.expect("disk source must produce a reader");
|
||||
let mut actual = Vec::new();
|
||||
source.read_to_end(&mut actual).await.expect("read the opened source");
|
||||
// Unbuffered local streams expose the file remainder; buffered streams
|
||||
// must stop at the current encoded window, including after an advance.
|
||||
assert_eq!(
|
||||
actual.as_slice(),
|
||||
&payload[advance * shard_size..expected_end],
|
||||
"wrong buffering boundary for shard {shard_size}, advance {advance}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "hotpath")]
|
||||
async fn remote_test_disk(transport: TestRemoteDataTransport) -> (DiskStore, TestRemoteDataTransport) {
|
||||
use crate::disk::endpoint::Endpoint;
|
||||
@@ -2188,28 +1762,6 @@ mod tests {
|
||||
assert!(result.unwrap().is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bounded_inline_bitrot_writer_counts_each_frame_header() {
|
||||
let mut writer = create_bitrot_writer(true, None, "bucket", "object", 6, 4, HashAlgorithm::HighwayHash256S)
|
||||
.await
|
||||
.expect("bounded inline writer");
|
||||
writer.write(b"abcd").await.expect("first full frame");
|
||||
writer.write(b"ef").await.expect("final partial frame");
|
||||
let raw = writer.into_inline_data().expect("export framed bytes");
|
||||
assert_eq!(raw.len(), 6 + 2 * 32);
|
||||
let mut exact = create_bitrot_writer(true, None, "bucket", "object", 4, 4, HashAlgorithm::HighwayHash256S)
|
||||
.await
|
||||
.expect("exact bounded writer");
|
||||
exact.write(b"abcd").await.expect("fill admitted frame");
|
||||
assert!(exact.write(b"e").await.is_err());
|
||||
assert_eq!(exact.into_inline_data().expect("intact first frame").len(), 4 + 32);
|
||||
assert!(
|
||||
create_bitrot_writer(true, None, "bucket", "object", 1, 0, HashAlgorithm::HighwayHash256S)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_create_bitrot_writer_inline() {
|
||||
use rustfs_utils::HashAlgorithm;
|
||||
|
||||
@@ -1,441 +0,0 @@
|
||||
//! Request and storage contracts for the per-object erasure-code block size.
|
||||
//!
|
||||
//! The request hint is deliberately kept separate from persisted file
|
||||
//! metadata. Application code resolves the hint and writes the selected
|
||||
//! value to the transient marker below; the set layer consumes that marker
|
||||
//! before constructing `FileInfo`. Readers use the block size persisted in
|
||||
//! `FileInfo::erasure`, never the request metadata.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::fmt;
|
||||
|
||||
/// The historical RustFS erasure block size.
|
||||
pub const DEFAULT_EC_BLOCK_SIZE: usize = 1024 * 1024;
|
||||
|
||||
/// The only block sizes accepted for new objects.
|
||||
pub const EC_BLOCK_SIZE_CANDIDATES: [usize; 4] = [64 * 1024, 256 * 1024, 1024 * 1024, 4 * 1024 * 1024];
|
||||
|
||||
/// User metadata key after the `x-amz-meta-` prefix has been removed.
|
||||
pub const EC_BLOCK_SIZE_HINT_METADATA_KEY: &str = "rustfs-ec-block-size-hint";
|
||||
|
||||
/// Internal marker passed from the request layer to the storage writer.
|
||||
/// It is always removed before user metadata is persisted.
|
||||
pub const EC_BLOCK_SIZE_HINT_INTERNAL_SUFFIX: &str = "ec-block-size-hint";
|
||||
pub const EC_BLOCK_SIZE_HINT_INTERNAL_KEY: &str = "x-rustfs-internal-ec-block-size-hint";
|
||||
|
||||
/// Internal projection copied into `ObjectInfo` for response construction.
|
||||
/// The projection is filtered from S3 user metadata.
|
||||
pub const EC_BLOCK_SIZE_INFO_INTERNAL_SUFFIX: &str = "ec-block-size";
|
||||
pub const EC_BLOCK_SIZE_INFO_INTERNAL_KEY: &str = "x-rustfs-internal-ec-block-size";
|
||||
|
||||
pub const EC_BLOCK_SIZE_RESPONSE_HEADER: &str = "x-rustfs-effective-ec-block-size";
|
||||
pub const EC_BLOCK_SIZE_HINT_STATUS_HEADER: &str = "x-rustfs-layout-hint-status";
|
||||
pub const EC_BLOCK_SIZE_HINT_REASON_HEADER: &str = "x-rustfs-layout-hint-reason";
|
||||
|
||||
/// Persisted marker for the per-shard bitrot frame size used by new EC
|
||||
/// objects. The value is internal metadata and is always written through the
|
||||
/// RustFS/MinIO compatibility-key pair.
|
||||
pub const EC_READ_QUANTUM_INFO_INTERNAL_SUFFIX: &str = "ec-read-quantum";
|
||||
pub const EC_READ_QUANTUM_INFO_INTERNAL_KEY: &str = "x-rustfs-internal-ec-read-quantum";
|
||||
|
||||
/// Keep the physical read quantum bounded even when an object chooses a large
|
||||
/// layout block. The smaller entries cover the candidate layouts whose shard
|
||||
/// size is below the target quantum.
|
||||
pub const EC_READ_QUANTUM_CANDIDATES: [usize; 9] = [
|
||||
1024 * 1024,
|
||||
512 * 1024,
|
||||
256 * 1024,
|
||||
128 * 1024,
|
||||
64 * 1024,
|
||||
32 * 1024,
|
||||
16 * 1024,
|
||||
8 * 1024,
|
||||
4 * 1024,
|
||||
];
|
||||
|
||||
const EC_READ_QUANTUM_LEGACY_CANDIDATES: [usize; 7] =
|
||||
[256 * 1024, 128 * 1024, 64 * 1024, 32 * 1024, 16 * 1024, 8 * 1024, 4 * 1024];
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum EcReadQuantumMetadataError {
|
||||
ConflictingValues,
|
||||
InvalidValue,
|
||||
UnsupportedGeometry,
|
||||
}
|
||||
|
||||
impl fmt::Display for EcReadQuantumMetadataError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(match self {
|
||||
Self::ConflictingValues => "EC read quantum metadata has conflicting compatibility values",
|
||||
Self::InvalidValue => "EC read quantum metadata is not a supported decimal value",
|
||||
Self::UnsupportedGeometry => "EC read quantum metadata does not match the object geometry",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for EcReadQuantumMetadataError {}
|
||||
|
||||
/// A syntactically valid request value. It may still be outside the
|
||||
/// supported candidate set.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct ClientEcBlockSizeHint {
|
||||
pub requested_bytes: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum EcBlockSizeHintError {
|
||||
Empty,
|
||||
TooLong,
|
||||
NonAsciiDecimal,
|
||||
Zero,
|
||||
Overflow,
|
||||
}
|
||||
|
||||
impl fmt::Display for EcBlockSizeHintError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
let message = match self {
|
||||
Self::Empty => "value is empty",
|
||||
Self::TooLong => "value has more than 20 digits",
|
||||
Self::NonAsciiDecimal => "value must contain only ASCII decimal digits",
|
||||
Self::Zero => "value must be greater than zero",
|
||||
Self::Overflow => "value does not fit in an unsigned 64-bit integer",
|
||||
};
|
||||
f.write_str(message)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for EcBlockSizeHintError {}
|
||||
|
||||
/// Parse the strict wire representation required by the SPEC.
|
||||
pub fn parse_ec_block_size_hint(value: &str) -> Result<ClientEcBlockSizeHint, EcBlockSizeHintError> {
|
||||
if value.is_empty() {
|
||||
return Err(EcBlockSizeHintError::Empty);
|
||||
}
|
||||
if value.len() > 20 {
|
||||
return Err(EcBlockSizeHintError::TooLong);
|
||||
}
|
||||
if !value.bytes().all(|byte| byte.is_ascii_digit()) {
|
||||
return Err(EcBlockSizeHintError::NonAsciiDecimal);
|
||||
}
|
||||
let requested_bytes = value.parse::<u64>().map_err(|_| EcBlockSizeHintError::Overflow)?;
|
||||
if requested_bytes == 0 {
|
||||
return Err(EcBlockSizeHintError::Zero);
|
||||
}
|
||||
Ok(ClientEcBlockSizeHint { requested_bytes })
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum EcBlockSizeHintIgnoreReason {
|
||||
Disabled,
|
||||
FleetNotConfirmed,
|
||||
UnsupportedSize,
|
||||
IneligibleWrite,
|
||||
}
|
||||
|
||||
impl EcBlockSizeHintIgnoreReason {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Disabled => "disabled",
|
||||
Self::FleetNotConfirmed => "fleet-unconfirmed",
|
||||
Self::UnsupportedSize => "unsupported-size",
|
||||
Self::IneligibleWrite => "unsupported-write-mode",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum EcBlockSizeHintOutcome {
|
||||
NoHint,
|
||||
Applied,
|
||||
Ignored(EcBlockSizeHintIgnoreReason),
|
||||
}
|
||||
|
||||
impl EcBlockSizeHintOutcome {
|
||||
pub const fn status_header(self) -> Option<&'static str> {
|
||||
match self {
|
||||
Self::NoHint => None,
|
||||
Self::Applied => Some("applied"),
|
||||
Self::Ignored(_) => Some("ignored"),
|
||||
}
|
||||
}
|
||||
|
||||
pub const fn reason_header(self) -> Option<&'static str> {
|
||||
match self {
|
||||
Self::Ignored(reason) => Some(reason.as_str()),
|
||||
Self::NoHint | Self::Applied => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct ResolvedEcBlockSize {
|
||||
pub effective_block_size: usize,
|
||||
pub outcome: EcBlockSizeHintOutcome,
|
||||
}
|
||||
|
||||
/// Resolve a request hint after authentication and eligibility checks.
|
||||
///
|
||||
/// Disabled and fleet-unconfirmed requests deliberately do not parse the
|
||||
/// value. In those rollout phases the header remains ordinary user metadata,
|
||||
/// including when its value would be malformed.
|
||||
pub fn resolve_ec_block_size_hint(
|
||||
raw_hint: Option<&str>,
|
||||
feature_enabled: bool,
|
||||
fleet_confirmed: bool,
|
||||
eligible_write: bool,
|
||||
) -> Result<ResolvedEcBlockSize, EcBlockSizeHintError> {
|
||||
let Some(raw_hint) = raw_hint else {
|
||||
return Ok(ResolvedEcBlockSize {
|
||||
effective_block_size: DEFAULT_EC_BLOCK_SIZE,
|
||||
outcome: EcBlockSizeHintOutcome::NoHint,
|
||||
});
|
||||
};
|
||||
|
||||
if !feature_enabled {
|
||||
return Ok(ResolvedEcBlockSize {
|
||||
effective_block_size: DEFAULT_EC_BLOCK_SIZE,
|
||||
outcome: EcBlockSizeHintOutcome::Ignored(EcBlockSizeHintIgnoreReason::Disabled),
|
||||
});
|
||||
}
|
||||
if !fleet_confirmed {
|
||||
return Ok(ResolvedEcBlockSize {
|
||||
effective_block_size: DEFAULT_EC_BLOCK_SIZE,
|
||||
outcome: EcBlockSizeHintOutcome::Ignored(EcBlockSizeHintIgnoreReason::FleetNotConfirmed),
|
||||
});
|
||||
}
|
||||
|
||||
let hint = parse_ec_block_size_hint(raw_hint)?;
|
||||
let Some(effective_block_size) = usize::try_from(hint.requested_bytes)
|
||||
.ok()
|
||||
.filter(|size| EC_BLOCK_SIZE_CANDIDATES.contains(size))
|
||||
else {
|
||||
return Ok(ResolvedEcBlockSize {
|
||||
effective_block_size: DEFAULT_EC_BLOCK_SIZE,
|
||||
outcome: EcBlockSizeHintOutcome::Ignored(EcBlockSizeHintIgnoreReason::UnsupportedSize),
|
||||
});
|
||||
};
|
||||
|
||||
if !eligible_write {
|
||||
return Ok(ResolvedEcBlockSize {
|
||||
effective_block_size: DEFAULT_EC_BLOCK_SIZE,
|
||||
outcome: EcBlockSizeHintOutcome::Ignored(EcBlockSizeHintIgnoreReason::IneligibleWrite),
|
||||
});
|
||||
}
|
||||
|
||||
Ok(ResolvedEcBlockSize {
|
||||
effective_block_size,
|
||||
outcome: EcBlockSizeHintOutcome::Applied,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn is_supported_ec_block_size(value: usize) -> bool {
|
||||
EC_BLOCK_SIZE_CANDIDATES.contains(&value)
|
||||
}
|
||||
|
||||
/// Return whether the storage writer rollout is enabled and fleet-confirmed.
|
||||
/// Read it on the write path so each new object uses the active process
|
||||
/// environment values.
|
||||
pub fn ec_read_quantum_write_enabled() -> bool {
|
||||
rustfs_utils::get_env_bool(rustfs_config::ENV_EC_READ_QUANTUM_ENABLE, rustfs_config::DEFAULT_EC_READ_QUANTUM_ENABLE)
|
||||
&& rustfs_utils::get_env_bool(
|
||||
rustfs_config::ENV_EC_READ_QUANTUM_FLEET_CONFIRMED,
|
||||
rustfs_config::DEFAULT_EC_READ_QUANTUM_FLEET_CONFIRMED,
|
||||
)
|
||||
}
|
||||
|
||||
/// Select a frame size that preserves exact EC geometry. A candidate must be
|
||||
/// a divisor of both the physical shard size and the logical layout block
|
||||
/// divided across data shards; this keeps every complete layout stripe aligned
|
||||
/// and leaves only the final object stripe short.
|
||||
pub fn select_ec_read_quantum(block_size: usize, data_shards: usize, shard_size: usize) -> Option<usize> {
|
||||
if block_size == 0 || data_shards == 0 || shard_size == 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Small layouts benefit from half-shard frames because their common range
|
||||
// requests otherwise pull a complete data-plus-parity stripe. Larger
|
||||
// layouts use a coarser target to cap authenticated-frame count during
|
||||
// upload: the 1 MiB layout fits one frame per shard and the 4 MiB layout
|
||||
// uses two 1 MiB frames per shard. A 4 MiB range still avoids the legacy
|
||||
// multi-megabyte stripe read while PUT avoids paying for eight hashes per
|
||||
// shard.
|
||||
let target = if block_size <= 256 * 1024 {
|
||||
shard_size / 2
|
||||
} else {
|
||||
1024 * 1024
|
||||
};
|
||||
EC_READ_QUANTUM_CANDIDATES.into_iter().find(|candidate| {
|
||||
*candidate <= target && ec_read_quantum_matches_geometry(*candidate, block_size, data_shards, shard_size)
|
||||
})
|
||||
}
|
||||
|
||||
fn select_legacy_ec_read_quantum(block_size: usize, data_shards: usize, shard_size: usize) -> Option<usize> {
|
||||
if block_size == 0 || data_shards == 0 || shard_size == 0 {
|
||||
return None;
|
||||
}
|
||||
EC_READ_QUANTUM_LEGACY_CANDIDATES
|
||||
.into_iter()
|
||||
.find(|candidate| ec_read_quantum_matches_geometry(*candidate, block_size, data_shards, shard_size))
|
||||
}
|
||||
|
||||
fn ec_read_quantum_matches_geometry(quantum: usize, block_size: usize, data_shards: usize, shard_size: usize) -> bool {
|
||||
quantum > 0
|
||||
&& quantum <= shard_size
|
||||
&& EC_READ_QUANTUM_CANDIDATES.contains(&quantum)
|
||||
&& shard_size.is_multiple_of(quantum)
|
||||
&& quantum
|
||||
.checked_mul(data_shards)
|
||||
.is_some_and(|logical_quantum| block_size.is_multiple_of(logical_quantum))
|
||||
}
|
||||
|
||||
/// Resolve a persisted marker. Absence is the legacy layout-frame format;
|
||||
/// presence with a malformed or conflicting value is an error so a damaged
|
||||
/// metadata record cannot silently reinterpret the shard stream.
|
||||
pub fn persisted_ec_read_quantum(
|
||||
metadata: &HashMap<String, String>,
|
||||
block_size: usize,
|
||||
data_shards: usize,
|
||||
shard_size: usize,
|
||||
) -> Result<Option<usize>, EcReadQuantumMetadataError> {
|
||||
let present = rustfs_utils::http::metadata_compat::contains_key_str(metadata, EC_READ_QUANTUM_INFO_INTERNAL_SUFFIX);
|
||||
let Some(value) = rustfs_utils::http::metadata_compat::get_consistent_str(metadata, EC_READ_QUANTUM_INFO_INTERNAL_SUFFIX)
|
||||
else {
|
||||
return if present {
|
||||
Err(EcReadQuantumMetadataError::ConflictingValues)
|
||||
} else {
|
||||
Ok(None)
|
||||
};
|
||||
};
|
||||
let quantum = value.parse::<usize>().map_err(|_| EcReadQuantumMetadataError::InvalidValue)?;
|
||||
// Accept markers written by the previous selector as well as the current
|
||||
// smaller target. Existing 64 KiB and 256 KiB objects may carry a
|
||||
// full-shard marker and must remain readable after the rollout changes.
|
||||
let current = select_ec_read_quantum(block_size, data_shards, shard_size);
|
||||
let legacy = select_legacy_ec_read_quantum(block_size, data_shards, shard_size);
|
||||
if current != Some(quantum) && legacy != Some(quantum) {
|
||||
return Err(EcReadQuantumMetadataError::UnsupportedGeometry);
|
||||
}
|
||||
Ok(Some(quantum))
|
||||
}
|
||||
|
||||
pub fn insert_ec_read_quantum(metadata: &mut HashMap<String, String>, quantum: usize) {
|
||||
rustfs_utils::http::metadata_compat::insert_str(metadata, EC_READ_QUANTUM_INFO_INTERNAL_SUFFIX, quantum.to_string());
|
||||
}
|
||||
|
||||
/// Consume a trusted transient marker from storage options.
|
||||
pub fn take_ec_block_size_hint(metadata: &mut std::collections::HashMap<String, String>) -> Option<usize> {
|
||||
let value = rustfs_utils::http::metadata_compat::get_consistent_str(metadata, EC_BLOCK_SIZE_HINT_INTERNAL_SUFFIX)
|
||||
.map(ToOwned::to_owned);
|
||||
rustfs_utils::http::metadata_compat::remove_str(metadata, EC_BLOCK_SIZE_HINT_INTERNAL_SUFFIX);
|
||||
let value = value?;
|
||||
let parsed = value.parse::<usize>().ok()?;
|
||||
is_supported_ec_block_size(parsed).then_some(parsed)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parser_accepts_decimal_values_and_rejects_wire_variants() {
|
||||
assert_eq!(parse_ec_block_size_hint("65536").unwrap().requested_bytes, 65_536);
|
||||
for value in ["", "0", "+65536", "65_536", "65536 ", "1.0", "128"] {
|
||||
assert!(parse_ec_block_size_hint(value).is_err(), "{value:?} must be rejected");
|
||||
}
|
||||
assert_eq!(parse_ec_block_size_hint("18446744073709551615").unwrap().requested_bytes, u64::MAX);
|
||||
assert!(matches!(
|
||||
parse_ec_block_size_hint("18446744073709551616"),
|
||||
Err(EcBlockSizeHintError::Overflow)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolver_applies_each_candidate_only_when_rollout_and_write_are_eligible() {
|
||||
for candidate in EC_BLOCK_SIZE_CANDIDATES {
|
||||
let result = resolve_ec_block_size_hint(Some(&candidate.to_string()), true, true, true).unwrap();
|
||||
assert_eq!(result.effective_block_size, candidate);
|
||||
assert_eq!(result.outcome, EcBlockSizeHintOutcome::Applied);
|
||||
}
|
||||
assert_eq!(
|
||||
resolve_ec_block_size_hint(Some("131072"), true, true, true).unwrap().outcome,
|
||||
EcBlockSizeHintOutcome::Ignored(EcBlockSizeHintIgnoreReason::UnsupportedSize)
|
||||
);
|
||||
let ineligible = resolve_ec_block_size_hint(Some("65536"), true, true, false).unwrap();
|
||||
assert_eq!(
|
||||
ineligible.outcome,
|
||||
EcBlockSizeHintOutcome::Ignored(EcBlockSizeHintIgnoreReason::IneligibleWrite)
|
||||
);
|
||||
assert_eq!(ineligible.outcome.reason_header(), Some("unsupported-write-mode"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disabled_and_unconfirmed_rollouts_do_not_turn_malformed_metadata_into_errors() {
|
||||
for (enabled, fleet, reason) in [
|
||||
(false, false, EcBlockSizeHintIgnoreReason::Disabled),
|
||||
(true, false, EcBlockSizeHintIgnoreReason::FleetNotConfirmed),
|
||||
] {
|
||||
let result = resolve_ec_block_size_hint(Some("not-a-number"), enabled, fleet, true).unwrap();
|
||||
assert_eq!(result.effective_block_size, DEFAULT_EC_BLOCK_SIZE);
|
||||
assert_eq!(result.outcome, EcBlockSizeHintOutcome::Ignored(reason));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transient_marker_is_consumed_only_for_supported_candidates() {
|
||||
let mut metadata = std::collections::HashMap::new();
|
||||
rustfs_utils::http::metadata_compat::insert_str(&mut metadata, EC_BLOCK_SIZE_HINT_INTERNAL_SUFFIX, "262144".to_string());
|
||||
assert_eq!(take_ec_block_size_hint(&mut metadata), Some(262_144));
|
||||
assert!(!metadata.contains_key(EC_BLOCK_SIZE_HINT_INTERNAL_KEY));
|
||||
assert!(!metadata.contains_key("x-minio-internal-ec-block-size-hint"));
|
||||
|
||||
let mut unsupported =
|
||||
std::collections::HashMap::from([(EC_BLOCK_SIZE_HINT_INTERNAL_KEY.to_string(), "131072".to_string())]);
|
||||
assert_eq!(take_ec_block_size_hint(&mut unsupported), None);
|
||||
assert!(!unsupported.contains_key(EC_BLOCK_SIZE_HINT_INTERNAL_KEY));
|
||||
|
||||
let mut conflicting = std::collections::HashMap::from([
|
||||
(EC_BLOCK_SIZE_HINT_INTERNAL_KEY.to_string(), "65536".to_string()),
|
||||
("x-minio-internal-ec-block-size-hint".to_string(), "262144".to_string()),
|
||||
]);
|
||||
assert_eq!(take_ec_block_size_hint(&mut conflicting), None);
|
||||
assert!(conflicting.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_quantum_stays_aligned_with_layout_geometry() {
|
||||
// Two data shards produce the same compact geometry used by the
|
||||
// persisted 64 KiB, 256 KiB, 1 MiB and 4 MiB candidates. Large
|
||||
// layouts deliberately use coarser frames to keep PUT overhead bounded.
|
||||
assert_eq!(select_ec_read_quantum(64 * 1024, 2, 32 * 1024), Some(16 * 1024));
|
||||
assert_eq!(select_ec_read_quantum(256 * 1024, 2, 128 * 1024), Some(64 * 1024));
|
||||
assert_eq!(select_ec_read_quantum(1024 * 1024, 2, 512 * 1024), Some(512 * 1024));
|
||||
assert_eq!(select_ec_read_quantum(4 * 1024 * 1024, 2, 2 * 1024 * 1024), Some(1024 * 1024));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn persisted_read_quantum_rejects_conflicts_and_bad_geometry() {
|
||||
let mut metadata = HashMap::new();
|
||||
insert_ec_read_quantum(&mut metadata, 256 * 1024);
|
||||
assert_eq!(
|
||||
persisted_ec_read_quantum(&metadata, 4 * 1024 * 1024, 2, 2 * 1024 * 1024),
|
||||
Ok(Some(256 * 1024))
|
||||
);
|
||||
|
||||
metadata.clear();
|
||||
insert_ec_read_quantum(&mut metadata, 1024 * 1024);
|
||||
assert_eq!(
|
||||
persisted_ec_read_quantum(&metadata, 4 * 1024 * 1024, 2, 2 * 1024 * 1024),
|
||||
Ok(Some(1024 * 1024))
|
||||
);
|
||||
|
||||
metadata.insert("x-minio-internal-ec-read-quantum".to_owned(), "128".to_owned());
|
||||
assert_eq!(
|
||||
persisted_ec_read_quantum(&metadata, 4 * 1024 * 1024, 2, 2 * 1024 * 1024),
|
||||
Err(EcReadQuantumMetadataError::ConflictingValues)
|
||||
);
|
||||
|
||||
metadata.clear();
|
||||
insert_ec_read_quantum(&mut metadata, 32 * 1024);
|
||||
assert_eq!(persisted_ec_read_quantum(&metadata, 64 * 1024, 2, 32 * 1024), Ok(Some(32 * 1024)));
|
||||
}
|
||||
}
|
||||
@@ -99,7 +99,6 @@ pub(crate) fn legacy_encrypted_range_seek_enabled() -> bool {
|
||||
}
|
||||
|
||||
mod body_cache_hook;
|
||||
pub mod ec_block_layout;
|
||||
mod encryption;
|
||||
mod hook_slot;
|
||||
mod multipart_completion_binding;
|
||||
@@ -124,7 +123,6 @@ pub use body_cache_hook::{
|
||||
pub(crate) use body_cache_hook::{
|
||||
get_object_body_cache_hook, get_object_body_cache_hook_suppressed, without_get_object_body_cache_hook,
|
||||
};
|
||||
pub use ec_block_layout::*;
|
||||
pub use encryption::{
|
||||
EncryptionResolutionError, EncryptionResolutionErrorKind, ObjectEncryptionResolver, ReadEncryptionMaterial,
|
||||
ReadEncryptionMode, ReadEncryptionRequest,
|
||||
|
||||
@@ -1218,8 +1218,6 @@ impl std::fmt::Debug for GetObjectSink {
|
||||
|
||||
#[derive(Default, Clone)]
|
||||
pub struct ObjectOptions {
|
||||
/// Part copies retain source locks and cannot queue for capped upload staging.
|
||||
pub multipart_copy_source_locked: bool,
|
||||
#[doc(hidden)]
|
||||
pub delete_recovery_work_budget: Option<Arc<DeleteRecoveryWorkBudget>>,
|
||||
/// Caller-provided destination for a GET body: when set, the legacy
|
||||
@@ -1234,9 +1232,6 @@ pub struct ObjectOptions {
|
||||
pub mod_time: Option<OffsetDateTime>,
|
||||
pub part_number: Option<usize>,
|
||||
|
||||
/// Exact logical versions authorized under the recursive delete bucket write lock.
|
||||
/// An empty request plan must not dispatch any disk mutation. Never serialized.
|
||||
pub recursive_delete_targets: Option<Arc<[(String, Uuid)]>>,
|
||||
pub delete_prefix: bool,
|
||||
pub delete_prefix_object: bool,
|
||||
pub version_id: Option<String>,
|
||||
@@ -1285,10 +1280,6 @@ pub struct ObjectOptions {
|
||||
/// True when an upper layer already holds the object read lock before
|
||||
/// forwarding a no_lock read to the set layer.
|
||||
pub metadata_cache_safe: bool,
|
||||
/// Read-quorum proof captured from the live distributed namespace guard.
|
||||
/// This transient value is never serialized or persisted.
|
||||
#[doc(hidden)]
|
||||
pub metadata_cache_fence_proof: Option<Arc<rustfs_lock::CacheFenceProof>>,
|
||||
|
||||
pub versioned: bool,
|
||||
pub version_suspended: bool,
|
||||
@@ -1397,10 +1388,6 @@ pub struct ObjectOptions {
|
||||
#[doc(hidden)]
|
||||
pub multipart_completion_authorization:
|
||||
Option<Arc<dyn crate::storage_api_contracts::multipart::MultipartCompletionAuthorization<Error = Error>>>,
|
||||
/// In-process authorization against the selected upload, while Abort holds its write lock.
|
||||
#[doc(hidden)]
|
||||
pub multipart_abort_authorization:
|
||||
Option<Arc<dyn crate::storage_api_contracts::multipart::MultipartAbortAuthorization<Error = Error>>>,
|
||||
pub eval_metadata: Option<HashMap<String, String>>,
|
||||
/// Internal compare-and-set condition for replication workers publishing
|
||||
/// terminal status after remote I/O. Storage validates it while holding
|
||||
@@ -1514,7 +1501,6 @@ impl std::fmt::Debug for ObjectOptions {
|
||||
.field("expected_bucket_incarnation_id", &self.expected_bucket_incarnation_id)
|
||||
.field("no_lock", &self.no_lock)
|
||||
.field("metadata_cache_safe", &self.metadata_cache_safe)
|
||||
.field("metadata_cache_fence_proof", &self.metadata_cache_fence_proof.is_some())
|
||||
.field("versioned", &self.versioned)
|
||||
.field("version_suspended", &self.version_suspended)
|
||||
.field("incl_free_versions", &self.incl_free_versions)
|
||||
@@ -1594,7 +1580,6 @@ impl ObjectOptions {
|
||||
let mut opts = self.clone();
|
||||
opts.no_lock = true;
|
||||
opts.metadata_cache_safe = false;
|
||||
opts.metadata_cache_fence_proof = None;
|
||||
opts.include_part_checksums = true;
|
||||
opts
|
||||
}
|
||||
@@ -1665,11 +1650,6 @@ impl ObjectOptions {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn add_metadata_cache_read_guard(&mut self, guard: &rustfs_lock::NamespaceLockGuard) {
|
||||
self.add_namespace_lock_guard(guard);
|
||||
self.metadata_cache_fence_proof = guard.cache_fence_proof().cloned().map(Arc::new);
|
||||
}
|
||||
|
||||
pub fn add_bucket_lifecycle_lock_guard(&mut self, guard: &rustfs_lock::NamespaceLockGuard) {
|
||||
let fence = self.bucket_lifecycle_lock_fence.get_or_insert_with(NamespaceLockFence::new);
|
||||
if let Some(signal) = guard.lock_lost_signal() {
|
||||
@@ -1953,17 +1933,6 @@ impl Clone for ObjectInfo {
|
||||
}
|
||||
|
||||
impl ObjectInfo {
|
||||
/// Return the persisted EC block size exposed by the response-side
|
||||
/// projection. Callers must not infer this value from request metadata.
|
||||
pub fn effective_ec_block_size(&self) -> Option<usize> {
|
||||
rustfs_utils::http::metadata_compat::get_consistent_str(
|
||||
&self.user_defined,
|
||||
crate::object_api::EC_BLOCK_SIZE_INFO_INTERNAL_SUFFIX,
|
||||
)
|
||||
.and_then(|value| value.parse::<usize>().ok())
|
||||
.filter(|value| *value > 0)
|
||||
}
|
||||
|
||||
pub(crate) fn shard_integrity_write_mode(&self) -> ShardIntegrityWriteMode {
|
||||
// Any declaration requires protection. Malformed declarations remain
|
||||
// errors in the source reader and must never select the legacy path.
|
||||
@@ -2277,10 +2246,9 @@ impl ObjectInfo {
|
||||
|
||||
let inlined = fi.inline_data();
|
||||
let free_version = fi.tier_free_version();
|
||||
let is_remote = fi.is_remote();
|
||||
let metadata = std::mem::take(&mut fi.metadata);
|
||||
let parts = std::mem::take(&mut fi.parts);
|
||||
Self::from_file_info_parts(&fi, inlined, free_version, is_remote, metadata, parts, bucket, object, version_id)
|
||||
Self::from_file_info_parts(&fi, inlined, free_version, metadata, parts, bucket, object, version_id)
|
||||
}
|
||||
|
||||
pub(crate) fn from_file_info_with_version_id(
|
||||
@@ -2293,7 +2261,6 @@ impl ObjectInfo {
|
||||
fi,
|
||||
fi.inline_data(),
|
||||
fi.tier_free_version(),
|
||||
fi.is_remote(),
|
||||
fi.metadata.clone(),
|
||||
fi.parts.clone(),
|
||||
bucket,
|
||||
@@ -2307,7 +2274,6 @@ impl ObjectInfo {
|
||||
fi: &FileInfo,
|
||||
inlined: bool,
|
||||
free_version: bool,
|
||||
is_remote: bool,
|
||||
mut metadata: HashMap<String, String>,
|
||||
parts: Vec<ObjectPartInfo>,
|
||||
bucket: &str,
|
||||
@@ -2383,17 +2349,6 @@ impl ObjectInfo {
|
||||
};
|
||||
|
||||
clean_metadata(&mut metadata);
|
||||
// The frame quantum is a storage-layout detail. Keep it in
|
||||
// FileInfo for the reader, but never expose the internal marker as
|
||||
// ordinary S3 metadata.
|
||||
rustfs_utils::http::metadata_compat::remove_str(&mut metadata, crate::object_api::EC_READ_QUANTUM_INFO_INTERNAL_SUFFIX);
|
||||
if !fi.deleted && !is_remote && fi.erasure.block_size > 0 {
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut metadata,
|
||||
crate::object_api::EC_BLOCK_SIZE_INFO_INTERNAL_SUFFIX,
|
||||
fi.erasure.block_size.to_string(),
|
||||
);
|
||||
}
|
||||
|
||||
let storage_class = Some(
|
||||
storageclass::effective_class(
|
||||
@@ -3367,54 +3322,6 @@ mod tests {
|
||||
assert_eq!(info.version_id, None, "a normalized absent version must not be rewritten to nil");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn from_file_info_projects_persisted_ec_block_size_for_responses() {
|
||||
let mut fi = FileInfo::new("bucket/object", 4, 2);
|
||||
fi.size = 1;
|
||||
fi.erasure.block_size = 64 * 1024;
|
||||
|
||||
let info = ObjectInfo::from_file_info(&fi, "bucket", "object", false);
|
||||
|
||||
assert_eq!(info.effective_ec_block_size(), Some(64 * 1024));
|
||||
assert_eq!(
|
||||
info.user_defined
|
||||
.get(crate::object_api::EC_BLOCK_SIZE_INFO_INTERNAL_KEY)
|
||||
.map(String::as_str),
|
||||
Some("65536")
|
||||
);
|
||||
|
||||
fi.erasure.block_size = 131_072;
|
||||
let info = ObjectInfo::from_file_info(&fi, "bucket", "object", false);
|
||||
assert_eq!(info.effective_ec_block_size(), Some(131_072));
|
||||
|
||||
fi.transition_status = rustfs_filemeta::TRANSITION_COMPLETE.to_string();
|
||||
let info = ObjectInfo::from_file_info(&fi, "bucket", "remote-object", false);
|
||||
assert_eq!(info.effective_ec_block_size(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn from_owned_file_info_preserves_restored_ec_block_size_projection() {
|
||||
let mut fi = FileInfo::new("bucket/object", 4, 2);
|
||||
fi.size = 1;
|
||||
fi.erasure.block_size = 64 * 1024;
|
||||
fi.transition_status = rustfs_filemeta::TRANSITION_COMPLETE.to_string();
|
||||
let restore = rustfs_filemeta::RestoreStatus {
|
||||
is_restore_in_progress: Some(false),
|
||||
restore_expiry_date: Some(OffsetDateTime::now_utc() + time::Duration::days(1)),
|
||||
};
|
||||
fi.metadata.insert(metadata_keys::RESTORE.to_string(), restore.to_string());
|
||||
assert!(!fi.is_remote(), "the restored source has a local copy");
|
||||
|
||||
let borrowed = ObjectInfo::from_file_info(&fi, "bucket", "object", false);
|
||||
let owned = ObjectInfo::from_owned_file_info(fi, "bucket", "object", false);
|
||||
|
||||
assert_eq!(owned.effective_ec_block_size(), Some(64 * 1024));
|
||||
assert_eq!(owned.user_defined, borrowed.user_defined);
|
||||
for key in ["x-rustfs-internal-ec-block-size", "x-minio-internal-ec-block-size"] {
|
||||
assert_eq!(owned.user_defined.get(key).map(String::as_str), Some("65536"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn from_file_info_reports_effective_storage_class_for_legacy_metadata() {
|
||||
for legacy_label in [
|
||||
|
||||
@@ -604,10 +604,10 @@ pub(crate) async fn initialize_local_disk_maps(
|
||||
|
||||
let disk = match new_disk(ep, opt).await {
|
||||
Ok(disk) => disk,
|
||||
Err(crate::disk::error::DiskError::FileAccessDenied | crate::disk::error::DiskError::VolumeNotFound) => {
|
||||
// Preserve the configured slot when a local volume is unavailable.
|
||||
// LocalDisk logs the cause; format loading retains authority
|
||||
// over fatal errors and quorum. Never create a missing root.
|
||||
Err(crate::disk::error::DiskError::FileAccessDenied) => {
|
||||
// LocalDisk already logged the filesystem cause and repair
|
||||
// hint. Keep this slot unavailable so ECStore initialization
|
||||
// can apply its existing fatal-error and quorum rules.
|
||||
global_local_disk_map.insert(ep.to_string(), None);
|
||||
continue;
|
||||
}
|
||||
@@ -888,95 +888,6 @@ mod tests {
|
||||
disk_map.keys().collect::<Vec<_>>()
|
||||
);
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn missing_local_volume_preserves_offline_slot_without_creating_root() {
|
||||
use crate::disk::{DiskOption, error::DiskError};
|
||||
use crate::layout::endpoints::{EndpointServerPools, PoolEndpoints};
|
||||
|
||||
for drive_count in [2, 3, 4] {
|
||||
let directory = tempfile::tempdir().expect("create volume parent");
|
||||
let mut endpoints = Vec::new();
|
||||
for index in 0..drive_count {
|
||||
let path = directory.path().join(format!("disk{index}"));
|
||||
if index != 0 {
|
||||
std::fs::create_dir(&path).expect("create surviving volume");
|
||||
}
|
||||
let mut endpoint = Endpoint::try_from(path.to_str().expect("UTF-8 path")).expect("local endpoint");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(index);
|
||||
endpoints.push(endpoint);
|
||||
}
|
||||
let missing = endpoints[0].to_string();
|
||||
let pools = EndpointServerPools(vec![PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: drive_count,
|
||||
endpoints: endpoints.into(),
|
||||
cmd_line: "missing-volume".into(),
|
||||
platform: "test".into(),
|
||||
}]);
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let options = DiskOption {
|
||||
cleanup: true,
|
||||
health_check: false,
|
||||
};
|
||||
super::initialize_local_disk_maps(&ctx, pools.clone(), &options)
|
||||
.await
|
||||
.expect("missing volume should remain offline for quorum evaluation");
|
||||
let map = ctx.local_disk_map();
|
||||
let map = map.read().await;
|
||||
assert_eq!(map.len(), drive_count);
|
||||
assert!(matches!(map.get(&missing), Some(None)));
|
||||
assert_eq!(map.values().flatten().count(), drive_count - 1);
|
||||
drop(map);
|
||||
let drives = ctx.local_disk_set_drives();
|
||||
let drives = drives.read().await;
|
||||
assert_eq!(drives[0][0].len(), drive_count);
|
||||
assert!(drives[0][0][0].is_none());
|
||||
assert!(drives[0][0][1..].iter().all(Option::is_some));
|
||||
drop(drives);
|
||||
|
||||
let (disks, errors) = crate::store::init_format::init_disks(&ctx, &pools.0[0].endpoints, &options).await;
|
||||
assert!(disks[0].is_none());
|
||||
assert_eq!(errors[0], Some(DiskError::VolumeNotFound));
|
||||
assert!(disks[1..].iter().all(Option::is_some));
|
||||
assert!(errors[1..].iter().all(Option::is_none));
|
||||
assert!(!directory.path().join("disk0").exists(), "startup must not create a missing mount root");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn invalid_local_volume_still_fails_registry_initialization() {
|
||||
use crate::disk::DiskOption;
|
||||
use crate::layout::endpoints::{EndpointServerPools, PoolEndpoints};
|
||||
|
||||
let directory = tempfile::tempdir().expect("create volume parent");
|
||||
let path = directory.path().join("regular-file");
|
||||
std::fs::write(&path, b"preserve existing file").expect("create invalid volume");
|
||||
let mut endpoint = Endpoint::try_from(path.to_str().expect("UTF-8 path")).expect("local endpoint");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(0);
|
||||
let pools = EndpointServerPools(vec![PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: 1,
|
||||
endpoints: vec![endpoint].into(),
|
||||
cmd_line: "invalid-volume".into(),
|
||||
platform: "test".into(),
|
||||
}]);
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let options = DiskOption {
|
||||
cleanup: true,
|
||||
health_check: false,
|
||||
};
|
||||
super::initialize_local_disk_maps(&ctx, pools, &options)
|
||||
.await
|
||||
.expect_err("a non-directory volume must retain its initialization error");
|
||||
assert_eq!(std::fs::read(&path).expect("read existing file"), b"preserve existing file");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_invalid_registry_pool_index_has_no_authority_or_registry_effect() {
|
||||
use crate::disk::DiskOption;
|
||||
|
||||
@@ -15,6 +15,8 @@
|
||||
use crate::diagnostics::admin_server_info::get_local_server_property;
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use crate::storage_api_contracts::admin::StorageAdminApi;
|
||||
#[cfg(test)]
|
||||
use chrono::Utc;
|
||||
use jiff::Timestamp;
|
||||
use rustfs_heal_contracts::heal_channel::DriveState;
|
||||
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
|
||||
@@ -598,6 +600,10 @@ mod test {
|
||||
use serial_test::serial;
|
||||
use std::time::Duration;
|
||||
|
||||
fn chrono_to_jiff_timestamp(timestamp: chrono::DateTime<Utc>) -> jiff::Timestamp {
|
||||
jiff::Timestamp::try_from(std::time::SystemTime::from(timestamp)).expect("test timestamp should fit in jiff")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tes_types() {
|
||||
let t = MetricType::ALL;
|
||||
@@ -679,10 +685,10 @@ mod test {
|
||||
|
||||
#[test]
|
||||
fn scanner_metrics_mapping_preserves_partial_source_status() {
|
||||
let current_started = Timestamp::now() - jiff::SignedDuration::from_secs(5);
|
||||
let current_started = Utc::now() - chrono::Duration::seconds(5);
|
||||
let scanner = to_madmin_scanner_metrics(rustfs_scanner_metrics::metrics::ScannerMetricsReport {
|
||||
current_cycle_active: true,
|
||||
current_started,
|
||||
current_started: chrono_to_jiff_timestamp(current_started),
|
||||
last_cycle_partial_source: "usage".to_string(),
|
||||
last_cycle_partial_source_code: 1,
|
||||
cycle_timeout_total: 3,
|
||||
@@ -697,7 +703,7 @@ mod test {
|
||||
});
|
||||
|
||||
assert_eq!(scanner.current_cycle_active, Some(true));
|
||||
assert_eq!(scanner.current_started, current_started);
|
||||
assert_eq!(scanner.current_started, chrono_to_jiff_timestamp(current_started));
|
||||
assert_eq!(scanner.last_cycle_partial_source, "usage");
|
||||
assert_eq!(scanner.last_cycle_partial_source_code, 1);
|
||||
assert_eq!(scanner.cycle_timeout_total, 3);
|
||||
@@ -717,8 +723,8 @@ mod test {
|
||||
async fn collect_local_metrics_preserves_scanner_cycle_started_time() {
|
||||
let previous_init_time = *rustfs_scanner_metrics::GLOBAL_INIT_TIME.read().await;
|
||||
let previous_cycle = global_metrics().get_cycle().await;
|
||||
let init_time = Timestamp::now() - jiff::SignedDuration::from_hours(1);
|
||||
let cycle_started = Timestamp::now() - jiff::SignedDuration::from_secs(5);
|
||||
let init_time = Utc::now() - chrono::Duration::hours(1);
|
||||
let cycle_started = Utc::now() - chrono::Duration::seconds(5);
|
||||
*rustfs_scanner_metrics::GLOBAL_INIT_TIME.write().await = Some(init_time);
|
||||
let cycle = CurrentCycle {
|
||||
current: 0,
|
||||
@@ -742,7 +748,7 @@ mod test {
|
||||
aggregated.merge(decoded);
|
||||
let scanner = aggregated.aggregated.scanner.expect("scanner metrics");
|
||||
assert_eq!(scanner.current_cycle_active, Some(true));
|
||||
assert_eq!(scanner.current_started, cycle_started);
|
||||
assert_eq!(scanner.current_started, chrono_to_jiff_timestamp(cycle_started));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user