mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-20 15:07:34 +00:00
Compare commits
17 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 739699426b | |||
| 0d2a0ea345 | |||
| c1f97d27f6 | |||
| 1066ab4d1d | |||
| b0e824437b | |||
| 913a5df515 | |||
| f483cfedc8 | |||
| f2177505da | |||
| a758b58bf2 | |||
| 9075131b23 | |||
| 050be16dc7 | |||
| 8efa1a96c0 | |||
| 444dbd9956 | |||
| 33e0fe8c6f | |||
| 205227b441 | |||
| 3e5d37742d | |||
| 8e6265907a |
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=9ccb03f4139e99ec11a63810523d504df39191b86e74b28a72dd9a075682293b
|
||||
sha256-linux=5ef052b03a03afbd00ebfcbbdcb754bbde414b7e6151aae3e2ee78024ce1d0b6
|
||||
sha256-darwin=8fef326a3fddd8a548e2cbcc6ea8159f831113595fb003eeebbd9aeb4ac23e48
|
||||
sha256-linux=8b4381578377704ac8b1b5156128b83f036ec4aedc9d863fff25df15112420fc
|
||||
|
||||
@@ -1 +1 @@
|
||||
sha256=fd5683273bba1eae3b4402a4e854a5d168ace845bd0e5fc801f8e336485a4394
|
||||
sha256=061d0a697967d2dc8f472ff885886e069b1cf4aeaa5f230e177b7236017c23b6
|
||||
|
||||
@@ -39,11 +39,9 @@ script-tests: ## Run shell script tests
|
||||
./scripts/test_entrypoint_directories.sh
|
||||
./scripts/test_internode_grpc_ab_bench.sh
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_internode_grpc_window.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_internode_rpc_timeout_boundary.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_internode_msgpack_peer_canary.py
|
||||
./scripts/test_internode_msgpack_peer_canary_runner.sh
|
||||
./scripts/test_object_batch_bench_enhanced.sh
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_inline_docker_lab.py
|
||||
./scripts/test_package_service_scripts.sh
|
||||
./scripts/test_hotpath_warp_ab_gate.sh
|
||||
./scripts/test_hotpath_warp_abba.sh
|
||||
@@ -103,6 +101,8 @@ script-tests: ## Run shell script tests
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_retain_validation_evidence.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_timing_report.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_junit_diff.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_dual_stack.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_mint_stack_diff.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_conformance_rulings.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/compat_matrix_report.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
|
||||
|
||||
@@ -141,9 +141,8 @@ setup = 'ecstore-large-stack'
|
||||
filter = '(package(rustfs) & test(/^(admin::handlers::replication::target_repair_tests::[a-z_]+|admin::handlers::plugin_attempt::v2_tests::delegation_v2_persists_and_vends_with_real_parent_intersection|admin::handlers::site_replication::tests::durable_iam_state_apply_iam_item_rejects_a_stale_recreate_after_a_replicated_delete|admin::service::federated_identity::tests::(binding_keeps_explicit_iam_mapping_strict|markerless_legacy_oidc_session_crosses_auth_and_iam_deletion_boundaries)|storage::scanner_dirty_journal::tests::(clear_callback_keeps_unknown_coverage_until_all_pending_work_is_confirmed|owned_journals_keep_peer_records_after_interleaved_writes_and_clear))$/)) | (package(rustfs-heal) & ((binary(heal_b5_versioned_regression_test) & test(/^serial_tests::(test_heal_unversioned_bucket_e2e|test_historical_null_recursive_heal_restores_physical_versions|test_recursive_deep_heal_reports_stale_delete_marker_repair)$/)) | (binary(usage_observation_read_repair_test) & test(=usage_observation_read_repair_survives_snapshot_cleanup)))) | (package(rustfs-test-utils) & test(=data_usage_snapshot_tests::data_usage_snapshot_recovery_wires_primary_backup_and_negative_cache))'
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
# The multipart refusal fixture composes real-store setup, abort, quota and legal-hold futures.
|
||||
[[profile.default.scripts]]
|
||||
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::complete::tests::completion_absent_binding_refusal_order_baseline|app::multipart_usecase::complete::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access_owned::tests::generation_guards::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
|
||||
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::complete::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
# These handler/store fixtures and the relocated-pool GET compose deep storage
|
||||
@@ -466,7 +465,7 @@ filter = '(package(rustfs) & test(/^(admin::handlers::replication::target_repair
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
[[profile.ci.scripts]]
|
||||
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::complete::tests::completion_absent_binding_refusal_order_baseline|app::multipart_usecase::complete::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access_owned::tests::generation_guards::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
|
||||
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::complete::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
[[profile.ci.scripts]]
|
||||
@@ -723,8 +722,9 @@ test-group = 'ecstore-serial-flaky'
|
||||
# fixture as the existing smoke modules.
|
||||
#
|
||||
# Dual-stack differential (rustfs/backlog#2740): `dual_stack_differential::`
|
||||
# starts two single-node servers on random ports with isolated temp
|
||||
# dirs, and runs one SDK script covering every S3 operation against both. Its
|
||||
# starts two single-node servers from the same binary, one per
|
||||
# RUSTFS_S3_STACK value, on random ports with isolated temp dirs, and runs one
|
||||
# SDK script covering every operation of `impl S3 for FS` against both. Its
|
||||
# comparator unit tests need no server. It meets the admission criteria
|
||||
# unchanged; the module clause admits a new differential test without another
|
||||
# filter edit.
|
||||
|
||||
@@ -9,9 +9,9 @@
|
||||
# Adding a target is one more [[targets]] table, for example:
|
||||
#
|
||||
# [[targets]]
|
||||
# name = "candidate-tuned"
|
||||
# name = "candidate-gateway"
|
||||
# binary = "candidate"
|
||||
# env = { RUSTFS_EXAMPLE_SETTING = "on" }
|
||||
# env = { RUSTFS_S3_STACK = "gateway" }
|
||||
|
||||
schema = 1
|
||||
|
||||
|
||||
@@ -114,14 +114,8 @@
|
||||
}
|
||||
],
|
||||
"upgrade_sources": {
|
||||
"basis": "The three previous-release sources pinned in .github/workflows/e2e-upgrade.yml are exercised today (source.version / source.asset / source.sha256). No other N-1/N-2 compatibility is claimed, and these digests are not inferred from tags.",
|
||||
"basis": "Exactly the two previous-release sources pinned in .github/workflows/e2e-upgrade.yml are exercised today (source.version / source.asset / source.sha256). No other N-1/N-2 compatibility is claimed, and these digests are not inferred from tags.",
|
||||
"sources": [
|
||||
{
|
||||
"version": "1.0.5",
|
||||
"role": "gateway compatibility baseline",
|
||||
"artifact_name": "rustfs-linux-x86_64-gnu-v1.0.5.zip",
|
||||
"artifact_sha256": "5addae2f74756bf80104d0975e6347cda843bc2daffdfcd1f0cb80118b95cbd0"
|
||||
},
|
||||
{
|
||||
"version": "1.0.1",
|
||||
"role": "N-1",
|
||||
@@ -136,6 +130,6 @@
|
||||
}
|
||||
],
|
||||
"rollback_supported": false,
|
||||
"rollback_basis": "The release upgrade lane checks bucket-config rollback. The hosted compatibility workflow also checks in-flight multipart rollback to 1.0.5. Full data-plane rollback is not certified and is never a default claim."
|
||||
"rollback_basis": "Only the bucket-config-rollback case (rollback_to_previous_release_reads_current_bucket_metadata) is exercised. Full data-plane rollback is not certified and is never a default claim."
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,6 +28,8 @@ rustfs/src/app/storage_api.rs
|
||||
rustfs/src/admin/router.rs
|
||||
rustfs/src/app/metadata_route.rs
|
||||
rustfs/src/server/http.rs
|
||||
rustfs/src/server/layer.rs
|
||||
rustfs/src/server/hybrid.rs
|
||||
rustfs/src/protocols/client.rs
|
||||
|
||||
# e2e oracle: the test target that verifies S3 behaviour against s3s
|
||||
|
||||
@@ -13,6 +13,6 @@
|
||||
# s3_error_lines lines holding s3_error!, same scope, minus those the script
|
||||
# header's rule clears (rustfs_s3_types's macro); fails closed
|
||||
# ecstore_files Rust files referencing s3s paths under crates/ecstore/src
|
||||
files=33
|
||||
s3_error_lines=89
|
||||
files=63
|
||||
s3_error_lines=209
|
||||
ecstore_files=0
|
||||
|
||||
@@ -22,7 +22,7 @@ rustfs-connect-symbol-catalog 6 0
|
||||
rustfs-credentials 24 0
|
||||
rustfs-crypto 67 0
|
||||
rustfs-data-usage 58 0
|
||||
rustfs-ecstore 7858 3
|
||||
rustfs-ecstore 7858 2
|
||||
rustfs-extension-schema 23 0
|
||||
rustfs-filemeta 361 0
|
||||
rustfs-gateway-adapter 74 0
|
||||
|
||||
@@ -35,7 +35,6 @@ services:
|
||||
- RUSTFS_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_LOCK_ACQUIRE_TIMEOUT:-5}
|
||||
- RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
|
||||
- RUSTFS_DURABILITY_MODE=${RUSTFS_DURABILITY_MODE:-strict}
|
||||
- RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE=${RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS:-0}
|
||||
# Internode gRPC optimization knobs (grpc-optimization P0-P3). Defaults match the binary
|
||||
@@ -47,8 +46,6 @@ services:
|
||||
- RUSTFS_INTERNODE_CHANNEL_ISOLATION=${RUSTFS_INTERNODE_CHANNEL_ISOLATION:-false}
|
||||
- RUSTFS_INTERNODE_BULK_CHANNELS=${RUSTFS_INTERNODE_BULK_CHANNELS:-2}
|
||||
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY:-false}
|
||||
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED:-false}
|
||||
- RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED=${RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED:-false}
|
||||
- RUSTFS_INTERNODE_PREWARM=${RUSTFS_INTERNODE_PREWARM:-false}
|
||||
- RUSTFS_INTERNODE_OFFLINE_BYPASS=${RUSTFS_INTERNODE_OFFLINE_BYPASS:-false}
|
||||
- RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS=${RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS:-5}
|
||||
@@ -62,7 +59,7 @@ services:
|
||||
- node1_data_2:/data/rustfs2
|
||||
- node1_data_3:/data/rustfs3
|
||||
ports:
|
||||
- "${RUSTFS_NODE1_PORT:-9000}:9000"
|
||||
- "9000:9000"
|
||||
networks:
|
||||
- rustfs-cluster-net
|
||||
|
||||
@@ -88,7 +85,6 @@ services:
|
||||
- RUSTFS_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_LOCK_ACQUIRE_TIMEOUT:-5}
|
||||
- RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
|
||||
- RUSTFS_DURABILITY_MODE=${RUSTFS_DURABILITY_MODE:-strict}
|
||||
- RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE=${RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS:-0}
|
||||
# Internode gRPC optimization knobs (grpc-optimization P0-P3). Defaults match the binary
|
||||
@@ -100,8 +96,6 @@ services:
|
||||
- RUSTFS_INTERNODE_CHANNEL_ISOLATION=${RUSTFS_INTERNODE_CHANNEL_ISOLATION:-false}
|
||||
- RUSTFS_INTERNODE_BULK_CHANNELS=${RUSTFS_INTERNODE_BULK_CHANNELS:-2}
|
||||
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY:-false}
|
||||
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED:-false}
|
||||
- RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED=${RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED:-false}
|
||||
- RUSTFS_INTERNODE_PREWARM=${RUSTFS_INTERNODE_PREWARM:-false}
|
||||
- RUSTFS_INTERNODE_OFFLINE_BYPASS=${RUSTFS_INTERNODE_OFFLINE_BYPASS:-false}
|
||||
- RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS=${RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS:-5}
|
||||
@@ -115,7 +109,7 @@ services:
|
||||
- node2_data_2:/data/rustfs2
|
||||
- node2_data_3:/data/rustfs3
|
||||
ports:
|
||||
- "${RUSTFS_NODE2_PORT:-9001}:9000"
|
||||
- "9001:9000"
|
||||
networks:
|
||||
- rustfs-cluster-net
|
||||
|
||||
@@ -141,7 +135,6 @@ services:
|
||||
- RUSTFS_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_LOCK_ACQUIRE_TIMEOUT:-5}
|
||||
- RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
|
||||
- RUSTFS_DURABILITY_MODE=${RUSTFS_DURABILITY_MODE:-strict}
|
||||
- RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE=${RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS:-0}
|
||||
# Internode gRPC optimization knobs (grpc-optimization P0-P3). Defaults match the binary
|
||||
@@ -153,8 +146,6 @@ services:
|
||||
- RUSTFS_INTERNODE_CHANNEL_ISOLATION=${RUSTFS_INTERNODE_CHANNEL_ISOLATION:-false}
|
||||
- RUSTFS_INTERNODE_BULK_CHANNELS=${RUSTFS_INTERNODE_BULK_CHANNELS:-2}
|
||||
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY:-false}
|
||||
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED:-false}
|
||||
- RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED=${RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED:-false}
|
||||
- RUSTFS_INTERNODE_PREWARM=${RUSTFS_INTERNODE_PREWARM:-false}
|
||||
- RUSTFS_INTERNODE_OFFLINE_BYPASS=${RUSTFS_INTERNODE_OFFLINE_BYPASS:-false}
|
||||
- RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS=${RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS:-5}
|
||||
@@ -168,7 +159,7 @@ services:
|
||||
- node3_data_2:/data/rustfs2
|
||||
- node3_data_3:/data/rustfs3
|
||||
ports:
|
||||
- "${RUSTFS_NODE3_PORT:-9002}:9000"
|
||||
- "9002:9000"
|
||||
networks:
|
||||
- rustfs-cluster-net
|
||||
|
||||
@@ -194,7 +185,6 @@ services:
|
||||
- RUSTFS_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_LOCK_ACQUIRE_TIMEOUT:-5}
|
||||
- RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
|
||||
- RUSTFS_DURABILITY_MODE=${RUSTFS_DURABILITY_MODE:-strict}
|
||||
- RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE=${RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS:-0}
|
||||
# Internode gRPC optimization knobs (grpc-optimization P0-P3). Defaults match the binary
|
||||
@@ -206,8 +196,6 @@ services:
|
||||
- RUSTFS_INTERNODE_CHANNEL_ISOLATION=${RUSTFS_INTERNODE_CHANNEL_ISOLATION:-false}
|
||||
- RUSTFS_INTERNODE_BULK_CHANNELS=${RUSTFS_INTERNODE_BULK_CHANNELS:-2}
|
||||
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY:-false}
|
||||
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED:-false}
|
||||
- RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED=${RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED:-false}
|
||||
- RUSTFS_INTERNODE_PREWARM=${RUSTFS_INTERNODE_PREWARM:-false}
|
||||
- RUSTFS_INTERNODE_OFFLINE_BYPASS=${RUSTFS_INTERNODE_OFFLINE_BYPASS:-false}
|
||||
- RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS=${RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS:-5}
|
||||
@@ -221,7 +209,7 @@ services:
|
||||
- node4_data_2:/data/rustfs2
|
||||
- node4_data_3:/data/rustfs3
|
||||
ports:
|
||||
- "${RUSTFS_NODE4_PORT:-9003}:9000"
|
||||
- "9003:9000"
|
||||
networks:
|
||||
- rustfs-cluster-net
|
||||
|
||||
|
||||
@@ -1,58 +0,0 @@
|
||||
# syntax=docker/dockerfile:1.6
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Server-only Linux validation image. The build context supplies a Git bundle.
|
||||
FROM rust:1.100.0-trixie AS build-tools
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates clang lld libssl-dev pkg-config protobuf-compiler python3 python3-boto3 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
ENV RUSTUP_TOOLCHAIN=1.100.0 \
|
||||
CARGO_BUILD_JOBS=4 \
|
||||
CARGO_NET_GIT_FETCH_WITH_CLI=true \
|
||||
CARGO_TARGET_DIR=/work/rustfs/target \
|
||||
RUSTFLAGS="-C link-arg=-fuse-ld=lld"
|
||||
|
||||
FROM build-tools AS build
|
||||
ARG SOURCE_REVISION
|
||||
COPY repository.bundle /tmp/repository.bundle
|
||||
COPY inspect_geometry.rs /tmp/inspect_geometry.rs
|
||||
RUN git clone /tmp/repository.bundle /work/rustfs \
|
||||
&& git -C /work/rustfs checkout --detach "$SOURCE_REVISION" \
|
||||
&& test "$(git -C /work/rustfs rev-parse HEAD)" = "$SOURCE_REVISION"
|
||||
WORKDIR /work/rustfs
|
||||
RUN --mount=type=cache,id=rustfs-inline-registry,target=/usr/local/cargo/registry \
|
||||
--mount=type=cache,id=rustfs-inline-git,target=/usr/local/cargo/git \
|
||||
--mount=type=cache,id=rustfs-inline-linux-target,target=/work/rustfs/target \
|
||||
cargo build --locked --release --bin rustfs \
|
||||
&& install -Dm755 target/release/rustfs /out/rustfs
|
||||
|
||||
# Build the diagnostic tool after preserving the clean-source server binary.
|
||||
RUN cp /tmp/inspect_geometry.rs crates/filemeta/examples/inline_lab_inspect.rs
|
||||
RUN --mount=type=cache,id=rustfs-inline-registry,target=/usr/local/cargo/registry \
|
||||
--mount=type=cache,id=rustfs-inline-git,target=/usr/local/cargo/git \
|
||||
--mount=type=cache,id=rustfs-inline-linux-target,target=/work/rustfs/target \
|
||||
cargo build --locked --release -p rustfs-filemeta --example inline_lab_inspect \
|
||||
&& install -Dm755 target/release/examples/inline_lab_inspect /out/inline_lab_inspect
|
||||
|
||||
FROM debian:trixie-slim AS runtime
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates curl libssl3t64 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
ARG SOURCE_REVISION
|
||||
LABEL org.opencontainers.image.revision="$SOURCE_REVISION" \
|
||||
rustfs.validation="inline"
|
||||
COPY --from=build /out/rustfs /usr/local/bin/rustfs
|
||||
COPY --from=build /out/inline_lab_inspect /usr/local/bin/inline_lab_inspect
|
||||
ENTRYPOINT ["/usr/local/bin/rustfs"]
|
||||
@@ -1,5 +0,0 @@
|
||||
FROM debian:trixie-slim
|
||||
COPY mc /usr/bin/mc
|
||||
RUN chmod 0755 /usr/bin/mc && /usr/bin/mc --version
|
||||
LABEL rustfs.validation="inline"
|
||||
ENTRYPOINT ["/usr/bin/mc"]
|
||||
@@ -1,133 +0,0 @@
|
||||
# Inline geometry Linux lab
|
||||
|
||||
**Use this when** validating inline placement, block geometry, Range reads and reader rollback in a task-owned local Docker cluster.
|
||||
|
||||
The adapter reuses the repository's four-node compose topology with 16 logical Docker volumes and EC12+4. It binds one S3 port to loopback, keeps strict durability and checks running binary hashes and persisted object geometry. Docker volumes share the host storage substrate. Performance observations describe this local virtualized lab.
|
||||
|
||||
## Build a frozen server image
|
||||
|
||||
Use the current stable toolchain pinned in the lab Dockerfile. Supply a Git bundle and the metadata inspector source as the build context. The server is compiled from a clean checkout before the diagnostic example is added.
|
||||
|
||||
```bash
|
||||
set -euo pipefail
|
||||
LAB_BUILD_DIR="$(mktemp -d)"
|
||||
LAB_REVISION="$(git rev-parse HEAD)"
|
||||
git bundle create "$LAB_BUILD_DIR/repository.bundle" HEAD
|
||||
cp crates/filemeta/examples/inline_lab_inspect.rs "$LAB_BUILD_DIR/inspect_geometry.rs"
|
||||
docker --context "$LAB_CONTEXT" build --target build-tools \
|
||||
-t rustfs-inline-build-tools:1.100 \
|
||||
-f .docker/inline-lab/Dockerfile .docker/inline-lab
|
||||
docker --context "$LAB_CONTEXT" build \
|
||||
--build-arg SOURCE_REVISION="$LAB_REVISION" \
|
||||
-t rustfs-inline-validation:local \
|
||||
-f .docker/inline-lab/Dockerfile "$LAB_BUILD_DIR"
|
||||
LAB_EXPORT_ID="$(docker --context "$LAB_CONTEXT" create \
|
||||
--label rustfs.validation=inline rustfs-inline-validation:local)"
|
||||
docker --context "$LAB_CONTEXT" cp \
|
||||
"$LAB_EXPORT_ID:/usr/local/bin/rustfs" "$LAB_BUILD_DIR/rustfs-linux"
|
||||
docker --context "$LAB_CONTEXT" rm "$LAB_EXPORT_ID"
|
||||
```
|
||||
|
||||
## Configure and probe
|
||||
|
||||
Select a new compose project and a private state directory. The adapter generates test credentials there. Existing resources require matching ownership labels. Configure checks the image's source revision and pins immutable image IDs. Each operation takes an exclusive state lock. Deployment waits for `/health/ready` on every node, covering storage, IAM, lock quorum and peer dependencies.
|
||||
|
||||
```bash
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" configure \
|
||||
--context "$LAB_CONTEXT" --project "$LAB_PROJECT" \
|
||||
--image rustfs-inline-validation:local --revision "$LAB_REVISION" \
|
||||
--binary "$LAB_BUILD_DIR/rustfs-linux" --port 19080
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" deploy \
|
||||
--bucket inline-geometry-probe --shard-bytes 0
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" probe \
|
||||
--bucket inline-geometry-probe
|
||||
```
|
||||
|
||||
The probe verifies complete bytes, HEAD sizes, Range contents, EC12+4 and the actual persisted block. It records inline markers for 4 KiB, 1 MiB, 1 MiB + 1 byte, 1.25 MiB, 1,572,856 bytes, 1,572,857 bytes and 1.5 MiB objects in `probe.json`. The server-only image includes the existing filemeta decoder through a small geometry inspector. Console assets and the administration CLI are outside this lab image.
|
||||
|
||||
## ABBA deployment hook
|
||||
|
||||
Use `inline_docker_lab.py ... deploy` in the existing `run_hotpath_warp_abba.sh` external deploy hook. Export `RUSTFS_ERASURE_SHARD_BYTES` according to `HOTPATH_ABBA_PHASE`. The adapter uses the supplied leg, phase, bucket, binary and evidence-file context, resets only owned lab volumes, checks running configuration and binary hashes, creates the fresh bucket and writes the runner's deployment evidence.
|
||||
|
||||
Use the same frozen binary for both sides of a geometry experiment. Fix the inline budget and encoding mode, then compare zero with one candidate shard value. Include the CPU, memory and image identities from every node in each result. The state `config.json` supplies the generated test credentials for the load driver. Set `DOCKER_CONTEXT` to the selected context when the driver captures Docker resource snapshots.
|
||||
|
||||
For content compatibility across an image or geometry change, configure the next selected image and artifact using the same state, then deploy with `--keep-data` and an existing bucket. This retains owned volumes. Compare existing objects and version IDs with the manifest prepared before the change.
|
||||
|
||||
## Cleanup
|
||||
|
||||
```bash
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" down --purge
|
||||
```
|
||||
|
||||
Cleanup checks ownership for containers, volumes and networks. A configuration cannot adopt another project's resources. Raw measurements, state, Git bundles and build outputs stay in task artifact directories.
|
||||
|
||||
Unit checks run with `scripts/python_bin.sh scripts/test_inline_docker_lab.py` and are included in `make script-tests`.
|
||||
|
||||
## Linux content-probe client
|
||||
|
||||
The PUT after-probe requires a working `mc`. For a macOS client affected by cgo initialization failures, compile the same frozen public mc module for Linux with CGO disabled, build it with `Dockerfile.mc`, and use `scripts/inline_docker_mc.sh` as the process-local `mc` command. Record the module version, checksum and resulting image ID with the run.
|
||||
|
||||
The adapter rewrites only the lab loopback alias endpoint to the private `node1` endpoint. Alias configuration stays in the probe's own `MC_CONFIG_DIR`, and `cat` streams raw bytes. The existing HEAD, GET and hash checks continue to execute.
|
||||
|
||||
```bash
|
||||
export RUSTFS_INLINE_LAB_STATE="$LAB_STATE"
|
||||
export RUSTFS_INLINE_MC_IMAGE="$LAB_MC_IMAGE"
|
||||
mkdir -p "$LAB_TOOLS"
|
||||
ln -s "$PWD/scripts/inline_docker_mc.sh" "$LAB_TOOLS/mc"
|
||||
export PATH="$LAB_TOOLS:$PATH"
|
||||
```
|
||||
|
||||
## Compare transport and placement independently
|
||||
|
||||
`deploy --rpc-mode json-compat` explicitly clears both MessagePack-only flags on every node. `deploy --rpc-mode msgpack-only` enables both flags for the selected homogeneous lab fleet. Deployment evidence records the mode and checks both running flags. The default is `json-compat`. The ABBA hook may select the same mode through `RUSTFS_INLINE_LAB_RPC_MODE`.
|
||||
|
||||
First compare the base and bounded-multi-block binaries at `--shard-bytes 0` in JSON compatibility mode. Then repeat with MessagePack-only on both sides. A same-binary transport ABBA isolates the encoding effect. Keep the 128 KiB budget, CPU sets, strict durability, load and object set fixed. Check actual markers and block sizes at the boundary sizes before timing. Retain baseline-drift and candidate gates, error logs and raw request distributions.
|
||||
|
||||
For a rolling downgrade, redeploy the current binary with `--keep-data --rpc-mode json-compat` before introducing an older node. Check old and new reads, version IDs and batch deletes against the persisted manifest. A running MessagePack-only flag pair establishes the lab configuration. Compatibility with a particular old reader requires the explicit reader test.
|
||||
|
||||
## Preserve a compatibility manifest
|
||||
|
||||
Run the seed once in a fresh lab bucket. It records full byte oracles and the observed version inventory, including any additional version matching a known body, and leaves a 5 MiB multipart part ready for the next reader. Each bucket has its own private `compat-<bucket>.json` manifest. Keep the same state and volumes across image changes. A fresh deployment or cleanup that resets volumes moves existing compatibility manifests into `compat-history`. Retained-data deployments preserve the active manifests.
|
||||
|
||||
```bash
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" compat-seed --bucket inline-compat
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" compat-check --bucket inline-compat
|
||||
# Configure the next frozen image and binary, then retain the data.
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" deploy \
|
||||
--bucket inline-compat --shard-bytes 0 --rpc-mode json-compat --keep-data
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" compat-check --bucket inline-compat
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" compat-finish-mpu --bucket inline-compat
|
||||
```
|
||||
|
||||
The check compares every persisted version ID, HEAD length, complete SHA256, cross-block Range and per-version geometry from `xl.meta`. The seed includes zero-length objects, null versions, the 192 KiB versioned admission boundary and the accumulated 128 KiB shard boundary. Seed a second bucket on the new writer before the downgrade, then check both manifests using the selected old reader. Complete an old upload on the new binary and a new upload on the old binary. After returning to the current fleet, `compat-delete --bucket <bucket>` deletes the recorded versions in one S3 batch and checks the exact remaining inventory and bytes.
|
||||
|
||||
Successful multipart-completion and batch-delete receipts are saved before subsequent content and geometry checks. If either check fails, rerun the same command with the same state directory to resume verification from the saved completion state. After a seed check fails with a saved manifest, use `compat-check` to resume verification.
|
||||
|
||||
For mixed-version testing, retain JSON compatibility mode throughout the rolling replacement. Verify each node's immutable image and binary, then perform manifest checks and new writes through both coordinator versions using `--coordinator node1` and `--coordinator node4`. The manifest records the actual writer coordinator revision from the running image. Stop one owned node to check read availability at EC12+4, restore it and check the manifests again. Record the exact oldest tested source revision, toolchain and build profile with the evidence. The version boundary is established by these executed cells.
|
||||
|
||||
An owned lab can replace just its fourth node through a compose image override. Select the frozen old image first and verify its revision label. Keep `compose.json` in JSON compatibility mode for this cell.
|
||||
|
||||
```bash
|
||||
set -euo pipefail
|
||||
OLD_IMAGE_ID="$(docker --context "$LAB_CONTEXT" image inspect "$OLD_IMAGE" --format '{{.Id}}')"
|
||||
test "$(docker --context "$LAB_CONTEXT" image inspect "$OLD_IMAGE_ID" --format '{{index .Config.Labels "org.opencontainers.image.revision"}}')" = "$OLD_REVISION"
|
||||
INLINE_OLD_IMAGE_ID="$OLD_IMAGE_ID" scripts/python_bin.sh - <<'PYTHON' > "$LAB_STATE/mixed-image.json"
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
override = {"services": {"node4": {"image": os.environ["INLINE_OLD_IMAGE_ID"]}}}
|
||||
json.dump(override, sys.stdout)
|
||||
PYTHON
|
||||
docker --context "$LAB_CONTEXT" compose --project-name "$LAB_PROJECT" \
|
||||
--file "$LAB_STATE/compose.json" --file "$LAB_STATE/mixed-image.json" \
|
||||
up --detach --no-deps --no-build --pull never node4
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" compat-check \
|
||||
--bucket inline-compat --coordinator node4
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" compat-seed \
|
||||
--bucket inline-mixed-writer --coordinator node4
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" compat-check \
|
||||
--bucket inline-mixed-writer --coordinator node1
|
||||
```
|
||||
|
||||
Inspect all four running image IDs and binary hashes before recording the mixed-fleet result. Redeploy the configured current image with `--keep-data --rpc-mode json-compat` to restore a homogeneous fleet. The same manifest checks apply after stopping and restarting one owned node through the base compose file.
|
||||
@@ -34,10 +34,6 @@ If you want the Kafka-backed HA Tempo path, use `docker-compose-example-for-rust
|
||||
- **High Performance**: Optimized configurations for batching, compression, and memory management.
|
||||
- **Standardized Protocols**: Built entirely on OpenTelemetry standards.
|
||||
|
||||
## HTTP Request Latency Percentiles
|
||||
|
||||
The main RustFS dashboard shows request latency percentiles in milliseconds. The HTTP server request, response-body chunk, and response-body stream histograms export bucket boundaries in seconds. During a rolling upgrade or rollback that changes the bucket layout, compute percentiles from nodes that share one layout and from a query window entirely on that layout. The bundled dashboard and recording rules use a five-minute window. The mean from `_sum` divided by `_count` remains usable across the change.
|
||||
|
||||
## GET Performance Optimization Dashboards
|
||||
|
||||
Three pre-built Grafana dashboards are included for monitoring RustFS GET performance optimization rollout:
|
||||
|
||||
@@ -34,10 +34,6 @@
|
||||
- **高性能**: 针对批处理、压缩和内存管理进行了优化配置。
|
||||
- **标准化协议**: 完全基于 OpenTelemetry 标准构建。
|
||||
|
||||
## HTTP 请求延迟分位数
|
||||
|
||||
RustFS 主仪表盘以毫秒显示请求延迟分位数。HTTP 服务的请求耗时、响应体分块延迟和响应体流传输耗时直方图以秒为单位导出桶边界。滚动升级或回滚导致桶边界变化时,只对采用同一套桶边界的节点和时间窗口计算分位数。自带仪表盘和记录规则使用五分钟窗口。变更期间仍可用 `_sum` 除以 `_count` 计算平均值。
|
||||
|
||||
## GET 性能优化仪表盘
|
||||
|
||||
包含三个预构建的 Grafana 仪表盘,用于监控 RustFS GET 性能优化发布:
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
# Copyright 2026 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Both-stack leg
|
||||
description: Prove which S3 stack one e2e leg's servers selected and upload the leg for the summary (call with if always)
|
||||
|
||||
inputs:
|
||||
profile:
|
||||
description: The nextest profile the leg ran
|
||||
required: true
|
||||
stack:
|
||||
description: The stack the leg requested, legacy or gateway
|
||||
required: true
|
||||
logs:
|
||||
description: Directory or file holding the servers' output
|
||||
required: true
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Prove which stack the servers selected
|
||||
shell: bash
|
||||
env:
|
||||
PROFILE: ${{ inputs.profile }}
|
||||
STACK: ${{ inputs.stack }}
|
||||
LOGS: ${{ inputs.logs }}
|
||||
run: |
|
||||
# Outside the workspace: e2e_binary.py refuses a run whose source tree changed.
|
||||
mkdir -p "$RUNNER_TEMP/leg"
|
||||
junit="target/nextest/${PROFILE}/junit.xml"
|
||||
if [ -s "$junit" ]; then cp "$junit" "$RUNNER_TEMP/leg/junit.xml"; fi
|
||||
# A gateway leg stopped by its preflight has no output; the preflight's stack.json stays.
|
||||
if [ -e "$LOGS" ]; then python3 scripts/dual_stack.py stack --expect "$STACK" --out "$RUNNER_TEMP/leg/stack.json" "$LOGS"; fi
|
||||
|
||||
- name: Upload the leg
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: dual-stack-leg-${{ inputs.profile }}-${{ inputs.stack }}
|
||||
overwrite: true
|
||||
path: ${{ runner.temp }}/leg
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
@@ -0,0 +1,56 @@
|
||||
# Copyright 2026 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Both-stack summary
|
||||
description: Diff the legacy and gateway legs of this run (scripts/junit_diff.py) and upload one dated summary
|
||||
|
||||
inputs:
|
||||
profiles:
|
||||
description: Comma-separated nextest profiles whose two legs this run uploaded
|
||||
required: true
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Download the legs
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
pattern: dual-stack-leg-*
|
||||
path: legs
|
||||
|
||||
- name: Diff every profile and write the summary
|
||||
id: summary
|
||||
shell: bash
|
||||
env:
|
||||
PROFILES: ${{ inputs.profiles }}
|
||||
run: |
|
||||
# The shell runs with -e; the exit status is reported after the summary is written.
|
||||
set +e -u -o pipefail
|
||||
date="$(date -u +%F)"
|
||||
echo "date=${date}" >> "$GITHUB_OUTPUT"
|
||||
mkdir -p legs
|
||||
python3 scripts/dual_stack.py summary --profiles "$PROFILES" --legs legs --date "$date" --out-dir dual-stack
|
||||
status=$?
|
||||
if [ -f dual-stack/summary.md ]; then cat dual-stack/summary.md >> "$GITHUB_STEP_SUMMARY"; fi
|
||||
exit "$status"
|
||||
|
||||
- name: Upload the dated summary
|
||||
if: always() && steps.summary.outputs.date != ''
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: dual-stack-summary-${{ steps.summary.outputs.date }}
|
||||
overwrite: true
|
||||
path: dual-stack
|
||||
if-no-files-found: error
|
||||
retention-days: 30
|
||||
@@ -814,7 +814,7 @@
|
||||
"monitor": "workflow",
|
||||
"max_age_hours": 36,
|
||||
"result_owner": "dual-stack-nightly.yml",
|
||||
"disposition": "nightly run of the two e2e profiles only ci.yml runs, on the gateway stack, with the JUnit reports retained for 30 days",
|
||||
"disposition": "both-stack JUnit diff of the two profiles only ci.yml runs; the gateway legs are report-only until T2.10",
|
||||
"failure_owner": "watchdog",
|
||||
"origin": "added"
|
||||
},
|
||||
|
||||
+175
-13
@@ -92,6 +92,7 @@ jobs:
|
||||
connect: ${{ steps.scope.outputs.connect || 'true' }}
|
||||
packages: ${{ steps.scope.outputs.packages }}
|
||||
full_validation: ${{ steps.scope.outputs.full_validation || 'true' }}
|
||||
gateway_smoke: ${{ steps.scope.outputs.gateway_smoke || 'true' }}
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
@@ -118,7 +119,7 @@ jobs:
|
||||
echo "Base CI policy unavailable; running the full matrix."
|
||||
fi
|
||||
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=8$' "$GITHUB_OUTPUT"; then
|
||||
printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
|
||||
printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'gateway_smoke=true' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
|
||||
echo "Base policy does not cover this workflow contract; retaining the full matrix."
|
||||
fi
|
||||
- name: Retain verification plan
|
||||
@@ -675,6 +676,54 @@ jobs:
|
||||
retention-days: 3
|
||||
if-no-files-found: warn
|
||||
|
||||
test-and-lint-gateway-stack:
|
||||
name: Test and Lint (gateway-stack)
|
||||
# The gateway S3 stack (rustfs/backlog#2734) sits behind the rustfs
|
||||
# `gateway-stack` feature, which no other pull-request lane lints or tests.
|
||||
# Unlike the dormant feature lanes above, this one runs for every code
|
||||
# change: each operation slice puts its handler tests and its refusal-matrix
|
||||
# row behind the feature, and a lane that ran only on demand would let them
|
||||
# rot behind a green pull request.
|
||||
if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
# Self-hosted for build scale and cache ownership: a workspace-scale Cargo
|
||||
# build that reads the ci-dev cache written on this image.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: ci-dev-ea7eec74fe14
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
# A feature that does not compile fails here; nothing below is skipped
|
||||
# into a pass.
|
||||
- name: Run clippy with gateway-stack
|
||||
run: cargo clippy --locked -p rustfs --all-targets --features gateway-stack -- -D warnings
|
||||
|
||||
# The gateway stack's own tests: the assembly, the operation table and
|
||||
# its completeness, the shared handler sequence, the stack selection, and
|
||||
# the embedded server driven through the whole middleware stack. The
|
||||
# filter names modules, never operations.
|
||||
- name: Run tests with gateway-stack
|
||||
env:
|
||||
# The same bounded link concurrency as the other feature lanes.
|
||||
CARGO_BUILD_JOBS: "2"
|
||||
run: |
|
||||
cargo nextest run --profile ci --locked -p rustfs --features gateway-stack --no-tests fail \
|
||||
--status-level all --final-status-level all \
|
||||
-E 'test(/gateway_stack|gateway_handlers|server::stack::/) | binary(embedded_gateway_stack_test)'
|
||||
|
||||
build-rustfs-debug-binary:
|
||||
name: Build RustFS Debug Binary
|
||||
if: contains(fromJSON('["full", "e2e"]'), needs.classify-changes.outputs.mode) && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
@@ -937,6 +986,8 @@ jobs:
|
||||
# Retain the producer image and fixture paths until the runtime pilot passes.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
RUSTFS_S3_STACK: legacy
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -1057,6 +1108,125 @@ jobs:
|
||||
path: ${{ runner.temp }}/rustfs-e2e-*/rustfs.log
|
||||
retention-days: 3
|
||||
|
||||
# The same e2e-smoke suite against the same debug binary with the gateway
|
||||
# S3 stack selected (rustfs/backlog#2734). Report-only: it is outside the
|
||||
# Test and Lint aggregate and becomes required at T2.10. The test-binary
|
||||
# download, identity, membership, run and completeness steps must stay
|
||||
# identical to e2e-tests; scripts/ci_gate.py enforces that.
|
||||
#
|
||||
# When it runs: every push to main or gateway/integration, merge group,
|
||||
# nightly schedule and manual dispatch, and every pull request that changes a
|
||||
# gateway-stack input (GATEWAY_SMOKE_PATHS and the rustfs-gateway pins in
|
||||
# scripts/ci_gate.py, published by Select CI scope as gateway_smoke).
|
||||
# Why it is skipped on every other pull request: until T2.10 the gateway
|
||||
# stack does not serve the whole smoke suite, so the lane fails on any
|
||||
# revision. A pull request that touches none of its inputs can neither cause
|
||||
# nor fix that failure, and a red check there says nothing about the change.
|
||||
# Skipped is the honest state; a failure must never be rendered green, so no
|
||||
# step here may carry continue-on-error. A selection that is missing or
|
||||
# unreadable runs the lane. At T2.10, when the lane becomes required, it
|
||||
# must run on every pull request again: delete the selection, do not extend
|
||||
# it.
|
||||
e2e-tests-gateway:
|
||||
name: e2e-smoke (gateway, report-only until T2.10)
|
||||
needs: [ classify-changes, build-rustfs-debug-binary ]
|
||||
if: github.event_name != 'pull_request' || needs.classify-changes.outputs.gateway_smoke != 'false'
|
||||
runs-on: sm-standard-2
|
||||
timeout-minutes: 30
|
||||
continue-on-error: true
|
||||
env:
|
||||
RUSTFS_S3_STACK: gateway
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install verification runtime
|
||||
uses: ./.github/actions/test-runtime
|
||||
|
||||
- name: Download E2E test binaries
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: e2e-test-binaries
|
||||
path: ${{ runner.temp }}
|
||||
|
||||
- name: Verify E2E archive identity
|
||||
run: python3 scripts/ci_test_artifact.py verify --configuration e2e --archive "$RUNNER_TEMP/e2e-tests.tar.zst"
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install awscurl
|
||||
run: |
|
||||
python3 -m pip install --user --upgrade pip "awscurl==0.44"
|
||||
echo "AWSCURL_PATH=$HOME/.local/bin/awscurl" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Verify awscurl
|
||||
run: test -x "$AWSCURL_PATH"
|
||||
|
||||
# Download after the cache restore so the freshly built binary from the
|
||||
# build job always wins over anything restored into target/debug.
|
||||
- name: Download debug binary
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: rustfs-debug-binary
|
||||
path: target/debug
|
||||
|
||||
- name: Make binary executable
|
||||
run: chmod +x ./target/debug/rustfs
|
||||
|
||||
# Build the e2e test graph once. The archive is reused by the smoke
|
||||
# selection guard, security exact-count check, and run below, avoiding a
|
||||
# second compile of the same e2e_test target on cold runners (backlog#1645).
|
||||
- name: Verify e2e smoke test membership
|
||||
env:
|
||||
NEXTEST_ARCHIVE: ${{ runner.temp }}/e2e-tests.tar.zst
|
||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-smoke-list.json
|
||||
run: |
|
||||
cargo nextest list --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" --workspace-remap "$GITHUB_WORKSPACE" --extract-to "$GITHUB_WORKSPACE" --extract-overwrite --message-format json > "${NEXTEST_LISTING}"
|
||||
python3 ./scripts/check_test_wiring.py --check-profile e2e-smoke "${NEXTEST_LISTING}"
|
||||
./scripts/check_security_smoke_count.sh check "${NEXTEST_LISTING}"
|
||||
|
||||
# PR smoke subset of the in-repo e2e suite (backlog#1149 ci-4). The
|
||||
# profile.e2e-smoke default-filter in .config/nextest.toml is the single
|
||||
# wiring mechanism for e2e tests in CI — extend that filter instead of
|
||||
# adding new e2e jobs here. Each test spawns its own rustfs server on a
|
||||
# random port and reuses the downloaded debug binary above.
|
||||
- name: Run e2e smoke suite
|
||||
env:
|
||||
NEXTEST_ARCHIVE: ${{ runner.temp }}/e2e-tests.tar.zst
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-smoke-logs
|
||||
SAMPLER_INTERVAL_SECS: "15"
|
||||
run: |
|
||||
./scripts/ci/resource_sampler.sh start e2e-smoke
|
||||
trap './scripts/ci/resource_sampler.sh stop' EXIT
|
||||
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-smoke --cargo-metadata target/nextest/cargo-metadata.json --binaries-metadata target/nextest/binaries-metadata.json --workspace-remap "$GITHUB_WORKSPACE" --target-dir-remap "$GITHUB_WORKSPACE/target" \
|
||||
--status-level all --final-status-level all --failure-output final
|
||||
|
||||
- name: Verify e2e smoke execution completeness
|
||||
run: python3 scripts/ci_test_artifact.py evidence "$RUNNER_TEMP/rustfs-e2e-smoke-list.json" target/nextest/e2e-smoke/junit.xml
|
||||
|
||||
- name: Upload e2e smoke diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-smoke-gateway-diagnostics-${{ github.run_number }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
${{ runner.temp }}/rustfs-e2e-smoke-logs/
|
||||
${{ runner.temp }}/rustfs-e2e-smoke-list.json
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Upload e2e smoke JUnit report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-smoke-gateway-junit-${{ github.run_number }}-${{ github.run_attempt }}
|
||||
path: target/nextest/e2e-smoke/junit.xml
|
||||
if-no-files-found: warn
|
||||
|
||||
e2e-full:
|
||||
name: End-to-End Tests (full merge gate)
|
||||
# Merge gate only (backlog#1149 ci-5): the never-automated user-visible
|
||||
@@ -1091,12 +1261,9 @@ jobs:
|
||||
- name: Verify E2E archive identity
|
||||
run: python3 scripts/ci_test_artifact.py verify --configuration e2e --archive "$RUNNER_TEMP/e2e-tests.tar.zst"
|
||||
|
||||
- name: Install network fault-injection tools and the persistence test clock
|
||||
- name: Install network fault-injection tools
|
||||
run: |
|
||||
sudo apt-get install -y iptables libfaketime
|
||||
clock_library="$(dpkg-query -L libfaketime | grep -E '/libfaketime[.]so[.]1$')"
|
||||
test -f "$clock_library"
|
||||
echo "RUSTFS_PERSISTENCE_CLOCK_LIBRARY=$clock_library" >> "$GITHUB_ENV"
|
||||
sudo apt-get install -y iptables
|
||||
sudo -n iptables --version
|
||||
# The endpoint-blackhole heal scenario needs CAP_NET_ADMIN. Containerised
|
||||
# runners can run iptables but not touch the rule set; the test then logs
|
||||
@@ -1203,14 +1370,8 @@ jobs:
|
||||
# default-filter in .config/nextest.toml is the single wiring mechanism —
|
||||
# extend that filter, never add ad-hoc e2e jobs here. Reuses the downloaded
|
||||
# debug binary; each test spawns its own rustfs server on a random port.
|
||||
- name: Download pinned release for persistence byte parity
|
||||
id: persistence-reference
|
||||
uses: ./.github/actions/reference-binary
|
||||
|
||||
- name: Run e2e full suite
|
||||
env:
|
||||
RUSTFS_PERSISTENCE_REFERENCE_BINARY: ${{ steps.persistence-reference.outputs.path }}
|
||||
RUSTFS_PERSISTENCE_XML_DIR: ${{ runner.temp }}/rustfs-persistence-xml
|
||||
CARGO_BIN_EXE_rustfs: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs
|
||||
RUSTFS_E2E_STARTUP_CAS_BINARY: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs
|
||||
RUSTFS_E2E_STARTUP_CAS_BUILD_MANIFEST: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs.e2e-startup-cas-build.json
|
||||
@@ -1229,7 +1390,6 @@ jobs:
|
||||
path: |
|
||||
target/nextest/e2e-full/junit.xml
|
||||
${{ runner.temp }}/rustfs-e2e-full-list.json
|
||||
${{ runner.temp }}/rustfs-persistence-xml
|
||||
retention-days: 7
|
||||
|
||||
- name: Upload startup CAS evidence
|
||||
@@ -1452,6 +1612,7 @@ jobs:
|
||||
- fips-provider
|
||||
- test-and-lint-rio-v2
|
||||
- test-and-lint-protocols
|
||||
- test-and-lint-gateway-stack
|
||||
- build-rustfs-debug-binary
|
||||
- uring-integration
|
||||
- e2e-tests
|
||||
@@ -1487,6 +1648,7 @@ jobs:
|
||||
- core-contract
|
||||
- test-and-lint-rio-v2
|
||||
- test-and-lint-protocols
|
||||
- test-and-lint-gateway-stack
|
||||
- build-rustfs-debug-binary
|
||||
- build-rustfs-debug-binary-rio-v2
|
||||
- uring-integration
|
||||
|
||||
@@ -12,18 +12,21 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Nightly run of the two e2e profiles only ci.yml runs, e2e-smoke and e2e-full
|
||||
# (rustfs/backlog#2772, T3.1), on the gateway stack, which is the only S3 stack.
|
||||
# The workflow keeps its file and registry name from the period when it diffed two
|
||||
# stacks. Each leg uploads its JUnit report and server logs for 30 days; the one
|
||||
# comparison against the last legacy record (the `dual-stack-leg-<profile>-legacy`
|
||||
# artifacts of the final both-stack run) is made by hand with scripts/junit_diff.py.
|
||||
# Nightly both-stack diff of the two e2e profiles only ci.yml runs, e2e-smoke
|
||||
# and e2e-full (rustfs/backlog#2772, T3.1). The other profiles carry a `stack`
|
||||
# dimension in their own workflows, where the existing lane is the legacy leg.
|
||||
#
|
||||
# Left out: `dual_stack_differential::` (servers with an explicit
|
||||
# One build serves every leg. One preflight starts that binary with
|
||||
# RUSTFS_S3_STACK=gateway (scripts/dual_stack.py preflight); without a proven
|
||||
# gateway start the gateway legs do not run and the summary says `skipped:
|
||||
# gateway stack refuses startup: <why>`. Which stack a leg ran is read from the
|
||||
# servers' own start-up line. The gateway legs are report-only until T2.10.
|
||||
#
|
||||
# Left out of both stacks: `dual_stack_differential::` (servers with an explicit
|
||||
# stack), `fake_s3_target::`, the upgrade write-readiness cases and one deadline
|
||||
# case (no server), and `distributed_startup_regression_test::` (needs ci.yml's
|
||||
# hooks-binary manifest). e2e-full uses ci.yml's pinned mc and Vault binaries
|
||||
# and network fault-injection tools.
|
||||
# hooks-binary manifest). Both e2e-full stacks use ci.yml's pinned mc and
|
||||
# Vault binaries and network fault-injection tools.
|
||||
|
||||
name: dual-stack-nightly
|
||||
|
||||
@@ -47,11 +50,13 @@ defaults:
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Build once
|
||||
name: Build once and probe the gateway stack
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
# The runner and cache of ci.yml's build-rustfs-debug-binary.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 60
|
||||
outputs:
|
||||
stacks: ${{ steps.preflight.outputs.stacks }}
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
@@ -81,15 +86,47 @@ jobs:
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
# Without a proven gateway start there is no gateway leg to run.
|
||||
- name: Gateway preflight
|
||||
id: preflight
|
||||
run: |
|
||||
if python3 scripts/dual_stack.py preflight --binary target/debug/rustfs --out "$RUNNER_TEMP/preflight/stack.json"; then
|
||||
echo 'stacks=["legacy","gateway"]' >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo 'stacks=["legacy"]' >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# The evidence stands in for a gateway leg that is skipped; a leg that runs overwrites it.
|
||||
- name: Upload the preflight evidence for e2e-smoke
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: dual-stack-leg-e2e-smoke-gateway
|
||||
path: ${{ runner.temp }}/preflight
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
|
||||
- name: Upload the preflight evidence for e2e-full
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: dual-stack-leg-e2e-full-gateway
|
||||
path: ${{ runner.temp }}/preflight
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
|
||||
leg:
|
||||
name: ${{ matrix.profile }}
|
||||
name: ${{ matrix.profile }} (${{ matrix.stack }}${{ matrix.stack == 'gateway' && ', report-only until T2.10' || '' }})
|
||||
needs: [ build ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 120
|
||||
# Report-only until T2.10: the gateway stack cannot serve the suites yet.
|
||||
continue-on-error: ${{ matrix.stack == 'gateway' }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
profile: [ e2e-smoke, e2e-full ]
|
||||
stack: ${{ fromJSON(needs.build.outputs.stacks) }}
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
@@ -171,9 +208,10 @@ jobs:
|
||||
run: |
|
||||
"$RUSTFS_TEST_VAULT_BIN" version
|
||||
|
||||
- name: Run the profile
|
||||
- name: Run the profile on the ${{ matrix.stack }} stack
|
||||
env:
|
||||
PROFILE: ${{ matrix.profile }}
|
||||
RUSTFS_S3_STACK: ${{ matrix.stack }}
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/server-logs
|
||||
run: |
|
||||
chmod +x target/debug/rustfs
|
||||
@@ -182,14 +220,23 @@ jobs:
|
||||
filter="$filter & not test(=list_objects_v2_pagination_test::tests::test_list_objects_v2_deadline_rejects_unresponsive_peer)"
|
||||
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile "$PROFILE" -p e2e_test -E "$filter"
|
||||
|
||||
- name: Upload the leg
|
||||
- name: Prove the stack and upload the leg
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
uses: ./.github/actions/dual-stack-leg
|
||||
with: { profile: "${{ matrix.profile }}", stack: "${{ matrix.stack }}", logs: "${{ runner.temp }}/server-logs" }
|
||||
|
||||
summary:
|
||||
name: Both-stack diff summary
|
||||
needs: [ build, leg ]
|
||||
if: github.repository == 'rustfs/rustfs' && always()
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
name: e2e-nightly-${{ matrix.profile }}
|
||||
overwrite: true
|
||||
path: |
|
||||
target/nextest/${{ matrix.profile }}/junit.xml
|
||||
${{ runner.temp }}/server-logs
|
||||
if-no-files-found: warn
|
||||
retention-days: 30
|
||||
persist-credentials: false
|
||||
|
||||
- uses: ./.github/actions/dual-stack-summary
|
||||
with:
|
||||
profiles: e2e-smoke,e2e-full
|
||||
|
||||
@@ -53,11 +53,14 @@ concurrency:
|
||||
jobs:
|
||||
distributed:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Distributed 4-node 4-disk e2e
|
||||
name: Distributed 4-node 4-disk e2e (${{ matrix.stack }}${{ matrix.stack == 'gateway' && ', report-only until T2.10' || '' }})
|
||||
# GitHub-hosted VM: loop and tmpfs mounts work here. sm-standard-4 is an
|
||||
# ARC pod and rejects both (`mount -o loop` ENOENT, tmpfs "read-only").
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 180
|
||||
# The gateway leg is report-only until T2.10.
|
||||
continue-on-error: ${{ matrix.stack == 'gateway' }}
|
||||
strategy: { fail-fast: false, matrix: { stack: [legacy, gateway] } }
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
NO_PROXY: 127.0.0.1,localhost
|
||||
@@ -129,6 +132,11 @@ jobs:
|
||||
run: |
|
||||
python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks
|
||||
|
||||
# A gateway binary that refuses to start skips the suite; the summary says why.
|
||||
- name: Gateway preflight
|
||||
if: matrix.stack == 'gateway'
|
||||
run: python3 scripts/dual_stack.py preflight --binary target/debug/rustfs --out "$RUNNER_TEMP/leg/stack.json"
|
||||
|
||||
- name: Verify distributed e2e membership
|
||||
env:
|
||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-distributed-list.json
|
||||
@@ -140,6 +148,7 @@ jobs:
|
||||
env:
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-distributed-logs
|
||||
FILTER: ${{ inputs.filter }}
|
||||
RUSTFS_S3_STACK: ${{ matrix.stack }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
rm -f target/nextest/e2e-distributed/junit.xml
|
||||
@@ -150,11 +159,19 @@ jobs:
|
||||
fi
|
||||
test -s target/nextest/e2e-distributed/junit.xml
|
||||
|
||||
- name: Prove the stack and upload the leg
|
||||
if: always()
|
||||
uses: ./.github/actions/dual-stack-leg
|
||||
with:
|
||||
profile: e2e-distributed
|
||||
stack: ${{ matrix.stack }}
|
||||
logs: ${{ runner.temp }}/rustfs-e2e-distributed-logs
|
||||
|
||||
- name: Upload distributed e2e diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-distributed-${{ github.run_number }}
|
||||
name: e2e-distributed-${{ github.run_number }}-${{ matrix.stack }}
|
||||
path: |
|
||||
target/nextest/e2e-distributed/junit.xml
|
||||
${{ runner.temp }}/rustfs-e2e-distributed-list.json
|
||||
@@ -174,6 +191,20 @@ jobs:
|
||||
fi
|
||||
done
|
||||
|
||||
dual-stack-summary:
|
||||
name: Both-stack diff summary
|
||||
needs: [distributed]
|
||||
if: github.repository == 'rustfs/rustfs' && always()
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: ./.github/actions/dual-stack-summary
|
||||
with:
|
||||
profiles: e2e-distributed
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [distributed]
|
||||
|
||||
@@ -54,15 +54,18 @@ concurrency:
|
||||
jobs:
|
||||
repl-nightly:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Replication e2e (nightly)
|
||||
name: Replication e2e (nightly, ${{ matrix.stack }}${{ matrix.stack == 'gateway' && ', report-only until T2.10' || '' }})
|
||||
# dual-node tests spawn two full rustfs servers each; the extra headroom
|
||||
# over the per-PR sm-standard-2 e2e job keeps the parallel process fan-out
|
||||
# from starving the runner. Mirrors the ILM serial lane's runner class.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 45
|
||||
# The gateway leg is report-only until T2.10.
|
||||
continue-on-error: ${{ matrix.stack == 'gateway' }}
|
||||
strategy: { fail-fast: false, matrix: { stack: [legacy, gateway] } }
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
RUSTFS_BUILD_FEATURES: ""
|
||||
RUSTFS_BUILD_FEATURES: ${{ matrix.stack == 'gateway' && 'gateway-stack' || '' }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -96,6 +99,11 @@ jobs:
|
||||
run: |
|
||||
python3 scripts/e2e_binary.py build --bins --features "$RUSTFS_BUILD_FEATURES"
|
||||
|
||||
# A gateway binary that refuses to start skips the suite; the summary says why.
|
||||
- name: Gateway preflight
|
||||
if: matrix.stack == 'gateway'
|
||||
run: python3 scripts/dual_stack.py preflight --binary target/debug/rustfs --out "$RUNNER_TEMP/leg/stack.json"
|
||||
|
||||
- name: Verify replication e2e membership
|
||||
env:
|
||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-repl-nightly-list.json
|
||||
@@ -106,13 +114,22 @@ jobs:
|
||||
- name: Run replication e2e nightly suite
|
||||
env:
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-repl-nightly-logs
|
||||
RUSTFS_S3_STACK: ${{ matrix.stack }}
|
||||
run: python3 scripts/e2e_binary.py run --features "$RUSTFS_BUILD_FEATURES" -- cargo nextest run --profile e2e-repl-nightly -p e2e_test
|
||||
|
||||
- name: Prove the stack and upload the leg
|
||||
if: always()
|
||||
uses: ./.github/actions/dual-stack-leg
|
||||
with:
|
||||
profile: e2e-repl-nightly
|
||||
stack: ${{ matrix.stack }}
|
||||
logs: ${{ runner.temp }}/rustfs-e2e-repl-nightly-logs
|
||||
|
||||
- name: Upload nextest junit report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-replication-nightly-junit-${{ github.run_number }}
|
||||
name: e2e-replication-nightly-junit-${{ github.run_number }}-${{ matrix.stack }}
|
||||
path: |
|
||||
target/nextest/e2e-repl-nightly/junit.xml
|
||||
${{ runner.temp }}/rustfs-e2e-repl-nightly-list.json
|
||||
@@ -122,9 +139,12 @@ jobs:
|
||||
|
||||
cluster-nightly:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Cluster fault e2e (nightly)
|
||||
name: Cluster fault e2e (nightly, ${{ matrix.stack }}${{ matrix.stack == 'gateway' && ', report-only until T2.10' || '' }})
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
# The gateway leg is report-only until T2.10.
|
||||
continue-on-error: ${{ matrix.stack == 'gateway' }}
|
||||
strategy: { fail-fast: false, matrix: { stack: [legacy, gateway] } }
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
@@ -145,6 +165,11 @@ jobs:
|
||||
run: |
|
||||
python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks
|
||||
|
||||
# A gateway binary that refuses to start skips the suite; the summary says why.
|
||||
- name: Gateway preflight
|
||||
if: matrix.stack == 'gateway'
|
||||
run: python3 scripts/dual_stack.py preflight --binary target/debug/rustfs --out "$RUNNER_TEMP/leg/stack.json"
|
||||
|
||||
- name: Verify cluster fault e2e membership
|
||||
env:
|
||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-nightly-list.json
|
||||
@@ -156,13 +181,22 @@ jobs:
|
||||
env:
|
||||
CARGO_BIN_EXE_rustfs: ${{ github.workspace }}/target/debug/rustfs
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-nightly-logs
|
||||
RUSTFS_S3_STACK: ${{ matrix.stack }}
|
||||
run: python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-nightly -p e2e_test
|
||||
|
||||
- name: Prove the stack and upload the leg
|
||||
if: always()
|
||||
uses: ./.github/actions/dual-stack-leg
|
||||
with:
|
||||
profile: e2e-nightly
|
||||
stack: ${{ matrix.stack }}
|
||||
logs: ${{ runner.temp }}/rustfs-e2e-nightly-logs
|
||||
|
||||
- name: Upload cluster fault diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-cluster-nightly-${{ github.run_number }}
|
||||
name: e2e-cluster-nightly-${{ github.run_number }}-${{ matrix.stack }}
|
||||
path: |
|
||||
target/nextest/e2e-nightly/junit.xml
|
||||
${{ runner.temp }}/rustfs-e2e-nightly-list.json
|
||||
@@ -172,12 +206,15 @@ jobs:
|
||||
|
||||
protocols-nightly:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Protocol e2e (nightly)
|
||||
name: Protocol e2e (nightly, ${{ matrix.stack }}${{ matrix.stack == 'gateway' && ', report-only until T2.10' || '' }})
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
# The gateway leg is report-only until T2.10.
|
||||
continue-on-error: ${{ matrix.stack == 'gateway' }}
|
||||
strategy: { fail-fast: false, matrix: { stack: [legacy, gateway] } }
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
RUSTFS_BUILD_FEATURES: ftps,webdav,sftp
|
||||
RUSTFS_BUILD_FEATURES: ${{ matrix.stack == 'gateway' && 'ftps,webdav,sftp,gateway-stack' || 'ftps,webdav,sftp' }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -201,6 +238,11 @@ jobs:
|
||||
- name: Build protocol server
|
||||
run: python3 scripts/e2e_binary.py build --features "$RUSTFS_BUILD_FEATURES"
|
||||
|
||||
# A gateway binary that refuses to start skips the suite; the summary says why.
|
||||
- name: Gateway preflight
|
||||
if: matrix.stack == 'gateway'
|
||||
run: python3 scripts/dual_stack.py preflight --binary target/debug/rustfs --out "$RUNNER_TEMP/leg/stack.json"
|
||||
|
||||
# The suite owns fixed protocol ports and serializes its internal cases.
|
||||
- name: Verify protocol e2e membership
|
||||
env:
|
||||
@@ -212,16 +254,28 @@ jobs:
|
||||
- name: Run protocol e2e nightly suite
|
||||
env:
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-protocol-e2e-logs
|
||||
RUSTFS_S3_STACK: ${{ matrix.stack }}
|
||||
# These servers set no log level of their own; the stack proof reads their start-up line.
|
||||
RUST_LOG: rustfs=info
|
||||
# The servers inherit nextest's stdout (--no-capture): keep it where the stack proof reads it.
|
||||
run: |
|
||||
set -o pipefail
|
||||
mkdir -p "$RUSTFS_E2E_LOG_DIR"
|
||||
python3 scripts/e2e_binary.py run --features "$RUSTFS_BUILD_FEATURES" -- cargo nextest run -j 1 --profile e2e-protocols -p e2e_test --no-capture 2>&1 | tee "$RUSTFS_E2E_LOG_DIR/nextest.log"
|
||||
|
||||
- name: Prove the stack and upload the leg
|
||||
if: always()
|
||||
uses: ./.github/actions/dual-stack-leg
|
||||
with:
|
||||
profile: e2e-protocols
|
||||
stack: ${{ matrix.stack }}
|
||||
logs: ${{ runner.temp }}/rustfs-protocol-e2e-logs
|
||||
|
||||
- name: Upload protocol diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-protocol-nightly-${{ github.run_number }}
|
||||
name: e2e-protocol-nightly-${{ github.run_number }}-${{ matrix.stack }}
|
||||
path: |
|
||||
target/nextest/e2e-protocols/junit.xml
|
||||
${{ runner.temp }}/rustfs-e2e-protocols-list.json
|
||||
@@ -229,6 +283,20 @@ jobs:
|
||||
retention-days: 7
|
||||
if-no-files-found: warn
|
||||
|
||||
dual-stack-summary:
|
||||
name: Both-stack diff summary
|
||||
needs: [repl-nightly, cluster-nightly, protocols-nightly]
|
||||
if: github.repository == 'rustfs/rustfs' && always()
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: ./.github/actions/dual-stack-summary
|
||||
with:
|
||||
profiles: e2e-repl-nightly,e2e-nightly,e2e-protocols
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [repl-nightly, cluster-nightly, protocols-nightly]
|
||||
|
||||
@@ -90,16 +90,11 @@ jobs:
|
||||
- version: 1.0.1
|
||||
asset: rustfs-linux-x86_64-gnu-v1.0.1.zip
|
||||
sha256: 9742e183cb0dfb0d48fa909797af572fe47b7f827569036baa4156e0d3958ee5
|
||||
- version: 1.0.5
|
||||
asset: rustfs-linux-x86_64-gnu-v1.0.5.zip
|
||||
sha256: 5addae2f74756bf80104d0975e6347cda843bc2daffdfcd1f0cb80118b95cbd0
|
||||
revision: 34e6449ed90e3d3a106f6655342e0b3127f33082
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
UPGRADE_SOURCE_VERSION: ${{ matrix.source.version }}
|
||||
UPGRADE_SOURCE_ASSET: ${{ matrix.source.asset }}
|
||||
UPGRADE_SOURCE_SHA256: ${{ matrix.source.sha256 }}
|
||||
UPGRADE_SOURCE_REVISION: ${{ matrix.source.revision }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -137,13 +132,6 @@ jobs:
|
||||
unzip -q "$archive" -d "$SOURCE_DIR"
|
||||
chmod +x "$SOURCE_DIR/rustfs"
|
||||
test -x "$SOURCE_DIR/rustfs"
|
||||
if [[ -n "$UPGRADE_SOURCE_REVISION" ]]; then
|
||||
mkdir -p "$RUNNER_TEMP/rustfs-upgrade-logs"
|
||||
version_log="$RUNNER_TEMP/rustfs-upgrade-logs/source-version.txt"
|
||||
"$SOURCE_DIR/rustfs" --version | tee "$version_log"
|
||||
grep -Fx "rustfs $UPGRADE_SOURCE_VERSION" "$version_log"
|
||||
grep -Eq "^git commit[[:space:]]*:[[:space:]]*$UPGRADE_SOURCE_REVISION$" "$version_log"
|
||||
fi
|
||||
echo "RUSTFS_UPGRADE_SOURCE_BINARY=$SOURCE_DIR/rustfs" >> "$GITHUB_ENV"
|
||||
echo "RUSTFS_E2E_LOG_DIR=$RUNNER_TEMP/rustfs-upgrade-logs" >> "$GITHUB_ENV"
|
||||
|
||||
@@ -163,14 +151,12 @@ jobs:
|
||||
multipart-layout-upgrade:direct_upgrade_from_previous_release_preserves_multipart_layouts
|
||||
multipart-layout-baseline:previous_release_baseline_replicates_multipart_layouts
|
||||
)
|
||||
case "$UPGRADE_SOURCE_VERSION" in
|
||||
1.0.0-rc.5)
|
||||
cases+=(odm-config-rollback:rc5_rollback_requires_restoring_odm_configuration)
|
||||
cases+=(checkpoint-upgrade:rc5_checkpoint_upgrade_rewinds_signed_progress_without_losing_objects) ;;
|
||||
1.0.1) cases+=(legacy-delete-reservation:stable_upgrade_deletes_foreign_reservations_without_losing_live_objects) ;;
|
||||
1.0.5) ;; # The bucket rollback case also completes an in-flight multipart upload.
|
||||
*) echo "::error::Unknown upgrade source $UPGRADE_SOURCE_VERSION"; exit 1 ;;
|
||||
esac
|
||||
if [[ "$UPGRADE_SOURCE_VERSION" == "1.0.0-rc.5" ]]; then
|
||||
cases+=(odm-config-rollback:rc5_rollback_requires_restoring_odm_configuration)
|
||||
cases+=(checkpoint-upgrade:rc5_checkpoint_upgrade_rewinds_signed_progress_without_losing_objects)
|
||||
else
|
||||
cases+=(legacy-delete-reservation:stable_upgrade_deletes_foreign_reservations_without_losing_live_objects)
|
||||
fi
|
||||
status=0
|
||||
for entry in "${cases[@]}"; do
|
||||
artifact="${entry%%:*}"
|
||||
@@ -192,7 +178,7 @@ jobs:
|
||||
exit "$status"
|
||||
|
||||
- name: Run mixed-version MPU recovery
|
||||
if: ${{ !cancelled() && steps.verify_source.outcome == 'success' && matrix.source.version != '1.0.5' }}
|
||||
if: ${{ !cancelled() && steps.verify_source.outcome == 'success' }}
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
NO_PROXY: 127.0.0.1,localhost,::1
|
||||
@@ -213,24 +199,6 @@ jobs:
|
||||
"$mpu_test" -- --exact --nocapture --test-threads=1 \
|
||||
2>&1 | tee "$RUSTFS_E2E_LOG_DIR/test.log"
|
||||
|
||||
- name: Verify gateway baseline G09 evidence
|
||||
if: ${{ success() && matrix.source.version == '1.0.5' }}
|
||||
shell: bash
|
||||
run: |
|
||||
for field in mixed_version_reader_evidence mixed_version_writer_evidence rollback_payload_evidence; do
|
||||
directory=mixed-version-upgrade
|
||||
if [[ "$field" == rollback_payload_evidence ]]; then directory=bucket-config-rollback; fi
|
||||
jq -e --arg field "$field" --arg current "$GITHUB_SHA" --arg previous "$UPGRADE_SOURCE_REVISION" \
|
||||
'.gate == "G09" and .field == $field and .current_revision == $current and .previous_revision == $previous
|
||||
and ($field != "rollback_payload_evidence" or .rollback_payload_replayed == true)' \
|
||||
"$RUNNER_TEMP/rustfs-upgrade-g09-evidence/$directory/G09-$field.json"
|
||||
done
|
||||
jq -e --arg current "$GITHUB_SHA" --arg previous "$UPGRADE_SOURCE_REVISION" \
|
||||
'.current_revision == $current and .previous_revision == $previous
|
||||
and .part_sizes == [5242880, 5242880, 4096] and .bytes == (.part_sizes | add)
|
||||
and (.rollback_read_sha256 | test("^[a-f0-9]{64}$")) and .rollback_read_sha256 == .roll_forward_read_sha256' \
|
||||
"$RUNNER_TEMP/rustfs-upgrade-g09-evidence/bucket-config-rollback/inflight-multipart-rollback.json"
|
||||
|
||||
- name: Upload scanner/heal G09 evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
|
||||
@@ -15,20 +15,31 @@
|
||||
# Gateway black-box acceptance lanes (rustfs/backlog#2775 T3.4, #2776 T3.5).
|
||||
#
|
||||
# Runs two suites of the rustfs/gateway repository against the candidate binary,
|
||||
# one server:
|
||||
# once per S3 stack (RUSTFS_S3_STACK=legacy and =gateway), one server per stack:
|
||||
#
|
||||
# - the conformance corpus, `--profile rustfs`, judged by
|
||||
# scripts/check_conformance_rulings.py against .config/conformance-rulings.toml
|
||||
# (red: an unruled failure, a ruled case failing differently, a stale ruling,
|
||||
# a skip on the server's own answer);
|
||||
# - the client matrix, ci/compat/run_matrix.sh --external, judged by
|
||||
# - the conformance corpus, `--profile rustfs`. That profile describes the
|
||||
# gateway edge, so scripts/check_conformance_rulings.py judges the gateway
|
||||
# stack against .config/conformance-rulings.toml (red: an unruled failure, a
|
||||
# ruled case failing differently, a stale ruling, a skip on the server's own
|
||||
# answer) and publishes the legacy run as a reference that is never judged:
|
||||
# it only has to serve, stay up and complete;
|
||||
# - the client matrix, ci/compat/run_matrix.sh --external, judged per stack by
|
||||
# scripts/compat_matrix_report.py (no `fail`, no `sut-unregistered`, every
|
||||
# client ran; a client that could not be installed is reported as skipped).
|
||||
# The legacy matrix is the baseline: every cell that passes on legacy must
|
||||
# pass on gateway (`compare`).
|
||||
#
|
||||
# The suites are checked out at the rustfs/gateway rev Cargo.toml pins, so the
|
||||
# suite and the framework under test never drift apart. The verdict is computed
|
||||
# from what the run left on disk (startup record, runner exit code, liveness
|
||||
# after the corpus, report), never from the exit status of an earlier step.
|
||||
# suite and the framework under test never drift apart. The verdict of each
|
||||
# stack is computed from what the run left on disk (startup record, the stack
|
||||
# the server's own log names, runner exit code, liveness after the corpus,
|
||||
# report), never from the exit status of an earlier step. Needs
|
||||
# scripts/dual_stack.py and the `s3_stack` startup field of rustfs/rustfs#9367.
|
||||
#
|
||||
# The gateway stack is report-only until T2.10: its job may fail without failing
|
||||
# the workflow, and its summary says what it observed (did not start, or how many
|
||||
# cases ran, passed, failed and were answered 501). The stack comparison is
|
||||
# report-only for the same reason; its conformance half is information only.
|
||||
#
|
||||
# Not a pull-request gate. Scheduled failures are owned by the scheduled-validation
|
||||
# watchdog (.github/scheduled-validations.json).
|
||||
@@ -155,15 +166,21 @@ jobs:
|
||||
retention-days: 1
|
||||
|
||||
acceptance:
|
||||
name: Acceptance
|
||||
name: ${{ matrix.stack == 'gateway' && 'Acceptance (gateway, report-only until T2.10)' || 'Acceptance (legacy)' }}
|
||||
needs: [ plan, build-candidate, build-gateway-tools ]
|
||||
# GitHub-hosted like the s3-tests targets of reference-parity.yml: the job
|
||||
# only runs prebuilt binaries and the pinned clients, none of which needs Docker.
|
||||
runs-on: ubuntu-latest
|
||||
# About 3x the 15 minutes the job took in run 43325101234 (client install about
|
||||
# About 3x the 15 minutes the legacy job took in run 43325101234 (client install about
|
||||
# 8, matrix about 3, corpus about 3), so a hung server costs 45 runner minutes, not more.
|
||||
timeout-minutes: 45
|
||||
continue-on-error: ${{ matrix.stack == 'gateway' }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
stack: [ legacy, gateway ]
|
||||
env:
|
||||
STACK: ${{ matrix.stack }}
|
||||
ENDPOINT: http://127.0.0.1:9000
|
||||
# The conformance suite signs with these fixed fixture credentials (the AWS
|
||||
# documentation example pair), so the candidate is started with them and the
|
||||
@@ -196,8 +213,10 @@ jobs:
|
||||
name: gateway-acceptance-tools
|
||||
path: tools
|
||||
|
||||
# Records `ready`, or what the process said when it exited, in lane/startup.txt.
|
||||
- name: Start candidate
|
||||
# Records `ready`, or what the process said when it exited, in lane/startup.txt. `ready`
|
||||
# also needs the server's own `server_starting` log event to name the requested stack
|
||||
# (its `s3_stack` field comes from rustfs/rustfs#9367; scripts/dual_stack.py reads it).
|
||||
- name: Start candidate (${{ matrix.stack }} stack)
|
||||
run: |
|
||||
set -uo pipefail
|
||||
chmod +x target/debug/rustfs tools/*
|
||||
@@ -205,7 +224,7 @@ jobs:
|
||||
target/debug/rustfs --version | tee lane/candidate-version.txt
|
||||
# The four directories share the runner's one disk; the bypass is the CI case the
|
||||
# physical-disk guard sanctions (as in mint.yml).
|
||||
RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true RUST_LOG=info target/debug/rustfs --address 127.0.0.1:9000 \
|
||||
RUSTFS_S3_STACK="$STACK" RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true RUST_LOG=info target/debug/rustfs --address 127.0.0.1:9000 \
|
||||
--access-key "$ACCESS_KEY" --secret-key "$SECRET_KEY" "$RUNNER_TEMP"/data/rustfs{0..3} \
|
||||
> lane/rustfs.log 2>&1 &
|
||||
pid=$!
|
||||
@@ -220,8 +239,13 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
if curl -sf "$ENDPOINT/health/ready" >/dev/null; then
|
||||
echo ready > lane/startup.txt
|
||||
exit 0
|
||||
if python3 scripts/dual_stack.py stack --expect "$STACK" --out lane/stack.json lane/rustfs.log > /dev/null; then
|
||||
echo ready > lane/startup.txt
|
||||
exit 0
|
||||
fi
|
||||
echo "did not start: stack not observed" > lane/startup.txt
|
||||
cat lane/startup.txt
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
@@ -314,23 +338,58 @@ jobs:
|
||||
run: |
|
||||
set -uo pipefail
|
||||
status=0
|
||||
python3 scripts/check_conformance_rulings.py --run-dir lane | tee -a "$GITHUB_STEP_SUMMARY" || status=1
|
||||
python3 scripts/compat_matrix_report.py check --run-dir lane | tee -a "$GITHUB_STEP_SUMMARY" || status=1
|
||||
python3 scripts/check_conformance_rulings.py --stack "$STACK" --run-dir lane | tee -a "$GITHUB_STEP_SUMMARY" || status=1
|
||||
python3 scripts/compat_matrix_report.py check --stack "$STACK" --run-dir lane | tee -a "$GITHUB_STEP_SUMMARY" || status=1
|
||||
exit "$status"
|
||||
|
||||
- name: Upload lane results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: gateway-acceptance-lane
|
||||
name: gateway-acceptance-lane-${{ matrix.stack }}
|
||||
overwrite: true
|
||||
path: |
|
||||
lane/*.txt
|
||||
lane/*.log
|
||||
lane/report.json
|
||||
lane/stack.json
|
||||
lane/compat/matrix.json
|
||||
lane/compat/probe.jsonl
|
||||
lane/compat/sut.log
|
||||
lane/compat/results/**
|
||||
if-no-files-found: error
|
||||
retention-days: 14
|
||||
|
||||
compare:
|
||||
name: Compare stacks (report-only until T2.10)
|
||||
needs: [ acceptance ]
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download lane results
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
pattern: gateway-acceptance-lane-*
|
||||
path: lanes
|
||||
|
||||
- name: Diff legacy and gateway
|
||||
run: |
|
||||
set -uo pipefail
|
||||
status=0
|
||||
{
|
||||
python3 scripts/check_conformance_rulings.py \
|
||||
--diff lanes/gateway-acceptance-lane-legacy lanes/gateway-acceptance-lane-gateway || status=1
|
||||
echo
|
||||
python3 scripts/compat_matrix_report.py diff \
|
||||
--legacy lanes/gateway-acceptance-lane-legacy --gateway lanes/gateway-acceptance-lane-gateway || status=1
|
||||
} > diff.md
|
||||
cat diff.md
|
||||
cat diff.md >> "$GITHUB_STEP_SUMMARY"
|
||||
exit "$status"
|
||||
|
||||
@@ -44,6 +44,13 @@
|
||||
# well inside the 120-minute budget and buys deterministic infra. The
|
||||
# docker-capable self-hosted `dind-sm-standard-2` label was the alternative but
|
||||
# has fewer cores and reintroduces fleet-state risk for no reliability gain.
|
||||
#
|
||||
# Stack dimension (rustfs/backlog#2774, T3.3): one image, run with RUSTFS_S3_STACK=legacy and
|
||||
# =gateway. `compare` (scripts/mint_stack_diff.py) holds the gate that every case legacy passes
|
||||
# passes on gateway with no new failure. The gateway leg and `compare` are report-only until
|
||||
# T2.10. A gateway server that refuses to start stops its leg at "Wait for RustFS ready", before
|
||||
# mint runs; the summary says `skipped: gateway stack refuses startup: <why>`. The stack is read
|
||||
# from the server's own start-up line (scripts/dual_stack.py stack).
|
||||
|
||||
name: mint
|
||||
|
||||
@@ -138,8 +145,16 @@ jobs:
|
||||
retention-days: 1
|
||||
|
||||
mint:
|
||||
name: mint
|
||||
name: mint (${{ matrix.stack }}${{ matrix.stack == 'gateway' && ', report-only until T2.10' || '' }})
|
||||
needs: [image]
|
||||
# Report-only until T2.10: the gateway stack cannot serve the suites yet.
|
||||
continue-on-error: ${{ matrix.stack == 'gateway' }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
stack: [legacy, gateway]
|
||||
env:
|
||||
STACK: ${{ matrix.stack }}
|
||||
# GitHub-hosted: reliably provides a running Docker daemon + buildx +
|
||||
# python3. See the header note (ci-2) for why the self-hosted sm-standard-4
|
||||
# label was abandoned (no docker.sock on the pod its only run landed on).
|
||||
@@ -179,6 +194,7 @@ jobs:
|
||||
-e RUSTFS_SECRET_KEY="${S3_SECRET_KEY}" \
|
||||
-e RUSTFS_VOLUMES="/data/rustfs{0...3}" \
|
||||
-e RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true \
|
||||
-e RUSTFS_S3_STACK="${STACK}" \
|
||||
-e RUST_LOG=rustfs=info \
|
||||
-v /tmp/rustfs-mint:/data \
|
||||
rustfs-ci
|
||||
@@ -275,13 +291,47 @@ jobs:
|
||||
mkdir -p artifacts/rustfs-mint
|
||||
docker logs rustfs-mint > artifacts/rustfs-mint/rustfs.log 2>&1 || true
|
||||
|
||||
- name: Prove which stack the server selected
|
||||
if: always()
|
||||
run: python3 scripts/dual_stack.py stack --expect "${STACK}" --out artifacts/stack.json artifacts/rustfs-mint
|
||||
|
||||
- name: Upload artifacts
|
||||
if: always() && env.ACT != 'true'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: mint
|
||||
name: mint-${{ matrix.stack }}
|
||||
path: artifacts/**
|
||||
|
||||
compare:
|
||||
name: Compare stacks (report-only until T2.10)
|
||||
needs: [mint]
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download both legs
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
pattern: mint-{legacy,gateway}
|
||||
path: legs
|
||||
|
||||
# A leg that never uploaded leaves no directory: create both so that the
|
||||
# comparison reports "no result" for it instead of failing to find it.
|
||||
- name: Compare the stacks
|
||||
run: |
|
||||
# The shell runs with -e; the exit status is reported after the summary is written.
|
||||
set +e -u -o pipefail
|
||||
mkdir -p legs/mint-legacy legs/mint-gateway
|
||||
python3 scripts/mint_stack_diff.py legs/mint-legacy legs/mint-gateway | tee report.txt
|
||||
status="${PIPESTATUS[0]}"
|
||||
{ echo '## Mint: gateway versus legacy'; echo; echo '```'; cat report.txt; echo '```'; } >> "$GITHUB_STEP_SUMMARY"
|
||||
exit "$status"
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [image, mint]
|
||||
|
||||
@@ -68,11 +68,14 @@ env:
|
||||
|
||||
jobs:
|
||||
minio-source:
|
||||
name: MinIO source (read-through, list-through, backfill)
|
||||
name: MinIO source (read-through, list-through, backfill, ${{ matrix.stack }}${{ matrix.stack == 'gateway' && ', report-only until T2.10' || '' }})
|
||||
# Keep this scheduled interoperability lane scoped to the upstream repo.
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 90
|
||||
# The gateway leg is report-only until T2.10.
|
||||
continue-on-error: ${{ matrix.stack == 'gateway' }}
|
||||
strategy: { fail-fast: false, matrix: { stack: [legacy, gateway] } }
|
||||
env:
|
||||
NO_PROXY: 127.0.0.1,localhost
|
||||
# Fixed credentials of the container this job starts and throws away;
|
||||
@@ -137,6 +140,11 @@ jobs:
|
||||
- name: Build the RustFS binary under test
|
||||
run: python3 scripts/e2e_binary.py build --bins
|
||||
|
||||
# A gateway binary that refuses to start skips the suite; the summary says why.
|
||||
- name: Gateway preflight
|
||||
if: matrix.stack == 'gateway'
|
||||
run: python3 scripts/dual_stack.py preflight --binary target/debug/rustfs --out "$RUNNER_TEMP/leg/stack.json"
|
||||
|
||||
# The lane selects tests by module, so a rename would quietly shrink it.
|
||||
# The committed digest in .config/e2e-odm-interop-selection.txt fails
|
||||
# closed on that.
|
||||
@@ -147,9 +155,18 @@ jobs:
|
||||
|
||||
- name: Run the interop cases against MinIO
|
||||
env:
|
||||
RUSTFS_S3_STACK: ${{ matrix.stack }}
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-logs
|
||||
run: python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-odm-interop -p e2e_test --no-tests=fail
|
||||
|
||||
- name: Prove the stack and upload the leg
|
||||
if: always()
|
||||
uses: ./.github/actions/dual-stack-leg
|
||||
with:
|
||||
profile: e2e-odm-interop
|
||||
stack: ${{ matrix.stack }}
|
||||
logs: ${{ runner.temp }}/rustfs-e2e-logs
|
||||
|
||||
- name: Build the MinIO interop report
|
||||
if: always()
|
||||
uses: ./.github/actions/odm-interop-report
|
||||
@@ -173,7 +190,7 @@ jobs:
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: odm-interop-minio-${{ github.run_number }}-${{ github.run_attempt }}
|
||||
name: odm-interop-minio-${{ github.run_number }}-${{ github.run_attempt }}-${{ matrix.stack }}
|
||||
path: |
|
||||
artifacts/odm-interop/minio
|
||||
target/nextest/e2e-odm-interop/junit.xml
|
||||
@@ -299,6 +316,20 @@ jobs:
|
||||
target/nextest/e2e-odm-interop/junit.xml
|
||||
retention-days: 14
|
||||
|
||||
dual-stack-summary:
|
||||
name: Both-stack diff summary
|
||||
needs: [minio-source]
|
||||
if: github.repository == 'rustfs/rustfs' && always()
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: ./.github/actions/dual-stack-summary
|
||||
with:
|
||||
profiles: e2e-odm-interop
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [minio-source, cloud-source]
|
||||
|
||||
@@ -32,9 +32,9 @@
|
||||
#
|
||||
# The `differential` job adds the 73-operation dual-stack differential
|
||||
# (crates/e2e_test/src/dual_stack_differential, rustfs/backlog#2740) against the
|
||||
# same pinned reference: side A is the released binary (RUSTFS_E2E_REFERENCE_BINARY),
|
||||
# side B is the candidate built above.
|
||||
# It reuses the candidate build artifact, never builds the candidate
|
||||
# same pinned reference: side A is the released binary (RUSTFS_E2E_REFERENCE_BINARY,
|
||||
# started without RUSTFS_S3_STACK), side B is the candidate built above on its
|
||||
# legacy stack. It reuses the candidate build artifact, never builds the candidate
|
||||
# twice, and prints reference_side, version_a and version_b from the report the test
|
||||
# wrote; it fails when side A was not the reference or both sides report the same
|
||||
# version. Every (op, field, a, b) difference goes to the job summary. Report-only
|
||||
@@ -310,7 +310,7 @@ jobs:
|
||||
chmod +x target/debug/rustfs
|
||||
test -x "$REFERENCE_PATH"
|
||||
mkdir -p "$RUSTFS_E2E_LOG_DIR"
|
||||
# Side A: the released binary. Side B: the candidate.
|
||||
# Side A: the released binary. Side B: the candidate on its legacy stack.
|
||||
export RUSTFS_E2E_REFERENCE_BINARY="$REFERENCE_PATH"
|
||||
test_name='dual_stack_differential::tests::both_stacks_answer_every_operation_identically'
|
||||
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- \
|
||||
|
||||
@@ -13,8 +13,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Deployment
|
||||
|
||||
- **The S3 protocol stack is now rustfs/gateway**, and it is the only S3 stack. The legacy s3s-based stack is removed, and so are the `RUSTFS_S3_STACK` environment variable and the `gateway-stack` cargo feature. There is no on-disk format change. `RUSTFS_S3_STACK` no longer selects anything: a server started with it set, to any value, refuses to start and prints `RUSTFS_S3_STACK is no longer supported; the gateway stack is the only S3 stack since <version>`, where `<version>` is the release's version. Remove the variable from service units, container environments, Helm values and orchestration files before upgrading. To roll back, follow the [S3 stack rollback procedure](docs/operations/s3-stack-rollback.md).
|
||||
|
||||
- Before upgrading, verify that the configured root access key is at least 3 bytes and the root secret key is at least 8 bytes. RustFS now rejects shorter values at startup and identifies the setting to update. Version 1.0.5 accepted these settings at startup and returned HTTP 500 when root requested STS credentials. Coordinate root credential changes across all nodes and clients. See [root credential rotation considerations](docs/operations/two-factor-auth.md#root-credentials-cannot-be-changed-at-runtime).
|
||||
|
||||
- Existing volumes created by older root-running container images must be writable by the non-root runtime identity before upgrading. The default container UID/GID is `10001:10001`; stop the affected instances, verify host mount paths, and for ordinary bind mounts run `sudo chown -R 10001:10001 /actual/data /actual/logs` on each RustFS data/log mount. Account for ACLs, NFS, Kubernetes, and rootless UID mappings. Image-build ownership changes do not migrate mounted data. See the Docker Quick Start in [README.md](README.md).
|
||||
|
||||
Generated
+392
-452
File diff suppressed because it is too large
Load Diff
+32
-39
@@ -71,7 +71,6 @@ members = [
|
||||
"crates/zip", # ZIP file handling and compression
|
||||
"crates/zstd-compat", # Fixed codec for dependencies requiring zstd 0.13
|
||||
]
|
||||
exclude = ["vendor/hyper"]
|
||||
resolver = "3"
|
||||
|
||||
[workspace.package]
|
||||
@@ -152,16 +151,14 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.5" }
|
||||
# engine crates move onto from s3s (rustfs/backlog#2745). The one pin of the
|
||||
# gateway in this workspace; no crate may enable a `compat-s3s*` feature except
|
||||
# the s3s-facing edge of the rustfs binary.
|
||||
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "60dcd2a55713dbcc9dbd6e165922e1a0df4473d7", version = "0.42.0", default-features = false }
|
||||
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "60dcd2a55713dbcc9dbd6e165922e1a0df4473d7", version = "0.2.0", default-features = false }
|
||||
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "2a6b8d1f7663db978d8784ab3ea331cade19237b", version = "0.40.0", default-features = false }
|
||||
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "2a6b8d1f7663db978d8784ab3ea331cade19237b", version = "0.2.0", default-features = false }
|
||||
# The gateway framework itself, consumed for its extension traits by the ring-2
|
||||
# adapter. `default-features = false` keeps the `server` feature, and with it the
|
||||
# server crate, out of the build.
|
||||
rustfs-gateway = { git = "https://github.com/rustfs/gateway.git", rev = "60dcd2a55713dbcc9dbd6e165922e1a0df4473d7", version = "0.66.0", default-features = false }
|
||||
rustfs-gateway = { git = "https://github.com/rustfs/gateway.git", rev = "2a6b8d1f7663db978d8784ab3ea331cade19237b", version = "0.63.0", default-features = false }
|
||||
# Test-only: builds the `EffectiveHost` a host-resolution test queries with.
|
||||
rustfs-gateway-http = { git = "https://github.com/rustfs/gateway.git", rev = "60dcd2a55713dbcc9dbd6e165922e1a0df4473d7", default-features = false }
|
||||
rustfs-gateway-core = { git = "https://github.com/rustfs/gateway.git", rev = "60dcd2a55713dbcc9dbd6e165922e1a0df4473d7", version = "0.63.0", default-features = false }
|
||||
rustfs-gateway-dialect-rustfs-admin = { git = "https://github.com/rustfs/gateway.git", rev = "60dcd2a55713dbcc9dbd6e165922e1a0df4473d7", version = "0.13.0", default-features = false }
|
||||
rustfs-gateway-http = { git = "https://github.com/rustfs/gateway.git", rev = "2a6b8d1f7663db978d8784ab3ea331cade19237b", default-features = false }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-channel = "2.5.0"
|
||||
@@ -169,13 +166,13 @@ async_zip = { default-features = false, version = "0.0.19" }
|
||||
mysql_async = { default-features = false, version = "0.37.1" }
|
||||
async-compression = { version = "0.4.52" }
|
||||
async-trait = "0.1.92"
|
||||
async-nats = { version = "0.53.0", default-features = false }
|
||||
async-nats = { version = "0.52.0", default-features = false }
|
||||
axum = "0.8.9"
|
||||
futures = "0.3.34"
|
||||
futures-core = "0.3.34"
|
||||
futures-lite = "2.6.1"
|
||||
futures-util = "0.3.34"
|
||||
pollster = "1.0.2"
|
||||
pollster = "1.0.1"
|
||||
pulsar = { default-features = false, version = "=6.9.0" }
|
||||
lapin = { default-features = false, version = "4.12.3" }
|
||||
hyper = { version = "1.12.0" }
|
||||
@@ -189,7 +186,6 @@ memchr = "2.8.3"
|
||||
minlz = "1.2.3"
|
||||
reqwest = "0.13.5"
|
||||
rustfs-kafka-async = { version = "1.4.1" }
|
||||
slab = { version = "0.4.12" }
|
||||
socket2 = { version = "0.6.5" }
|
||||
tokio = { version = "1.54.1" }
|
||||
tokio-rustls = { default-features = false, version = "0.26.6" }
|
||||
@@ -210,8 +206,8 @@ quinn = { version = "0.11.12", default-features = false, features = [
|
||||
] }
|
||||
|
||||
# Serialization and Data Formats
|
||||
apache-avro = { version = "0.23.0", features = ["snappy", "zstandard"] }
|
||||
bytes = { version = "1.12.2" }
|
||||
apache-avro = { version = "0.22.0", features = ["snappy", "zstandard"] }
|
||||
bytes = { version = "1.12.1" }
|
||||
csv-core = "0.1.13"
|
||||
bytesize = "2.7.0"
|
||||
byteorder = "1.5.0"
|
||||
@@ -222,7 +218,6 @@ quick-xml = "0.42.0"
|
||||
rmp = { version = "0.8.15" }
|
||||
rmp-serde = { version = "1.3.1" }
|
||||
serde = { version = "1.0.229" }
|
||||
serde-value = "0.7.0"
|
||||
serde_ignored = { version = "0.1" }
|
||||
serde_with = { version = "3", default-features = false, features = ["macros", "std"] }
|
||||
serde_json = { version = "1.0.154" }
|
||||
@@ -244,7 +239,7 @@ jsonwebtoken = { version = "12.0.0" }
|
||||
openidconnect = { default-features = false, version = "4.1.0" }
|
||||
pbkdf2 = "0.13.0"
|
||||
p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
|
||||
rsa = { version = "0.10.0", features = ["pkcs1v15"] }
|
||||
rsa = { version = "=0.10.0-rc.18" }
|
||||
rustls = { default-features = false, version = "0.23.46" }
|
||||
rustls-native-certs = "0.8"
|
||||
rustls-pki-types = "1.16.0"
|
||||
@@ -252,13 +247,13 @@ x509-parser = "0.18.1"
|
||||
sha1 = "0.11.0"
|
||||
sha2 = "0.11.1"
|
||||
subtle = "2.6"
|
||||
zeroize = { version = "1.9.2" }
|
||||
zeroize = { version = "1.9.1" }
|
||||
proptest = "1"
|
||||
|
||||
# Time and Date
|
||||
chrono = { version = "0.4.45" }
|
||||
humantime = "2.4.0"
|
||||
jiff = { version = "1.0.1" }
|
||||
jiff = { version = "0.2.43" }
|
||||
time = { version = "0.3.55" }
|
||||
|
||||
# Database
|
||||
@@ -275,18 +270,18 @@ tar-codec = "0.0.16"
|
||||
tar-framing = "0.0.16"
|
||||
atoi = "3.1.1"
|
||||
atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.13.3" }
|
||||
aws-config = { version = "1.13.2" }
|
||||
aws-credential-types = { version = "1.3.1" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.133.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.160.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.127.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.131.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.159.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.126.0" }
|
||||
aws-smithy-async = { version = "1.3.1" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.5.3" }
|
||||
aws-smithy-runtime-api = { version = "1.22.0" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.5.2" }
|
||||
aws-smithy-runtime-api = { version = "1.21.0" }
|
||||
aws-smithy-types = { version = "1.9.4" }
|
||||
base64-simd = "0.8.0"
|
||||
brotli = "9.1.3"
|
||||
clap = { version = "4.6.8" }
|
||||
brotli = "9.1.1"
|
||||
clap = { version = "4.6.7" }
|
||||
const-str = { version = "1.1.0" }
|
||||
convert_case = "0.12.0"
|
||||
criterion = { version = "0.8" }
|
||||
@@ -294,7 +289,7 @@ crossbeam-queue = "0.3.14"
|
||||
crossbeam-channel = "0.5.17"
|
||||
crossbeam-deque = "0.8.8"
|
||||
crossbeam-utils = "0.8.23"
|
||||
datafusion = { default-features = false, version = "55.3.0" }
|
||||
datafusion = { default-features = false, version = "55.2.0" }
|
||||
derive_builder = "0.20.2"
|
||||
ed25519-dalek = "3.0.0"
|
||||
enumset = "1.1.15"
|
||||
@@ -313,14 +308,14 @@ hostname = "0.4.2"
|
||||
ipnetwork = { version = "0.21.1" }
|
||||
lazy_static = "1.5.1"
|
||||
libc = "0.2.191"
|
||||
cuobject = { version = "0.1.0", default-features = false }
|
||||
cuobject = "0.1.0"
|
||||
libsystemd = "0.7.2"
|
||||
local-ip-address = "0.6.13"
|
||||
log = "0.4"
|
||||
memmap2 = "0.9.12"
|
||||
nixl-sys = { git = "https://github.com/houseme/nixl", rev = "cc636c83f13d692e85b67e55aed092413d16fb41" }
|
||||
lz4 = "1.28.1"
|
||||
lz4_flex = { version = "0.14.2", default-features = false, features = ["std", "frame"] }
|
||||
lz4_flex = { version = "=0.14.0", default-features = false, features = ["std", "frame"] }
|
||||
matchit = "0.9.2"
|
||||
md-5 = "0.11.0"
|
||||
mime_guess = "2.0.5"
|
||||
@@ -339,14 +334,14 @@ percent-encoding = "2.3.2"
|
||||
qrcode-rs = { version = "2.2.0", default-features = false, features = ["std", "svg"] }
|
||||
pin-project-lite = "0.2.17"
|
||||
pretty_assertions = "1.4.1"
|
||||
rand = { version = "0.10.4" }
|
||||
rand = { version = "0.10.3" }
|
||||
ratelimit = "2.0.0"
|
||||
rayon = "1.12.0"
|
||||
rustfs-erasure-codec = { version = "9.1.0" }
|
||||
reed-solomon-simd = "3.1.0"
|
||||
regex = { version = "1.13.2" }
|
||||
regex = { version = "1.13.1" }
|
||||
rumqttc = { package = "rumqttc-next", version = "0.34.0" }
|
||||
redis = { version = "1.8.2" }
|
||||
redis = { version = "1.8.1" }
|
||||
rustify = { version = "0.7", default-features = false }
|
||||
rustix = { version = "1.1.6" }
|
||||
rust-embed = { version = "8.14.0" }
|
||||
@@ -366,7 +361,7 @@ snap = "1.1.2"
|
||||
starshard = { version = "2.4.1" }
|
||||
strum = { version = "0.28.0" }
|
||||
sync_wrapper = "1.0.2"
|
||||
sysinfo = "0.40.4"
|
||||
sysinfo = "0.40.3"
|
||||
temp-env = "0.3.6"
|
||||
tempfile = "3.27.0"
|
||||
test-case = "3.4.1"
|
||||
@@ -380,15 +375,15 @@ tracing-subscriber = { version = "0.3.23" }
|
||||
transform-stream = "0.3.1"
|
||||
url = "2.5.8"
|
||||
urlencoding = "2.1.3"
|
||||
uuid = { version = "1.29.3" }
|
||||
uuid = { version = "1.29.2" }
|
||||
vaultrs = { version = "0.8.0" }
|
||||
tar = "0.4.46"
|
||||
walkdir = "2.5.0"
|
||||
winapi-util = "0.1.11"
|
||||
windows = { version = "0.62.2" }
|
||||
windows-sys = "0.61.2"
|
||||
xxhash-rust = { version = "0.8.21" }
|
||||
zip = "9.7.0"
|
||||
xxhash-rust = { version = "0.8.19" }
|
||||
zip = "9.5.1"
|
||||
zstd = "0.14.4"
|
||||
|
||||
# Observability and Metrics
|
||||
@@ -407,9 +402,9 @@ pyroscope = { version = "2.1.5" }
|
||||
# FTP and SFTP
|
||||
libunftp = { version = "0.25.0" }
|
||||
unftp-core = "0.1.1"
|
||||
suppaftp = { version = "13.0.0" }
|
||||
suppaftp = { version = "12.2.1" }
|
||||
rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
russh = { version = "0.65.2" }
|
||||
russh = { version = "0.65.1" }
|
||||
russh-sftp = "3.0.1"
|
||||
|
||||
# WebDAV
|
||||
@@ -418,7 +413,7 @@ dav-server = "0.12.0"
|
||||
# Performance Analysis and Memory Profiling
|
||||
rustfs-mimalloc = { version = "0.7.4" }
|
||||
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
|
||||
hotpath = { version = "0.32.1", default-features = false }
|
||||
hotpath = { version = "0.31.2", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.49" }
|
||||
|
||||
@@ -458,6 +453,4 @@ strip = "none"
|
||||
|
||||
# Pin upstream fixes until compatible crates.io releases are available.
|
||||
[patch.crates-io]
|
||||
# Remove after Hyper releases HTTP/2 Expect support (hyperium/hyper#4217).
|
||||
hyper = { path = "vendor/hyper" }
|
||||
zstd = { path = "crates/zstd-compat" }
|
||||
|
||||
@@ -302,8 +302,6 @@ Drive timeout profile preset:
|
||||
|
||||
RustFS production guidance remains direct-attached local POSIX filesystems. Network-mounted filesystems (for example `nfs`, `cifs`, `smb2`, and `fuse.*`) are treated as unsupported by this startup guard. On Windows, endpoints on network drives (UNC shares such as `\\server\share` and mapped drive letters) are treated the same way, even though Windows reports the server's filesystem (usually `NTFS`) for them.
|
||||
|
||||
A local POSIX filesystem bind-mounted into a gVisor sandbox with directfs is a supported storage path. gVisor presents that bind mount as `9p`/`V9FS`, so the host filesystem type is not visible; the startup guard recognizes mountinfo options `trans=fd` and `directfs` and does not treat the mount as unsupported. Genuine 9p, virtio, and NFS shares remain unsupported.
|
||||
|
||||
## 📄 License
|
||||
|
||||
This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.
|
||||
|
||||
@@ -152,33 +152,6 @@ pub const DEFAULT_SHARD_INTEGRITY_WRITE: bool = false;
|
||||
pub const ENV_SHARD_INTEGRITY_FLEET_CONFIRMED: &str = "RUSTFS_SHARD_INTEGRITY_FLEET_CONFIRMED";
|
||||
pub const DEFAULT_SHARD_INTEGRITY_FLEET_CONFIRMED: bool = false;
|
||||
|
||||
/// Request parsing switch for the per-object erasure-code block-size hint.
|
||||
/// The default keeps the feature inert during rolling upgrades.
|
||||
pub const ENV_EC_BLOCK_SIZE_HINT_ENABLE: &str = "RUSTFS_EC_BLOCK_SIZE_HINT_ENABLE";
|
||||
pub const DEFAULT_EC_BLOCK_SIZE_HINT_ENABLE: bool = false;
|
||||
|
||||
/// Operator confirmation that every writer and reader in the fleet understands
|
||||
/// the persisted erasure block-size geometry.
|
||||
pub const ENV_EC_BLOCK_SIZE_HINT_FLEET_CONFIRMED: &str = "RUSTFS_EC_BLOCK_SIZE_HINT_FLEET_CONFIRMED";
|
||||
pub const DEFAULT_EC_BLOCK_SIZE_HINT_FLEET_CONFIRMED: bool = false;
|
||||
|
||||
/// Write new EC shards with a bounded bitrot frame size so Range GETs do not
|
||||
/// inherit the full persisted layout block as their physical read quantum.
|
||||
/// The marker is only emitted after the fleet confirmation gate is enabled;
|
||||
/// readers without this feature continue to consume legacy objects unchanged.
|
||||
pub const ENV_EC_READ_QUANTUM_ENABLE: &str = "RUSTFS_EC_READ_QUANTUM_ENABLE";
|
||||
pub const DEFAULT_EC_READ_QUANTUM_ENABLE: bool = false;
|
||||
|
||||
/// Operator confirmation that every reader and writer understands the
|
||||
/// persisted EC read-quantum marker and its per-frame bitrot layout.
|
||||
pub const ENV_EC_READ_QUANTUM_FLEET_CONFIRMED: &str = "RUSTFS_EC_READ_QUANTUM_FLEET_CONFIRMED";
|
||||
pub const DEFAULT_EC_READ_QUANTUM_FLEET_CONFIRMED: bool = false;
|
||||
|
||||
const _: () = assert!(!DEFAULT_EC_BLOCK_SIZE_HINT_ENABLE);
|
||||
const _: () = assert!(!DEFAULT_EC_BLOCK_SIZE_HINT_FLEET_CONFIRMED);
|
||||
const _: () = assert!(!DEFAULT_EC_READ_QUANTUM_ENABLE);
|
||||
const _: () = assert!(!DEFAULT_EC_READ_QUANTUM_FLEET_CONFIRMED);
|
||||
|
||||
const _: () = assert!(!DEFAULT_SHARD_INTEGRITY_WRITE);
|
||||
const _: () = assert!(!DEFAULT_SHARD_INTEGRITY_FLEET_CONFIRMED);
|
||||
|
||||
|
||||
@@ -48,7 +48,7 @@ pub const ENV_RUNTIME_DIAL9_S3_PREFIX: &str = "RUSTFS_RUNTIME_DIAL9_S3_PREFIX";
|
||||
pub const DEFAULT_WORKER_THREADS: usize = 16;
|
||||
pub const DEFAULT_MAX_BLOCKING_THREADS: usize = 1024;
|
||||
pub const DEFAULT_THREAD_PRINT_ENABLED: bool = false;
|
||||
pub const DEFAULT_THREAD_STACK_SIZE: usize = 2 * MI_B; // 2 MiB
|
||||
pub const DEFAULT_THREAD_STACK_SIZE: usize = MI_B; // 1 MiB
|
||||
pub const DEFAULT_THREAD_KEEP_ALIVE: u64 = 60; // seconds
|
||||
pub const DEFAULT_GLOBAL_QUEUE_INTERVAL: u32 = 31;
|
||||
pub const DEFAULT_THREAD_NAME: &str = "rustfs-worker";
|
||||
|
||||
@@ -36,7 +36,7 @@ jsonwebtoken = { workspace = true, features = ["aws_lc_rs"] }
|
||||
base64-simd = { workspace = true }
|
||||
pbkdf2 = { workspace = true, optional = true }
|
||||
rand = { workspace = true, features = ["serde"] }
|
||||
rsa = { workspace = true }
|
||||
rsa = { workspace = true, features = ["sha2"] }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
sha2 = { workspace = true, optional = true }
|
||||
thiserror.workspace = true
|
||||
|
||||
@@ -13,9 +13,9 @@
|
||||
// limitations under the License.
|
||||
|
||||
use rsa::{
|
||||
Pkcs1v15Encrypt, PrivateKey, PublicKey,
|
||||
Pkcs1v15Encrypt, RsaPrivateKey, RsaPublicKey,
|
||||
pkcs8::{DecodePrivateKey, DecodePublicKey},
|
||||
pss::{Signature, SigningKey, VerifyingKey},
|
||||
pss::{BlindedSigningKey, Signature, VerifyingKey},
|
||||
sha2::Sha256,
|
||||
signature::{RandomizedSigner, Verifier},
|
||||
traits::PublicKeyParts,
|
||||
@@ -37,7 +37,7 @@ pub struct Token {
|
||||
pub fn gencode(token: &Token, key: &str) -> Result<String> {
|
||||
let data = serde_json::to_vec(token)?;
|
||||
let mut rng = rand::rng();
|
||||
let public_key = PublicKey::from_public_key_pem(key).map_err(Error::other)?;
|
||||
let public_key = RsaPublicKey::from_public_key_pem(key).map_err(Error::other)?;
|
||||
let encrypted_data = public_key.encrypt(&mut rng, Pkcs1v15Encrypt, &data).map_err(Error::other)?;
|
||||
Ok(base64_simd::URL_SAFE_NO_PAD.encode_to_string(&encrypted_data))
|
||||
}
|
||||
@@ -51,7 +51,7 @@ pub fn parse(token: &str, key: &str) -> Result<Token> {
|
||||
let encrypted_data = base64_simd::URL_SAFE_NO_PAD
|
||||
.decode_to_vec(token.as_bytes())
|
||||
.map_err(Error::other)?;
|
||||
let private_key = PrivateKey::from_pkcs8_pem(key).map_err(Error::other)?;
|
||||
let private_key = RsaPrivateKey::from_pkcs8_pem(key).map_err(Error::other)?;
|
||||
let decrypted_data = private_key.decrypt(Pkcs1v15Encrypt, &encrypted_data).map_err(Error::other)?;
|
||||
serde_json::from_slice(&decrypted_data).map_err(Error::other)
|
||||
}
|
||||
@@ -63,8 +63,8 @@ pub fn parse(token: &str, key: &str) -> Result<Token> {
|
||||
pub fn sign_license_token(token: &Token, private_key_pem: &str) -> Result<String> {
|
||||
let payload = serde_json::to_vec(token)?;
|
||||
let mut rng = rand::rng();
|
||||
let private_key = PrivateKey::from_pkcs8_pem(private_key_pem).map_err(Error::other)?;
|
||||
let signing_key = SigningKey::<Sha256>::new(private_key);
|
||||
let private_key = RsaPrivateKey::from_pkcs8_pem(private_key_pem).map_err(Error::other)?;
|
||||
let signing_key = BlindedSigningKey::<Sha256>::new(private_key);
|
||||
let signature: Signature = signing_key.try_sign_with_rng(&mut rng, &payload).map_err(Error::other)?;
|
||||
let signature: Box<[u8]> = signature.into();
|
||||
|
||||
@@ -80,7 +80,7 @@ pub fn parse_signed_license_token(token: &str, public_key_pem: &str) -> Result<T
|
||||
let signed_payload = base64_simd::URL_SAFE_NO_PAD
|
||||
.decode_to_vec(token.as_bytes())
|
||||
.map_err(Error::other)?;
|
||||
let public_key = PublicKey::from_public_key_pem(public_key_pem).map_err(Error::other)?;
|
||||
let public_key = RsaPublicKey::from_public_key_pem(public_key_pem).map_err(Error::other)?;
|
||||
let signature_len = public_key.size();
|
||||
|
||||
if signed_payload.len() <= signature_len {
|
||||
@@ -103,7 +103,7 @@ pub fn parse_license_with_public_key(license: &str, public_key: &str) -> Result<
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rsa::{
|
||||
PrivateKey,
|
||||
RsaPrivateKey,
|
||||
pkcs8::{EncodePrivateKey, EncodePublicKey, LineEnding},
|
||||
};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
@@ -112,8 +112,8 @@ mod tests {
|
||||
fn test_sign_license_token_and_parse_signed_license_token() {
|
||||
let mut rng = rand::rng();
|
||||
let bits = 2048;
|
||||
let private_key = PrivateKey::new(&mut rng, bits).expect("Failed to generate private key");
|
||||
let public_key = PublicKey::from(&private_key);
|
||||
let private_key = RsaPrivateKey::new(&mut rng, bits).expect("Failed to generate private key");
|
||||
let public_key = RsaPublicKey::from(&private_key);
|
||||
|
||||
let private_key_pem = private_key
|
||||
.to_pkcs8_pem(LineEnding::LF)
|
||||
@@ -144,8 +144,8 @@ mod tests {
|
||||
fn test_legacy_gencode_and_parse_roundtrip() {
|
||||
let mut rng = rand::rng();
|
||||
let bits = 2048;
|
||||
let private_key = PrivateKey::new(&mut rng, bits).expect("Failed to generate private key");
|
||||
let public_key = PublicKey::from(&private_key);
|
||||
let private_key = RsaPrivateKey::new(&mut rng, bits).expect("Failed to generate private key");
|
||||
let public_key = RsaPublicKey::from(&private_key);
|
||||
|
||||
let private_key_pem = private_key
|
||||
.to_pkcs8_pem(LineEnding::LF)
|
||||
@@ -173,8 +173,8 @@ mod tests {
|
||||
#[test]
|
||||
fn test_parse_signed_license_token_rejects_tampered_payload() {
|
||||
let mut rng = rand::rng();
|
||||
let private_key = PrivateKey::new(&mut rng, 2048).expect("Failed to generate private key");
|
||||
let public_key = PublicKey::from(&private_key);
|
||||
let private_key = RsaPrivateKey::new(&mut rng, 2048).expect("Failed to generate private key");
|
||||
let public_key = RsaPublicKey::from(&private_key);
|
||||
let private_key_pem = private_key
|
||||
.to_pkcs8_pem(LineEnding::LF)
|
||||
.expect("failed to encode private key pem");
|
||||
@@ -206,8 +206,8 @@ mod tests {
|
||||
#[test]
|
||||
fn test_parse_signed_license_token_rejects_invalid_token() {
|
||||
let mut rng = rand::rng();
|
||||
let private_key = PrivateKey::new(&mut rng, 2048).expect("Failed to generate private key");
|
||||
let public_key = PublicKey::from(&private_key);
|
||||
let private_key = RsaPrivateKey::new(&mut rng, 2048).expect("Failed to generate private key");
|
||||
let public_key = RsaPublicKey::from(&private_key);
|
||||
let public_key_pem = public_key
|
||||
.to_public_key_pem(LineEnding::LF)
|
||||
.expect("failed to encode public key pem");
|
||||
|
||||
@@ -66,12 +66,6 @@ pub const LEGACY_DATA_USAGE_OBJECT_NAME: &str = ".usage.json";
|
||||
/// stale tier names from growing persisted per-tier maps without bound.
|
||||
pub const UNKNOWN_TIER: &str = "UNKNOWN_TIER";
|
||||
|
||||
/// Whether a remote tier name can be represented in scanner accounting.
|
||||
/// Local storage classes are checked separately by the configuration owner.
|
||||
pub fn is_scannable_tier_name(name: &str) -> bool {
|
||||
!name.is_empty() && name.len() <= 256 && !name.bytes().any(|byte| byte.is_ascii_control()) && name != UNKNOWN_TIER
|
||||
}
|
||||
|
||||
/// Returns true when `existing_last_update` is ahead of `now` by more than
|
||||
/// [`USAGE_LAST_UPDATE_FUTURE_TOLERANCE`], i.e. the persisted timestamp cannot be
|
||||
/// trusted for staleness comparisons and a fresh snapshot save must be allowed.
|
||||
|
||||
@@ -98,6 +98,7 @@ rustfs-madmin.workspace = true
|
||||
rustfs-filemeta.workspace = true
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
|
||||
aws-sdk-sts = { workspace = true, default-features = false, features = ["default-https-client", "rt-tokio"] }
|
||||
aws-smithy-http-client = { workspace = true, default-features = false, features = ["rustls-aws-lc"] }
|
||||
aws-smithy-types.workspace = true
|
||||
async-trait = { workspace = true }
|
||||
@@ -123,12 +124,12 @@ urlencoding.workspace = true
|
||||
walkdir.workspace = true
|
||||
base64-simd = { workspace = true }
|
||||
rand = { workspace = true, features = ["serde"] }
|
||||
jiff = { workspace = true, features = ["serde"] }
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
hex-simd = { workspace = true }
|
||||
md-5 = { workspace = true }
|
||||
prost.workspace = true
|
||||
sha2 = { workspace = true }
|
||||
rustfs-s3-types = { workspace = true }
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
zstd.workspace = true
|
||||
time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }
|
||||
rcgen.workspace = true
|
||||
@@ -138,11 +139,8 @@ rustls = { workspace = true, default-features = false, features = ["aws-lc-rs",
|
||||
clap = { workspace = true, features = ["derive", "env"] }
|
||||
|
||||
[dev-dependencies]
|
||||
serde-value.workspace = true
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
async-compression = { workspace = true, features = ["tokio", "bzip2", "lz4", "xz"] }
|
||||
aws-config = { workspace = true }
|
||||
aws-sdk-sts = { workspace = true, default-features = false, features = ["default-https-client", "rt-tokio"] }
|
||||
opentelemetry-proto = { workspace = true }
|
||||
russh = { workspace = true, features = ["serde"] }
|
||||
russh-sftp = { workspace = true }
|
||||
|
||||
@@ -29,7 +29,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
|
||||
| **distributed 4×4** | [`src/distributed/`](src/distributed) | Nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
|
||||
| **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
|
||||
| **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
|
||||
| **dual-stack differential** | `dual_stack_differential` | Two single-node servers answer one SDK script for all 73 S3 operations and every differing `(op, field, a, b)` is reported. Both sides run the binary under test on its only (gateway) stack unless `RUSTFS_E2E_REFERENCE_BINARY` names a released `rustfs` binary, which then runs side A. A missing, non-file or non-executable path fails the run; it never skips or falls back to the same-binary mode. `RUSTFS_E2E_STACK_B_REGION` starts side B in another region to inject a difference. The report in `<target>/dual-stack/<run>.json` records `reference_side` and each side's `--version` output |
|
||||
| **dual-stack differential** | `dual_stack_differential` | Two single-node servers answer one SDK script for all 73 S3 operations and every differing `(op, field, a, b)` is reported. Side A is the binary under test on `RUSTFS_S3_STACK=legacy`; side B runs `RUSTFS_E2E_STACK_B`. Set `RUSTFS_E2E_REFERENCE_BINARY` to a released `rustfs` binary to make side A that binary instead (started without `RUSTFS_S3_STACK`, which older releases do not know). A missing, non-file or non-executable path fails the run; it never skips or falls back to the same-binary mode. The report in `<target>/dual-stack/<run>.json` records `reference_side` and each side's `--version` output |
|
||||
|
||||
The external-tool `storage_metric_ownership_test` validates the OTLP/Collector/Prometheus path, including a rolling upgrade and node failures. See the [storage metrics guide](../../docs/operations/storage-metrics.md) for its required binaries and focused command.
|
||||
|
||||
@@ -211,8 +211,6 @@ the wiring source of truth. Committed test-ID digests under
|
||||
|
||||
**Endpoint blackhole scenario skipped** — `heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_after_target_endpoint_blackhole` installs a loopback `iptables` DROP rule and therefore needs `CAP_NET_ADMIN` (root or passwordless `sudo -n iptables`). A host where `iptables` is missing or cannot read the OUTPUT chain (typical inside an unprivileged container, where the nf_tables backend reports "Permission denied" even under `sudo`) logs a `heal_interruption_skipped` warning and returns without exercising heal. Set `RUSTFS_E2E_REQUIRE_NET_FAULT_INJECTION=1` on lanes that do provision the capability so a broken runner fails instead of skipping.
|
||||
|
||||
The full suite compares persisted bucket configurations and object records against the release pinned in `.config/reference-binary.toml`. Set `RUSTFS_PERSISTENCE_REFERENCE_BINARY` to the verified executable and `RUSTFS_PERSISTENCE_CLOCK_LIBRARY` to the absolute path of libfaketime (`libfaketime.so.1` on Linux or `libfaketime.1.dylib` on macOS). To retain the two sets of 13 XML documents for the gateway golden corpus, set `RUSTFS_PERSISTENCE_XML_DIR` to an output directory. CI includes them in the full suite's JUnit artifact.
|
||||
|
||||
**Reproduce a CI failure locally** — run the exact profile/lane:
|
||||
|
||||
```bash
|
||||
|
||||
@@ -21,8 +21,8 @@ use crate::common::{
|
||||
};
|
||||
use http::header::{AUTHORIZATION, HOST};
|
||||
use reqwest::StatusCode;
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::{constants::UNSIGNED_PAYLOAD, sign_v4};
|
||||
use s3s::Body;
|
||||
use serde_json::Value;
|
||||
use std::{error::Error, time::Duration};
|
||||
|
||||
|
||||
@@ -146,11 +146,7 @@ async fn create_service_account_for(
|
||||
Some(serde_json::json!({ "targetUser": parent }).to_string()),
|
||||
)
|
||||
.await?;
|
||||
service_account_keys(&response)
|
||||
}
|
||||
|
||||
fn service_account_keys(response: &str) -> Result<(String, String), Box<dyn Error + Send + Sync>> {
|
||||
let response: serde_json::Value = serde_json::from_str(response)?;
|
||||
let response: serde_json::Value = serde_json::from_str(&response)?;
|
||||
let access_key = response["credentials"]["accessKey"]
|
||||
.as_str()
|
||||
.ok_or("service account response should contain credentials.accessKey")?
|
||||
@@ -348,134 +344,6 @@ async fn test_update_service_account_enforces_owner_and_parent_scope() -> TestRe
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_delegated_service_account_creation_through_console_sessions() -> TestResult {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_without_cleanup(vec![]).await?;
|
||||
let bucket = "delegated-backups";
|
||||
let target = "backup-user";
|
||||
let operator = "backup-operator";
|
||||
let operator_secret = "operatorSecret123";
|
||||
let wildcard_admin = "backup-admin";
|
||||
let wildcard_secret = "adminSecret123";
|
||||
let s3 = env.create_s3_client();
|
||||
s3.create_bucket().bucket(bucket).send().await?;
|
||||
for (name, secret) in [
|
||||
(target, "backupSecret123"),
|
||||
(operator, operator_secret),
|
||||
(wildcard_admin, wildcard_secret),
|
||||
("other-user", "otherUserSecret123"),
|
||||
] {
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::PUT,
|
||||
&format!("/rustfs/admin/v3/add-user?accessKey={name}"),
|
||||
Some(serde_json::json!({"secretKey":secret,"status":"enabled"}).to_string()),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
let delegation = serde_json::json!({
|
||||
"Version":"2012-10-17",
|
||||
"Statement":[
|
||||
{"Effect":"Allow","Action":"admin:CreateServiceAccount"},
|
||||
{"Effect":"Allow","Action":"sts:AssumeRole","Resource":"arn:aws:s3:::*"},
|
||||
{"Effect":"Allow","Action":"admin:CreateServiceAccountForUser","Condition":{"StringEquals":{"svc:TargetUser":target}}}
|
||||
]
|
||||
});
|
||||
let wildcard = serde_json::json!({
|
||||
"Version":"2012-10-17",
|
||||
"Statement":[
|
||||
{"Effect":"Allow","Action":"admin:*"},
|
||||
{"Effect":"Allow","Action":"sts:AssumeRole","Resource":"arn:aws:s3:::*"}
|
||||
]
|
||||
});
|
||||
for (user, name, policy) in [
|
||||
(wildcard_admin, "backup-admin-wildcard", wildcard.to_string()),
|
||||
(target, "backup-writer", bucket_rw_policy(bucket)),
|
||||
(operator, "backup-key-manager", delegation.to_string()),
|
||||
] {
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::PUT,
|
||||
&format!("/rustfs/admin/v3/add-canned-policy?name={name}"),
|
||||
Some(policy),
|
||||
)
|
||||
.await?;
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::POST,
|
||||
"/rustfs/admin/v3/idp/builtin/policy/attach",
|
||||
Some(serde_json::json!({"policies":[name],"user":user}).to_string()),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
for (name, secret) in [
|
||||
(operator, operator_secret),
|
||||
(wildcard_admin, wildcard_secret),
|
||||
(env.access_key.as_str(), env.secret_key.as_str()),
|
||||
] {
|
||||
let session = build_test_sts_client(&env.url, name, secret, None, "delegated-create")
|
||||
.assume_role()
|
||||
.role_arn("arn:aws:iam::123456789012:role/console")
|
||||
.role_session_name("console")
|
||||
.send()
|
||||
.await?;
|
||||
let caller = session.credentials().ok_or("missing console session")?;
|
||||
let create = |user: &str| {
|
||||
admin_request_with_session_token(
|
||||
&env.url,
|
||||
http::Method::PUT,
|
||||
"/rustfs/admin/v3/add-service-account",
|
||||
Some(serde_json::json!({"targetUser":user}).to_string()),
|
||||
caller.access_key_id(),
|
||||
caller.secret_access_key(),
|
||||
Some(caller.session_token()),
|
||||
)
|
||||
};
|
||||
let (status, response) = create(target).await?;
|
||||
assert_eq!(status, StatusCode::OK, "create for target as {name}");
|
||||
let (key, secret) = service_account_keys(&response)?;
|
||||
let client = build_s3_client(&env.url, &key, &secret);
|
||||
wait_for_s3_put(&client, bucket, "backup", Duration::from_secs(10)).await?;
|
||||
let denied = client
|
||||
.create_bucket()
|
||||
.bucket("outside-backup-policy")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("target cannot create unrelated buckets");
|
||||
assert_eq!(denied.as_service_error().and_then(|e| e.code()), Some("AccessDenied"));
|
||||
if name != env.access_key {
|
||||
let (status, _) = create(&env.access_key).await?;
|
||||
assert_eq!(status, StatusCode::FORBIDDEN, "delegated targets exclude root");
|
||||
}
|
||||
if name == operator {
|
||||
let (status, _) = create("other-user").await?;
|
||||
assert_eq!(status, StatusCode::FORBIDDEN, "outside delegated target");
|
||||
}
|
||||
}
|
||||
let restricted = build_test_sts_client(&env.url, &env.access_key, &env.secret_key, None, "restricted-root")
|
||||
.assume_role()
|
||||
.role_arn("arn:aws:iam::123456789012:role/console")
|
||||
.role_session_name("restricted-console")
|
||||
.policy(r#"{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"admin:CreateServiceAccount"}]}"#)
|
||||
.send()
|
||||
.await?;
|
||||
let caller = restricted.credentials().ok_or("missing restricted root session")?;
|
||||
let (status, _) = admin_request_with_session_token(
|
||||
&env.url,
|
||||
http::Method::PUT,
|
||||
"/rustfs/admin/v3/add-service-account",
|
||||
Some(serde_json::json!({"targetUser":target}).to_string()),
|
||||
caller.access_key_id(),
|
||||
caller.secret_access_key(),
|
||||
Some(caller.session_token()),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(status, StatusCode::FORBIDDEN, "restricted root session requires delegated permission");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Full user -> policy -> service-account lifecycle, proving each management
|
||||
/// call takes effect on the data plane, not just that the endpoint answers 200.
|
||||
#[tokio::test]
|
||||
|
||||
@@ -34,15 +34,12 @@
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::common::{
|
||||
AdminTransport, RustFSTestEnvironment, admin_attach_user_policy_via, admin_create_user, admin_request_with_session_token,
|
||||
init_logging, local_http_client, parse_assume_role_credentials,
|
||||
};
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use hmac::{Hmac, KeyInit as _, Mac};
|
||||
use http::header::HOST;
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use sha1::Sha1;
|
||||
use std::error::Error;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
@@ -399,54 +396,6 @@ mod tests {
|
||||
let (status, _) = signed_request(&env.url, http::Method::GET, ACCOUNT_INFO_PATH, None, user_ak, old_sk).await?;
|
||||
assert!(status.is_client_error(), "the old secret must stop working, status: {status}");
|
||||
|
||||
env.stop_server();
|
||||
drop(env);
|
||||
console_session_rotates_the_iam_password_and_revokes_the_old_session().await
|
||||
}
|
||||
|
||||
async fn console_session_rotates_the_iam_password_and_revokes_the_old_session() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let user = "console-password-user";
|
||||
let old_password = uuid::Uuid::new_v4().to_string();
|
||||
let new_password = uuid::Uuid::new_v4().to_string();
|
||||
admin_create_user(&env, user, &old_password).await?;
|
||||
admin_attach_user_policy_via(AdminTransport::Signed, &env.url, &env.access_key, &env.secret_key, "readonly", user)
|
||||
.await?;
|
||||
|
||||
let (status, response) = assume_role(&env.url, user, &old_password, None).await?;
|
||||
assert_eq!(status, reqwest::StatusCode::OK, "the current password must sign in");
|
||||
let (session_key, session_secret, token) = parse_assume_role_credentials(&response)?;
|
||||
let session_request = |method, path, body| {
|
||||
admin_request_with_session_token(&env.url, method, path, body, &session_key, &session_secret, Some(&token))
|
||||
};
|
||||
let password_request = |current: &str| {
|
||||
Some(serde_json::json!({ "current_secret_key": current, "new_secret_key": new_password }).to_string())
|
||||
};
|
||||
|
||||
let (status, body) =
|
||||
session_request(http::Method::POST, ACCOUNT_PASSWORD_PATH, password_request(&session_secret)).await?;
|
||||
assert_eq!(
|
||||
status,
|
||||
reqwest::StatusCode::BAD_REQUEST,
|
||||
"the session secret must not be accepted as the IAM password"
|
||||
);
|
||||
assert!(body.contains("InvalidRequest") && body.contains("the current secret key is incorrect"));
|
||||
|
||||
let (status, body) = session_request(http::Method::POST, ACCOUNT_PASSWORD_PATH, password_request(&old_password)).await?;
|
||||
assert_eq!(status, reqwest::StatusCode::OK, "the same session must accept the current IAM password");
|
||||
assert_eq!(serde_json::from_str::<serde_json::Value>(&body)?["sessions_revoked"], 1);
|
||||
|
||||
let (status, _) = session_request(http::Method::GET, ACCOUNT_INFO_PATH, None).await?;
|
||||
assert_eq!(status, reqwest::StatusCode::FORBIDDEN, "the pre-rotation session must be revoked");
|
||||
let (status, _) = assume_role(&env.url, user, &old_password, None).await?;
|
||||
assert_eq!(status, reqwest::StatusCode::FORBIDDEN, "the old password must no longer sign in");
|
||||
let (status, response) = assume_role(&env.url, user, &new_password, None).await?;
|
||||
assert_eq!(status, reqwest::StatusCode::OK, "the new password must sign in");
|
||||
parse_assume_role_credentials(&response)?;
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -15,9 +15,9 @@
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use http::header::HOST;
|
||||
use reqwest::StatusCode;
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use serde::Deserialize;
|
||||
use std::error::Error;
|
||||
|
||||
|
||||
@@ -17,9 +17,9 @@ use aws_sdk_s3::primitives::ByteStream;
|
||||
use http::header::HOST;
|
||||
use reqwest::StatusCode;
|
||||
use rustfs_madmin::{ITEM_OFFLINE, InfoMessage, StorageInfo};
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use serde::Deserialize;
|
||||
use std::error::Error;
|
||||
use tokio::time::{Duration, sleep, timeout};
|
||||
|
||||
@@ -21,9 +21,9 @@ mod tests {
|
||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use reqwest::StatusCode;
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::{pre_sign_v4, sign_v4};
|
||||
use s3s::Body;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::error::Error;
|
||||
use std::io::{Cursor, Write};
|
||||
|
||||
@@ -137,40 +137,6 @@ async fn test_bucket_policy_authenticated_user() -> Result<(), Box<dyn std::erro
|
||||
.await
|
||||
.map_err(|e| format!("DeleteObject failed: {}", e))?;
|
||||
|
||||
for (allowed, may_read) in [("s3:GetBucketPolicy", true), ("s3:PutBucketPolicy", false)] {
|
||||
let policy = serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Principal": { "AWS": [user_access] },
|
||||
"Action": allowed,
|
||||
"Resource": format!("arn:aws:s3:::{bucket_name}")
|
||||
}]
|
||||
})
|
||||
.to_string();
|
||||
admin_client
|
||||
.put_bucket_policy()
|
||||
.bucket(bucket_name)
|
||||
.policy(&policy)
|
||||
.send()
|
||||
.await?;
|
||||
let read = user_client.get_bucket_policy().bucket(bucket_name).send().await;
|
||||
let write = user_client
|
||||
.put_bucket_policy()
|
||||
.bucket(bucket_name)
|
||||
.policy(&policy)
|
||||
.send()
|
||||
.await;
|
||||
if may_read {
|
||||
read?;
|
||||
let denied = write.expect_err("GetBucketPolicy must not grant PutBucketPolicy");
|
||||
assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
|
||||
} else {
|
||||
let denied = read.expect_err("PutBucketPolicy must not grant GetBucketPolicy");
|
||||
assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
|
||||
write?;
|
||||
}
|
||||
}
|
||||
info!("Test Passed!");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -37,9 +37,9 @@
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, local_http_client};
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::error::Error;
|
||||
@@ -170,15 +170,6 @@ impl DiskFaultHarness {
|
||||
/// Start (or restart after [`Self::kill_server`]) the RustFS server with
|
||||
/// all volume directories and the same address.
|
||||
pub async fn start_server(&mut self) -> ChaosResult<()> {
|
||||
self.start_server_with_readiness(false).await
|
||||
}
|
||||
|
||||
/// Start a process once its node read-readiness probe passes.
|
||||
pub async fn start_server_at_read_quorum(&mut self) -> ChaosResult<()> {
|
||||
self.start_server_with_readiness(true).await
|
||||
}
|
||||
|
||||
async fn start_server_with_readiness(&mut self, read_quorum_only: bool) -> ChaosResult<()> {
|
||||
// The common helper always appends `env.temp_dir` as the final storage
|
||||
// path: point it at the last disk for startup and pass the remaining
|
||||
// disks explicitly, then restore it to the root so Drop cleans up
|
||||
@@ -190,13 +181,7 @@ impl DiskFaultHarness {
|
||||
let arg_refs: Vec<&str> = disk_args.iter().map(String::as_str).collect();
|
||||
let env_refs: Vec<(&str, &str)> = self.extra_env.iter().map(|(k, v)| (k.as_str(), v.as_str())).collect();
|
||||
|
||||
let result = if read_quorum_only {
|
||||
self.env
|
||||
.start_rustfs_server_with_env_at_read_quorum(arg_refs, &env_refs)
|
||||
.await
|
||||
} else {
|
||||
self.env.start_rustfs_server_with_env(arg_refs, &env_refs).await
|
||||
};
|
||||
let result = self.env.start_rustfs_server_with_env(arg_refs, &env_refs).await;
|
||||
self.env.temp_dir = self.root.to_string_lossy().to_string();
|
||||
result
|
||||
}
|
||||
|
||||
@@ -29,9 +29,9 @@ use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use reqwest::Client as HttpClient;
|
||||
use reqwest::StatusCode;
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use serde_json;
|
||||
use std::fs as stdfs;
|
||||
use std::io::{self, ErrorKind};
|
||||
@@ -722,7 +722,7 @@ impl RustFSTestEnvironment {
|
||||
cleanup_existing: bool,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let binary_path = rustfs_binary_path();
|
||||
self.start_rustfs_server_inner_with_binary(&binary_path, extra_args, extra_env, cleanup_existing, false)
|
||||
self.start_rustfs_server_inner_with_binary(&binary_path, extra_args, extra_env, cleanup_existing)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -732,7 +732,6 @@ impl RustFSTestEnvironment {
|
||||
extra_args: Vec<&str>,
|
||||
extra_env: &[(&str, &str)],
|
||||
cleanup_existing: bool,
|
||||
read_quorum_only: bool,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
if cleanup_existing {
|
||||
self.cleanup_existing_processes().await?;
|
||||
@@ -757,7 +756,7 @@ impl RustFSTestEnvironment {
|
||||
self.process = Some(process);
|
||||
|
||||
// Wait for server to be ready
|
||||
self.wait_for_server_ready_with_probe(read_quorum_only).await?;
|
||||
self.wait_for_server_ready().await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -771,7 +770,7 @@ impl RustFSTestEnvironment {
|
||||
extra_args: Vec<&str>,
|
||||
extra_env: &[(&str, &str)],
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, true, false)
|
||||
self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, true)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -783,7 +782,7 @@ impl RustFSTestEnvironment {
|
||||
extra_args: Vec<&str>,
|
||||
extra_env: &[(&str, &str)],
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, false, false)
|
||||
self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, false)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -801,18 +800,6 @@ impl RustFSTestEnvironment {
|
||||
self.start_rustfs_server_inner(extra_args, extra_env, true).await
|
||||
}
|
||||
|
||||
/// Start a degraded read-serving process using the node readiness probe.
|
||||
/// ListBuckets retains its namespace write-quorum requirement.
|
||||
pub async fn start_rustfs_server_with_env_at_read_quorum(
|
||||
&mut self,
|
||||
extra_args: Vec<&str>,
|
||||
extra_env: &[(&str, &str)],
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let binary_path = rustfs_binary_path();
|
||||
self.start_rustfs_server_inner_with_binary(&binary_path, extra_args, extra_env, true, true)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Start RustFS server without cleaning up other running RustFS processes.
|
||||
///
|
||||
/// This is useful for tests that need multiple independent RustFS instances
|
||||
@@ -841,16 +828,8 @@ impl RustFSTestEnvironment {
|
||||
/// Fails immediately if the spawned server process has already exited
|
||||
/// (e.g. a port bind failure) instead of polling out the full timeout.
|
||||
pub async fn wait_for_server_ready(&mut self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
self.wait_for_server_ready_with_probe(false).await
|
||||
}
|
||||
|
||||
async fn wait_for_server_ready_with_probe(
|
||||
&mut self,
|
||||
read_quorum_only: bool,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
info!("Waiting for RustFS server to be ready on {}", self.address);
|
||||
let client = self.create_s3_client();
|
||||
let health_client = HttpClient::builder().no_proxy().timeout(Duration::from_secs(2)).build()?;
|
||||
|
||||
for i in 0..60 {
|
||||
if let Some(process) = self.process.as_mut()
|
||||
@@ -859,16 +838,7 @@ impl RustFSTestEnvironment {
|
||||
return Err(format!("RustFS server process exited before becoming ready: {status}").into());
|
||||
}
|
||||
|
||||
let ready = if read_quorum_only {
|
||||
health_client
|
||||
.get(format!("{}/minio/health/ready", self.url))
|
||||
.send()
|
||||
.await
|
||||
.is_ok_and(|response| response.status().is_success())
|
||||
} else {
|
||||
TcpStream::connect(&self.address).await.is_ok() && client.list_buckets().send().await.is_ok()
|
||||
};
|
||||
if ready {
|
||||
if TcpStream::connect(&self.address).await.is_ok() && client.list_buckets().send().await.is_ok() {
|
||||
info!("✅ RustFS server is ready after {} attempts", i + 1);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
@@ -13,8 +13,8 @@
|
||||
// limitations under the License.
|
||||
|
||||
//! Over-the-wire smoke net for the embedded console listener (backlog#1154
|
||||
//! peri-4). The console is a second HTTP listener for the control-plane and
|
||||
//! unauthenticated console routes under `/rustfs/console`;
|
||||
//! peri-4). The console is a second HTTP listener that serves the full S3/admin
|
||||
//! surface plus the unauthenticated console routes under `/rustfs/console`;
|
||||
//! before this test its behaviour was only covered by in-process router tests
|
||||
//! (`rustfs/src/admin/console.rs`), so a misconfiguration that exposed
|
||||
//! credentials through the public console endpoints or fail-opened the
|
||||
@@ -26,13 +26,13 @@
|
||||
//! * `/rustfs/console/license` exposes only the coarse `licensed` flag.
|
||||
//! * the console SPA prefix dispatches to the static handler, never to the
|
||||
//! S3 API (no S3 error XML), whether or not console assets are embedded.
|
||||
//! * unauthenticated admin API requests are denied with 403 AccessDenied.
|
||||
//! * S3 root/object requests on the console listener return an empty 404,
|
||||
//! including authenticated requests, before S3 dispatch.
|
||||
//! * unauthenticated requests to the admin API and the S3 root on the console
|
||||
//! listener are denied (403 AccessDenied) — the extra listener does not
|
||||
//! fail-open the protected surface.
|
||||
//! * a listener with the console disabled (the main S3 port here) does not
|
||||
//! serve the unauthenticated console endpoints at all.
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client, signed_request_with_client};
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use std::error::Error;
|
||||
use tokio::time::{Duration, sleep};
|
||||
|
||||
@@ -166,39 +166,13 @@ async fn test_console_over_the_wire_smoke() -> TestResult {
|
||||
let admin_body = admin_response.text().await?;
|
||||
assert!(admin_body.contains("AccessDenied"), "admin denial must be AccessDenied: {admin_body}");
|
||||
|
||||
let signed_data_root = signed_request_with_client(
|
||||
&client,
|
||||
http::Method::GET,
|
||||
&format!("{}/", env.url),
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
let s3_root_response = client.get(format!("{console_base}/")).send().await?;
|
||||
assert_eq!(
|
||||
signed_data_root.status(),
|
||||
reqwest::StatusCode::OK,
|
||||
"the same credentials must reach S3 on the data listener"
|
||||
s3_root_response.status(),
|
||||
reqwest::StatusCode::FORBIDDEN,
|
||||
"unauthenticated S3 root on the console listener must be denied"
|
||||
);
|
||||
|
||||
for path in ["/", "/console-object-bucket/key"] {
|
||||
let url = format!("{console_base}{path}");
|
||||
for signed in [false, true] {
|
||||
let response = if signed {
|
||||
signed_request_with_client(&client, http::Method::GET, &url, &env.access_key, &env.secret_key, None, None).await?
|
||||
} else {
|
||||
client.get(&url).send().await?
|
||||
};
|
||||
assert_eq!(
|
||||
response.status(),
|
||||
reqwest::StatusCode::NOT_FOUND,
|
||||
"S3 route {path}, signed={signed} must be isolated from the console listener"
|
||||
);
|
||||
assert!(response.bytes().await?.is_empty(), "isolated S3 routes must not reach a handler");
|
||||
}
|
||||
}
|
||||
|
||||
// --- console disabled on a listener means no console surface at all ------
|
||||
// The main S3 listener of this same process runs with the console disabled,
|
||||
// so its console paths must not answer with the unauthenticated console
|
||||
|
||||
@@ -44,9 +44,9 @@ mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use http::header::HOST;
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use std::error::Error;
|
||||
|
||||
/// Signed raw `PUT` copy request with an explicit copy-source conditional
|
||||
|
||||
@@ -21,9 +21,9 @@ mod tests {
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
||||
use http::header::HOST;
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use std::error::Error;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpStream;
|
||||
|
||||
@@ -26,7 +26,7 @@ use aws_sdk_s3::types::{
|
||||
DefaultRetention, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHoldStatus, ObjectLockRetentionMode,
|
||||
ObjectLockRule,
|
||||
};
|
||||
use jiff::{SignedDuration, Timestamp};
|
||||
use chrono::{Duration as ChronoDuration, Utc};
|
||||
|
||||
fn delete_denied(error: &SdkError<DeleteObjectError>, context: &str) -> TestResult {
|
||||
let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
|
||||
@@ -66,7 +66,7 @@ async fn four_node_four_drive_object_lock_worm_blocks_delete() -> TestResult {
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let retain_until = Timestamp::now() + SignedDuration::from_hours(24);
|
||||
let retain_until = Utc::now() + ChronoDuration::days(1);
|
||||
|
||||
let compliance_key = "compliance.bin";
|
||||
let compliance_version = put_object_with_retention(
|
||||
@@ -176,10 +176,7 @@ async fn four_node_default_retention_is_visible_and_non_lock_bucket_rejects_conf
|
||||
let retain_until = retention
|
||||
.retain_until_date()
|
||||
.ok_or("default retention omitted retain-until date")?;
|
||||
assert!(
|
||||
retain_until.secs() > Timestamp::now().as_second(),
|
||||
"default retention is not in the future"
|
||||
);
|
||||
assert!(retain_until.secs() > Utc::now().timestamp(), "default retention is not in the future");
|
||||
|
||||
let versioning = reader.get_bucket_versioning().bucket(&bucket).send().await?;
|
||||
assert_eq!(versioning.status().map(|status| status.as_str()), Some("Enabled"));
|
||||
|
||||
@@ -38,8 +38,8 @@ use aws_sdk_s3::types::{
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, ObjectLockLegalHoldStatus, ObjectLockRetentionMode,
|
||||
VersioningConfiguration,
|
||||
};
|
||||
use chrono::{DateTime, Utc};
|
||||
use http::{Method, StatusCode};
|
||||
use jiff::Timestamp;
|
||||
use serde_json::Value;
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::path::{Path, PathBuf};
|
||||
@@ -526,8 +526,8 @@ async fn put_multipart(client: &Client, bucket: &str, key: &str) -> TestResult<V
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
fn retention_deadline() -> TestResult<Timestamp> {
|
||||
Ok(Timestamp::now() + jiff::SignedDuration::from_hours(24))
|
||||
fn retention_deadline() -> TestResult<DateTime<Utc>> {
|
||||
DateTime::<Utc>::from_timestamp(Utc::now().timestamp() + 24 * 3600, 0).ok_or_else(|| "retention deadline out of range".into())
|
||||
}
|
||||
|
||||
/// Seed the deterministic inventory into the seed pool and record it.
|
||||
@@ -654,7 +654,7 @@ pub(crate) async fn seed_manifest(run: &TopologyRun) -> TestResult<TopologyManif
|
||||
let mut entry = ManifestEntry::body(&locked, &key, "compliance", Some(version_id.clone()), Some(version_id), &body);
|
||||
entry.retention = Some(RetentionRecord {
|
||||
mode: "COMPLIANCE".to_string(),
|
||||
retain_until_unix: retain_until.as_second(),
|
||||
retain_until_unix: retain_until.timestamp(),
|
||||
});
|
||||
entries.push(entry);
|
||||
}
|
||||
|
||||
@@ -31,7 +31,7 @@ use super::harness::{
|
||||
use crate::common::{
|
||||
AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, init_logging,
|
||||
};
|
||||
use crate::upgrade_compatibility_test::{UpgradeSourceCapabilities, wait_for_upgrade_write_readiness};
|
||||
use crate::upgrade_compatibility_test::wait_for_upgrade_write_readiness;
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use std::path::{Path, PathBuf};
|
||||
@@ -427,12 +427,7 @@ async fn assert_history_and_iam(dist: &DistCluster, seed: &UpgradeSeed, context:
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn compare_degraded_legacy_list_parts(
|
||||
dist: &mut DistCluster,
|
||||
seed: &UpgradeSeed,
|
||||
previous: &Path,
|
||||
source: &UpgradeSourceCapabilities,
|
||||
) -> TestResult {
|
||||
async fn compare_degraded_legacy_list_parts(dist: &mut DistCluster, seed: &UpgradeSeed, previous: &Path) -> TestResult {
|
||||
let decoded = String::from_utf8(base64_simd::URL_SAFE_NO_PAD.decode_to_vec(seed.multipart_upload.as_bytes())?)?;
|
||||
let (_, upload_dir) = decoded.split_once('.').ok_or("legacy upload ID lacks deployment delimiter")?;
|
||||
let hash = sha256_hex(format!("{}/legacy-multipart.bin", seed.history_bucket).as_bytes());
|
||||
@@ -494,22 +489,16 @@ async fn compare_degraded_legacy_list_parts(
|
||||
// Three online disks retain metadata quorum but have no part directory;
|
||||
// the fourth disk is offline. A legacy empty response is not a proof that
|
||||
// the unavailable participant has no part.
|
||||
let mut quorum_errors =
|
||||
vec![current.expect_err("current coordinator must not infer empty from missing directories and offline disks")];
|
||||
if source.requires_list_parts_read_quorum {
|
||||
quorum_errors.push(legacy.expect_err("previous coordinator must require evidence from unavailable disks"));
|
||||
} else {
|
||||
assert!(
|
||||
legacy
|
||||
.expect("previous coordinator should expose its legacy empty-list behavior")
|
||||
.parts()
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
for error in quorum_errors {
|
||||
assert_eq!(error.raw_response().expect("S3 response").status().as_u16(), 503);
|
||||
assert_eq!(error.as_service_error().and_then(|error| error.meta().code()), Some("ServiceUnavailable"));
|
||||
}
|
||||
let legacy = legacy.expect("previous coordinator should expose its legacy empty-list behavior");
|
||||
assert!(legacy.parts().is_empty());
|
||||
let current = current.expect_err("current coordinator must not infer empty from missing directories and offline disks");
|
||||
assert_eq!(current.raw_response().expect("current S3 response").status().as_u16(), 503);
|
||||
// Since #9149, SlowDownRead is reserved for GET/HEAD object reads; a
|
||||
// ListParts read-quorum failure is reported as ServiceUnavailable.
|
||||
assert_eq!(
|
||||
current.as_service_error().and_then(|error| error.meta().code()),
|
||||
Some("ServiceUnavailable")
|
||||
);
|
||||
dist.cluster.start_node_from_binary(3, previous).await?;
|
||||
wait_for_upgrade_write_readiness(&clients, &seed.history_bucket, "mixed-list-restored", CREDENTIAL_TIMEOUT).await?;
|
||||
for (path, before) in original_files {
|
||||
@@ -531,7 +520,7 @@ async fn compare_degraded_legacy_list_parts(
|
||||
assert_eq!(listed.parts()[0].e_tag(), Some(seed.multipart_etag.as_str()));
|
||||
}
|
||||
eprintln!(
|
||||
"MPU mixed degraded: one offline disk, three missing part directories; checked each coordinator against its release contract; original part restored on both"
|
||||
"MPU mixed degraded: one offline disk, three missing part directories; previous returned empty, current returned 503 SlowDownRead; original part restored on both"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -603,7 +592,6 @@ async fn replace_node_and_wait_for_write_readiness(dist: &mut DistCluster, seed:
|
||||
async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> TestResult {
|
||||
init_logging();
|
||||
let previous = source_binary()?;
|
||||
let source = UpgradeSourceCapabilities::from_binary(&previous).await?;
|
||||
let mut dist = DistCluster::new_stopped(DistLayout::FourNodeFourDisk).await?;
|
||||
dist.cluster.set_env("RUSTFS_STORAGE_CLASS_RRS", "EC:0");
|
||||
capture_upgrade_logs(&mut dist, "rolling-upgrade")?;
|
||||
@@ -613,7 +601,7 @@ async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> Te
|
||||
|
||||
replace_node_and_wait_for_write_readiness(&mut dist, &seed, 0).await?;
|
||||
assert_history_and_iam(&dist, &seed, "one-current-node").await?;
|
||||
compare_degraded_legacy_list_parts(&mut dist, &seed, &previous, &source).await?;
|
||||
compare_degraded_legacy_list_parts(&mut dist, &seed, &previous).await?;
|
||||
|
||||
for node_idx in [1, 2] {
|
||||
replace_node_and_wait_for_write_readiness(&mut dist, &seed, node_idx).await?;
|
||||
|
||||
@@ -12,22 +12,21 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Differential: two `rustfs` processes, one SDK script (rustfs/backlog#2740).
|
||||
//! Dual-stack differential: one `rustfs` binary, two processes, one SDK script (rustfs/backlog#2740).
|
||||
//!
|
||||
//! Side B is the binary under test. Side A is the binary under test too, unless
|
||||
//! `RUSTFS_E2E_REFERENCE_BINARY` names a released binary: then A is that binary, and a bad path
|
||||
//! fails the run instead of falling back (see `reference`). The report names which side is the
|
||||
//! reference and the `--version` each side ran. `RUSTFS_E2E_STACK_B_REGION`, when set, starts B
|
||||
//! in another region so that a difference can be injected by hand. Neither side is given the
|
||||
//! retired stack selector: the binary under test refuses to start with it, and the gateway stack
|
||||
//! is its only S3 stack. Both servers
|
||||
//! Stack A runs `RUSTFS_S3_STACK=legacy` of the binary under test, unless `RUSTFS_E2E_REFERENCE_BINARY`
|
||||
//! names a released binary: then A is that binary, started without `RUSTFS_S3_STACK` (older
|
||||
//! releases do not know it), and a bad path fails the run instead of falling back (see
|
||||
//! `reference`). The report names which side is the reference and the `--version` each side ran. Stack B runs the value of `RUSTFS_E2E_STACK_B`
|
||||
//! (default `legacy` until the gateway stack is assembled), and `RUSTFS_E2E_STACK_B_REGION`, when
|
||||
//! set, starts B in another region so that a difference can be injected by hand. Both servers
|
||||
//! answer the script in `scenarios`, which covers every operation of `impl S3 for FS`; `compare`
|
||||
//! reports each `(op, field, a, b)` on which the answers differ, and every run writes its report
|
||||
//! to `<target>/dual-stack/<run>.json`. Two processes, not two embedded servers: bucket metadata
|
||||
//! is process-global, so two stacks in one process would read each other's buckets.
|
||||
|
||||
mod compare;
|
||||
pub(crate) mod reference;
|
||||
mod reference;
|
||||
mod scenarios;
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, rustfs_binary_path, workspace_root};
|
||||
@@ -38,18 +37,20 @@ use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
type BoxError = Box<dyn std::error::Error + Send + Sync>;
|
||||
|
||||
/// Label of a side that runs the binary under test.
|
||||
const UNDER_TEST: &str = "under-test";
|
||||
const STACK_A: &str = "legacy";
|
||||
/// Label of side A in the report when it is the released reference binary.
|
||||
const STACK_A_REFERENCE: &str = "reference";
|
||||
const ENV_STACK_B: &str = "RUSTFS_E2E_STACK_B";
|
||||
const ENV_STACK_B_REGION: &str = "RUSTFS_E2E_STACK_B_REGION";
|
||||
/// Becomes `gateway` once rustfs/backlog#2734 T2.5 assembles the gateway stack.
|
||||
const DEFAULT_STACK_B: &str = "legacy";
|
||||
/// The region the negative control gives stack A explicitly, so an inherited `RUSTFS_REGION`
|
||||
/// cannot erase the injected difference. Not `us-east-1`: that region answers GetBucketLocation
|
||||
/// with an empty LocationConstraint and re-creating an owned bucket with 200, so it would not
|
||||
/// report its own name and would add CreateBucket differences to the control.
|
||||
const CONTROL_REGION_A: &str = "us-west-2";
|
||||
const CONTROL_REGION_B: &str = "eu-west-1";
|
||||
/// `normal` cases side A must answer with a 2xx. The comparison alone cannot tell a
|
||||
/// `normal` cases the legacy stack must answer with a 2xx. The comparison alone cannot tell a
|
||||
/// script that exercises real behavior from one in which both stacks fail the same way. Five of
|
||||
/// the 73 fail on legacy by design of this script: PutBucketReplication (no registered remote
|
||||
/// target), GetBucketReplication (so no configuration), PutBucketNotificationConfiguration (no
|
||||
@@ -59,6 +60,7 @@ const MIN_NORMAL_SUCCESSES: usize = 68;
|
||||
|
||||
struct PairConfig {
|
||||
side_a: SideA,
|
||||
stack_b: String,
|
||||
region_a: Option<String>,
|
||||
region_b: Option<String>,
|
||||
}
|
||||
@@ -67,6 +69,7 @@ impl PairConfig {
|
||||
fn from_env() -> Result<Self, BoxError> {
|
||||
Ok(Self {
|
||||
side_a: SideA::from_env()?,
|
||||
stack_b: std::env::var(ENV_STACK_B).unwrap_or_else(|_| DEFAULT_STACK_B.to_owned()),
|
||||
region_a: None,
|
||||
region_b: std::env::var(ENV_STACK_B_REGION).ok(),
|
||||
})
|
||||
@@ -74,16 +77,16 @@ impl PairConfig {
|
||||
|
||||
fn stack_a_label(&self) -> &'static str {
|
||||
match self.side_a {
|
||||
SideA::SameBinary => UNDER_TEST,
|
||||
SideA::SameBinary => STACK_A,
|
||||
SideA::Reference(_) => STACK_A_REFERENCE,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Starts one server of the binary under test, overriding its region only when asked.
|
||||
async fn start_under_test(region: Option<&str>) -> Result<RustFSTestEnvironment, BoxError> {
|
||||
/// Starts one server of the binary under test on `stack`, overriding its region only when asked.
|
||||
async fn start_stack(stack: &str, region: Option<&str>) -> Result<RustFSTestEnvironment, BoxError> {
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
let mut vars = Vec::new();
|
||||
let mut vars = vec![("RUSTFS_S3_STACK", stack)];
|
||||
if let Some(region) = region {
|
||||
vars.push(("RUSTFS_REGION", region));
|
||||
}
|
||||
@@ -91,7 +94,8 @@ async fn start_under_test(region: Option<&str>) -> Result<RustFSTestEnvironment,
|
||||
Ok(env)
|
||||
}
|
||||
|
||||
/// Starts one server of a released binary.
|
||||
/// Starts one server of a released binary. `RUSTFS_S3_STACK` is not passed: older releases do not
|
||||
/// know it and have exactly one stack.
|
||||
async fn start_reference(binary: &Path, region: Option<&str>) -> Result<RustFSTestEnvironment, BoxError> {
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
let mut vars = Vec::new();
|
||||
@@ -107,11 +111,11 @@ async fn start_reference(binary: &Path, region: Option<&str>) -> Result<RustFSTe
|
||||
async fn spawn_pair(config: &PairConfig) -> Result<(RustFSTestEnvironment, RustFSTestEnvironment), BoxError> {
|
||||
let side_a = async {
|
||||
match &config.side_a {
|
||||
SideA::SameBinary => start_under_test(config.region_a.as_deref()).await,
|
||||
SideA::SameBinary => start_stack(STACK_A, config.region_a.as_deref()).await,
|
||||
SideA::Reference(binary) => start_reference(binary, config.region_a.as_deref()).await,
|
||||
}
|
||||
};
|
||||
tokio::try_join!(side_a, start_under_test(config.region_b.as_deref()))
|
||||
tokio::try_join!(side_a, start_stack(&config.stack_b, config.region_b.as_deref()))
|
||||
}
|
||||
|
||||
fn report_directory() -> PathBuf {
|
||||
@@ -156,7 +160,7 @@ async fn run_differential(label: &str, config: &PairConfig) -> Run {
|
||||
schema: 2,
|
||||
run: format!("{label}-{millis}-{}", std::process::id()),
|
||||
stack_a: config.stack_a_label().to_owned(),
|
||||
stack_b: UNDER_TEST.to_owned(),
|
||||
stack_b: config.stack_b.clone(),
|
||||
reference_side: matches!(config.side_a, SideA::Reference(_)).then_some("a"),
|
||||
version_a,
|
||||
version_b,
|
||||
@@ -231,6 +235,7 @@ mod tests {
|
||||
init_logging();
|
||||
let config = PairConfig {
|
||||
side_a: SideA::SameBinary,
|
||||
stack_b: STACK_A.to_owned(),
|
||||
region_a: Some(CONTROL_REGION_A.to_owned()),
|
||||
region_b: Some(CONTROL_REGION_B.to_owned()),
|
||||
};
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
//! Which binary side A of the differential runs, and what it reports about itself.
|
||||
//!
|
||||
//! `RUSTFS_E2E_REFERENCE_BINARY` names a released `rustfs` binary to use as the reference. Unset,
|
||||
//! side A is the binary under test. Set, it must resolve to an
|
||||
//! side A is the binary under test on `RUSTFS_S3_STACK=legacy`. Set, it must resolve to an
|
||||
//! executable file or the run fails: a reference that was asked for and silently replaced by the
|
||||
//! binary under test would report an agreement with itself as agreement with the release.
|
||||
|
||||
@@ -28,9 +28,9 @@ pub(super) const ENV_REFERENCE_BINARY: &str = "RUSTFS_E2E_REFERENCE_BINARY";
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(super) enum SideA {
|
||||
/// Both sides are the binary under test.
|
||||
/// Both sides are the binary under test; A runs `RUSTFS_S3_STACK=legacy`.
|
||||
SameBinary,
|
||||
/// A is this released binary.
|
||||
/// A is this released binary, started without `RUSTFS_S3_STACK`.
|
||||
Reference(PathBuf),
|
||||
}
|
||||
|
||||
@@ -78,7 +78,7 @@ fn is_executable(_metadata: &std::fs::Metadata) -> bool {
|
||||
|
||||
/// What `<binary> --version` printed, trimmed. Fails when the binary does not run, exits
|
||||
/// non-zero, or prints nothing, because an empty version would report nothing as observed.
|
||||
pub(crate) async fn binary_version(binary: &Path) -> Result<String, BoxError> {
|
||||
pub(super) async fn binary_version(binary: &Path) -> Result<String, BoxError> {
|
||||
let output = tokio::process::Command::new(binary).arg("--version").output().await?;
|
||||
if !output.status.success() {
|
||||
return Err(format!("{} --version failed with {}", binary.display(), output.status).into());
|
||||
|
||||
@@ -29,11 +29,10 @@ use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::ServerSideEncryption;
|
||||
use base64_simd::STANDARD as BASE64;
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use jiff::Timestamp;
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use serde_json;
|
||||
use std::process::{Child, Command};
|
||||
use std::time::Duration;
|
||||
@@ -387,7 +386,7 @@ pub async fn create_key_with_specific_id(key_dir: &str, key_id: &str) -> Result<
|
||||
"usage": "EncryptDecrypt",
|
||||
"status": "Active",
|
||||
"metadata": HashMap::<String, String>::new(),
|
||||
"created_at": format!("{}[UTC]", Timestamp::now().strftime("%Y-%m-%dT%H:%M:%S%.fZ")),
|
||||
"created_at": format!("{}[UTC]", chrono::Utc::now().to_rfc3339()),
|
||||
"rotated_at": serde_json::Value::Null,
|
||||
"created_by": "e2e-test",
|
||||
"encrypted_key_material": BASE64.encode_to_string(key_data),
|
||||
|
||||
@@ -31,15 +31,12 @@ pub mod chaos;
|
||||
#[cfg(test)]
|
||||
pub mod fake_s3_target;
|
||||
|
||||
// Differential (rustfs/backlog#2740): two processes, the binary under test against
|
||||
// itself or a released reference binary, the same SDK script for every S3 operation,
|
||||
// Dual-stack differential (rustfs/backlog#2740): one binary, two processes on
|
||||
// different RUSTFS_S3_STACK values, the same SDK script for every S3 operation,
|
||||
// compared on status, error code, allowlisted headers and typed body fields.
|
||||
#[cfg(test)]
|
||||
mod dual_stack_differential;
|
||||
|
||||
#[cfg(test)]
|
||||
mod persistence_parity;
|
||||
|
||||
// On-demand migration (backlog#2147): shared two-server environment, admin
|
||||
// wrappers, and the harness self-test (backlog#2151). Behavior scenarios are
|
||||
// added by later ODM tasks.
|
||||
|
||||
@@ -24,9 +24,9 @@
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use http::header::HOST;
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use std::error::Error;
|
||||
|
||||
/// Sends a SigV4-signed `GET` where the signature is computed over `sign_path`
|
||||
|
||||
@@ -21,9 +21,9 @@ mod tests {
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
||||
use http::header::HOST;
|
||||
use reqwest::StatusCode;
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use std::error::Error;
|
||||
use tracing::info;
|
||||
|
||||
|
||||
@@ -20,9 +20,9 @@ mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use http::header::HOST;
|
||||
use reqwest::StatusCode;
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use std::error::Error;
|
||||
use tracing::info;
|
||||
|
||||
|
||||
@@ -23,17 +23,17 @@ use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
ServerSideEncryption, ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule,
|
||||
};
|
||||
use chrono::{Duration as ChronoDuration, Utc};
|
||||
use flate2::{
|
||||
Compression,
|
||||
write::{GzEncoder, ZlibEncoder},
|
||||
};
|
||||
use http::HeaderValue;
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use jiff::{SignedDuration, Timestamp};
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use std::collections::HashMap;
|
||||
use std::error::Error;
|
||||
use std::io::Cursor;
|
||||
@@ -42,8 +42,8 @@ use tokio::io::AsyncWriteExt;
|
||||
use uuid::Uuid;
|
||||
|
||||
fn encode_post_policy(conditions: Vec<serde_json::Value>) -> String {
|
||||
let expiration = (Timestamp::now() + SignedDuration::from_hours(1))
|
||||
.strftime("%Y-%m-%dT%H:%M:%S.000Z")
|
||||
let expiration = (Utc::now() + ChronoDuration::hours(1))
|
||||
.format("%Y-%m-%dT%H:%M:%S.000Z")
|
||||
.to_string();
|
||||
let policy = serde_json::json!({
|
||||
"expiration": expiration,
|
||||
@@ -1543,30 +1543,6 @@ async fn test_anonymous_post_object_honors_success_action_status() -> Result<(),
|
||||
let uploaded = get_out.body.collect().await?.into_bytes();
|
||||
assert_eq!(uploaded.as_ref(), expected_body.as_slice(), "uploaded object body should match");
|
||||
|
||||
// An empty redirect is rejected when rendering the response, after storage succeeds.
|
||||
for (field, value, expected_status) in [
|
||||
("success_action_status", "200", reqwest::StatusCode::OK),
|
||||
("success_action_status", "204", reqwest::StatusCode::NO_CONTENT),
|
||||
("success_action_redirect", "", reqwest::StatusCode::BAD_REQUEST),
|
||||
] {
|
||||
let key = format!("{field}-{value}.txt");
|
||||
let form = reqwest::multipart::Form::new()
|
||||
.text("key", key.clone())
|
||||
.text(field, value)
|
||||
.part("file", reqwest::multipart::Part::bytes(expected_body.clone()).file_name("upload.txt"));
|
||||
let response = local_http_client()
|
||||
.post(format!("{}/{bucket}", env.url))
|
||||
.multipart(form)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(response.status(), expected_status);
|
||||
if value.is_empty() {
|
||||
assert!(response.text().await?.contains("<Code>InvalidArgument</Code>"));
|
||||
}
|
||||
let stored = admin_client.get_object().bucket(bucket).key(key).send().await?;
|
||||
assert_eq!(stored.body.collect().await?.into_bytes().as_ref(), expected_body.as_slice());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -283,7 +283,6 @@ async fn test_cold_degraded_object_mutations_require_only_storage_quorum() -> Te
|
||||
#[tokio::test]
|
||||
async fn test_degraded_cluster_read_quorum_follows_erasure_layout() -> TestResult {
|
||||
crate::common::init_logging();
|
||||
assert_single_node_cold_start_with_missing_volume().await?;
|
||||
|
||||
for (node_count, parity) in [(3, 1), (4, 2), (6, 3), (6, 2), (4, 0)] {
|
||||
let read_quorum = node_count - parity;
|
||||
@@ -431,138 +430,6 @@ async fn test_degraded_cluster_read_quorum_follows_erasure_layout() -> TestResul
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_single_node_cold_start_with_missing_volume() -> TestResult {
|
||||
for drive_count in [2, 3, 4] {
|
||||
let mut server = crate::chaos::DiskFaultHarness::new(drive_count).await?;
|
||||
server.set_env("RUSTFS_STORAGE_CLASS_STANDARD", format!("EC:{}", drive_count / 2));
|
||||
server.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false");
|
||||
server.set_env("RUSTFS_OBS_METRICS_EXPORT_ENABLED", "false");
|
||||
server.set_env("RUSTFS_HEALTH_ENDPOINT_ENABLE", "true");
|
||||
server.set_env("RUST_LOG", "warn");
|
||||
server.start_server_at_read_quorum().await?;
|
||||
let client = server.env.create_s3_client();
|
||||
let client = Client::from_conf(
|
||||
client
|
||||
.config()
|
||||
.to_builder()
|
||||
.retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(1))
|
||||
.build(),
|
||||
);
|
||||
let bucket = format!("missing-volume-{drive_count}");
|
||||
client.create_bucket().bucket(&bucket).send().await?;
|
||||
let small = vec![17; 4096];
|
||||
let large = vec![73; 8 * 1024 * 1024];
|
||||
for (key, body) in [("small", &small), ("large", &large)] {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.body(Bytes::copy_from_slice(body).into())
|
||||
.send()
|
||||
.await?;
|
||||
for disk in 0..drive_count {
|
||||
let census = crate::chaos::census_object_version_on_disk(server.disk_path(disk), &bucket, key, None)?;
|
||||
assert!(census.is_complete(), "seed shard must exist before restart: {census:?}");
|
||||
}
|
||||
}
|
||||
server.kill_server();
|
||||
let missing = drive_count - 1;
|
||||
server.take_disk_offline(missing)?;
|
||||
server.start_server_at_read_quorum().await?;
|
||||
|
||||
for (key, body) in [("small", &small), ("large", &large)] {
|
||||
assert_quorum_object_body(&client, &bucket, key, body).await?;
|
||||
let head = client.head_object().bucket(&bucket).key(key).send().await?;
|
||||
assert_eq!(head.content_length(), Some(i64::try_from(body.len())?));
|
||||
}
|
||||
let listing = client.list_objects_v2().bucket(&bucket).send().await?;
|
||||
let keys: Vec<_> = listing.contents().iter().filter_map(|object| object.key()).collect();
|
||||
assert_eq!(keys, ["large", "small"]);
|
||||
|
||||
let put = client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key("degraded-write")
|
||||
.body(Bytes::from_static(b"degraded write").into())
|
||||
.send()
|
||||
.await;
|
||||
if drive_count == 2 {
|
||||
let error = put.expect_err("one of two drives cannot commit a new object");
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(503));
|
||||
let error = client
|
||||
.delete_object()
|
||||
.bucket(&bucket)
|
||||
.key("large")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("one of two drives cannot commit an object deletion");
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(503));
|
||||
assert_quorum_object_body(&client, &bucket, "large", &large).await?;
|
||||
} else {
|
||||
put?;
|
||||
assert_quorum_object_body(&client, &bucket, "degraded-write", b"degraded write").await?;
|
||||
client.delete_object().bucket(&bucket).key("degraded-write").send().await?;
|
||||
}
|
||||
assert!(
|
||||
!server.disk_path(missing).exists(),
|
||||
"degraded startup and S3 operations must leave the root absent"
|
||||
);
|
||||
|
||||
if drive_count == 2 {
|
||||
server.kill_server();
|
||||
std::fs::create_dir(server.disk_path(missing))?;
|
||||
server.start_server_at_read_quorum().await?;
|
||||
for (key, body) in [("small", &small), ("large", &large)] {
|
||||
assert_quorum_object_body(&client, &bucket, key, body).await?;
|
||||
client.head_object().bucket(&bucket).key(key).send().await?;
|
||||
}
|
||||
assert_eq!(client.list_objects_v2().bucket(&bucket).send().await?.contents().len(), 2);
|
||||
let error = client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key("empty-root-write")
|
||||
.body(Bytes::from_static(b"must remain unavailable").into())
|
||||
.send()
|
||||
.await
|
||||
.expect_err("an empty replacement cannot restore write quorum");
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(503));
|
||||
assert!(
|
||||
!server
|
||||
.disk_path(missing)
|
||||
.join(crate::storage_api::RUSTFS_META_BUCKET)
|
||||
.join("format.json")
|
||||
.exists(),
|
||||
"read admission must not format the empty replacement"
|
||||
);
|
||||
}
|
||||
server.kill_server();
|
||||
server.bring_disk_online(missing)?;
|
||||
server.start_server_at_read_quorum().await?;
|
||||
for (key, body) in [("small", &small), ("large", &large)] {
|
||||
assert_quorum_object_body(&client, &bucket, key, body).await?;
|
||||
}
|
||||
for key in ["degraded-write", "empty-root-write"] {
|
||||
let error = client
|
||||
.head_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("rejected or deleted degraded write must stay absent after recovery");
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(404));
|
||||
}
|
||||
client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key("recovery-write")
|
||||
.body(Bytes::from_static(b"recovered write").into())
|
||||
.send()
|
||||
.await?;
|
||||
assert_quorum_object_body(&client, &bucket, "recovery-write", b"recovered write").await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cold_start_read_quorum_follows_erasure_layout() -> TestResult {
|
||||
crate::common::init_logging();
|
||||
|
||||
@@ -17,9 +17,9 @@ use aws_sdk_s3::primitives::ByteStream;
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use reqwest::StatusCode;
|
||||
use rustfs_config::{ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, ENV_NOTIFY_ENABLE};
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::pre_sign_v4;
|
||||
use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS;
|
||||
use s3s::Body;
|
||||
use std::collections::HashMap;
|
||||
use std::error::Error;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
@@ -29,7 +29,7 @@ use aws_sdk_s3::types::{
|
||||
DefaultRetention, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHold, ObjectLockLegalHoldStatus, ObjectLockMode,
|
||||
ObjectLockRetention, ObjectLockRetentionMode, ObjectLockRule,
|
||||
};
|
||||
use jiff::{SignedDuration, Timestamp};
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use tracing::info;
|
||||
|
||||
use crate::common::RustFSTestEnvironment;
|
||||
@@ -126,10 +126,10 @@ pub async fn put_object_with_retention(
|
||||
key: &str,
|
||||
data: &[u8],
|
||||
mode: ObjectLockRetentionMode,
|
||||
retain_until: Timestamp,
|
||||
retain_until: DateTime<Utc>,
|
||||
) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
|
||||
// AWS SDK requires UTC time without timezone offset (e.g., "2026-01-24T11:20:14Z")
|
||||
let retain_until_str = retain_until.strftime("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||||
let retain_until_str = retain_until.format("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||||
let retain_until_datetime =
|
||||
aws_sdk_s3::primitives::DateTime::from_str(&retain_until_str, aws_sdk_s3::primitives::DateTimeFormat::DateTime)?;
|
||||
|
||||
@@ -181,10 +181,10 @@ pub async fn put_object_retention(
|
||||
key: &str,
|
||||
version_id: Option<&str>,
|
||||
mode: ObjectLockRetentionMode,
|
||||
retain_until: Timestamp,
|
||||
retain_until: DateTime<Utc>,
|
||||
bypass_governance: bool,
|
||||
) -> Result<(), Box<SdkError<PutObjectRetentionError>>> {
|
||||
let retain_until_datetime = aws_sdk_s3::primitives::DateTime::from_secs(retain_until.as_second());
|
||||
let retain_until_datetime = aws_sdk_s3::primitives::DateTime::from_secs(retain_until.timestamp());
|
||||
|
||||
let retention = ObjectLockRetention::builder()
|
||||
.mode(mode.clone())
|
||||
@@ -253,6 +253,6 @@ pub async fn delete_object_with_bypass(
|
||||
}
|
||||
|
||||
/// Calculate a future retain_until date
|
||||
pub fn future_retain_until(days: i64) -> Timestamp {
|
||||
Timestamp::now() + SignedDuration::from_hours(days * 24)
|
||||
pub fn future_retain_until(days: i64) -> DateTime<Utc> {
|
||||
Utc::now() + Duration::days(days)
|
||||
}
|
||||
|
||||
@@ -34,7 +34,7 @@ use aws_sdk_s3::types::{
|
||||
CompletedMultipartUpload, CompletedPart, Delete, MetadataDirective, ObjectIdentifier, ObjectLockLegalHoldStatus,
|
||||
ObjectLockMode, ObjectLockRetentionMode,
|
||||
};
|
||||
use jiff::{SignedDuration, Timestamp};
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use tracing::info;
|
||||
|
||||
/// Initialize test logging
|
||||
@@ -67,7 +67,7 @@ async fn put_bucket_deny_policy(
|
||||
}
|
||||
|
||||
fn retention_timestamp(days: i64) -> aws_sdk_s3::primitives::DateTime {
|
||||
let retain_until = future_retain_until(days).strftime("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||||
let retain_until = future_retain_until(days).format("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||||
aws_sdk_s3::primitives::DateTime::from_str(&retain_until, aws_sdk_s3::primitives::DateTimeFormat::DateTime)
|
||||
.expect("retention timestamp should parse")
|
||||
}
|
||||
@@ -118,14 +118,13 @@ where
|
||||
);
|
||||
}
|
||||
|
||||
fn parse_s3_datetime(value: &aws_sdk_s3::primitives::DateTime) -> Timestamp {
|
||||
fn parse_s3_datetime(value: &aws_sdk_s3::primitives::DateTime) -> DateTime<Utc> {
|
||||
let formatted = value
|
||||
.fmt(DateTimeFormat::DateTime)
|
||||
.expect("S3 timestamp should format as RFC3339");
|
||||
let normalized = formatted
|
||||
.strip_suffix('Z')
|
||||
.map_or(formatted.clone(), |prefix| format!("{prefix}+00:00"));
|
||||
Timestamp::strptime("%Y-%m-%dT%H:%M:%S%.f%:z", normalized).expect("S3 timestamp should parse as RFC3339")
|
||||
DateTime::parse_from_rfc3339(&formatted)
|
||||
.expect("S3 timestamp should parse as RFC3339")
|
||||
.with_timezone(&Utc)
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
@@ -471,7 +470,7 @@ async fn test_get_object_retention_returns_configured_values() {
|
||||
let bucket = "test-get-retention";
|
||||
let key = "retained-object";
|
||||
let retain_until = future_retain_until(30);
|
||||
let retain_until_expected = retain_until.strftime("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||||
let retain_until_expected = retain_until.format("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||||
|
||||
env.create_object_lock_bucket(bucket).await.unwrap();
|
||||
|
||||
@@ -1925,7 +1924,7 @@ async fn test_copy_object_retention_uses_destination_policy() {
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let copy_started = Timestamp::now();
|
||||
let copy_started = Utc::now();
|
||||
client
|
||||
.copy_object()
|
||||
.copy_source(format!("{src_bucket}/{src_key}"))
|
||||
@@ -1948,7 +1947,7 @@ async fn test_copy_object_retention_uses_destination_policy() {
|
||||
assert_eq!(retention.mode().map(|value| value.as_str()), Some("GOVERNANCE"));
|
||||
let retain_until = parse_s3_datetime(retention.retain_until_date().expect("retain-until date should be present"));
|
||||
assert!(
|
||||
retain_until < copy_started + SignedDuration::from_hours(3 * 24),
|
||||
retain_until < copy_started + Duration::days(3),
|
||||
"destination default retention should not inherit the source's longer retention"
|
||||
);
|
||||
|
||||
@@ -2009,7 +2008,7 @@ async fn test_copy_object_retention_uses_destination_policy() {
|
||||
.expect("explicit retain-until date should be present"),
|
||||
);
|
||||
assert!(
|
||||
explicit_retain_until > Timestamp::now() + SignedDuration::from_hours(20 * 24),
|
||||
explicit_retain_until > Utc::now() + Duration::days(20),
|
||||
"explicit retention should override the shorter bucket default"
|
||||
);
|
||||
|
||||
@@ -2089,7 +2088,7 @@ async fn test_multipart_default_retention_fixed_at_create() {
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let create_started = Timestamp::now();
|
||||
let create_started = Utc::now();
|
||||
let create_output = client.create_multipart_upload().bucket(bucket).key(key).send().await.unwrap();
|
||||
let upload_id = create_output.upload_id().unwrap();
|
||||
|
||||
@@ -2132,7 +2131,7 @@ async fn test_multipart_default_retention_fixed_at_create() {
|
||||
assert_eq!(retention.mode().map(|value| value.as_str()), Some("GOVERNANCE"));
|
||||
let retain_until = parse_s3_datetime(retention.retain_until_date().expect("retain-until date should be present"));
|
||||
assert!(
|
||||
retain_until < create_started + SignedDuration::from_hours(3 * 24),
|
||||
retain_until < create_started + Duration::days(3),
|
||||
"CompleteMultipartUpload should keep the default retention calculated at create time"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -693,7 +693,7 @@ impl OdmTestEnv {
|
||||
.uri(uri)
|
||||
.header(http::header::HOST, authority)
|
||||
.header("x-amz-content-sha256", rustfs_signer::constants::UNSIGNED_PAYLOAD)
|
||||
.body(rustfs_s3_types::Body::empty())?;
|
||||
.body(s3s::Body::empty())?;
|
||||
let signed = rustfs_signer::sign_v4(request, 0, &self.rustfs.access_key, &self.rustfs.secret_key, "", "us-east-1");
|
||||
|
||||
let mut builder =
|
||||
|
||||
@@ -1,449 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Compare bytes produced by S3 writes to independent, single-drive volumes.
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, admin_request, rustfs_binary_path, signed_s3_request_with_headers};
|
||||
use crate::dual_stack_differential::reference::binary_version;
|
||||
use crate::fake_s3_target::{FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target};
|
||||
use crate::on_demand_migration::common::{ODM_ADMIN_ROUTE, ODM_MODULE_SWITCH_ENV, OdmSourceSpec};
|
||||
use crate::replication_extension_test::send_set_replication_target_request;
|
||||
use crate::storage_api::persistence_parity::{BucketMetadata, RUSTFS_META_BUCKET};
|
||||
use crate::upgrade_compatibility_test::{set_bucket_quota, write_multipart};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{BucketCannedAcl, Tag, Tagging};
|
||||
use http::{HeaderMap, Method};
|
||||
use md5::{Digest, Md5};
|
||||
use rustfs_filemeta::{FileMeta, MetaObject};
|
||||
use serde_value::Value;
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
type Error = Box<dyn std::error::Error + Send + Sync>;
|
||||
const CONFIGS: &str = "persisted-configs";
|
||||
const OBJECTS: &str = "persisted-objects";
|
||||
const FIRST_PART_SIZE: usize = 5 * 1024 * 1024;
|
||||
const LOGS: &str = "rustfs-access-logs";
|
||||
|
||||
fn bucket_metadata(volume: &Path, bucket: &str) -> Result<BucketMetadata, Error> {
|
||||
let key = BucketMetadata::new(bucket).save_file_path();
|
||||
let object = volume.join(RUSTFS_META_BUCKET).join(&key);
|
||||
let meta = FileMeta::load(&std::fs::read(object.join("xl.meta"))?)?;
|
||||
let info = meta.into_fileinfo(RUSTFS_META_BUCKET, &key, "", true, false, true)?;
|
||||
info.validate_for_metadata_read()?;
|
||||
let [part] = info.parts.as_slice() else { return Err("bucket metadata must have one part".into()) };
|
||||
let size = usize::try_from(info.size)?;
|
||||
if info.erasure.data_blocks != 1 || size != part.size || size > info.erasure.block_size {
|
||||
return Err("bucket metadata exceeds the single-drive, single-stripe fixture".into());
|
||||
}
|
||||
let framed = if info.inline_data() {
|
||||
info.data.ok_or("missing inline bucket metadata")?.to_vec()
|
||||
} else {
|
||||
let directory = info.data_dir.ok_or("missing bucket metadata data directory")?;
|
||||
std::fs::read(object.join(directory.to_string()).join(format!("part.{}", part.number)))?
|
||||
};
|
||||
let algorithm = info.erasure.get_checksum_info(part.number).algorithm;
|
||||
let (digest, payload) = framed.split_at_checked(algorithm.size()).ok_or("truncated bucket metadata")?;
|
||||
if digest.is_empty() || payload.len() != size || algorithm.hash_encode(payload).as_ref() != digest {
|
||||
return Err("invalid bucket metadata shard checksum or size".into());
|
||||
}
|
||||
BucketMetadata::check_header(payload)?;
|
||||
let fields = &payload[4..];
|
||||
if bucket == CONFIGS {
|
||||
// The production decoder defaults a missing field to empty. Require the ACL field on disk
|
||||
// so dropping it cannot look like the expected zero-byte document.
|
||||
let keys: BTreeMap<&str, serde::de::IgnoredAny> = rmp_serde::from_slice(fields)?;
|
||||
if !keys.contains_key("BucketAclConfigJSON") {
|
||||
return Err("missing BucketAclConfigJSON field".into());
|
||||
}
|
||||
}
|
||||
Ok(BucketMetadata::unmarshal(fields)?)
|
||||
}
|
||||
|
||||
type VersionFields = BTreeMap<String, Value>;
|
||||
|
||||
fn read_object(volume: &Path, key: &str) -> Result<(MetaObject, VersionFields), Error> {
|
||||
let meta = FileMeta::load(&std::fs::read(volume.join(OBJECTS).join(key).join("xl.meta"))?)?;
|
||||
let [version] = meta.versions.as_slice() else {
|
||||
return Err(format!("{key}: expected one version").into());
|
||||
};
|
||||
let object = version.parse_version_meta()?.object.ok_or("missing V2 object")?;
|
||||
if object.mod_time.is_none() || object.data_dir.is_none_or(|id| id.is_nil()) {
|
||||
return Err(format!("{key}: missing modification time or data directory").into());
|
||||
}
|
||||
Ok((object, nonvolatile_fields(&version.meta, key)?))
|
||||
}
|
||||
|
||||
fn nonvolatile_fields(raw: &[u8], key: &str) -> Result<VersionFields, Error> {
|
||||
// The production reader skips unknown keys and supplies defaults for some missing keys.
|
||||
// Keep the raw maps so both the version envelope and V2Obj preserve every field and value.
|
||||
let mut fields: VersionFields = rmp_serde::from_slice(raw)?;
|
||||
let Some(Value::Map(object_fields)) = fields.get_mut("V2Obj") else {
|
||||
return Err(format!("{key}: missing raw V2Obj map").into());
|
||||
};
|
||||
for volatile in ["MTime", "DDir"] {
|
||||
let value = object_fields
|
||||
.get_mut(&Value::String(volatile.to_owned()))
|
||||
.ok_or_else(|| format!("{key}: missing {volatile}"))?;
|
||||
*value = Value::Unit;
|
||||
}
|
||||
Ok(fields)
|
||||
}
|
||||
|
||||
async fn put_xml(env: &RustFSTestEnvironment, query: &str, xml: &str) -> Result<(), Error> {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("content-md5", base64_simd::STANDARD.encode_to_string(Md5::digest(xml)).parse()?);
|
||||
let response = signed_s3_request_with_headers(
|
||||
Method::PUT,
|
||||
&format!("{}/{CONFIGS}?{query}", env.url),
|
||||
Some(xml.to_owned()),
|
||||
Some("application/xml"),
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
&headers,
|
||||
)
|
||||
.await?;
|
||||
if !response.status().is_success() {
|
||||
return Err(format!("PUT ?{query} failed with {}", response.status()).into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
struct Snapshot {
|
||||
configs: BTreeMap<&'static str, Vec<u8>>,
|
||||
objects: BTreeMap<&'static str, VersionFields>,
|
||||
multipart_files: BTreeMap<String, Vec<u8>>,
|
||||
}
|
||||
|
||||
async fn capture(binary: &Path, target: &FakeS3Target, clock: &str, clock_library: &str) -> Result<Snapshot, Error> {
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
let queue = tempfile::tempdir()?;
|
||||
let clock_loader = if cfg!(target_os = "macos") {
|
||||
"DYLD_INSERT_LIBRARIES"
|
||||
} else {
|
||||
"LD_PRELOAD"
|
||||
};
|
||||
let options = [
|
||||
(clock_loader, clock_library),
|
||||
("DYLD_FORCE_FLAT_NAMESPACE", "1"),
|
||||
("FAKETIME", clock),
|
||||
("FAKETIME_DONT_FAKE_MONOTONIC", "1"),
|
||||
("TZ", "UTC"),
|
||||
("RUSTFS_NOTIFY_ENABLE", "true"),
|
||||
("RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY", "on"),
|
||||
("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY", target.endpoint()),
|
||||
(
|
||||
"RUSTFS_NOTIFY_WEBHOOK_QUEUE_DIR_PRIMARY",
|
||||
queue.path().to_str().ok_or("non-UTF8 queue path")?,
|
||||
),
|
||||
("RUSTFS_OUTBOUND_ALLOW_ORIGINS", target.endpoint()),
|
||||
("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", "true"),
|
||||
(ODM_MODULE_SWITCH_ENV, "true"),
|
||||
("RUSTFS_NEW_BUCKET_DURABILITY_MODE", "strict"),
|
||||
("RUSTFS_SCANNER_CYCLE", "1"),
|
||||
("RUSTFS_SCANNER_START_DELAY_SECS", "0"),
|
||||
];
|
||||
env.start_rustfs_server_from_binary_without_cleanup(binary, vec![], &options)
|
||||
.await?;
|
||||
let client = env.create_s3_client();
|
||||
write_configs(&env, &client, target).await?;
|
||||
write_objects(&client).await?;
|
||||
env.stop_server();
|
||||
read_snapshot(Path::new(&env.temp_dir))
|
||||
}
|
||||
|
||||
async fn write_configs(env: &RustFSTestEnvironment, client: &Client, target: &FakeS3Target) -> Result<(), Error> {
|
||||
env.create_test_bucket(LOGS).await?;
|
||||
env.create_test_bucket(OBJECTS).await?;
|
||||
env.create_test_bucket("persistence-warehouse").await?;
|
||||
env.create_test_bucket("persistence-migration").await?;
|
||||
client
|
||||
.create_bucket()
|
||||
.bucket(CONFIGS)
|
||||
.object_lock_enabled_for_bucket(true)
|
||||
.send()
|
||||
.await?;
|
||||
macro_rules! golden {
|
||||
($path:literal) => {
|
||||
include_str!(concat!("../../ecstore/tests/fixtures/bucket-config-goldens/", $path, ".in.xml"))
|
||||
};
|
||||
}
|
||||
let target_request = serde_json::json!({
|
||||
"endpoint": target.address(), "credentials": {"accessKey": FAKE_ACCESS_KEY, "secretKey": FAKE_SECRET_KEY},
|
||||
"targetbucket": "interop-dr", "type": "replication", "secure": false,
|
||||
"deploymentID": "0b6f3c2a-5e6a-4d2b-9c1e-7a8f9d0e1b2c"
|
||||
});
|
||||
let response = send_set_replication_target_request(env, CONFIGS, false, target_request).await?;
|
||||
if !response.status().is_success() {
|
||||
return Err(format!("set remote target failed with {}", response.status()).into());
|
||||
}
|
||||
let target_arn: String = response.json().await?;
|
||||
let replication = golden!("replication/rustfs").replace("arn:rustfs:replication:", "arn:minio:replication:");
|
||||
if !replication.contains(&format!("<Bucket>{target_arn}</Bucket>")) {
|
||||
return Err("registered target does not match the golden replication destination".into());
|
||||
}
|
||||
for (query, xml) in [
|
||||
("versioning", golden!("versioning/minio")),
|
||||
("notification", golden!("notification/rustfs")),
|
||||
("replication", replication.as_str()),
|
||||
("lifecycle", golden!("lifecycle/minio")),
|
||||
("object-lock", golden!("object_lock/rustfs")),
|
||||
("encryption", golden!("encryption/minio")),
|
||||
("tagging", golden!("tagging/rustfs")),
|
||||
("cors", golden!("cors/rustfs")),
|
||||
("logging", golden!("logging/rustfs")),
|
||||
("website", golden!("website/rustfs")),
|
||||
("accelerate", golden!("accelerate/rustfs")),
|
||||
("requestPayment", golden!("request_payment/rustfs")),
|
||||
("publicAccessBlock", golden!("public_access_block/rustfs")),
|
||||
] {
|
||||
put_xml(env, query, xml).await?;
|
||||
}
|
||||
client
|
||||
.put_bucket_acl()
|
||||
.bucket(CONFIGS)
|
||||
.acl(BucketCannedAcl::Private)
|
||||
.send()
|
||||
.await?;
|
||||
let acl = client.get_bucket_acl().bucket(CONFIGS).send().await?;
|
||||
let owner = acl.owner().and_then(|owner| owner.id()).ok_or("ACL owner missing")?;
|
||||
let [grant] = acl.grants() else { return Err("expected exactly one ACL grant".into()) };
|
||||
let grantee = grant.grantee().ok_or("missing ACL grantee")?;
|
||||
if owner.is_empty()
|
||||
|| grant.permission().map(|p| p.as_str()) != Some("FULL_CONTROL")
|
||||
|| grantee.r#type().as_str() != "CanonicalUser"
|
||||
|| grantee.id().is_some()
|
||||
{
|
||||
return Err("GET ACL changed its canonical owner grant".into());
|
||||
}
|
||||
let policy = serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Deny", "Principal": "*", "Action": "s3:DeleteObject",
|
||||
"Resource": format!("arn:aws:s3:::{CONFIGS}/protected/*"),
|
||||
}],
|
||||
});
|
||||
client
|
||||
.put_bucket_policy()
|
||||
.bucket(CONFIGS)
|
||||
.policy(policy.to_string())
|
||||
.send()
|
||||
.await?;
|
||||
set_bucket_quota(env, CONFIGS, 16 * 1024 * 1024).await?;
|
||||
let migration = OdmSourceSpec::for_fake_source(target, "migration-source");
|
||||
for (path, body) in [
|
||||
("/iceberg/v1/buckets/persistence-warehouse".to_owned(), None),
|
||||
(format!("{ODM_ADMIN_ROUTE}/persistence-migration"), Some(migration.to_json().to_string())),
|
||||
] {
|
||||
let (status, _) = admin_request(&env.url, Method::PUT, &path, body, &env.access_key, &env.secret_key).await?;
|
||||
if !status.is_success() {
|
||||
return Err(format!("configuring {path} failed with {status}").into());
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn write_objects(client: &Client) -> Result<(), Error> {
|
||||
for key in ["put", "tagged"] {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(OBJECTS)
|
||||
.key(key)
|
||||
.metadata("purpose", "byte-parity")
|
||||
.body(ByteStream::from_static(b"persisted object contents"))
|
||||
.send()
|
||||
.await?;
|
||||
}
|
||||
client
|
||||
.copy_object()
|
||||
.bucket(OBJECTS)
|
||||
.key("copy")
|
||||
.copy_source(format!("{OBJECTS}/put"))
|
||||
.send()
|
||||
.await?;
|
||||
let tags = Tagging::builder()
|
||||
.tag_set(Tag::builder().key("type").value("persisted").build()?)
|
||||
.build()?;
|
||||
client
|
||||
.put_object_tagging()
|
||||
.bucket(OBJECTS)
|
||||
.key("tagged")
|
||||
.tagging(tags)
|
||||
.send()
|
||||
.await?;
|
||||
write_multipart(client, OBJECTS, "multipart", &[vec![0x31; FIRST_PART_SIZE], vec![0x32; 100]]).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn read_snapshot(volume: &Path) -> Result<Snapshot, Error> {
|
||||
let b = bucket_metadata(volume, CONFIGS)?;
|
||||
let configs = BTreeMap::from([
|
||||
("acl", b.bucket_acl_config_json),
|
||||
("accelerate", b.accelerate_config_xml),
|
||||
("cors", b.cors_config_xml),
|
||||
("encryption", b.encryption_config_xml),
|
||||
("lifecycle", b.lifecycle_config_xml),
|
||||
("logging", b.logging_config_xml),
|
||||
("notification", b.notification_config_xml),
|
||||
("object_lock", b.object_lock_config_xml),
|
||||
("public_access_block", b.public_access_block_config_xml),
|
||||
("request_payment", b.request_payment_config_xml),
|
||||
("tagging", b.tagging_config_xml),
|
||||
("versioning", b.versioning_config_xml),
|
||||
("website", b.website_config_xml),
|
||||
("durability", b.durability_config_json),
|
||||
("policy", b.policy_config_json),
|
||||
("quota", b.quota_config_json),
|
||||
("replication", b.replication_config_xml),
|
||||
("targets", b.bucket_targets_config_json),
|
||||
("table", bucket_metadata(volume, "persistence-warehouse")?.table_bucket_config_json),
|
||||
("odm", bucket_metadata(volume, "persistence-migration")?.on_demand_migration_config_json),
|
||||
]);
|
||||
let mut objects = BTreeMap::new();
|
||||
for key in ["put", "copy", "tagged"] {
|
||||
let (_, fields) = read_object(volume, key)?;
|
||||
objects.insert(key, fields);
|
||||
}
|
||||
let (multipart, _) = read_object(volume, "multipart")?;
|
||||
if multipart.part_numbers != [1, 2] || multipart.part_sizes != [FIRST_PART_SIZE, 100] {
|
||||
return Err("xl.meta does not describe both uploaded parts".into());
|
||||
}
|
||||
let directory = volume
|
||||
.join(OBJECTS)
|
||||
.join("multipart")
|
||||
.join(multipart.data_dir.ok_or("missing multipart data directory")?.to_string());
|
||||
let mut multipart_files = BTreeMap::new();
|
||||
for entry in std::fs::read_dir(directory)? {
|
||||
let entry = entry?;
|
||||
let name = entry.file_name().into_string().map_err(|_| "invalid multipart file name")?;
|
||||
if name.starts_with("part.") {
|
||||
let bytes = std::fs::read(entry.path())?;
|
||||
if bytes.is_empty() {
|
||||
return Err(format!("empty multipart file {name}").into());
|
||||
}
|
||||
multipart_files.insert(name, bytes);
|
||||
}
|
||||
}
|
||||
for name in ["part.1", "part.2", "part.1.meta", "part.2.meta"] {
|
||||
if !multipart_files.contains_key(name) {
|
||||
return Err(format!("missing multipart file {name}").into());
|
||||
}
|
||||
}
|
||||
Ok(Snapshot {
|
||||
configs,
|
||||
objects,
|
||||
multipart_files,
|
||||
})
|
||||
}
|
||||
|
||||
fn save_xml_samples(root: &Path, side: &str, snapshot: &Snapshot) -> Result<(), Error> {
|
||||
let directory = root.join(side);
|
||||
std::fs::create_dir_all(&directory)?;
|
||||
let families: [&str; 13] = [
|
||||
"accelerate",
|
||||
"cors",
|
||||
"encryption",
|
||||
"lifecycle",
|
||||
"logging",
|
||||
"notification",
|
||||
"object_lock",
|
||||
"public_access_block",
|
||||
"replication",
|
||||
"request_payment",
|
||||
"tagging",
|
||||
"versioning",
|
||||
"website",
|
||||
];
|
||||
for family in families {
|
||||
let bytes = snapshot
|
||||
.configs
|
||||
.get(family)
|
||||
.ok_or_else(|| format!("missing XML family {family}"))?;
|
||||
std::fs::write(directory.join(format!("{family}.xml")), bytes)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn persistence_byte_parity() -> Result<(), Error> {
|
||||
let reference = PathBuf::from(
|
||||
std::env::var_os("RUSTFS_PERSISTENCE_REFERENCE_BINARY").ok_or("missing pinned RUSTFS_PERSISTENCE_REFERENCE_BINARY")?,
|
||||
);
|
||||
let candidate = rustfs_binary_path();
|
||||
if binary_version(&reference).await? == binary_version(&candidate).await? {
|
||||
return Err("reference and candidate binaries report identical build identities".into());
|
||||
}
|
||||
// The quota writer records the process wall clock inside its JSON document. Keep raw bytes
|
||||
// comparable while the SDK's signature clock and both servers' monotonic timers advance normally.
|
||||
let clock_library = std::env::var("RUSTFS_PERSISTENCE_CLOCK_LIBRARY")?;
|
||||
if !Path::new(&clock_library).is_absolute() || !Path::new(&clock_library).is_file() {
|
||||
return Err("RUSTFS_PERSISTENCE_CLOCK_LIBRARY must name an absolute libfaketime path".into());
|
||||
}
|
||||
let clock = time::OffsetDateTime::now_utc()
|
||||
.format(time::macros::format_description!("[year]-[month]-[day] [hour]:[minute]:[second]"))?;
|
||||
let target = FakeS3Target::start().await?;
|
||||
target.create_bucket("migration-source");
|
||||
target.create_bucket_with_object_lock("interop-dr");
|
||||
let reference = capture(&reference, &target, &clock, &clock_library).await?;
|
||||
let candidate = capture(&candidate, &target, &clock, &clock_library).await?;
|
||||
if let Some(directory) = std::env::var_os("RUSTFS_PERSISTENCE_XML_DIR") {
|
||||
save_xml_samples(Path::new(&directory), "reference", &reference)?;
|
||||
save_xml_samples(Path::new(&directory), "candidate", &candidate)?;
|
||||
}
|
||||
if reference.configs.len() != 20 || candidate.configs.len() != 20 {
|
||||
return Err("expected 13 XML and seven JSON fields on each side".into());
|
||||
}
|
||||
for (&field, expected) in &reference.configs {
|
||||
// Canned ACLs acknowledge the write and return one owner grant without persisting an ACL document.
|
||||
if expected.is_empty() != (field == "acl") || candidate.configs.get(field) != Some(expected) {
|
||||
return Err(format!("incorrect or different persisted bucket config: {field}").into());
|
||||
}
|
||||
}
|
||||
for (key, expected) in &reference.objects {
|
||||
if candidate.objects.get(key) != Some(expected) {
|
||||
return Err(format!("different persisted xl.meta fields: {key}").into());
|
||||
}
|
||||
}
|
||||
if candidate.multipart_files != reference.multipart_files {
|
||||
return Err("different multipart part files or part metadata".into());
|
||||
}
|
||||
eprintln!(
|
||||
"Compared {} bucket fields (canned ACL empty), {} full object records and multipart part files",
|
||||
reference.configs.len(),
|
||||
reference.objects.len()
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn version_comparison_preserves_unknown_fields_and_absent_values() -> Result<(), Error> {
|
||||
let mut sample = serde_json::json!({"Type": 1, "v": 1, "V2Obj": {"DDir": vec![1; 16], "MTime": 1}});
|
||||
let fields = |value: &serde_json::Value| nonvolatile_fields(&rmp_serde::to_vec(value)?, "fixture");
|
||||
let original = fields(&sample)?;
|
||||
sample["V2Obj"]["DDir"] = serde_json::json!(vec![2; 16]);
|
||||
sample["V2Obj"]["MTime"] = serde_json::json!(2);
|
||||
assert_eq!(fields(&sample)?, original);
|
||||
sample["V2Obj"]["Future"] = serde_json::Value::Null;
|
||||
let with_nil = fields(&sample)?;
|
||||
assert_ne!(with_nil, original);
|
||||
sample["V2Obj"]["Future"] = serde_json::json!(false);
|
||||
let with_value = fields(&sample)?;
|
||||
assert_ne!(with_value, with_nil);
|
||||
sample["FutureEnvelope"] = serde_json::json!(1);
|
||||
assert_ne!(fields(&sample)?, with_value);
|
||||
assert!(fields(&serde_json::json!({"V2Obj": {"MTime": 1}})).is_err());
|
||||
Ok(())
|
||||
}
|
||||
@@ -20,9 +20,9 @@
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use http::header::{CONTENT_LENGTH, HOST, TRANSFER_ENCODING};
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::sign_v4;
|
||||
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
|
||||
use s3s::Body;
|
||||
use std::error::Error;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpStream;
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
|
||||
use async_trait::async_trait;
|
||||
use rustfs_lock::{
|
||||
CacheFenceStamp, LockClient, LockError, LockId, LockInfo, LockRequest, LockResponse, LockStats, LockStatus, LockType, Result,
|
||||
LockClient, LockError, LockId, LockInfo, LockRequest, LockResponse, LockStats, LockStatus, LockType, Result,
|
||||
types::{LockMetadata, LockPriority},
|
||||
};
|
||||
use rustfs_protos::proto_gen::node_service::{BatchGenerallyLockRequest, GenerallyLockRequest, PingRequest};
|
||||
@@ -64,7 +64,6 @@ impl GrpcLockClient {
|
||||
deadlock_detection: false,
|
||||
suppress_contention_logs: false,
|
||||
refresh_interval: None,
|
||||
cache_fence_stamp_requested: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -125,15 +124,10 @@ impl LockClient for GrpcLockClient {
|
||||
|
||||
// Check if the lock acquisition was successful
|
||||
if resp.success {
|
||||
let info = Self::build_lock_info(request, resp.lock_info);
|
||||
match resp
|
||||
.cache_fence_stamp
|
||||
.as_deref()
|
||||
.and_then(|encoded| CacheFenceStamp::decode(encoded).ok())
|
||||
{
|
||||
Some(stamp) => Ok(LockResponse::success_with_cache_fence_stamp(info, std::time::Duration::ZERO, stamp)),
|
||||
None => Ok(LockResponse::success(info, std::time::Duration::ZERO)),
|
||||
}
|
||||
Ok(LockResponse::success(
|
||||
Self::build_lock_info(request, resp.lock_info),
|
||||
std::time::Duration::ZERO,
|
||||
))
|
||||
} else {
|
||||
// Lock acquisition failed
|
||||
Ok(LockResponse::failure(
|
||||
@@ -166,15 +160,7 @@ impl LockClient for GrpcLockClient {
|
||||
.enumerate()
|
||||
.map(|(idx, request)| match resp.results.get(idx) {
|
||||
Some(result) if result.success => {
|
||||
let info = Self::build_lock_info(request, result.lock_info.clone());
|
||||
match result
|
||||
.cache_fence_stamp
|
||||
.as_deref()
|
||||
.and_then(|encoded| CacheFenceStamp::decode(encoded).ok())
|
||||
{
|
||||
Some(stamp) => LockResponse::success_with_cache_fence_stamp(info, std::time::Duration::ZERO, stamp),
|
||||
None => LockResponse::success(info, std::time::Duration::ZERO),
|
||||
}
|
||||
LockResponse::success(Self::build_lock_info(request, result.lock_info.clone()), std::time::Duration::ZERO)
|
||||
}
|
||||
Some(result) => LockResponse::failure(
|
||||
result
|
||||
|
||||
@@ -43,7 +43,6 @@ fn lock_result_from_response(response: rustfs_lock::LockResponse) -> GenerallyLo
|
||||
success: response.success,
|
||||
error_info: response.error,
|
||||
lock_info: response.lock_info.and_then(|info| serde_json::to_string(&info).ok()),
|
||||
cache_fence_stamp: response.cache_fence_stamp.map(|stamp| stamp.encode().to_vec().into()),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
@@ -181,7 +180,6 @@ impl NodeService for MinimalLockNodeService {
|
||||
success: result.success,
|
||||
error_info: result.error,
|
||||
lock_info: lock_info_json,
|
||||
cache_fence_stamp: result.cache_fence_stamp.map(|stamp| stamp.encode().to_vec().into()),
|
||||
..Default::default()
|
||||
}))
|
||||
}
|
||||
@@ -573,15 +571,6 @@ impl NodeService for MinimalLockNodeService {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
type RenameDataBatchStream = ResponseStream<rustfs_protos::proto_gen::node_service::RenameDataBatchResult>;
|
||||
|
||||
async fn rename_data_batch(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::RenameDataBatchRequest>,
|
||||
) -> Result<Response<Self::RenameDataBatchStream>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn rename_multipart_data(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::RenameDataRequest>,
|
||||
|
||||
@@ -27,9 +27,9 @@ use http::Version;
|
||||
use http::header::HOST;
|
||||
use rcgen::generate_simple_self_signed;
|
||||
use reqwest::{Certificate, Client, Response, StatusCode};
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use std::error::Error;
|
||||
use std::path::Path;
|
||||
use std::process::Command;
|
||||
|
||||
@@ -59,6 +59,7 @@ use rustfs_madmin::{
|
||||
AddServiceAccountReq, ListServiceAccountsResp, PeerInfo, PeerSite, ReplicateAddStatus, ReplicateEditStatus,
|
||||
ReplicateRemoveStatus, SRRemoveReq, SRResyncOpStatus, SRStatusInfo, SiteReplicationInfo, SyncStatus,
|
||||
};
|
||||
use s3s::header::{X_AMZ_REPLICATION_STATUS, X_AMZ_TAGGING};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::BTreeMap;
|
||||
use std::convert::Infallible;
|
||||
@@ -77,8 +78,6 @@ use tokio::task::JoinHandle;
|
||||
use tokio::task::JoinSet;
|
||||
use tokio::time::{Duration, sleep, timeout};
|
||||
|
||||
const X_AMZ_REPLICATION_STATUS: http::HeaderName = http::HeaderName::from_static("x-amz-replication-status");
|
||||
const X_AMZ_TAGGING: http::HeaderName = http::HeaderName::from_static("x-amz-tagging");
|
||||
type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
|
||||
type BacklogMetricPoints = Arc<Mutex<BTreeMap<String, BTreeMap<String, (u64, f64)>>>>;
|
||||
|
||||
@@ -465,7 +464,7 @@ pub(crate) async fn set_replication_target_with_options(
|
||||
Ok(arn)
|
||||
}
|
||||
|
||||
pub(crate) async fn send_set_replication_target_request(
|
||||
async fn send_set_replication_target_request(
|
||||
source_env: &RustFSTestEnvironment,
|
||||
source_bucket: &str,
|
||||
update: bool,
|
||||
|
||||
@@ -32,9 +32,9 @@ mod tests {
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use http::StatusCode;
|
||||
use http::header::HOST;
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use std::error::Error;
|
||||
use tracing::{debug, info};
|
||||
|
||||
|
||||
@@ -57,12 +57,6 @@ pub(crate) mod replication_extension {
|
||||
pub(crate) use super::BucketTargetSys;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod persistence_parity {
|
||||
pub(crate) use super::RUSTFS_META_BUCKET;
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata::BucketMetadata;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod upgrade_compatibility {
|
||||
pub(crate) use super::{DiskOption, Endpoint, RUSTFS_META_MULTIPART_BUCKET, new_disk};
|
||||
|
||||
@@ -23,9 +23,9 @@ mod tests {
|
||||
use aws_sdk_s3::types::{ObjectAttributes, StorageClass};
|
||||
use http::header::HOST;
|
||||
use reqwest::StatusCode;
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use serde_json::Value;
|
||||
use std::error::Error;
|
||||
use std::path::Path;
|
||||
|
||||
@@ -46,12 +46,9 @@ type BoxError = Box<dyn std::error::Error + Send + Sync>;
|
||||
|
||||
const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY";
|
||||
const G09_EVIDENCE_DIR_ENV: &str = "RUSTFS_SCANNER_HEAL_G09_EVIDENCE_DIR";
|
||||
const RC2_COMMIT: &str = "e9f53180273b32a9c2b0aa8b5a384047bc7e875c";
|
||||
const RC5_COMMIT: &str = "40a2470feb567201165a5b809b7598bb4b1f68f5";
|
||||
const STABLE_1_0_1_COMMIT: &str = "6de965ae3c965a78ff819fbcd7acd4aa44177d92";
|
||||
const STABLE_1_0_4_COMMIT: &str = "46337942d55f3a8e6a97b147ca98d9ea8115555d";
|
||||
const STABLE_1_0_5_COMMIT: &str = "34e6449ed90e3d3a106f6655342e0b3127f33082";
|
||||
const STABLE_1_0_6_COMMIT: &str = "de9009d882644effc2628e777650025c3d8e7c65";
|
||||
const SSE_MASTER_KEY_ENV: &str = "RUSTFS_SSE_S3_MASTER_KEY";
|
||||
const SSE_MASTER_KEY: &str = "QkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkI=";
|
||||
const PLAIN_BUCKET: &str = "upgrade-plain-data";
|
||||
@@ -145,41 +142,6 @@ fn current_source_revision() -> Result<String, BoxError> {
|
||||
Ok(revision.to_string())
|
||||
}
|
||||
|
||||
pub(crate) struct UpgradeSourceCapabilities {
|
||||
supports_delete_recovery: bool,
|
||||
supports_multipart_publication: bool,
|
||||
pub(crate) requires_list_parts_read_quorum: bool,
|
||||
}
|
||||
|
||||
impl UpgradeSourceCapabilities {
|
||||
fn from_revision(revision: &str) -> Result<Self, BoxError> {
|
||||
// 1.0.4 supports the delete decision protocol (#8468). Version 1.0.5 also
|
||||
// shipped native MPU publication (#8919) and strict ListParts evidence (#9037, #9149).
|
||||
match revision {
|
||||
RC2_COMMIT | RC5_COMMIT | STABLE_1_0_1_COMMIT => Ok(Self {
|
||||
supports_delete_recovery: false,
|
||||
supports_multipart_publication: false,
|
||||
requires_list_parts_read_quorum: false,
|
||||
}),
|
||||
STABLE_1_0_4_COMMIT => Ok(Self {
|
||||
supports_delete_recovery: true,
|
||||
supports_multipart_publication: false,
|
||||
requires_list_parts_read_quorum: false,
|
||||
}),
|
||||
STABLE_1_0_5_COMMIT | STABLE_1_0_6_COMMIT => Ok(Self {
|
||||
supports_delete_recovery: true,
|
||||
supports_multipart_publication: true,
|
||||
requires_list_parts_read_quorum: true,
|
||||
}),
|
||||
_ => Err(format!("no pinned upgrade expectations for source revision {revision}").into()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn from_binary(binary: &Path) -> Result<Self, BoxError> {
|
||||
Self::from_revision(&binary_source_revision(binary).await?)
|
||||
}
|
||||
}
|
||||
|
||||
async fn binary_source_revision(binary: &Path) -> Result<String, BoxError> {
|
||||
let output = tokio::process::Command::new(binary).arg("--version").output().await?;
|
||||
if !output.status.success() {
|
||||
@@ -263,25 +225,6 @@ mod g09_evidence_tests {
|
||||
use super::*;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn pinned_upgrade_sources_keep_distinct_delete_and_list_parts_contracts() -> TestResult {
|
||||
for (revision, delete_recovery, multipart_publication, list_parts_quorum) in [
|
||||
(RC2_COMMIT, false, false, false),
|
||||
(RC5_COMMIT, false, false, false),
|
||||
(STABLE_1_0_1_COMMIT, false, false, false),
|
||||
(STABLE_1_0_4_COMMIT, true, false, false),
|
||||
(STABLE_1_0_5_COMMIT, true, true, true),
|
||||
(STABLE_1_0_6_COMMIT, true, true, true),
|
||||
] {
|
||||
let source = UpgradeSourceCapabilities::from_revision(revision)?;
|
||||
assert_eq!(source.supports_delete_recovery, delete_recovery, "{revision}");
|
||||
assert_eq!(source.supports_multipart_publication, multipart_publication, "{revision}");
|
||||
assert_eq!(source.requires_list_parts_read_quorum, list_parts_quorum, "{revision}");
|
||||
}
|
||||
assert!(UpgradeSourceCapabilities::from_revision("ffffffffffffffffffffffffffffffffffffffff").is_err());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn source_revision_parser_requires_lowercase_forty_byte_sha() -> TestResult {
|
||||
let binary = Path::new("rustfs");
|
||||
@@ -422,7 +365,7 @@ async fn complete_pending_multipart(
|
||||
.await?)
|
||||
}
|
||||
|
||||
pub(crate) async fn write_multipart(client: &Client, bucket: &str, key: &str, parts: &[Vec<u8>]) -> TestResult {
|
||||
async fn write_multipart(client: &Client, bucket: &str, key: &str, parts: &[Vec<u8>]) -> TestResult {
|
||||
let pending = stage_multipart(client, bucket, key, parts).await?;
|
||||
complete_pending_multipart(client, bucket, key, &pending).await?;
|
||||
Ok(())
|
||||
@@ -1068,7 +1011,8 @@ async fn exercise_mixed_cluster(
|
||||
vec![u8::try_from(MULTIPART_WORKERS - 1)?; 64 * 1024]
|
||||
);
|
||||
|
||||
// Use a separate old-writer key so the original seed still spans the entire rollout.
|
||||
// 1.0.4 already implements the delete decision protocol. Use a separate
|
||||
// old-writer key so the original seed still spans the entire rollout.
|
||||
if previous_has_delete_recovery {
|
||||
let key = format!(".upgrade-delete/{phase}/previous-writer");
|
||||
previous_client
|
||||
@@ -2349,7 +2293,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
let previous_binary = source_binary()?;
|
||||
// Bind the expected protocol to the verified release source, never to the
|
||||
// DELETE result itself. The older CI source pins retain their 503 contract.
|
||||
let source = UpgradeSourceCapabilities::from_binary(&previous_binary).await?;
|
||||
let previous_has_delete_recovery = binary_source_revision(&previous_binary).await? == STABLE_1_0_4_COMMIT;
|
||||
let evidence_context = g09_evidence_context(&previous_binary).await?;
|
||||
let current_binary = rustfs_binary_path();
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(MIXED_NODE_COUNT).await?;
|
||||
@@ -2371,21 +2315,17 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
cluster.stop_node(0)?;
|
||||
cluster.start_node_from_binary(0, ¤t_binary).await?;
|
||||
let mut mixed_deleted_keys = Vec::new();
|
||||
if let Some(key) = exercise_mixed_cluster(&cluster, "one-current-node", 0, 1, source.supports_delete_recovery).await? {
|
||||
if let Some(key) = exercise_mixed_cluster(&cluster, "one-current-node", 0, 1, previous_has_delete_recovery).await? {
|
||||
mixed_deleted_keys.push(key);
|
||||
}
|
||||
let mixed_completion = if source.supports_multipart_publication {
|
||||
let clients = cluster.create_all_clients()?;
|
||||
Some(finish_pending_multipart(&clients, &pending, pending_parts.concat()).await?)
|
||||
} else {
|
||||
assert_mixed_pending_refusal(&cluster, &pending, &sources).await?;
|
||||
None
|
||||
};
|
||||
// One controlled out-of-policy attempt proves fail-closed behavior below
|
||||
// native write quorum; positive MPU writers remain paused while mixed.
|
||||
assert_mixed_pending_refusal(&cluster, &pending, &sources).await?;
|
||||
|
||||
for (node_idx, phase) in [(1, "two-current-nodes"), (2, "one-previous-node")] {
|
||||
cluster.stop_node(node_idx)?;
|
||||
cluster.start_node_from_binary(node_idx, ¤t_binary).await?;
|
||||
if let Some(key) = exercise_mixed_cluster(&cluster, phase, 0, 3, source.supports_delete_recovery).await? {
|
||||
if let Some(key) = exercise_mixed_cluster(&cluster, phase, 0, 3, previous_has_delete_recovery).await? {
|
||||
mixed_deleted_keys.push(key);
|
||||
}
|
||||
}
|
||||
@@ -2395,13 +2335,6 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
let clients = cluster.create_all_clients()?;
|
||||
wait_for_drive_readmission(&clients, MIXED_BUCKET, "homogeneous-current", DRIVE_READMISSION_TIMEOUT).await?;
|
||||
let completed_pending = finish_pending_multipart(&clients, &pending, pending_parts.concat()).await?;
|
||||
if let Some(mixed) = mixed_completion {
|
||||
assert_eq!(
|
||||
completed_pending.version_id, mixed.version_id,
|
||||
"full upgrade changed the completed version"
|
||||
);
|
||||
assert_eq!(completed_pending.etag, mixed.etag, "full upgrade changed the completed ETag");
|
||||
}
|
||||
exercise_homogeneous_multipart(&cluster, "homogeneous-current").await?;
|
||||
verify_load_bodies(&clients, "homogeneous-previous", "multipart").await?;
|
||||
assert_mixed_deletes_visible(&clients, &mixed_deleted_keys).await?;
|
||||
@@ -2518,7 +2451,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
"phases": phases,
|
||||
"objects_per_phase": objects_per_phase,
|
||||
"mixed_load": "plain-put",
|
||||
"mixed_delete_protocol_supported": source.supports_delete_recovery,
|
||||
"mixed_delete_protocol_supported": previous_has_delete_recovery,
|
||||
"mixed_deleted_keys": mixed_deleted_keys,
|
||||
"homogeneous_multipart_phases": ["homogeneous-previous", "homogeneous-current", "homogeneous-rollback"],
|
||||
"multipart_objects_per_homogeneous_phase": MULTIPART_WORKERS * MULTIPART_UPLOADS_PER_WORKER,
|
||||
@@ -2528,7 +2461,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
"all nodes list every mixed-version object after homogeneous-current convergence",
|
||||
"the previous-release seed survives the baseline restart and every upgrade phase",
|
||||
"previous readers read every current-native MPU body after quiescent rollback",
|
||||
"the same previous-release upload remains readable after full upgrade and rollback"
|
||||
"same legacy upload resumes with identical body, ETag and version after full upgrade"
|
||||
],
|
||||
}),
|
||||
)?;
|
||||
@@ -2550,13 +2483,9 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
"previous-release writer publishes objects readable by current node",
|
||||
"mixed phases retain 16x16 plain PUT writers while positive multipart writers are paused",
|
||||
"16x16 multipart load runs in homogeneous previous, current and rollback phases",
|
||||
if source.supports_multipart_publication {
|
||||
"one-current-node Complete and replay return the same version and ETag, and all readers agree on the body"
|
||||
} else {
|
||||
"one-current-node Complete returns typed 503 and preserves actual source authority and parts"
|
||||
},
|
||||
"fully upgraded readers see the same body, ETag and sole version after replay",
|
||||
if source.supports_delete_recovery {
|
||||
"one-current-node Complete returns typed 503 and preserves actual source authority and parts",
|
||||
"the same legacy upload completes and replays once fully upgraded without adding a version",
|
||||
if previous_has_delete_recovery {
|
||||
"mixed-version DELETE succeeds and stays absent through every reader after upgrade and rollback"
|
||||
} else {
|
||||
"modern DELETE fails closed and preserves every byte while old peers remain"
|
||||
@@ -2903,7 +2832,7 @@ async fn restart_from_binary(env: &mut RustFSTestEnvironment, binary: &Path, ser
|
||||
env.start_rustfs_server_from_binary(binary, vec![], server_env).await
|
||||
}
|
||||
|
||||
pub(crate) async fn set_bucket_quota(env: &RustFSTestEnvironment, bucket: &str, quota_bytes: u64) -> TestResult {
|
||||
async fn set_bucket_quota(env: &RustFSTestEnvironment, bucket: &str, quota_bytes: u64) -> TestResult {
|
||||
let path = format!("/rustfs/admin/v3/quota/{bucket}");
|
||||
let body = serde_json::json!({ "quota": quota_bytes, "quota_type": "HARD" }).to_string();
|
||||
let deadline = Instant::now() + QUOTA_READINESS_TIMEOUT;
|
||||
@@ -3457,8 +3386,6 @@ async fn direct_upgrade_from_previous_release_preserves_bucket_configuration() -
|
||||
#[tokio::test]
|
||||
#[ignore = "requires a pinned previous RustFS release binary"]
|
||||
async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestResult {
|
||||
use sha2::Digest;
|
||||
|
||||
init_logging();
|
||||
let previous_binary = source_binary()?;
|
||||
let evidence_context = g09_evidence_context(&previous_binary).await?;
|
||||
@@ -3496,16 +3423,6 @@ async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestRes
|
||||
let multipart_bytes = multipart_parts.concat();
|
||||
write_multipart(&new_client, ROLLBACK_BUCKET, multipart_key, &multipart_parts).await?;
|
||||
|
||||
// This release can finish an upload staged by the current gateway.
|
||||
let inflight = if binary_source_revision(&previous_binary).await? == STABLE_1_0_5_COMMIT {
|
||||
env.create_test_bucket(PENDING_MULTIPART_BUCKET).await?;
|
||||
let parts = [vec![1; LAYOUT_PART_SIZE], vec![2; LAYOUT_PART_SIZE], vec![3; 4096]];
|
||||
let pending = stage_multipart(&new_client, PENDING_MULTIPART_BUCKET, PENDING_MULTIPART_KEY, &parts).await?;
|
||||
Some((pending, parts.concat()))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
restart_from_binary(&mut env, &previous_binary, &server_env).await?;
|
||||
let old_client = env.create_s3_client();
|
||||
|
||||
@@ -3521,16 +3438,6 @@ async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestRes
|
||||
assert_eq!(encryption, Some(ServerSideEncryption::Aes256));
|
||||
assert_eq!(body, multipart_bytes, "the multipart object did not survive the rollback");
|
||||
|
||||
let rollback_multipart = if let Some((pending, expected)) = inflight {
|
||||
assert_pending_parts(&old_client, &pending).await?;
|
||||
complete_pending_multipart(&old_client, PENDING_MULTIPART_BUCKET, PENDING_MULTIPART_KEY, &pending).await?;
|
||||
let (_, body) = read_object(&old_client, PENDING_MULTIPART_BUCKET, PENDING_MULTIPART_KEY, None).await?;
|
||||
assert_eq!(body, expected);
|
||||
Some((pending.part_sizes, body))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// A PUT on the rolled-back release must still honour the encryption
|
||||
// configuration it decoded out of the current build's metadata blob.
|
||||
let post_rollback_key = "rollback/written-after-rollback";
|
||||
@@ -3571,25 +3478,6 @@ async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestRes
|
||||
post_rollback_bytes
|
||||
);
|
||||
|
||||
if let Some((part_sizes, rollback_read)) = rollback_multipart {
|
||||
let (_, current_read) = read_object(¤t_again, PENDING_MULTIPART_BUCKET, PENDING_MULTIPART_KEY, None).await?;
|
||||
assert_eq!(current_read, rollback_read);
|
||||
if let Some(context) = &evidence_context {
|
||||
let evidence = serde_json::json!({
|
||||
"previous_revision": context.previous_revision,
|
||||
"current_revision": context.current_revision,
|
||||
"part_sizes": part_sizes,
|
||||
"bytes": current_read.len(),
|
||||
"rollback_read_sha256": hex_simd::encode_to_string(sha2::Sha256::digest(&rollback_read), hex_simd::AsciiCase::Lower),
|
||||
"roll_forward_read_sha256": hex_simd::encode_to_string(sha2::Sha256::digest(¤t_read), hex_simd::AsciiCase::Lower),
|
||||
});
|
||||
std::fs::write(
|
||||
context.directory.join("inflight-multipart-rollback.json"),
|
||||
serde_json::to_vec_pretty(&evidence)?,
|
||||
)?;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(context) = evidence_context.as_ref() {
|
||||
write_g09_evidence(
|
||||
context,
|
||||
|
||||
@@ -126,7 +126,6 @@ test-util = []
|
||||
e2e-test-hooks = []
|
||||
|
||||
[dependencies]
|
||||
slab.workspace = true
|
||||
hashbrown.workspace = true
|
||||
hotpath.workspace = true
|
||||
rustfs-filemeta.workspace = true
|
||||
@@ -154,6 +153,7 @@ arc-swap.workspace = true
|
||||
async-trait.workspace = true
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
byteorder = { workspace = true }
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
jiff = { workspace = true, features = ["serde"] }
|
||||
thiserror.workspace = true
|
||||
flatbuffers.workspace = true
|
||||
@@ -258,6 +258,7 @@ windows-sys = { workspace = true, features = [
|
||||
windows-sys = { workspace = true, features = ["Win32_System_Ioctl"] }
|
||||
|
||||
[dev-dependencies]
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
aws-smithy-async.workspace = true
|
||||
rustfs-s3-ops.workspace = true
|
||||
tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util", "fs"] }
|
||||
|
||||
@@ -393,8 +393,7 @@ pub mod cluster {
|
||||
|
||||
pub mod compression {
|
||||
pub use crate::io_support::compress::{
|
||||
DISK_COMPRESSION_EXCLUDED_CONTENT_TYPES, MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible, is_disk_compression_enabled,
|
||||
is_multipart_disk_compression_enabled,
|
||||
MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible, is_disk_compression_enabled, is_multipart_disk_compression_enabled,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -615,21 +614,15 @@ pub mod object {
|
||||
};
|
||||
pub use crate::erasure::codec::shard_emit::ShardEmitter;
|
||||
pub use crate::object_api::{
|
||||
BLOCK_SIZE_V2, ClientEcBlockSizeHint, DEFAULT_EC_BLOCK_SIZE, EC_BLOCK_SIZE_CANDIDATES, EC_BLOCK_SIZE_HINT_INTERNAL_KEY,
|
||||
EC_BLOCK_SIZE_HINT_INTERNAL_SUFFIX, EC_BLOCK_SIZE_HINT_METADATA_KEY, EC_BLOCK_SIZE_HINT_REASON_HEADER,
|
||||
EC_BLOCK_SIZE_HINT_STATUS_HEADER, EC_BLOCK_SIZE_INFO_INTERNAL_KEY, EC_BLOCK_SIZE_INFO_INTERNAL_SUFFIX,
|
||||
EC_BLOCK_SIZE_RESPONSE_HEADER, EC_READ_QUANTUM_INFO_INTERNAL_KEY, EC_READ_QUANTUM_INFO_INTERNAL_SUFFIX,
|
||||
ERASURE_ALGORITHM, EcBlockSizeHintError, EcBlockSizeHintIgnoreReason, EcBlockSizeHintOutcome, EncryptionResolutionError,
|
||||
EncryptionResolutionErrorKind, GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup, GetObjectBodySource,
|
||||
GetObjectReadCondition, GetObjectReadMetadata, GetObjectReader, GetObjectSink, GetObjectSinkSegmentFactory,
|
||||
GetObjectSinkSegments, GetObjectSinkStream, GetObjectSinkWriter, NamespaceLockFence, ObjectEncryptionResolver,
|
||||
ObjectInfo, ObjectLockConfigSnapshot, ObjectMutationHook, ObjectOptions, PutObjReader, QuotaAdmission,
|
||||
RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest, ResolvedEcBlockSize,
|
||||
BLOCK_SIZE_V2, ERASURE_ALGORITHM, EncryptionResolutionError, EncryptionResolutionErrorKind, GetObjectBodyCacheHook,
|
||||
GetObjectBodyCacheHookLookup, GetObjectBodySource, GetObjectReadCondition, GetObjectReadMetadata, GetObjectReader,
|
||||
GetObjectSink, GetObjectSinkSegmentFactory, GetObjectSinkSegments, GetObjectSinkStream, GetObjectSinkWriter,
|
||||
NamespaceLockFence, ObjectEncryptionResolver, ObjectInfo, ObjectLockConfigSnapshot, ObjectMutationHook, ObjectOptions,
|
||||
PutObjReader, QuotaAdmission, RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest,
|
||||
SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, ScannerPublicationCommitScope, ScannerPublicationCommitStartError,
|
||||
ScannerPublicationCommitState, ShardIntegrityWriteMode, StreamConsumer, WriteCommitGuard, WriteCompletion,
|
||||
get_object_body_cache_plaintext_len, is_supported_ec_block_size, lookup_get_object_body_cache_hook,
|
||||
parse_ec_block_size_hint, register_get_object_body_cache_hook, register_object_mutation_hook, resolve_ec_block_size_hint,
|
||||
take_ec_block_size_hint, unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
|
||||
get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, register_get_object_body_cache_hook,
|
||||
register_object_mutation_hook, unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
|
||||
with_get_object_read_condition,
|
||||
};
|
||||
pub use crate::store::{
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -5591,7 +5591,7 @@ impl RestoreRequestOps for RestoreRequest {
|
||||
}
|
||||
|
||||
// For non-SELECT requests, days must be at least 1
|
||||
if !valid_restore_days(self) {
|
||||
if self.type_.is_none() && self.days.is_none_or(|d| d <= 0) {
|
||||
return Err(std::io::Error::other("restoration days should be at least 1"));
|
||||
}
|
||||
|
||||
@@ -5599,10 +5599,6 @@ impl RestoreRequestOps for RestoreRequest {
|
||||
}
|
||||
}
|
||||
|
||||
fn valid_restore_days(request: &RestoreRequest) -> bool {
|
||||
request.type_.as_deref() == Some("SELECT") || request.days.is_some_and(|days| days > 0)
|
||||
}
|
||||
|
||||
const _MAX_RESTORE_OBJECT_REQUEST_SIZE: i64 = 2 << 20;
|
||||
|
||||
/// Builds an exact-version lifecycle delete target with the concrete
|
||||
@@ -9377,18 +9373,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn standard_restore_days_are_positive_with_or_without_type() {
|
||||
let mut request = select_restore_request(None);
|
||||
for kind in [None, Some("STANDARD".to_owned())] {
|
||||
request.type_ = kind;
|
||||
for days in [None, Some(0), Some(-1), Some(1)] {
|
||||
request.days = days;
|
||||
assert_eq!(super::valid_restore_days(&request), days == Some(1));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn select_restore_s3_location_rejects_missing_s3_output_location() {
|
||||
let request = select_restore_request(Some(OutputLocation { s3: None }));
|
||||
|
||||
@@ -1136,14 +1136,6 @@ impl BucketMetadata {
|
||||
/// delivered late (backlog#2292). Only replication receivers should pass
|
||||
/// a foreign time; local edits keep [`Self::update_config`].
|
||||
pub fn update_config_at(&mut self, config_file: &str, data: Vec<u8>, updated: OffsetDateTime) -> Result<OffsetDateTime> {
|
||||
const MAX_BUCKET_CONFIG_BYTES: usize = 2 * 1024 * 1024;
|
||||
if data.len() > MAX_BUCKET_CONFIG_BYTES {
|
||||
return Err(Error::InvalidArgument(
|
||||
self.name.clone(),
|
||||
String::new(),
|
||||
"bucket configuration exceeds 2 MiB".to_owned(),
|
||||
));
|
||||
}
|
||||
match config_file {
|
||||
BUCKET_POLICY_CONFIG => {
|
||||
self.policy_config_json = data;
|
||||
@@ -1351,39 +1343,6 @@ impl BucketMetadata {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Parse only the configuration used for foreground write/delete admission.
|
||||
pub(super) fn parse_admission_configs(&mut self) {
|
||||
self.object_lock_config = None;
|
||||
if !self.object_lock_config_xml.is_empty()
|
||||
&& let Err(e) =
|
||||
deserialize::<ObjectLockConfiguration>(&self.object_lock_config_xml).map(|c| self.object_lock_config = Some(c))
|
||||
{
|
||||
tracing::warn!(
|
||||
event = "bucket_metadata_parse_failed",
|
||||
component = "ecstore",
|
||||
subsystem = "bucket_metadata",
|
||||
bucket = %self.name,
|
||||
config = "object_lock",
|
||||
error = %e,
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
}
|
||||
self.quota_config = None;
|
||||
if !self.quota_config_json.is_empty()
|
||||
&& let Err(e) = serde_json::from_slice(&self.quota_config_json).map(|c| self.quota_config = Some(c))
|
||||
{
|
||||
tracing::warn!(
|
||||
event = "bucket_metadata_parse_failed",
|
||||
component = "ecstore",
|
||||
subsystem = "bucket_metadata",
|
||||
bucket = %self.name,
|
||||
config = "quota",
|
||||
error = %e,
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode every stored sub-configuration into its typed field.
|
||||
///
|
||||
/// A decode failure never fails the whole load: this runs on every bucket
|
||||
@@ -1405,7 +1364,7 @@ impl BucketMetadata {
|
||||
/// | public access block | Fails closed in `get_public_access_block_config`: degrading grants the anonymous access the operator asked to block. |
|
||||
/// | quota | Fails closed in `get_quota_config`; the enforcement path in `quota::checker` already re-parses the raw JSON and refuses on error. |
|
||||
/// | lifecycle | `get_lifecycle_config` fails closed, so GetBucketLifecycle reports the fault instead of NoSuchLifecycleConfiguration. ILM, scanner and expiry-header consumers still degrade to "no rules": nothing is deleted or moved on the strength of an unreadable rule set, and the bucket keeps serving reads and writes. |
|
||||
/// | notification | `get_notification_config` fails closed, so GetBucketNotificationConfiguration reports the fault and startup skips that bucket's rule registration; metadata-bound server dispatch suppresses external delivery for the unreadable snapshot. Other buckets are unaffected. |
|
||||
/// | notification | `get_notification_config` fails closed, so GetBucketNotificationConfiguration reports the fault and startup leaves that one bucket's rules unchanged instead of clearing them; other buckets are unaffected. |
|
||||
/// | tagging, CORS, logging, website, accelerate, request payment | The getter fails closed so the matching GET API reports the fault instead of "not configured". Per-request consumers (CORS response headers) still degrade to "not configured", which is the restrictive direction. |
|
||||
/// | bucket ACL | Safe to degrade: it only shapes an optional response. |
|
||||
///
|
||||
@@ -1459,7 +1418,21 @@ impl BucketMetadata {
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
}
|
||||
self.parse_admission_configs();
|
||||
self.object_lock_config = None;
|
||||
if !self.object_lock_config_xml.is_empty()
|
||||
&& let Err(e) =
|
||||
deserialize::<ObjectLockConfiguration>(&self.object_lock_config_xml).map(|c| self.object_lock_config = Some(c))
|
||||
{
|
||||
tracing::warn!(
|
||||
event = "bucket_metadata_parse_failed",
|
||||
component = "ecstore",
|
||||
subsystem = "bucket_metadata",
|
||||
bucket = %self.name,
|
||||
config = "object_lock",
|
||||
error = %e,
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
}
|
||||
self.versioning_config = None;
|
||||
if !self.versioning_config_xml.is_empty()
|
||||
&& let Err(e) =
|
||||
@@ -1504,6 +1477,19 @@ impl BucketMetadata {
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
}
|
||||
if !self.quota_config_json.is_empty()
|
||||
&& let Err(e) = serde_json::from_slice(&self.quota_config_json).map(|c| self.quota_config = Some(c))
|
||||
{
|
||||
tracing::warn!(
|
||||
event = "bucket_metadata_parse_failed",
|
||||
component = "ecstore",
|
||||
subsystem = "bucket_metadata",
|
||||
bucket = %self.name,
|
||||
config = "quota",
|
||||
error = %e,
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
}
|
||||
self.replication_config = None;
|
||||
if !self.replication_config_xml.is_empty()
|
||||
&& let Err(e) = deserialize::<PersistedReplicationConfiguration>(&self.replication_config_xml)
|
||||
@@ -2089,33 +2075,6 @@ mod test {
|
||||
assert!(bm.object_locking());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bucket_config_size_limit_preserves_last_accepted_policy() {
|
||||
let mut metadata = BucketMetadata::new("policy-size-limit");
|
||||
let stamp = OffsetDateTime::now_utc();
|
||||
let mut policy = br#"{"Version":"2012-10-17","Statement":[]}"#.to_vec();
|
||||
policy.resize(2 * 1024 * 1024, b' ');
|
||||
metadata
|
||||
.update_config_at(BUCKET_POLICY_CONFIG, policy.clone(), stamp)
|
||||
.expect("accept a policy exactly at the size limit");
|
||||
let mut oversized = policy.clone();
|
||||
oversized.push(b' ');
|
||||
let result = metadata.update_config_at(BUCKET_POLICY_CONFIG, oversized, stamp + time::Duration::seconds(1));
|
||||
assert!(matches!(result, Err(Error::InvalidArgument(..))));
|
||||
assert!(metadata.policy_config_json == policy, "rejection must preserve the accepted policy");
|
||||
assert_eq!(metadata.policy_config_updated_at, stamp);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn admission_configs_leave_logging_unparsed() {
|
||||
let mut bm = BucketMetadata::new("bucket");
|
||||
bm.logging_config_xml = b"<BucketLoggingStatus/>".to_vec();
|
||||
bm.parse_admission_configs();
|
||||
assert!(bm.logging_config.is_none());
|
||||
bm.parse_all_configs().expect("parse logging config");
|
||||
assert!(bm.logging_config.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_all_configs_parses_stored_configs_without_store_dependency() {
|
||||
let mut bm = BucketMetadata::new("test-bucket");
|
||||
|
||||
@@ -1794,12 +1794,9 @@ async fn get_delete_and_quota_config_from_disk_in(
|
||||
if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() =>
|
||||
{
|
||||
if let Some(expected) = expiry_policy {
|
||||
let raw = &metadata.lifecycle_config_xml;
|
||||
let current = (!raw.is_empty())
|
||||
.then(|| super::utils::deserialize::<PersistedLifecycleConfiguration>(raw))
|
||||
.transpose()
|
||||
.map_err(|_| unreadable_config_error(bucket, BUCKET_LIFECYCLE_CONFIG, raw.len()))?;
|
||||
if !expected.matches_configuration(current.as_ref()) {
|
||||
let current = ConfigState::of(&metadata.lifecycle_config_xml, &metadata.lifecycle_config)
|
||||
.require(bucket, BUCKET_LIFECYCLE_CONFIG)?;
|
||||
if !expected.matches_configuration(current) {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
}
|
||||
@@ -1835,9 +1832,7 @@ pub(crate) async fn integrity_migration_metadata_in(
|
||||
BucketMetadataAuthority::Authoritative(metadata)
|
||||
if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() =>
|
||||
{
|
||||
let mut metadata = Arc::unwrap_or_clone(metadata);
|
||||
metadata.parse_all_configs()?;
|
||||
Ok(Arc::new(metadata))
|
||||
Ok(metadata)
|
||||
}
|
||||
BucketMetadataAuthority::MissingBucket => Err(Error::BucketNotFound(bucket.to_string())),
|
||||
_ => Err(Error::other("migration requires authoritative bucket metadata")),
|
||||
@@ -2612,10 +2607,6 @@ impl BucketMetadataSys {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn cached_metadata(&self) -> Vec<Arc<BucketMetadata>> {
|
||||
self.metadata_map.read().await.values().cloned().collect()
|
||||
}
|
||||
|
||||
pub async fn set(&self, bucket: String, bm: Arc<BucketMetadata>) {
|
||||
self.revoke_pending_bucket_creation(&bucket).await;
|
||||
bump_local_meta_mutation_gen();
|
||||
@@ -3517,7 +3508,7 @@ impl BucketMetadataSys {
|
||||
Some(guard),
|
||||
bucket,
|
||||
"bucket metadata authoritative snapshot",
|
||||
load_bucket_metadata_parse_with_presence(Arc::clone(&store), bucket, false),
|
||||
load_bucket_metadata_parse_with_presence(Arc::clone(&store), bucket, true),
|
||||
)),
|
||||
);
|
||||
match bucket_info_res {
|
||||
@@ -3526,12 +3517,11 @@ impl BucketMetadataSys {
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
|
||||
let (mut metadata, persisted) = metadata_res?;
|
||||
let (metadata, persisted) = metadata_res?;
|
||||
if persisted && metadata.lock_enabled && !metadata.bucket_creation_committed {
|
||||
return Err(Error::ErasureWriteQuorum);
|
||||
}
|
||||
if persisted {
|
||||
metadata.parse_admission_configs();
|
||||
let metadata = Arc::new(metadata);
|
||||
if can_use_local_authority_cache
|
||||
&& metadata.bucket_incarnation_sidecar
|
||||
@@ -5764,64 +5754,6 @@ mod tests {
|
||||
assert_eq!(tags.tag_set.len(), WRITERS, "every concurrent rewrite must survive: {tags:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn notification_snapshots_follow_periodic_refresh_after_lost_peer_signals() {
|
||||
let (dirs, store) = isolated_store_over_temp_disks().await;
|
||||
let writer = BucketMetadataSys::new(store.clone());
|
||||
let peer = Arc::new(RwLock::new(BucketMetadataSys::new(store.clone())));
|
||||
let bucket = "notification-periodic-snapshots";
|
||||
for dir in &dirs {
|
||||
std::fs::create_dir_all(dir.path().join(bucket)).expect("create empty fixture bucket");
|
||||
}
|
||||
let mut original = BucketMetadata::new(bucket);
|
||||
original.notification_config_xml = concat!(
|
||||
"<NotificationConfiguration><QueueConfiguration><Event>s3:ObjectCreated:Put</Event>",
|
||||
"<Queue>arn:rustfs:sqs:us-east-1:inert:webhook</Queue></QueueConfiguration></NotificationConfiguration>"
|
||||
)
|
||||
.as_bytes()
|
||||
.to_vec();
|
||||
original.parse_all_configs().expect("parse inert subscription");
|
||||
writer
|
||||
.persist_new_and_set(original)
|
||||
.await
|
||||
.expect("persist initial subscription");
|
||||
peer.read().await.reload_from_store(bucket).await.expect("prime peer cache");
|
||||
let old = peer.read().await.get(bucket).await.expect("cached original snapshot");
|
||||
|
||||
let mut removed = (*old).clone();
|
||||
removed.notification_config_xml = b"<NotificationConfiguration/>".to_vec();
|
||||
removed.parse_all_configs().expect("parse removed subscription");
|
||||
writer
|
||||
.persist_new_and_set(removed)
|
||||
.await
|
||||
.expect("persist removal without peer callback");
|
||||
assert!(Arc::ptr_eq(&old, &peer.read().await.get(bucket).await.expect("peer remains stale")));
|
||||
|
||||
// Exercise the timer's actual one-pass entry point, not the peer RPC.
|
||||
refresh_buckets_metadata_once(peer.clone()).await;
|
||||
let snapshots = peer.read().await.cached_metadata().await;
|
||||
assert_eq!(snapshots.len(), 1);
|
||||
assert!(!Arc::ptr_eq(&old, &snapshots[0]));
|
||||
assert_eq!(old.bucket_incarnation_id, snapshots[0].bucket_incarnation_id);
|
||||
assert_eq!(snapshots[0].notification_config_xml, b"<NotificationConfiguration/>");
|
||||
assert!(
|
||||
snapshots[0]
|
||||
.notification_config
|
||||
.as_ref()
|
||||
.expect("parsed removal")
|
||||
.behavior()
|
||||
.queue_routes
|
||||
.is_empty()
|
||||
);
|
||||
|
||||
for dir in &dirs {
|
||||
std::fs::remove_dir(dir.path().join(bucket)).expect("remove empty fixture bucket without peer callback");
|
||||
}
|
||||
refresh_buckets_metadata_once(peer.clone()).await;
|
||||
assert!(peer.read().await.cached_metadata().await.is_empty());
|
||||
assert!(peer.read().await.get(bucket).await.is_err());
|
||||
}
|
||||
|
||||
/// Pins the peer reload-notification contract (`reload_from_store`, the
|
||||
/// LoadBucketMetadata RPC path): only metadata actually read from
|
||||
/// persisted storage enters the cache. A load miss errors out and leaves
|
||||
|
||||
@@ -1607,7 +1607,6 @@ fn shard_object(ledger_object: &str, shard_index: u16, bucket_incarnation: Uuid,
|
||||
}
|
||||
|
||||
fn shard_index(object: &str) -> u16 {
|
||||
let object = rustfs_utils::path::encode_dir_object_ref(object);
|
||||
let digest = Sha256::digest(object.as_bytes());
|
||||
u16::from_be_bytes([digest[0], digest[1]]) % SHARDED_LEDGER_COUNT
|
||||
}
|
||||
@@ -2315,12 +2314,6 @@ fn log_deferred_settlement(data: &ReservationLedgerData, state: &'static str, er
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn quota_shard_agrees_with_directory_listing_name() {
|
||||
for listed in ["photos/", "reports/june/"] {
|
||||
assert_eq!(shard_index(listed), shard_index(&rustfs_utils::path::encode_dir_object(listed)));
|
||||
}
|
||||
}
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use serial_test::serial;
|
||||
|
||||
|
||||
@@ -256,64 +256,6 @@ pub(crate) fn remote_sdk_credentials(credentials: &RemoteCredentials, now: Syste
|
||||
Ok(builder.build())
|
||||
}
|
||||
|
||||
/// Bound the bytes Smithy can aggregate for a control document or service error.
|
||||
/// Successful GetObject responses are streamed directly to the caller.
|
||||
#[derive(Debug)]
|
||||
struct ControlResponseLimit(usize);
|
||||
|
||||
#[derive(Debug)]
|
||||
struct OversizedResponseRetryClassifier;
|
||||
|
||||
impl aws_smithy_runtime_api::client::retries::classifiers::ClassifyRetry for OversizedResponseRetryClassifier {
|
||||
fn classify_retry(
|
||||
&self,
|
||||
ctx: &aws_smithy_runtime_api::client::interceptors::context::InterceptorContext,
|
||||
) -> aws_smithy_runtime_api::client::retries::classifiers::RetryAction {
|
||||
use aws_smithy_runtime_api::client::retries::classifiers::RetryAction;
|
||||
if let Some(Err(error)) = ctx.output_or_error() {
|
||||
let mut cause = std::error::Error::source(error);
|
||||
while let Some(error) = cause {
|
||||
if error.is::<http_body_util::LengthLimitError>() {
|
||||
return RetryAction::RetryForbidden;
|
||||
}
|
||||
cause = error.source();
|
||||
}
|
||||
}
|
||||
RetryAction::NoActionIndicated
|
||||
}
|
||||
|
||||
fn name(&self) -> &'static str {
|
||||
"RustfsOversizedResponseRetryClassifier"
|
||||
}
|
||||
}
|
||||
|
||||
impl Intercept for ControlResponseLimit {
|
||||
fn name(&self) -> &'static str {
|
||||
"RustfsControlResponseLimit"
|
||||
}
|
||||
|
||||
fn modify_before_deserialization(
|
||||
&self,
|
||||
context: &mut BeforeDeserializationInterceptorContextMut<'_>,
|
||||
_runtime_components: &RuntimeComponents,
|
||||
cfg: &mut ConfigBag,
|
||||
) -> Result<(), BoxError> {
|
||||
let operation = cfg
|
||||
.load::<aws_smithy_runtime_api::client::orchestrator::Metadata>()
|
||||
.map(|m| m.name());
|
||||
bound_control_response(context.response_mut(), operation, self.0);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn bound_control_response(response: &mut HttpResponse, operation: Option<&str>, limit: usize) {
|
||||
if operation == Some("GetObject") && response.status().is_success() {
|
||||
return;
|
||||
}
|
||||
let body = std::mem::replace(response.body_mut(), SdkBody::taken());
|
||||
*response.body_mut() = SdkBody::from_body_1_x(http_body_util::Limited::new(body, limit));
|
||||
}
|
||||
|
||||
/// Only installed on non-streaming upload operations. The budget begins at
|
||||
/// response headers, independently of the potentially long request upload.
|
||||
#[derive(Debug)]
|
||||
@@ -433,8 +375,6 @@ pub async fn build_remote_s3_config(spec: &RemoteS3EndpointSpec) -> Result<aws_s
|
||||
.region(SdkRegion::new(spec.region.clone()))
|
||||
.behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
|
||||
.request_checksum_calculation(replication_request_checksum_calculation())
|
||||
.interceptor(ControlResponseLimit(16 * 1024 * 1024))
|
||||
.retry_classifier(OversizedResponseRetryClassifier)
|
||||
.retry_config(spec.retry.retry_config());
|
||||
|
||||
if spec.path_style.force_path_style() {
|
||||
@@ -845,33 +785,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remote_control_response_limit_preserves_object_streams() {
|
||||
use http_body_util::BodyExt;
|
||||
for (operation, status, bytes, allowed) in [
|
||||
(Some("ListObjectsV2"), 200, 4, true),
|
||||
(Some("ListObjectsV2"), 200, 5, false),
|
||||
(Some("GetObject"), 206, 5, true),
|
||||
(Some("GetObject"), 404, 5, false),
|
||||
(None, 200, 5, false),
|
||||
] {
|
||||
let mut response = HttpResponse::try_from(
|
||||
http::Response::builder()
|
||||
.status(status)
|
||||
.body(SdkBody::from(vec![1; bytes]))
|
||||
.expect("fixture response"),
|
||||
)
|
||||
.expect("Smithy response");
|
||||
bound_control_response(&mut response, operation, 4);
|
||||
let result = response.into_body().collect().await;
|
||||
if allowed {
|
||||
assert_eq!(result.expect("body is permitted").to_bytes().len(), bytes);
|
||||
} else {
|
||||
assert!(result.is_err(), "oversized control response must fail");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn build_remote_s3_client_rejects_loopback_and_metadata_endpoints() {
|
||||
// Default (no loopback opt-in): loopback in IPv4, IPv6 and hostname
|
||||
@@ -973,59 +886,6 @@ mod tests {
|
||||
wire_requests.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn sdk_remote_control_limit_stops_chunked_errors_and_preserves_downloads() {
|
||||
#[derive(Debug)]
|
||||
struct ChunkConnector(u16, Arc<AtomicUsize>);
|
||||
impl SmithyHttpConnector for ChunkConnector {
|
||||
fn call(&self, _: HttpRequest) -> HttpConnectorFuture {
|
||||
self.1.fetch_add(1, Ordering::SeqCst);
|
||||
let frames =
|
||||
[b"abc", b"def"].map(|s| Ok::<_, std::io::Error>(http_body::Frame::data(bytes::Bytes::from_static(s))));
|
||||
let body = http_body_util::StreamBody::new(futures::stream::iter(frames));
|
||||
HttpConnectorFuture::ready(Ok(HttpResponse::new(
|
||||
SmithyStatusCode::try_from(self.0).expect("fixture status"),
|
||||
SdkBody::from_body_1_x(body),
|
||||
)))
|
||||
}
|
||||
}
|
||||
for status in [200, 503] {
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let connector = SharedHttpConnector::new(ChunkConnector(status, calls.clone()));
|
||||
let mut target = spec("s3.example.com", true);
|
||||
target.retry = REPLICATION_TARGET_RETRY_POLICY;
|
||||
let config = build_remote_s3_config(&target)
|
||||
.await
|
||||
.expect("build remote config")
|
||||
.http_client(http_client_fn(move |_, _| connector.clone()))
|
||||
.interceptor(ControlResponseLimit(4))
|
||||
.build();
|
||||
let result = S3Client::from_conf(config)
|
||||
.get_object()
|
||||
.bucket("bucket")
|
||||
.key("key")
|
||||
.send()
|
||||
.await;
|
||||
if status == 200 {
|
||||
let bytes = result
|
||||
.expect("successful object stream")
|
||||
.body
|
||||
.collect()
|
||||
.await
|
||||
.expect("collect fixture object")
|
||||
.into_bytes();
|
||||
assert_eq!(&bytes[..], b"abcdef");
|
||||
} else {
|
||||
assert!(result.is_err(), "oversized error response must fail");
|
||||
}
|
||||
assert_eq!(
|
||||
calls.load(Ordering::SeqCst),
|
||||
1,
|
||||
"oversized control responses must not trigger SDK retries"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn retry_policy_decides_how_many_wire_requests_one_call_costs() {
|
||||
assert_eq!(
|
||||
@@ -1070,9 +930,8 @@ mod tests {
|
||||
.unwrap_or_else(|| panic!("signed request must contain {name}"))
|
||||
}
|
||||
|
||||
fn signing_time(headers: &[(String, String)]) -> jiff::Timestamp {
|
||||
let value = recorded_header(headers, "x-amz-date").trim_end_matches('Z');
|
||||
jiff::Timestamp::strptime("%Y%m%dT%H%M%S%z", format!("{value}+0000"))
|
||||
fn signing_time(headers: &[(String, String)]) -> chrono::NaiveDateTime {
|
||||
chrono::NaiveDateTime::parse_from_str(recorded_header(headers, "x-amz-date"), "%Y%m%dT%H%M%SZ")
|
||||
.expect("SDK signing timestamp must use the SigV4 format")
|
||||
}
|
||||
|
||||
@@ -1087,8 +946,8 @@ mod tests {
|
||||
.map(|(key, value)| (key.to_string(), value.to_string()))
|
||||
.collect(),
|
||||
);
|
||||
let server_time = jiff::Timestamp::try_from(self.clock.now()).expect("clock timestamp")
|
||||
+ jiff::SignedDuration::from_secs(self.skew_seconds);
|
||||
let server_time = chrono::DateTime::<chrono::Utc>::from(self.clock.now()).naive_utc()
|
||||
+ chrono::Duration::seconds(self.skew_seconds);
|
||||
let (status, body) = if headers.len() == 1 {
|
||||
(
|
||||
403,
|
||||
@@ -1099,7 +958,7 @@ mod tests {
|
||||
};
|
||||
let response = http::Response::builder()
|
||||
.status(status)
|
||||
.header("date", server_time.strftime("%a, %d %b %Y %H:%M:%S GMT").to_string())
|
||||
.header("date", server_time.format("%a, %d %b %Y %H:%M:%S GMT").to_string())
|
||||
.header("content-type", "application/xml")
|
||||
.header("content-length", body.len())
|
||||
.body(SdkBody::from(body))
|
||||
@@ -1137,7 +996,7 @@ mod tests {
|
||||
for error_code in ["RequestTimeTooSkewed", "SignatureDoesNotMatch"] {
|
||||
for skew_seconds in [-600, 600] {
|
||||
let (client, headers, clock) = clock_skew_client(error_code, skew_seconds, REPLICATION_TARGET_RETRY_POLICY).await;
|
||||
let initial = jiff::Timestamp::try_from(clock.now()).expect("clock timestamp");
|
||||
let initial = chrono::DateTime::<chrono::Utc>::from(clock.now()).naive_utc();
|
||||
client
|
||||
.get_object()
|
||||
.bucket("bucket")
|
||||
@@ -1164,12 +1023,12 @@ mod tests {
|
||||
assert_eq!(signing_time(&headers[0]), initial, "the first attempt must use the injected clock");
|
||||
assert_eq!(
|
||||
signing_time(&headers[1]),
|
||||
initial + jiff::SignedDuration::from_secs(skew_seconds),
|
||||
initial + chrono::Duration::seconds(skew_seconds),
|
||||
"{error_code}: retry must apply the measured offset exactly"
|
||||
);
|
||||
assert_eq!(
|
||||
signing_time(&headers[2]),
|
||||
initial + jiff::SignedDuration::from_secs(skew_seconds + 17),
|
||||
initial + chrono::Duration::seconds(skew_seconds + 17),
|
||||
"{error_code}: the next operation must apply cached skew to the advanced signing clock"
|
||||
);
|
||||
let signature = |index: usize| {
|
||||
|
||||
@@ -251,10 +251,10 @@ const METRIC_VERSION_IDENTITY_DRIFT_TOTAL: &str = "rustfs_replication_version_id
|
||||
/// the condition discoverable in any recent window.
|
||||
const VERSION_IDENTITY_DRIFT_LOG_INTERVAL: TokioDuration = TokioDuration::from_secs(600);
|
||||
|
||||
/// When each source bucket and ARN last reported version-identity drift. Throttling is
|
||||
/// When each target last reported version-identity drift, by ARN. Throttling is
|
||||
/// advisory only — the metric still counts every drifting PUT.
|
||||
type SourceTargetDriftTimes = HashMap<String, HashMap<String, Instant>>;
|
||||
static VERSION_IDENTITY_WARNED_ARNS: LazyLock<StdMutex<SourceTargetDriftTimes>> = LazyLock::new(|| StdMutex::new(HashMap::new()));
|
||||
static VERSION_IDENTITY_WARNED_ARNS: LazyLock<StdMutex<HashMap<String, Instant>>> =
|
||||
LazyLock::new(|| StdMutex::new(HashMap::new()));
|
||||
|
||||
/// Version purges a target refused for a reason a retry cannot change on its
|
||||
/// own. Two shapes today:
|
||||
@@ -395,13 +395,13 @@ fn audit_target_version_identity(tgt_client: &TargetClient, source_version_id: &
|
||||
// below (`replica_head_fallback`) knows whether a 404 on a
|
||||
// version-addressed HEAD can mean "replica missing" on this target.
|
||||
if let Some(capability) = version_identity_capability_from_put(source_version_id, assigned_version_id) {
|
||||
tgt_client.record_version_identity_capability(capability);
|
||||
ReplicationTargetStore::record_version_identity_capability(&tgt_client.arn, capability);
|
||||
}
|
||||
if !version_identity_drifted(source_version_id, assigned_version_id) {
|
||||
return;
|
||||
}
|
||||
counter!(METRIC_VERSION_IDENTITY_DRIFT_TOTAL).increment(1);
|
||||
if !version_identity_drift_log_due(&tgt_client.source_bucket, &tgt_client.arn, Instant::now()) {
|
||||
if !version_identity_drift_log_due(&tgt_client.arn, Instant::now()) {
|
||||
return;
|
||||
}
|
||||
// `error`, not `warn`: the target silently refuses the addressing scheme
|
||||
@@ -424,14 +424,14 @@ fn audit_target_version_identity(tgt_client: &TargetClient, source_version_id: &
|
||||
/// Whether this ARN's version-identity drift is due to be logged again at
|
||||
/// `now`, re-arming the throttle when it is. Split out from the audit so the
|
||||
/// interval policy is testable without a target client.
|
||||
fn version_identity_drift_log_due(bucket: &str, arn: &str, now: Instant) -> bool {
|
||||
fn version_identity_drift_log_due(arn: &str, now: Instant) -> bool {
|
||||
let mut warned = VERSION_IDENTITY_WARNED_ARNS
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
match warned.get(bucket).and_then(|entries| entries.get(arn)) {
|
||||
match warned.get(arn) {
|
||||
Some(last) if now.duration_since(*last) < VERSION_IDENTITY_DRIFT_LOG_INTERVAL => false,
|
||||
_ => {
|
||||
warned.entry(bucket.to_owned()).or_default().insert(arn.to_owned(), now);
|
||||
warned.insert(arn.to_string(), now);
|
||||
true
|
||||
}
|
||||
}
|
||||
@@ -697,7 +697,7 @@ async fn replica_head_fallback(
|
||||
if let Some(ledger_version_id) = ledger_version_id {
|
||||
// A ledger entry is direct evidence of the identity contract; it
|
||||
// survives a restart while the runtime verdict cache does not.
|
||||
tgt_client.record_version_identity_capability(VersionIdentityCapability::MintsOwn);
|
||||
ReplicationTargetStore::record_version_identity_capability(&tgt_client.arn, VersionIdentityCapability::MintsOwn);
|
||||
match head_object_for_worker(tgt_client, &tgt_client.bucket, object, Some(ledger_version_id.to_string())).await {
|
||||
Ok(head) => {
|
||||
return Some(Ok(Some(LocatedReplica {
|
||||
@@ -717,7 +717,7 @@ async fn replica_head_fallback(
|
||||
if !(encrypted && content_identity.is_some()) {
|
||||
return Some(head_object_fallback(tgt_client, object).await);
|
||||
}
|
||||
} else if !tgt_client.version_identity_capability().version_addressing_unreliable() {
|
||||
} else if !ReplicationTargetStore::version_identity_capability(&tgt_client.arn).version_addressing_unreliable() {
|
||||
return None;
|
||||
}
|
||||
let siblings = content_identity?;
|
||||
@@ -989,14 +989,14 @@ async fn sync_drifted_replica_metadata(
|
||||
/// (no PUT is ever sent — the object stays on the normal MRF retry channel
|
||||
/// and re-audits once the verdict's TTL expires or replication-check
|
||||
/// re-probes the target) and returns `None`.
|
||||
fn resolve_ssec_passthrough_gate(
|
||||
async fn resolve_ssec_passthrough_gate(
|
||||
ssec: bool,
|
||||
tgt_client: &TargetClient,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
rinfo: &mut ReplicatedTargetInfo,
|
||||
) -> Option<bool> {
|
||||
let (capability, expired) = tgt_client.ssec_passthrough_capability();
|
||||
let (capability, expired) = ReplicationTargetStore::ssec_passthrough_capability(&tgt_client.arn).await;
|
||||
match ssec_passthrough_gate(ssec, capability, expired) {
|
||||
SsecPassthroughGate::Proceed => Some(false),
|
||||
SsecPassthroughGate::ProceedWithAudit => Some(true),
|
||||
@@ -1023,7 +1023,7 @@ fn resolve_ssec_passthrough_gate(
|
||||
/// capability verdict for the target. Returns true when the SSE-C material
|
||||
/// provably survived; otherwise records `Unsupported` and settles `rinfo` as
|
||||
/// FAILED so the attempt never reports a silently unreadable COMPLETED.
|
||||
fn settle_ssec_passthrough_evidence(
|
||||
async fn settle_ssec_passthrough_evidence(
|
||||
head: &HeadObjectOutput,
|
||||
tgt_client: &TargetClient,
|
||||
bucket: &str,
|
||||
@@ -1031,10 +1031,10 @@ fn settle_ssec_passthrough_evidence(
|
||||
rinfo: &mut ReplicatedTargetInfo,
|
||||
) -> bool {
|
||||
if ssec_passthrough_evidence_present(head) {
|
||||
tgt_client.record_ssec_passthrough_capability(SsecPassthroughCapability::Supported);
|
||||
ReplicationTargetStore::record_ssec_passthrough_capability(&tgt_client.arn, SsecPassthroughCapability::Supported).await;
|
||||
return true;
|
||||
}
|
||||
tgt_client.record_ssec_passthrough_capability(SsecPassthroughCapability::Unsupported);
|
||||
ReplicationTargetStore::record_ssec_passthrough_capability(&tgt_client.arn, SsecPassthroughCapability::Unsupported).await;
|
||||
rinfo.replication_status = ReplicationStatusType::Failed;
|
||||
rinfo.error = Some(ERR_REPLICATION_SSEC_PASSTHROUGH_UNSUPPORTED.to_string());
|
||||
warn!(
|
||||
@@ -1070,7 +1070,7 @@ async fn audit_ssec_passthrough_replica(
|
||||
// version, so the audit HEAD does not 4xx-loop on those objects.
|
||||
let version_id = resolve_read_api_version_id(version_id);
|
||||
match head_object_for_worker(tgt_client.as_ref(), &tgt_client.bucket, object, version_id).await {
|
||||
Ok(head) => settle_ssec_passthrough_evidence(&head, tgt_client, bucket, object, rinfo),
|
||||
Ok(head) => settle_ssec_passthrough_evidence(&head, tgt_client, bucket, object, rinfo).await,
|
||||
Err(e) => {
|
||||
rinfo.replication_status = ReplicationStatusType::Failed;
|
||||
rinfo.error = Some(format!("SSE-C passthrough audit HEAD failed: {e}"));
|
||||
@@ -1967,7 +1967,8 @@ async fn verify_resync_head_result(
|
||||
// source marker id; verify the marker on the latest version.
|
||||
DeleteMarkerHeadOutcome::Ambiguous => {
|
||||
let mints_own = rejects_source_version_id(&err)
|
||||
|| target_client.version_identity_capability().version_addressing_unreliable();
|
||||
|| ReplicationTargetStore::version_identity_capability(&target_client.arn)
|
||||
.version_addressing_unreliable();
|
||||
!(mints_own && latest_target_version_is_delete_marker(target_client, &roi.name).await)
|
||||
}
|
||||
DeleteMarkerHeadOutcome::Exists | DeleteMarkerHeadOutcome::Missing => false,
|
||||
@@ -3664,10 +3665,10 @@ async fn resolve_purge_replica(
|
||||
return PurgeReplicaResolution::LedgerCorrupt;
|
||||
}
|
||||
if let Some(assigned) = source.ledger.get(&tgt_client.arn) {
|
||||
tgt_client.record_version_identity_capability(VersionIdentityCapability::MintsOwn);
|
||||
ReplicationTargetStore::record_version_identity_capability(&tgt_client.arn, VersionIdentityCapability::MintsOwn);
|
||||
return PurgeReplicaResolution::Resolved(assigned.clone());
|
||||
}
|
||||
if !force_lookup && !tgt_client.version_identity_capability().version_addressing_unreliable() {
|
||||
if !force_lookup && !ReplicationTargetStore::version_identity_capability(&tgt_client.arn).version_addressing_unreliable() {
|
||||
return PurgeReplicaResolution::SourceVersion;
|
||||
}
|
||||
let etag = source.etag.as_deref().filter(|etag| !etag.trim().is_empty());
|
||||
@@ -3827,7 +3828,8 @@ async fn replicate_delete_to_target(
|
||||
// A target that mints its own version ids never holds the source marker
|
||||
// id, so a version-addressed pre-check cannot succeed there; the marker
|
||||
// is created by the versionless DELETE below.
|
||||
let version_addressable = !tgt_client.version_identity_capability().version_addressing_unreliable();
|
||||
let version_addressable =
|
||||
!ReplicationTargetStore::version_identity_capability(&tgt_client.arn).version_addressing_unreliable();
|
||||
if version_addressable
|
||||
&& !is_version_purge
|
||||
&& dobj.delete_object.delete_marker
|
||||
@@ -3844,7 +3846,7 @@ async fn replicate_delete_to_target(
|
||||
// The verdict is not known yet (e.g. no PUT since restart) and the
|
||||
// target rejected the source id: learn it as the PUT path does.
|
||||
Err(e) if rejects_source_version_id(e.as_ref()) => {
|
||||
tgt_client.record_version_identity_capability(VersionIdentityCapability::MintsOwn);
|
||||
ReplicationTargetStore::record_version_identity_capability(&tgt_client.arn, VersionIdentityCapability::MintsOwn);
|
||||
}
|
||||
Ok(head) if head.delete_marker == Some(true) => {
|
||||
// Some targets expose an existing marker as a successful HEAD
|
||||
@@ -3920,7 +3922,7 @@ async fn replicate_delete_to_target(
|
||||
let object = &dobj.delete_object.object_name;
|
||||
match resolve_purge_replica(&tgt_client, object, source, true).await {
|
||||
PurgeReplicaResolution::Resolved(assigned) => {
|
||||
tgt_client.record_version_identity_capability(VersionIdentityCapability::MintsOwn);
|
||||
ReplicationTargetStore::record_version_identity_capability(&tgt_client.arn, VersionIdentityCapability::MintsOwn);
|
||||
counter!(METRIC_VERSION_PURGE_REPLICA_TOTAL, "resolution" => "resolved").increment(1);
|
||||
version_id = Some(assigned);
|
||||
removed = tgt_client
|
||||
@@ -4624,7 +4626,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
// N2 fail-closed: never PUT SSE-C ciphertext at a target known to drop
|
||||
// the passthrough transport headers, and never trust a convergence HEAD
|
||||
// against such a target — a previous broken replica matches by ETag.
|
||||
let Some(ssec_audit_required) = resolve_ssec_passthrough_gate(self.ssec, &tgt_client, &bucket, &object, &mut rinfo)
|
||||
let Some(ssec_audit_required) = resolve_ssec_passthrough_gate(self.ssec, &tgt_client, &bucket, &object, &mut rinfo).await
|
||||
else {
|
||||
send_local_event(EventArgs {
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
@@ -4745,7 +4747,9 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
// An SSE-C replica only counts as converged when the same
|
||||
// HEAD proves its decryption material survived; a broken
|
||||
// ciphertext copy from an earlier attempt matches by ETag.
|
||||
if ssec_audit_required && !settle_ssec_passthrough_evidence(&oi, &tgt_client, &bucket, &object, &mut rinfo) {
|
||||
if ssec_audit_required
|
||||
&& !settle_ssec_passthrough_evidence(&oi, &tgt_client, &bucket, &object, &mut rinfo).await
|
||||
{
|
||||
return rinfo;
|
||||
}
|
||||
rinfo.replication_status = ReplicationStatusType::Completed;
|
||||
@@ -4780,6 +4784,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
Ok(Some(located)) if located_replica_matches(identity.as_ref(), &located) => {
|
||||
if ssec_audit_required
|
||||
&& !settle_ssec_passthrough_evidence(&located.head, &tgt_client, &bucket, &object, &mut rinfo)
|
||||
.await
|
||||
{
|
||||
return rinfo;
|
||||
}
|
||||
@@ -4979,7 +4984,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
|
||||
// N2 fail-closed: see the gate in `replicate_object` — the same policy
|
||||
// applies to the metadata/existing-object transport.
|
||||
let Some(ssec_audit_required) = resolve_ssec_passthrough_gate(self.ssec, &tgt_client, &bucket, &object, &mut rinfo)
|
||||
let Some(ssec_audit_required) = resolve_ssec_passthrough_gate(self.ssec, &tgt_client, &bucket, &object, &mut rinfo).await
|
||||
else {
|
||||
send_local_event(EventArgs {
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
@@ -5352,7 +5357,7 @@ async fn resolve_replicate_all_action(
|
||||
// An SSE-C replica only counts as converged when the same HEAD
|
||||
// proves its decryption material survived; a broken ciphertext
|
||||
// copy from an earlier attempt matches by ETag.
|
||||
if ssec_audit_required && !settle_ssec_passthrough_evidence(&oi, tgt_client, bucket, object, rinfo) {
|
||||
if ssec_audit_required && !settle_ssec_passthrough_evidence(&oi, tgt_client, bucket, object, rinfo).await {
|
||||
rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs();
|
||||
return None;
|
||||
}
|
||||
@@ -5427,7 +5432,7 @@ async fn resolve_replicate_all_action(
|
||||
}
|
||||
if replica_matches
|
||||
&& ssec_audit_required
|
||||
&& !settle_ssec_passthrough_evidence(oi, tgt_client, bucket, object, rinfo)
|
||||
&& !settle_ssec_passthrough_evidence(oi, tgt_client, bucket, object, rinfo).await
|
||||
{
|
||||
rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs();
|
||||
return None;
|
||||
@@ -6701,8 +6706,6 @@ mod tests {
|
||||
.build();
|
||||
|
||||
Arc::new(TargetClient {
|
||||
source_bucket: "source".to_owned(),
|
||||
capabilities: Default::default(),
|
||||
endpoint,
|
||||
credentials: None,
|
||||
bucket: "target-bucket".to_string(),
|
||||
@@ -6907,7 +6910,7 @@ mod tests {
|
||||
async fn heal_redrive_locates_replica_by_etag_on_target_that_mints_own_version_ids() {
|
||||
let (endpoint, server) = spawn_drifted_target_server(3);
|
||||
let target = test_target_client(endpoint);
|
||||
target.record_version_identity_capability(VersionIdentityCapability::MintsOwn);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let (roi, object_info) = drifted_roi_and_object();
|
||||
let mut rinfo = replicate_all_target_info(&roi, &target);
|
||||
|
||||
@@ -6976,7 +6979,7 @@ mod tests {
|
||||
async fn resync_verification_counts_drifted_replica_as_replicated() {
|
||||
let (endpoint, server) = spawn_drifted_target_server(3);
|
||||
let target = test_target_client(endpoint);
|
||||
target.record_version_identity_capability(VersionIdentityCapability::MintsOwn);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let (roi, _) = drifted_roi_and_object();
|
||||
let mut st = TargetReplicationResyncStatus::default();
|
||||
|
||||
@@ -7090,7 +7093,7 @@ mod tests {
|
||||
static VERSIONS: &[(&str, i64)] = &[(DRIFTED_ASSIGNED_VERSION_ID, 4)];
|
||||
let (endpoint, server) = spawn_computed_etag_target(3, VERSIONS);
|
||||
let target = test_target_client(endpoint);
|
||||
target.record_version_identity_capability(VersionIdentityCapability::MintsOwn);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let (roi, object_info) = encrypted_roi_and_object(replica_written_secs() - 1);
|
||||
let mut rinfo = replicate_all_target_info(&roi, &target);
|
||||
|
||||
@@ -7112,7 +7115,7 @@ mod tests {
|
||||
static OTHER_SIZE: &[(&str, i64)] = &[(DRIFTED_ASSIGNED_VERSION_ID, 36)];
|
||||
let (endpoint, server) = spawn_computed_etag_target(2, OTHER_SIZE);
|
||||
let target = test_target_client(endpoint);
|
||||
target.record_version_identity_capability(VersionIdentityCapability::MintsOwn);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let (roi, object_info) = encrypted_roi_and_object(replica_written_secs() - 1);
|
||||
let mut rinfo = replicate_all_target_info(&roi, &target);
|
||||
let action = resolve_heal_action(&target, &roi, object_info, &mut rinfo).await;
|
||||
@@ -7123,7 +7126,7 @@ mod tests {
|
||||
static SAME_SIZE: &[(&str, i64)] = &[(DRIFTED_ASSIGNED_VERSION_ID, 4)];
|
||||
let (endpoint, server) = spawn_computed_etag_target(2, SAME_SIZE);
|
||||
let target = test_target_client(endpoint);
|
||||
target.record_version_identity_capability(VersionIdentityCapability::MintsOwn);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let (roi, object_info) = encrypted_roi_and_object(replica_written_secs() + 1);
|
||||
let mut rinfo = replicate_all_target_info(&roi, &target);
|
||||
let action = resolve_heal_action(&target, &roi, object_info, &mut rinfo).await;
|
||||
@@ -7136,7 +7139,7 @@ mod tests {
|
||||
static VERSIONS: &[(&str, i64)] = &[(DRIFTED_ASSIGNED_VERSION_ID, 4), ("another-minted-id", 4)];
|
||||
let (endpoint, server) = spawn_computed_etag_target(2, VERSIONS);
|
||||
let target = test_target_client(endpoint);
|
||||
target.record_version_identity_capability(VersionIdentityCapability::MintsOwn);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let (roi, object_info) = encrypted_roi_and_object(replica_written_secs() - 1);
|
||||
let mut rinfo = replicate_all_target_info(&roi, &target);
|
||||
|
||||
@@ -7158,7 +7161,7 @@ mod tests {
|
||||
static VERSIONS: &[(&str, i64)] = &[(DRIFTED_ASSIGNED_VERSION_ID, 4)];
|
||||
let (endpoint, server) = spawn_computed_etag_target(2, VERSIONS);
|
||||
let target = test_target_client(endpoint);
|
||||
target.record_version_identity_capability(VersionIdentityCapability::MintsOwn);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let (roi, mut object_info) = drifted_roi_and_object();
|
||||
object_info.mod_time = Some(OffsetDateTime::from_unix_timestamp(replica_written_secs() - 1).expect("valid timestamp"));
|
||||
let mut rinfo = replicate_all_target_info(&roi, &target);
|
||||
@@ -7176,7 +7179,7 @@ mod tests {
|
||||
static VERSIONS: &[(&str, i64)] = &[("null", 4)];
|
||||
let (endpoint, server) = spawn_computed_etag_target(2, VERSIONS);
|
||||
let target = test_target_client(endpoint);
|
||||
target.record_version_identity_capability(VersionIdentityCapability::MintsOwn);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let (roi, object_info) = encrypted_roi_and_object(replica_written_secs() - 1);
|
||||
let mut rinfo = replicate_all_target_info(&roi, &target);
|
||||
|
||||
@@ -7261,7 +7264,7 @@ mod tests {
|
||||
static VERSIONS: &[(&str, i64)] = &[(DRIFTED_ASSIGNED_VERSION_ID, 4)];
|
||||
let (endpoint, server) = spawn_computed_etag_target(3, VERSIONS);
|
||||
let target = test_target_client(endpoint);
|
||||
target.record_version_identity_capability(VersionIdentityCapability::MintsOwn);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let (roi, object_info) = encrypted_roi_and_object(replica_written_secs() - 1);
|
||||
let mut st = TargetReplicationResyncStatus::default();
|
||||
|
||||
@@ -7301,7 +7304,7 @@ mod tests {
|
||||
});
|
||||
let target = test_target_client(endpoint);
|
||||
register_test_target(&target).await;
|
||||
target.record_version_identity_capability(VersionIdentityCapability::MintsOwn);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let dobj = version_purge_dobj(&target.arn);
|
||||
|
||||
let rinfo = replicate_delete_to_target(&dobj, target.clone(), Some(encrypted_purge_source())).await;
|
||||
@@ -7327,7 +7330,7 @@ mod tests {
|
||||
});
|
||||
let target = test_target_client(endpoint);
|
||||
register_test_target(&target).await;
|
||||
target.record_version_identity_capability(VersionIdentityCapability::MintsOwn);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let dobj = version_purge_dobj(&target.arn);
|
||||
|
||||
let rinfo = replicate_delete_to_target(&dobj, target.clone(), Some(encrypted_purge_source())).await;
|
||||
@@ -7474,7 +7477,10 @@ mod tests {
|
||||
let rinfo = replicate_delete_to_target(&dobj, target.clone(), None).await;
|
||||
|
||||
assert_eq!(rinfo.replication_status, ReplicationStatusType::Completed, "{rinfo:?}");
|
||||
assert_eq!(target.version_identity_capability(), VersionIdentityCapability::MintsOwn);
|
||||
assert_eq!(
|
||||
ReplicationTargetStore::version_identity_capability(&target.arn),
|
||||
VersionIdentityCapability::MintsOwn
|
||||
);
|
||||
let seen = server.join().expect("test HTTP server should finish");
|
||||
assert_eq!(seen.len(), 2, "{seen:?}");
|
||||
assert!(seen[0].starts_with("HEAD "), "{seen:?}");
|
||||
@@ -7503,7 +7509,10 @@ mod tests {
|
||||
let rinfo = replicate_delete_to_target(&dobj, target.clone(), None).await;
|
||||
|
||||
assert_eq!(rinfo.replication_status, ReplicationStatusType::Failed, "{rinfo:?}");
|
||||
assert_ne!(target.version_identity_capability(), VersionIdentityCapability::MintsOwn);
|
||||
assert_ne!(
|
||||
ReplicationTargetStore::version_identity_capability(&target.arn),
|
||||
VersionIdentityCapability::MintsOwn
|
||||
);
|
||||
let seen = server.join().expect("test HTTP server should finish");
|
||||
assert_eq!(seen.len(), 1, "a denied HEAD must not trigger DELETE: {seen:?}");
|
||||
assert!(seen[0].starts_with("HEAD "), "{seen:?}");
|
||||
@@ -7522,7 +7531,7 @@ mod tests {
|
||||
});
|
||||
let target = test_target_client(endpoint);
|
||||
register_test_target(&target).await;
|
||||
target.record_version_identity_capability(VersionIdentityCapability::MintsOwn);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let dobj = marker_creation_dobj(&target.arn);
|
||||
|
||||
let rinfo = replicate_delete_to_target(&dobj, target.clone(), None).await;
|
||||
@@ -7884,7 +7893,7 @@ mod tests {
|
||||
seen[0]
|
||||
);
|
||||
assert!(
|
||||
target.version_identity_capability().version_addressing_unreliable(),
|
||||
ReplicationTargetStore::version_identity_capability(&target.arn).version_addressing_unreliable(),
|
||||
"a ledger entry proves the target mints its own ids"
|
||||
);
|
||||
}
|
||||
@@ -7902,7 +7911,7 @@ mod tests {
|
||||
});
|
||||
let target = test_target_client(endpoint);
|
||||
register_test_target(&target).await;
|
||||
target.record_version_identity_capability(VersionIdentityCapability::MintsOwn);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let dobj = version_purge_dobj(&target.arn);
|
||||
|
||||
let rinfo = replicate_delete_to_target(&dobj, target.clone(), Some(purge_source(&[]))).await;
|
||||
@@ -7929,7 +7938,7 @@ mod tests {
|
||||
});
|
||||
let target = test_target_client(endpoint);
|
||||
register_test_target(&target).await;
|
||||
target.record_version_identity_capability(VersionIdentityCapability::MintsOwn);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let dobj = version_purge_dobj(&target.arn);
|
||||
assert!(!purge_backoff_active(&dobj));
|
||||
|
||||
@@ -7957,7 +7966,7 @@ mod tests {
|
||||
});
|
||||
let target = test_target_client(endpoint);
|
||||
register_test_target(&target).await;
|
||||
target.record_version_identity_capability(VersionIdentityCapability::MintsOwn);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let dobj = version_purge_dobj(&target.arn);
|
||||
|
||||
let rinfo = replicate_delete_to_target(&dobj, target.clone(), Some(purge_source(&[]))).await;
|
||||
@@ -8029,7 +8038,7 @@ mod tests {
|
||||
assert!(seen[1].starts_with("GET "), "{}", seen[1]);
|
||||
assert!(seen[2].contains(DRIFTED_ASSIGNED_VERSION_ID), "{}", seen[2]);
|
||||
assert!(
|
||||
target.version_identity_capability().version_addressing_unreliable(),
|
||||
ReplicationTargetStore::version_identity_capability(&target.arn).version_addressing_unreliable(),
|
||||
"the located replica proves the target mints its own ids"
|
||||
);
|
||||
}
|
||||
@@ -8049,7 +8058,7 @@ mod tests {
|
||||
});
|
||||
let target = test_target_client(endpoint);
|
||||
register_test_target(&target).await;
|
||||
target.record_version_identity_capability(VersionIdentityCapability::MintsOwn);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let dobj = version_purge_dobj(&target.arn);
|
||||
let source = Arc::new(VersionPurgeSource {
|
||||
etag: Some(DRIFTED_ETAG.to_string()),
|
||||
@@ -8090,7 +8099,7 @@ mod tests {
|
||||
}
|
||||
});
|
||||
let target = test_target_client(endpoint);
|
||||
target.record_version_identity_capability(VersionIdentityCapability::MintsOwn);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let (roi, object_info) = drifted_roi_and_object();
|
||||
let mut rinfo = replicate_all_target_info(&roi, &target);
|
||||
|
||||
@@ -8241,12 +8250,21 @@ mod tests {
|
||||
let target = test_target_client("http://127.0.0.1:1".to_string());
|
||||
let source = Uuid::new_v4().to_string();
|
||||
audit_target_version_identity(&target, &source, Some(DRIFTED_ASSIGNED_VERSION_ID));
|
||||
assert_eq!(target.version_identity_capability(), VersionIdentityCapability::MintsOwn);
|
||||
assert_eq!(
|
||||
ReplicationTargetStore::version_identity_capability(&target.arn),
|
||||
VersionIdentityCapability::MintsOwn
|
||||
);
|
||||
audit_target_version_identity(&target, &source, Some(&source));
|
||||
assert_eq!(target.version_identity_capability(), VersionIdentityCapability::Adopts);
|
||||
assert_eq!(
|
||||
ReplicationTargetStore::version_identity_capability(&target.arn),
|
||||
VersionIdentityCapability::Adopts
|
||||
);
|
||||
// An unversioned write carries no contract and must not overwrite it.
|
||||
audit_target_version_identity(&target, "null", None);
|
||||
assert_eq!(target.version_identity_capability(), VersionIdentityCapability::Adopts);
|
||||
assert_eq!(
|
||||
ReplicationTargetStore::version_identity_capability(&target.arn),
|
||||
VersionIdentityCapability::Adopts
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -9432,13 +9450,13 @@ mod tests {
|
||||
let arn = "arn:replication::drift-throttle-test";
|
||||
let start = Instant::now();
|
||||
|
||||
assert!(version_identity_drift_log_due("source", arn, start), "first drift must be reported");
|
||||
assert!(version_identity_drift_log_due(arn, start), "first drift must be reported");
|
||||
assert!(
|
||||
!version_identity_drift_log_due("source", arn, start + VERSION_IDENTITY_DRIFT_LOG_INTERVAL / 2),
|
||||
!version_identity_drift_log_due(arn, start + VERSION_IDENTITY_DRIFT_LOG_INTERVAL / 2),
|
||||
"a second drift inside the interval must stay throttled"
|
||||
);
|
||||
assert!(
|
||||
version_identity_drift_log_due("source", arn, start + VERSION_IDENTITY_DRIFT_LOG_INTERVAL),
|
||||
version_identity_drift_log_due(arn, start + VERSION_IDENTITY_DRIFT_LOG_INTERVAL),
|
||||
"drift must become visible again once the interval elapses, instead of \
|
||||
going silent for the rest of the process lifetime"
|
||||
);
|
||||
@@ -9448,10 +9466,9 @@ mod tests {
|
||||
fn version_identity_drift_throttles_each_target_independently() {
|
||||
let now = Instant::now();
|
||||
|
||||
assert!(version_identity_drift_log_due("source", "arn:replication::drift-a", now));
|
||||
assert!(version_identity_drift_log_due("other-source", "arn:replication::drift-a", now));
|
||||
assert!(version_identity_drift_log_due("arn:replication::drift-a", now));
|
||||
assert!(
|
||||
version_identity_drift_log_due("source", "arn:replication::drift-b", now),
|
||||
version_identity_drift_log_due("arn:replication::drift-b", now),
|
||||
"one target's report must not silence another's"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -197,9 +197,31 @@ impl ReplicationTargetStore {
|
||||
BucketTargetSys::get().mark_target_offline(target_client).await
|
||||
}
|
||||
|
||||
/// Returns the cached verdict and whether it has outlived its TTL.
|
||||
pub(crate) async fn ssec_passthrough_capability(arn: &str) -> (SsecPassthroughCapability, bool) {
|
||||
BucketTargetSys::get().ssec_passthrough_capability(arn).await
|
||||
}
|
||||
|
||||
pub(crate) async fn record_ssec_passthrough_capability(arn: &str, capability: SsecPassthroughCapability) {
|
||||
BucketTargetSys::get()
|
||||
.record_ssec_passthrough_capability(arn, capability)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) fn version_identity_capability(arn: &str) -> VersionIdentityCapability {
|
||||
BucketTargetSys::get().version_identity_capability(arn)
|
||||
}
|
||||
|
||||
pub(crate) fn record_version_identity_capability(arn: &str, capability: VersionIdentityCapability) {
|
||||
BucketTargetSys::get().record_version_identity_capability(arn, capability)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn register_test_target(target_client: &Arc<TargetClient>) {
|
||||
BucketTargetSys::get().register_test_client(target_client).await;
|
||||
BucketTargetSys::get().arn_remotes_map.write().await.insert(
|
||||
target_client.arn.clone(),
|
||||
crate::bucket::bucket_target_sys::ArnTarget::with_client(target_client.clone()),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -28,7 +28,7 @@ const REDACTED_CREDENTIAL: &str = "<redacted>";
|
||||
const GO_YEAR_ONE_START_UNIX_SECONDS: i64 = -62_135_596_800;
|
||||
const GO_YEAR_TWO_START_UNIX_SECONDS: i64 = -62_104_060_800;
|
||||
|
||||
#[derive(Deserialize, Serialize, Default, Clone, PartialEq, Eq)]
|
||||
#[derive(Deserialize, Serialize, Default, Clone)]
|
||||
pub struct Credentials {
|
||||
#[serde(rename = "accessKey")]
|
||||
pub access_key: String,
|
||||
|
||||
@@ -73,11 +73,10 @@ use rustfs_protos::proto_gen::node_service::{
|
||||
PreparePartTransactionRequest, ReadAllRequest, ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse,
|
||||
ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RecoverDeleteTransactionRequest,
|
||||
RecoverMultipartSourceRequest as RecoverMultipartSourceRpcRequest,
|
||||
RecoverMultipartSourceResponse as RecoverMultipartSourceRpcResponse, RenameDataBatchItem, RenameDataBatchRequest,
|
||||
RenameDataBatchResult, RenameDataRequest, RenameFileRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest,
|
||||
SnapshotLeaseRenewRequest, SnapshotLeaseRequest, SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest,
|
||||
VerifyFileRequest, WriteAllRequest, WriteAllResponse, WriteFormatControlRequest, WriteMetadataRequest,
|
||||
node_service_client::NodeServiceClient,
|
||||
RecoverMultipartSourceResponse as RecoverMultipartSourceRpcResponse, RenameDataRequest, RenameFileRequest,
|
||||
SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest,
|
||||
SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteAllResponse,
|
||||
WriteFormatControlRequest, WriteMetadataRequest, node_service_client::NodeServiceClient,
|
||||
};
|
||||
use rustfs_protos::{
|
||||
ChannelClass, ChannelRebuildReason, ConnectionEvictionLogLevel, evict_failed_connection_with_reason, prewarm_channel_class,
|
||||
@@ -105,7 +104,7 @@ use tokio::{
|
||||
time::timeout,
|
||||
};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tonic::{Code, Request, Streaming, service::interceptor::InterceptedService};
|
||||
use tonic::{Code, Request, service::interceptor::InterceptedService};
|
||||
use tracing::{debug, trace, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -1046,7 +1045,6 @@ pub struct RemoteDisk {
|
||||
/// Optional per-peer gate held through complete bulk-unary decoding and validation.
|
||||
bulk_body_admission: Option<Arc<Semaphore>>,
|
||||
bulk_body_wait_timeout: Duration,
|
||||
rename_admission: Option<Arc<Semaphore>>,
|
||||
}
|
||||
|
||||
/// Shortest gap between two failure warnings of one kind from one remote disk.
|
||||
@@ -1062,251 +1060,6 @@ struct BulkBodyAdmissionRegistry {
|
||||
static BULK_BODY_ADMISSION_REGISTRY: std::sync::LazyLock<StdMutex<BulkBodyAdmissionRegistry>> =
|
||||
std::sync::LazyLock::new(|| StdMutex::new(BulkBodyAdmissionRegistry::default()));
|
||||
|
||||
const ENV_RUSTFS_EXPERIMENTAL_RENAME_STREAM_BATCH: &str = "RUSTFS_EXPERIMENTAL_RENAME_STREAM_BATCH";
|
||||
const RENAME_ADMISSION_REGISTRY_CAPACITY: usize = 512;
|
||||
|
||||
#[derive(Default)]
|
||||
struct RenameAdmissionRegistry {
|
||||
peers: HashMap<String, Weak<Semaphore>>,
|
||||
overflow: Option<Arc<Semaphore>>,
|
||||
}
|
||||
|
||||
static RENAME_ADMISSION_REGISTRY: std::sync::LazyLock<StdMutex<RenameAdmissionRegistry>> =
|
||||
std::sync::LazyLock::new(|| StdMutex::new(RenameAdmissionRegistry::default()));
|
||||
|
||||
fn peer_rename_admission(addr: &str, enabled: bool) -> Option<Arc<Semaphore>> {
|
||||
if !enabled {
|
||||
return None;
|
||||
}
|
||||
let mut registry = RENAME_ADMISSION_REGISTRY
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
if let Some(semaphore) = registry.peers.get(addr).and_then(Weak::upgrade) {
|
||||
return Some(semaphore);
|
||||
}
|
||||
if registry.peers.len() >= RENAME_ADMISSION_REGISTRY_CAPACITY {
|
||||
registry.peers.retain(|_, semaphore| semaphore.strong_count() > 0);
|
||||
}
|
||||
if registry.peers.len() >= RENAME_ADMISSION_REGISTRY_CAPACITY {
|
||||
return Some(Arc::clone(registry.overflow.get_or_insert_with(|| {
|
||||
Arc::new(Semaphore::new(rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT))
|
||||
})));
|
||||
}
|
||||
let semaphore = Arc::new(Semaphore::new(rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT));
|
||||
registry.peers.insert(addr.to_owned(), Arc::downgrade(&semaphore));
|
||||
Some(semaphore)
|
||||
}
|
||||
|
||||
struct RenameAdmissionGrant {
|
||||
permit: Option<OwnedSemaphorePermit>,
|
||||
batch: bool,
|
||||
}
|
||||
|
||||
impl RenameAdmissionGrant {
|
||||
fn release_one(&mut self) -> bool {
|
||||
self.permit.as_mut().is_some_and(|permit| permit.split(1).is_some())
|
||||
}
|
||||
|
||||
fn release_all(mut self) {
|
||||
drop(self.permit.take());
|
||||
}
|
||||
|
||||
fn quarantine_until(mut self, deadline: tokio::time::Instant) {
|
||||
let Some(permit) = self.permit.take() else {
|
||||
return;
|
||||
};
|
||||
let items = permit.num_permits();
|
||||
if items == 0 {
|
||||
return;
|
||||
}
|
||||
let hold = deadline.saturating_duration_since(tokio::time::Instant::now());
|
||||
rustfs_io_metrics::record_ecstore_rename_quarantine(self.batch, items, hold);
|
||||
let batch = self.batch;
|
||||
if let Ok(handle) = tokio::runtime::Handle::try_current() {
|
||||
handle.spawn(async move {
|
||||
tokio::time::sleep_until(deadline).await;
|
||||
drop(permit);
|
||||
rustfs_io_metrics::record_ecstore_rename_quarantine_released(batch, items);
|
||||
});
|
||||
} else {
|
||||
// Without a runtime there is no safe deadline task. Leaking until
|
||||
// process exit is safer than admitting work over a still-running mutation.
|
||||
rustfs_io_metrics::record_ecstore_rename_quarantine_no_runtime(batch, items);
|
||||
std::mem::forget(permit);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct RenameBatchCallGuard {
|
||||
admission: Option<RenameAdmissionGrant>,
|
||||
deadline: tokio::time::Instant,
|
||||
dispatch_state: RenameAdmissionDispatchState,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
enum RenameAdmissionDispatchState {
|
||||
NotDispatched,
|
||||
MayHavePublished,
|
||||
}
|
||||
|
||||
impl RenameBatchCallGuard {
|
||||
fn mark_dispatched(&mut self) {
|
||||
self.dispatch_state = RenameAdmissionDispatchState::MayHavePublished;
|
||||
}
|
||||
|
||||
fn complete(mut self) {
|
||||
debug_assert_eq!(self.dispatch_state, RenameAdmissionDispatchState::MayHavePublished);
|
||||
if let Some(admission) = self.admission.take() {
|
||||
admission.release_all();
|
||||
}
|
||||
}
|
||||
|
||||
fn release_without_quarantine(mut self) {
|
||||
if let Some(admission) = self.admission.take() {
|
||||
admission.release_all();
|
||||
}
|
||||
}
|
||||
|
||||
fn into_stream(mut self, inner: Streaming<RenameDataBatchResult>, expected_mask: u16) -> RemoteRenameBatchStream {
|
||||
debug_assert_eq!(self.dispatch_state, RenameAdmissionDispatchState::MayHavePublished);
|
||||
RemoteRenameBatchStream {
|
||||
inner,
|
||||
expected_mask,
|
||||
seen_mask: 0,
|
||||
admission: self.admission.take(),
|
||||
deadline: self.deadline,
|
||||
terminal: false,
|
||||
stream_started_at: rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now),
|
||||
first_item_at: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RenameBatchCallGuard {
|
||||
fn drop(&mut self) {
|
||||
if self.dispatch_state == RenameAdmissionDispatchState::MayHavePublished
|
||||
&& let Some(admission) = self.admission.take()
|
||||
{
|
||||
admission.quarantine_until(self.deadline);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct RenameBatchClientItem<'a> {
|
||||
pub slot: usize,
|
||||
pub disk: String,
|
||||
pub file_info: &'a FileInfo,
|
||||
}
|
||||
|
||||
pub(crate) enum RenameBatchStartError {
|
||||
NotDispatched(Error),
|
||||
UnsupportedBeforeStream,
|
||||
MayHavePublished(Error),
|
||||
}
|
||||
|
||||
pub(crate) struct RemoteRenameBatchOutcome {
|
||||
pub slot: usize,
|
||||
pub result: Result<RenameDataResp>,
|
||||
}
|
||||
|
||||
pub(crate) struct RemoteRenameBatchStream {
|
||||
inner: Streaming<RenameDataBatchResult>,
|
||||
expected_mask: u16,
|
||||
seen_mask: u16,
|
||||
admission: Option<RenameAdmissionGrant>,
|
||||
deadline: tokio::time::Instant,
|
||||
terminal: bool,
|
||||
stream_started_at: Option<Instant>,
|
||||
first_item_at: Option<Instant>,
|
||||
}
|
||||
|
||||
impl RemoteRenameBatchStream {
|
||||
fn terminate_ambiguous(&mut self, error: Error) -> Error {
|
||||
if !self.terminal {
|
||||
let missing = (self.expected_mask & !self.seen_mask).count_ones() as usize;
|
||||
rustfs_io_metrics::record_ecstore_rename_batch_terminal_missing(missing);
|
||||
}
|
||||
self.terminal = true;
|
||||
if let Some(admission) = self.admission.take() {
|
||||
admission.quarantine_until(self.deadline);
|
||||
}
|
||||
error
|
||||
}
|
||||
|
||||
pub(crate) async fn next_outcome(&mut self) -> Result<Option<RemoteRenameBatchOutcome>> {
|
||||
if self.terminal {
|
||||
return Ok(None);
|
||||
}
|
||||
let response = match tokio::time::timeout_at(self.deadline, self.inner.message()).await {
|
||||
Err(_) => return Err(self.terminate_ambiguous(DiskError::Timeout)),
|
||||
Ok(Err(status)) => return Err(self.terminate_ambiguous(status.into())),
|
||||
Ok(Ok(None)) if self.seen_mask == self.expected_mask => {
|
||||
self.terminal = true;
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Ok(None)) => return Err(self.terminate_ambiguous(DiskError::Unexpected)),
|
||||
Ok(Ok(Some(response))) => response,
|
||||
};
|
||||
let Ok(slot) = usize::try_from(response.item_index) else {
|
||||
return Err(self.terminate_ambiguous(DiskError::Unexpected));
|
||||
};
|
||||
if slot >= 16 {
|
||||
return Err(self.terminate_ambiguous(DiskError::Unexpected));
|
||||
}
|
||||
let bit = 1_u16 << slot;
|
||||
if self.expected_mask & bit == 0 || self.seen_mask & bit != 0 {
|
||||
return Err(self.terminate_ambiguous(DiskError::Unexpected));
|
||||
}
|
||||
self.seen_mask |= bit;
|
||||
if self.admission.as_mut().is_some_and(|admission| !admission.release_one()) {
|
||||
return Err(self.terminate_ambiguous(DiskError::Unexpected));
|
||||
}
|
||||
if let Some(stream_started_at) = self.stream_started_at {
|
||||
let completed_at = Instant::now();
|
||||
match self.first_item_at {
|
||||
Some(first_item_at) if self.seen_mask == self.expected_mask => {
|
||||
rustfs_io_metrics::record_ecstore_rename_batch_client_first_to_last(
|
||||
completed_at.saturating_duration_since(first_item_at),
|
||||
);
|
||||
}
|
||||
Some(_) => {}
|
||||
None => {
|
||||
rustfs_io_metrics::record_ecstore_rename_batch_client_stream_to_first(
|
||||
completed_at.saturating_duration_since(stream_started_at),
|
||||
);
|
||||
self.first_item_at = Some(completed_at);
|
||||
if self.seen_mask == self.expected_mask {
|
||||
rustfs_io_metrics::record_ecstore_rename_batch_client_first_to_last(Duration::ZERO);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
rustfs_io_metrics::record_ecstore_rename_batch_item_completed();
|
||||
let result = if response.success {
|
||||
if response.error.is_some() {
|
||||
Err(DiskError::FileCorrupt)
|
||||
} else {
|
||||
decode_msgpack_strict(&response.rename_data_resp_msgpack)
|
||||
}
|
||||
} else {
|
||||
Err(response.error.unwrap_or_default().into())
|
||||
};
|
||||
Ok(Some(RemoteRenameBatchOutcome { slot, result }))
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RemoteRenameBatchStream {
|
||||
fn drop(&mut self) {
|
||||
if !self.terminal {
|
||||
let missing = (self.expected_mask & !self.seen_mask).count_ones() as usize;
|
||||
rustfs_io_metrics::record_ecstore_rename_batch_terminal_missing(missing);
|
||||
}
|
||||
if let Some(admission) = self.admission.take() {
|
||||
admission.quarantine_until(self.deadline);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct BulkBodyPermit {
|
||||
_metrics: rustfs_io_metrics::internode_metrics::InternodeBulkBodyAdmissionGuard,
|
||||
_permit: OwnedSemaphorePermit,
|
||||
@@ -1473,10 +1226,6 @@ fn spawn_control_channel_prewarm(addr: String) {
|
||||
}
|
||||
|
||||
impl RemoteDisk {
|
||||
pub(crate) fn rename_stream_batch_enabled(&self) -> bool {
|
||||
self.rename_admission.is_some()
|
||||
}
|
||||
|
||||
async fn guarded_delete_versions_rpc(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -1802,8 +1551,6 @@ impl RemoteDisk {
|
||||
rustfs_utils::get_env_bool(ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING);
|
||||
let bulk_body_admission = peer_bulk_body_admission(&addr, bulk_body_concurrency_limit()?);
|
||||
let bulk_body_wait_timeout = bulk_body_wait_timeout();
|
||||
let rename_admission =
|
||||
peer_rename_admission(&addr, rustfs_utils::get_env_bool(ENV_RUSTFS_EXPERIMENTAL_RENAME_STREAM_BATCH, false));
|
||||
|
||||
let disk = Self {
|
||||
handle_id: Uuid::new_v4(),
|
||||
@@ -1826,7 +1573,6 @@ impl RemoteDisk {
|
||||
network_error_logs: rustfs_utils::LogThrottle::new(REMOTE_DISK_RPC_FAILURE_LOG_INTERVAL_MS),
|
||||
bulk_body_admission,
|
||||
bulk_body_wait_timeout,
|
||||
rename_admission,
|
||||
};
|
||||
record_drive_runtime_state(ep, RuntimeDriveHealthState::Online);
|
||||
|
||||
@@ -2562,54 +2308,6 @@ impl RemoteDisk {
|
||||
}
|
||||
}
|
||||
|
||||
async fn execute_peer_batch_with_timeout<T, F, Fut>(
|
||||
&self,
|
||||
op: &'static str,
|
||||
operation: F,
|
||||
timeout_duration: Duration,
|
||||
) -> Result<T>
|
||||
where
|
||||
F: FnOnce() -> Fut,
|
||||
Fut: std::future::Future<Output = Result<T>>,
|
||||
{
|
||||
if timeout_duration == Duration::ZERO {
|
||||
self.record_timeout(op, timeout_duration);
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
match time::timeout(timeout_duration, operation()).await {
|
||||
Ok(operation_result) => {
|
||||
if operation_result.is_ok() {
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
|
||||
}
|
||||
let evict = operation_result
|
||||
.as_ref()
|
||||
.err()
|
||||
.is_some_and(|error| !error.is_metacache_output_stream_closed() && is_network_like_disk_error(error));
|
||||
self.handle_network_like_error(op, timeout_duration, &operation_result, FailureHealthAction::IgnoreFailure)
|
||||
.await;
|
||||
if evict {
|
||||
evict_failed_connection_with_reason(
|
||||
&self.addr,
|
||||
ChannelRebuildReason::RpcFailure,
|
||||
ConnectionEvictionLogLevel::Debug,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
operation_result
|
||||
}
|
||||
Err(_) => {
|
||||
self.record_timeout(op, timeout_duration);
|
||||
evict_failed_connection_with_reason(
|
||||
&self.addr,
|
||||
ChannelRebuildReason::OperationTimeout,
|
||||
ConnectionEvictionLogLevel::Debug,
|
||||
)
|
||||
.await;
|
||||
Err(DiskError::Timeout)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn record_timeout(&self, op: &'static str, timeout_duration: Duration) {
|
||||
counter!(
|
||||
"rustfs_drive_op_timeout_total",
|
||||
@@ -2796,38 +2494,6 @@ impl RemoteDisk {
|
||||
}))
|
||||
}
|
||||
|
||||
async fn acquire_rename_admission_until(
|
||||
&self,
|
||||
item_count: usize,
|
||||
deadline: tokio::time::Instant,
|
||||
) -> Result<Option<RenameAdmissionGrant>> {
|
||||
let Some(semaphore) = &self.rename_admission else {
|
||||
return Ok(None);
|
||||
};
|
||||
if item_count == 0 || item_count > rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT {
|
||||
return Err(Error::other("invalid internode rename admission weight"));
|
||||
}
|
||||
if let Some(err) = self.offline_bypass_error() {
|
||||
return Err(err);
|
||||
}
|
||||
let weight = u32::try_from(item_count).map_err(Error::other)?;
|
||||
let started = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now);
|
||||
let (result, timed_out) = match tokio::time::timeout_at(deadline, Arc::clone(semaphore).acquire_many_owned(weight)).await
|
||||
{
|
||||
Ok(Ok(permit)) => (Ok(permit), false),
|
||||
Ok(Err(_)) => (Err(Error::RemoteClientUnavailable("internode rename admission closed".to_owned())), false),
|
||||
Err(_) => (Err(DiskError::DiskOngoingReq), true),
|
||||
};
|
||||
if let Some(started) = started {
|
||||
rustfs_io_metrics::record_ecstore_rename_admission(item_count > 1, item_count, started.elapsed(), timed_out);
|
||||
}
|
||||
let permit = result?;
|
||||
Ok(Some(RenameAdmissionGrant {
|
||||
permit: Some(permit),
|
||||
batch: item_count > 1,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn get_bulk_client(&self) -> Result<NodeRpcClient> {
|
||||
if let Some(err) = self.offline_bypass_error() {
|
||||
return Err(err);
|
||||
@@ -2893,44 +2559,9 @@ fn encode_file_info_versions_msgpack(value: &FileInfoVersions) -> Result<Vec<u8>
|
||||
encode_msgpack_with_capacity(value, capacity)
|
||||
}
|
||||
|
||||
fn encode_delete_versions_request(
|
||||
disk: String,
|
||||
volume: &str,
|
||||
versions: &[FileInfoVersions],
|
||||
opts: &DeleteOptions,
|
||||
) -> Result<DeleteVersionsRequest> {
|
||||
// Binary-only batches require fleet-confirmed receivers that traverse the
|
||||
// binary list. Capture that policy for the complete batch and its options.
|
||||
let msgpack_only = rustfs_protos::internode_rpc_msgpack_only();
|
||||
let opts_json = if msgpack_only {
|
||||
String::new()
|
||||
} else {
|
||||
serde_json::to_string(opts)?
|
||||
};
|
||||
let mut versions_json = if msgpack_only {
|
||||
Vec::new()
|
||||
} else {
|
||||
Vec::with_capacity(versions.len())
|
||||
};
|
||||
let mut versions_bin = Vec::with_capacity(versions.len());
|
||||
for version in versions {
|
||||
if !msgpack_only {
|
||||
versions_json.push(serde_json::to_string(version)?);
|
||||
}
|
||||
versions_bin.push(encode_file_info_versions_msgpack(version)?.into());
|
||||
}
|
||||
Ok(DeleteVersionsRequest {
|
||||
disk,
|
||||
volume: volume.to_owned(),
|
||||
versions: versions_json,
|
||||
opts: opts_json,
|
||||
versions_bin,
|
||||
opts_bin: encode_delete_options_msgpack(opts)?.into(),
|
||||
})
|
||||
}
|
||||
|
||||
/// JSON compatibility string for a dual-encoded request field. The captured
|
||||
/// send policy selects the encoding, and its generation guard binds dispatch.
|
||||
/// JSON compatibility string for a dual-encoded (`_bin` + text) request field. Returns an empty
|
||||
/// string only when msgpack-only mode and its explicit fleet confirmation guard are both enabled;
|
||||
/// otherwise the legacy JSON encoding is retained for old peers.
|
||||
fn compat_json<T: Serialize>(value: &T, policy: &MsgpackSendPolicy) -> Result<String> {
|
||||
if policy.omit_eligible_json() {
|
||||
return Ok(String::new());
|
||||
@@ -2968,18 +2599,6 @@ fn decode_msgpack_or_json<T: DeserializeOwned>(binary: &[u8], json: &str, value_
|
||||
}
|
||||
}
|
||||
|
||||
fn decode_msgpack_strict<T: DeserializeOwned>(binary: &[u8]) -> Result<T> {
|
||||
if binary.is_empty() {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
let mut cursor = Cursor::new(binary);
|
||||
let value = rmp_serde::from_read(&mut cursor).map_err(|_| DiskError::FileCorrupt)?;
|
||||
if cursor.position() != u64::try_from(binary.len()).map_err(Error::other)? {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
Ok(value)
|
||||
}
|
||||
|
||||
fn decode_multipart_publication_response(binary: &[u8]) -> Result<RenameDataResp> {
|
||||
let result = if binary.len() > MultipartSourceOperation::max_wire_payload_bytes() {
|
||||
Err(DiskError::FileCorrupt)
|
||||
@@ -3177,97 +2796,6 @@ impl RemoteDisk {
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub(crate) async fn start_rename_data_batch(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
items: Vec<RenameBatchClientItem<'_>>,
|
||||
) -> std::result::Result<RemoteRenameBatchStream, RenameBatchStartError> {
|
||||
if self.rename_admission.is_none() || items.is_empty() || items.len() > 16 {
|
||||
return Err(RenameBatchStartError::NotDispatched(DiskError::UnsupportedDisk));
|
||||
}
|
||||
let deadline = tokio::time::Instant::now() + get_max_timeout_duration();
|
||||
let mut expected_mask = 0_u16;
|
||||
let mut request_items = Vec::with_capacity(items.len());
|
||||
for item in items {
|
||||
if item.slot >= 16 {
|
||||
return Err(RenameBatchStartError::NotDispatched(DiskError::Unexpected));
|
||||
}
|
||||
let bit = 1_u16 << item.slot;
|
||||
if expected_mask & bit != 0 {
|
||||
return Err(RenameBatchStartError::NotDispatched(DiskError::Unexpected));
|
||||
}
|
||||
expected_mask |= bit;
|
||||
let item_index = u32::try_from(item.slot)
|
||||
.map_err(Error::other)
|
||||
.map_err(RenameBatchStartError::NotDispatched)?;
|
||||
let file_info_msgpack = encode_file_info_msgpack(item.file_info).map_err(RenameBatchStartError::NotDispatched)?;
|
||||
request_items.push(RenameDataBatchItem {
|
||||
item_index,
|
||||
disk: item.disk,
|
||||
file_info_msgpack: file_info_msgpack.into(),
|
||||
});
|
||||
}
|
||||
let mut request = Request::new(RenameDataBatchRequest {
|
||||
src_volume: src_volume.to_owned(),
|
||||
src_path: src_path.to_owned(),
|
||||
dst_volume: dst_volume.to_owned(),
|
||||
dst_path: dst_path.to_owned(),
|
||||
items: request_items,
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_rename_data_batch_request_body(request.get_ref())
|
||||
.map_err(Error::other)
|
||||
.map_err(RenameBatchStartError::NotDispatched)?;
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical_body)
|
||||
.map_err(Error::other)
|
||||
.map_err(RenameBatchStartError::NotDispatched)?;
|
||||
let mut client = match tokio::time::timeout_at(deadline, self.get_client()).await {
|
||||
Ok(result) => result.map_err(RenameBatchStartError::NotDispatched)?,
|
||||
Err(_) => return Err(RenameBatchStartError::NotDispatched(DiskError::Timeout)),
|
||||
};
|
||||
let admission = self
|
||||
.acquire_rename_admission_until(request.get_ref().items.len(), deadline)
|
||||
.await
|
||||
.map_err(RenameBatchStartError::NotDispatched)?;
|
||||
let guard = RenameBatchCallGuard {
|
||||
admission,
|
||||
deadline,
|
||||
dispatch_state: RenameAdmissionDispatchState::NotDispatched,
|
||||
};
|
||||
let call_timeout = deadline.saturating_duration_since(tokio::time::Instant::now());
|
||||
if call_timeout == Duration::ZERO {
|
||||
guard.release_without_quarantine();
|
||||
return Err(RenameBatchStartError::NotDispatched(DiskError::Timeout));
|
||||
}
|
||||
let mut guard = guard;
|
||||
guard.mark_dispatched();
|
||||
let response = match self
|
||||
.execute_peer_batch_with_timeout(
|
||||
"rename_data_batch",
|
||||
|| async {
|
||||
match client.rename_data_batch(request).await {
|
||||
Ok(response) => Ok(response),
|
||||
Err(status) if status.code() == Code::Unimplemented => Err(DiskError::UnsupportedDisk),
|
||||
Err(status) => Err(status.into()),
|
||||
}
|
||||
},
|
||||
call_timeout,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Err(DiskError::UnsupportedDisk) => {
|
||||
guard.release_without_quarantine();
|
||||
return Err(RenameBatchStartError::UnsupportedBeforeStream);
|
||||
}
|
||||
Err(error) => return Err(RenameBatchStartError::MayHavePublished(error)),
|
||||
Ok(response) => response,
|
||||
};
|
||||
Ok(guard.into_stream(response.into_inner(), expected_mask))
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub(crate) async fn rename_data_borrowed_with_fence(
|
||||
&self,
|
||||
@@ -3339,27 +2867,6 @@ impl RemoteDisk {
|
||||
let scanner_publication_lease = scanner_publication_lease_token
|
||||
.map(|token| Bytes::copy_from_slice(token.as_bytes()))
|
||||
.unwrap_or_default();
|
||||
let rename_deadline = tokio::time::Instant::now() + get_max_timeout_duration();
|
||||
let rename_admission = self.acquire_rename_admission_until(1, rename_deadline).await?;
|
||||
let mut rename_admission_guard = RenameBatchCallGuard {
|
||||
admission: rename_admission,
|
||||
deadline: rename_deadline,
|
||||
dispatch_state: RenameAdmissionDispatchState::NotDispatched,
|
||||
};
|
||||
let rpc_kind = if multipart_rename {
|
||||
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_MULTIPART
|
||||
} else if pool_meta_cas {
|
||||
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_POOL_META_CAS
|
||||
} else if quota_fenced {
|
||||
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_QUOTA_FENCE
|
||||
} else if incarnation_bound {
|
||||
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_INCARNATION
|
||||
} else if scanner_publication_lease_token.is_some() {
|
||||
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_SCANNER_LEASE
|
||||
} else {
|
||||
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_ORDINARY
|
||||
};
|
||||
rustfs_io_metrics::record_ecstore_rename_remote_rpc(rpc_kind);
|
||||
let rpc_started = rustfs_io_metrics::put_stage_timer();
|
||||
let mut retried_dual_write = false;
|
||||
let response = loop {
|
||||
@@ -3416,7 +2923,6 @@ impl RemoteDisk {
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_PREPARE,
|
||||
attempt_prepare_started,
|
||||
);
|
||||
rename_admission_guard.mark_dispatched();
|
||||
let result = rustfs_io_metrics::observe_put_stage_future(
|
||||
async {
|
||||
if multipart_rename {
|
||||
@@ -3447,7 +2953,6 @@ impl RemoteDisk {
|
||||
Err(status) => return Err(status.into()),
|
||||
}
|
||||
};
|
||||
rename_admission_guard.complete();
|
||||
let response_process_started = rustfs_io_metrics::put_stage_timer();
|
||||
let response_headers_started = rustfs_io_metrics::put_stage_timer();
|
||||
let _ = response.metadata().len();
|
||||
@@ -4042,10 +3547,52 @@ impl DiskAPI for RemoteDisk {
|
||||
return vec![Some(DiskError::FaultyDisk); versions.len()];
|
||||
}
|
||||
|
||||
let request = match encode_delete_versions_request(self.endpoint.to_string(), volume, &versions, &opts) {
|
||||
Ok(request) => request,
|
||||
Err(err) => return vec![Some(Error::other(err.to_string())); versions.len()],
|
||||
// `_bin` support for DeleteVersions is new (grpc-optimization P2); always dual-write JSON +
|
||||
// msgpack until its fallback counter has read zero across a release window.
|
||||
let opts_bin = match encode_delete_options_msgpack(&opts) {
|
||||
Ok(opts_bin) => opts_bin,
|
||||
Err(err) => {
|
||||
let mut errors = Vec::with_capacity(versions.len());
|
||||
for _ in 0..versions.len() {
|
||||
errors.push(Some(Error::other(err.to_string())));
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
};
|
||||
let opts = match serde_json::to_string(&opts) {
|
||||
Ok(opts) => opts,
|
||||
Err(err) => {
|
||||
let mut errors = Vec::with_capacity(versions.len());
|
||||
for _ in 0..versions.len() {
|
||||
errors.push(Some(Error::other(err.to_string())));
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
};
|
||||
let mut versions_str = Vec::with_capacity(versions.len());
|
||||
let mut versions_bin = Vec::with_capacity(versions.len());
|
||||
for file_info_versions in versions.iter() {
|
||||
versions_str.push(match serde_json::to_string(file_info_versions) {
|
||||
Ok(versions_str) => versions_str,
|
||||
Err(err) => {
|
||||
let mut errors = Vec::with_capacity(versions.len());
|
||||
for _ in 0..versions.len() {
|
||||
errors.push(Some(Error::other(err.to_string())));
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
});
|
||||
versions_bin.push(match encode_file_info_versions_msgpack(file_info_versions) {
|
||||
Ok(versions_bin) => Bytes::from(versions_bin),
|
||||
Err(err) => {
|
||||
let mut errors = Vec::with_capacity(versions.len());
|
||||
for _ in 0..versions.len() {
|
||||
errors.push(Some(Error::other(err.to_string())));
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
});
|
||||
}
|
||||
let mut client = match self.get_client().await {
|
||||
Ok(client) => client,
|
||||
Err(err) => {
|
||||
@@ -4057,7 +3604,14 @@ impl DiskAPI for RemoteDisk {
|
||||
}
|
||||
};
|
||||
|
||||
let mut request = Request::new(request);
|
||||
let mut request = Request::new(DeleteVersionsRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
volume: volume.to_string(),
|
||||
versions: versions_str,
|
||||
opts,
|
||||
versions_bin,
|
||||
opts_bin: opts_bin.into(),
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_delete_versions_request_body(request.get_ref());
|
||||
if let Err(err) = attach_mutation_body_digest(&mut request, canonical_body, "delete_versions") {
|
||||
let mut errors = Vec::with_capacity(versions.len());
|
||||
@@ -6783,94 +6337,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_versions_request_encoding_preserves_batches_and_fleet_guards() {
|
||||
use tonic_prost::prost::Message;
|
||||
let groups: Vec<_> = ["first", "second"]
|
||||
.into_iter()
|
||||
.map(|name| FileInfoVersions {
|
||||
volume: "bucket".to_owned(),
|
||||
name: name.to_owned(),
|
||||
versions: vec![sample_rename_data_file_info()],
|
||||
..Default::default()
|
||||
})
|
||||
.collect();
|
||||
let opts = DeleteOptions {
|
||||
undo_write: true,
|
||||
old_data_dir: Some(Uuid::from_u128(7)),
|
||||
..Default::default()
|
||||
};
|
||||
let mut binary_wire_size = 0;
|
||||
let mut compat_wire_size = 0;
|
||||
for (requested, confirmed, peer_capability, binary_only) in [
|
||||
(None, None, None, false),
|
||||
(Some("true"), None, None, false),
|
||||
(Some("false"), Some("true"), None, false),
|
||||
(Some("true"), Some("false"), None, false),
|
||||
(Some("true"), None, Some("true"), false),
|
||||
(Some("true"), Some("false"), Some("true"), false),
|
||||
(Some("true"), Some("true"), None, true),
|
||||
(Some("true"), Some("true"), Some("true"), true),
|
||||
] {
|
||||
with_internode_msgpack_env(
|
||||
[
|
||||
(rustfs_config::ENV_INTERNODE_RPC_MSGPACK_ONLY, requested),
|
||||
(rustfs_config::ENV_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED, confirmed),
|
||||
(rustfs_config::ENV_INTERNODE_RPC_MSGPACK_PEER_CAPABILITY, peer_capability),
|
||||
],
|
||||
|| {
|
||||
for batch in [&groups[..], &[]] {
|
||||
let disk = "disk".to_owned();
|
||||
let disk_buffer = disk.as_ptr();
|
||||
let request = encode_delete_versions_request(disk, "bucket", batch, &opts).expect("encode batch");
|
||||
assert_eq!(request.disk, "disk");
|
||||
assert_eq!(request.disk.as_ptr(), disk_buffer, "move the prepared endpoint into the request");
|
||||
assert_eq!(request.volume, "bucket");
|
||||
assert_eq!(request.versions_bin.len(), batch.len());
|
||||
assert_eq!(request.opts.is_empty(), binary_only);
|
||||
let decoded_opts: DeleteOptions = rmp_serde::from_slice(&request.opts_bin).expect("binary options");
|
||||
assert!(decoded_opts.undo_write);
|
||||
assert_eq!(decoded_opts.old_data_dir, opts.old_data_dir);
|
||||
for (encoded, expected) in request.versions_bin.iter().zip(batch) {
|
||||
let decoded: FileInfoVersions = rmp_serde::from_slice(encoded).expect("binary batch item");
|
||||
assert_eq!(decoded.name, expected.name);
|
||||
assert_eq!(decoded.versions, expected.versions);
|
||||
}
|
||||
if binary_only {
|
||||
assert!(request.versions.is_empty());
|
||||
} else {
|
||||
assert_eq!(request.opts, serde_json::to_string(&opts).expect("legacy options"));
|
||||
assert_eq!(request.versions.len(), batch.len());
|
||||
for (encoded, expected) in request.versions.iter().zip(batch) {
|
||||
let decoded: FileInfoVersions = serde_json::from_str(encoded).expect("legacy batch item");
|
||||
assert_eq!(decoded.name, expected.name);
|
||||
assert_eq!(decoded.versions, expected.versions);
|
||||
}
|
||||
}
|
||||
let canonical = rustfs_protos::canonical_delete_versions_request_body(&request).expect("bound batch");
|
||||
let mut modified = request.clone();
|
||||
modified.opts_bin = encode_delete_options_msgpack(&DeleteOptions::default())
|
||||
.expect("other options")
|
||||
.into();
|
||||
assert_ne!(
|
||||
canonical,
|
||||
rustfs_protos::canonical_delete_versions_request_body(&modified).expect("other bound batch")
|
||||
);
|
||||
if !batch.is_empty() {
|
||||
if binary_only {
|
||||
binary_wire_size = request.encoded_len();
|
||||
} else {
|
||||
compat_wire_size = request.encoded_len();
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
assert!(binary_wire_size > 0 && binary_wire_size < compat_wire_size);
|
||||
println!("delete batch encoded bytes compat={compat_wire_size} msgpack_only={binary_wire_size}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn msgpack_eligible_requests_bind_one_policy_and_bounded_local_retry() {
|
||||
let source = rustfs_protos::compat_manifest::production_source(include_str!("remote_disk.rs"), "remote_disk.rs");
|
||||
@@ -7821,10 +7287,7 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
fn with_internode_msgpack_env<R, const N: usize>(
|
||||
vars: [(&'static str, Option<&'static str>); N],
|
||||
f: impl FnOnce() -> R,
|
||||
) -> R {
|
||||
fn with_internode_msgpack_env<R>(vars: [(&'static str, Option<&'static str>); 2], f: impl FnOnce() -> R) -> R {
|
||||
temp_env::with_vars(vars, || {
|
||||
rustfs_protos::reset_internode_rpc_msgpack_only_cache();
|
||||
let result = f();
|
||||
@@ -13423,127 +12886,6 @@ mod tests {
|
||||
.expect("shared gate remains open");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rename_admission_is_weighted_shared_and_released_per_item() {
|
||||
use std::future::Future;
|
||||
use std::task::{Context, Poll, Waker};
|
||||
|
||||
let first = peer_rename_admission("http://rename-peer-a", true).expect("enabled rename gate");
|
||||
let same = peer_rename_admission("http://rename-peer-a", true).expect("same peer rename gate");
|
||||
let other = peer_rename_admission("http://rename-peer-b", true).expect("other peer rename gate");
|
||||
assert!(Arc::ptr_eq(&first, &same));
|
||||
assert!(!Arc::ptr_eq(&first, &other));
|
||||
assert!(peer_rename_admission("http://rename-disabled", false).is_none());
|
||||
assert_eq!(first.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
|
||||
let permit = Arc::clone(&first)
|
||||
.acquire_many_owned(4)
|
||||
.await
|
||||
.expect("weighted batch admission");
|
||||
let mut grant = RenameAdmissionGrant {
|
||||
permit: Some(permit),
|
||||
batch: true,
|
||||
};
|
||||
assert_eq!(first.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 4);
|
||||
assert!(grant.release_one());
|
||||
assert_eq!(first.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 3);
|
||||
drop(grant);
|
||||
assert_eq!(first.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
|
||||
let held = Arc::clone(&first)
|
||||
.acquire_many_owned(u32::try_from(rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 3).expect("small limit"))
|
||||
.await
|
||||
.expect("reserve all but three permits");
|
||||
let mut waiting = Box::pin(Arc::clone(&same).acquire_many_owned(4));
|
||||
assert!(matches!(waiting.as_mut().poll(&mut Context::from_waker(Waker::noop())), Poll::Pending));
|
||||
assert_eq!(first.available_permits(), 0, "the FIFO waiter reserves currently available permits");
|
||||
drop(waiting);
|
||||
assert_eq!(
|
||||
first.available_permits(),
|
||||
3,
|
||||
"cancelling a queued weighted acquire must return its reservation"
|
||||
);
|
||||
let waiting = Box::pin(Arc::clone(&same).acquire_many_owned(4));
|
||||
drop(held);
|
||||
let acquired = tokio::time::timeout(Duration::from_secs(1), waiting)
|
||||
.await
|
||||
.expect("weighted waiter wakes")
|
||||
.expect("rename gate remains open");
|
||||
drop(acquired);
|
||||
assert_eq!(first.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn rename_admission_quarantine_holds_until_absolute_deadline() {
|
||||
let gate = peer_rename_admission("http://rename-quarantine-peer", true).expect("enabled rename gate");
|
||||
let permit = Arc::clone(&gate)
|
||||
.acquire_many_owned(4)
|
||||
.await
|
||||
.expect("weighted batch admission");
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(1);
|
||||
RenameAdmissionGrant {
|
||||
permit: Some(permit),
|
||||
batch: true,
|
||||
}
|
||||
.quarantine_until(deadline);
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 4);
|
||||
|
||||
tokio::time::advance(Duration::from_millis(999)).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 4);
|
||||
tokio::time::advance(Duration::from_millis(1)).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn rename_call_guard_quarantines_only_after_dispatch() {
|
||||
let gate = peer_rename_admission("http://rename-call-guard-peer", true).expect("enabled rename gate");
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(1);
|
||||
let permit = Arc::clone(&gate).acquire_many_owned(4).await.expect("pre-dispatch permits");
|
||||
drop(RenameBatchCallGuard {
|
||||
admission: Some(RenameAdmissionGrant {
|
||||
permit: Some(permit),
|
||||
batch: true,
|
||||
}),
|
||||
deadline,
|
||||
dispatch_state: RenameAdmissionDispatchState::NotDispatched,
|
||||
});
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
|
||||
let permit = Arc::clone(&gate).acquire_many_owned(4).await.expect("dispatched permits");
|
||||
let mut guard = RenameBatchCallGuard {
|
||||
admission: Some(RenameAdmissionGrant {
|
||||
permit: Some(permit),
|
||||
batch: true,
|
||||
}),
|
||||
deadline,
|
||||
dispatch_state: RenameAdmissionDispatchState::NotDispatched,
|
||||
};
|
||||
guard.mark_dispatched();
|
||||
drop(guard);
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 4);
|
||||
tokio::time::advance(Duration::from_secs(1)).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
|
||||
let permit = Arc::clone(&gate)
|
||||
.acquire_many_owned(4)
|
||||
.await
|
||||
.expect("proven pre-mutation permits");
|
||||
let mut guard = RenameBatchCallGuard {
|
||||
admission: Some(RenameAdmissionGrant {
|
||||
permit: Some(permit),
|
||||
batch: true,
|
||||
}),
|
||||
deadline,
|
||||
dispatch_state: RenameAdmissionDispatchState::NotDispatched,
|
||||
};
|
||||
guard.mark_dispatched();
|
||||
guard.release_without_quarantine();
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remote_disk_builds_optional_complete_body_gate_from_configuration() {
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT, Some("3"))], async {
|
||||
|
||||
@@ -20,8 +20,8 @@ use async_trait::async_trait;
|
||||
use bytes::Bytes;
|
||||
use futures::{FutureExt, future::BoxFuture};
|
||||
use rustfs_lock::{
|
||||
BatchAcquireOutcome, CacheFenceStamp, LockCleanupOutcome, LockClient, LockError, LockInfo, LockRequest, LockResponse,
|
||||
LockStats, LockStatus, LockType, Result,
|
||||
BatchAcquireOutcome, LockCleanupOutcome, LockClient, LockError, LockInfo, LockRequest, LockResponse, LockStats, LockStatus,
|
||||
LockType, Result,
|
||||
types::{LockId, LockMetadata, LockPriority},
|
||||
};
|
||||
use rustfs_protos::proto_gen::node_service::{
|
||||
@@ -52,10 +52,6 @@ fn attach_lock_mutation_body_digest<T: rustfs_protos::CanonicalMutationBody>(req
|
||||
set_tonic_rolling_mutation_body_digest(request)
|
||||
}
|
||||
|
||||
fn decode_cache_fence_stamp(encoded: Option<&Bytes>) -> Option<CacheFenceStamp> {
|
||||
encoded.and_then(|encoded| CacheFenceStamp::decode(encoded).ok())
|
||||
}
|
||||
|
||||
/// Work to run if an RPC completes after its caller stops waiting.
|
||||
type LateCompletion<T> = Option<Box<dyn FnOnce(T) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send>>;
|
||||
type LateFailure = LateCompletion<()>;
|
||||
@@ -328,7 +324,6 @@ fn batch_acquire_outcome(lock_id: &LockId, lock_type: LockType, result: Option<&
|
||||
return BatchAcquireOutcome::Granted {
|
||||
lock_info,
|
||||
wait_time: Duration::ZERO,
|
||||
cache_fence_stamp: decode_cache_fence_stamp(result.cache_fence_stamp.as_ref()),
|
||||
};
|
||||
}
|
||||
if result.lock_info.is_some() {
|
||||
@@ -457,7 +452,6 @@ impl RemoteClient {
|
||||
deadlock_detection: false,
|
||||
suppress_contention_logs: false,
|
||||
refresh_interval: None,
|
||||
cache_fence_stamp_requested: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1579,11 +1573,10 @@ impl LockClient for RemoteClient {
|
||||
|
||||
// Check if the lock acquisition was successful
|
||||
if resp.success {
|
||||
let lock_info = Self::build_lock_info(request, resp.lock_info);
|
||||
match decode_cache_fence_stamp(resp.cache_fence_stamp.as_ref()) {
|
||||
Some(stamp) => Ok(LockResponse::success_with_cache_fence_stamp(lock_info, std::time::Duration::ZERO, stamp)),
|
||||
None => Ok(LockResponse::success(lock_info, std::time::Duration::ZERO)),
|
||||
}
|
||||
Ok(LockResponse::success(
|
||||
Self::build_lock_info(request, resp.lock_info),
|
||||
std::time::Duration::ZERO,
|
||||
))
|
||||
} else {
|
||||
// Lock acquisition failed
|
||||
Ok(LockResponse::failure(
|
||||
@@ -1601,19 +1594,12 @@ impl LockClient for RemoteClient {
|
||||
.iter()
|
||||
.zip(outcomes)
|
||||
.map(|(request, outcome)| match outcome {
|
||||
BatchAcquireOutcome::Granted {
|
||||
lock_info,
|
||||
wait_time,
|
||||
cache_fence_stamp,
|
||||
} => {
|
||||
let lock_info = lock_info
|
||||
BatchAcquireOutcome::Granted { lock_info, wait_time } => LockResponse::success(
|
||||
lock_info
|
||||
.map(|info| *info)
|
||||
.unwrap_or_else(|| Self::build_lock_info(request, None));
|
||||
match cache_fence_stamp {
|
||||
Some(stamp) => LockResponse::success_with_cache_fence_stamp(lock_info, wait_time, stamp),
|
||||
None => LockResponse::success(lock_info, wait_time),
|
||||
}
|
||||
}
|
||||
.unwrap_or_else(|| Self::build_lock_info(request, None)),
|
||||
wait_time,
|
||||
),
|
||||
BatchAcquireOutcome::Ambiguous {
|
||||
error: error @ LockError::Timeout { .. },
|
||||
..
|
||||
@@ -2381,38 +2367,6 @@ mod tests {
|
||||
assert!(matches!(classify(None), BatchAcquireOutcome::Ambiguous { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cache_fence_stamp_decoder_accepts_current_wire_shape_and_rejects_malformed_values() {
|
||||
let mut encoded = Vec::with_capacity(27);
|
||||
encoded.push(1);
|
||||
encoded.extend_from_slice(uuid::Uuid::from_u128(7).as_bytes());
|
||||
encoded.extend_from_slice(&17_u16.to_le_bytes());
|
||||
encoded.extend_from_slice(&42_u64.to_le_bytes());
|
||||
let encoded = Bytes::from(encoded);
|
||||
|
||||
let stamp = decode_cache_fence_stamp(Some(&encoded)).expect("current stamp wire shape");
|
||||
assert_eq!(stamp.voter_boot_id(), uuid::Uuid::from_u128(7));
|
||||
assert_eq!(stamp.shard_index(), 17);
|
||||
assert_eq!(stamp.generation(), 42);
|
||||
assert!(decode_cache_fence_stamp(None).is_none(), "legacy responses remain valid without proof");
|
||||
assert!(decode_cache_fence_stamp(Some(&Bytes::from_static(b"malformed"))).is_none());
|
||||
|
||||
let request = test_lock_request(Duration::from_secs(30));
|
||||
let result = GenerallyLockResult {
|
||||
success: true,
|
||||
cache_fence_stamp: Some(encoded),
|
||||
..Default::default()
|
||||
};
|
||||
let BatchAcquireOutcome::Granted {
|
||||
cache_fence_stamp: Some(batch_stamp),
|
||||
..
|
||||
} = batch_acquire_outcome(&request.lock_id, request.lock_type, Some(&result))
|
||||
else {
|
||||
panic!("valid batch grant must retain its cache-fence stamp");
|
||||
};
|
||||
assert_eq!(batch_stamp, stamp);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn batch_success_crossing_deadline_in_one_poll_runs_late_cleanup_once() {
|
||||
use crate::cluster::rpc::client::NoOpInterceptor;
|
||||
|
||||
@@ -101,6 +101,7 @@ const DEFAULT_RRS_STORAGE_CLASS: &str = "EC:1";
|
||||
const ZERO_SET_DRIVE_COUNT_ERROR: &str = "set drive count must be greater than zero";
|
||||
|
||||
pub static DEFAULT_INLINE_BLOCK: usize = 128 * 1024;
|
||||
const DEFAULT_INLINE_OBJECT_BUDGET: usize = 2 * DEFAULT_INLINE_BLOCK;
|
||||
|
||||
pub static DEFAULT_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
let kvs = vec![
|
||||
@@ -151,7 +152,6 @@ pub struct Config {
|
||||
inline_block: usize,
|
||||
initialized: bool,
|
||||
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
|
||||
// Preserve the explicit-limit marker when configurations round-trip through older readers.
|
||||
inline_block_explicit: bool,
|
||||
#[serde(skip)]
|
||||
standard_parities: Vec<PoolParity>,
|
||||
@@ -245,10 +245,8 @@ impl Config {
|
||||
return false;
|
||||
}
|
||||
|
||||
let Ok(shard_size) = usize::try_from(shard_size) else {
|
||||
return false;
|
||||
};
|
||||
let inline_block = self.inline_block();
|
||||
let shard_size = shard_size as usize;
|
||||
let inline_block = self.effective_inline_block(data_shards);
|
||||
|
||||
if versioned {
|
||||
shard_size <= inline_block / 8
|
||||
@@ -257,7 +255,27 @@ impl Config {
|
||||
}
|
||||
}
|
||||
|
||||
/// Per-shard inline data budget, defaulting to 128 KiB across EC layouts.
|
||||
/// Returns the per-shard inline budget used by both write admission and
|
||||
/// legacy read fallback.
|
||||
///
|
||||
/// The default budget is scaled by the number of data shards so a wider EC
|
||||
/// layout does not silently increase the maximum inline object size. An
|
||||
/// explicitly configured `inline_block` remains a fixed per-shard limit for
|
||||
/// compatibility with deployments that opted into the historical policy.
|
||||
pub(crate) fn effective_inline_block(&self, data_shards: usize) -> usize {
|
||||
if data_shards == 0 {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if self.initialized && self.inline_block_explicit {
|
||||
self.inline_block
|
||||
} else {
|
||||
// Keep the historical two-data-shard object budget while preventing
|
||||
// wider EC layouts from multiplying the maximum inline object size.
|
||||
DEFAULT_INLINE_OBJECT_BUDGET.div_ceil(data_shards).min(DEFAULT_INLINE_BLOCK)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn inline_block(&self) -> usize {
|
||||
if !self.initialized {
|
||||
DEFAULT_INLINE_BLOCK
|
||||
@@ -549,7 +567,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn should_inline_uses_fixed_default_threshold_for_every_ec_layout() {
|
||||
fn should_inline_scales_default_threshold_by_data_shards() {
|
||||
let config = lookup_config_for_pools_with_env(&KVS::new(), &[3, 12], no_env_overrides())
|
||||
.expect("default inline policy should resolve for EC2+1 and EC8+4");
|
||||
|
||||
@@ -558,14 +576,10 @@ mod tests {
|
||||
("EC2+1 unversioned above", 128 * 1024 + 1, 2, false, false),
|
||||
("EC2+1 versioned exact", 16 * 1024, 2, true, true),
|
||||
("EC2+1 versioned above", 16 * 1024 + 1, 2, true, false),
|
||||
("EC8+4 unversioned exact", 128 * 1024, 8, false, true),
|
||||
("EC8+4 unversioned above", 128 * 1024 + 1, 8, false, false),
|
||||
("EC8+4 versioned exact", 16 * 1024, 8, true, true),
|
||||
("EC8+4 versioned above", 16 * 1024 + 1, 8, true, false),
|
||||
("EC12+4 unversioned exact", 128 * 1024, 12, false, true),
|
||||
("EC12+4 unversioned above", 128 * 1024 + 1, 12, false, false),
|
||||
("EC12+4 versioned exact", 16 * 1024, 12, true, true),
|
||||
("EC12+4 versioned above", 16 * 1024 + 1, 12, true, false),
|
||||
("EC8+4 unversioned exact", 32 * 1024, 8, false, true),
|
||||
("EC8+4 unversioned above", 32 * 1024 + 1, 8, false, false),
|
||||
("EC8+4 versioned exact", 4 * 1024, 8, true, true),
|
||||
("EC8+4 versioned above", 4 * 1024 + 1, 8, true, false),
|
||||
("negative", -1, 2, false, false),
|
||||
("zero data shards", 0, 0, false, false),
|
||||
] {
|
||||
@@ -612,35 +626,37 @@ mod tests {
|
||||
let shard_size = erasure.shard_file_size(object_size);
|
||||
let inline = config.should_inline(shard_size, data_shards, false);
|
||||
|
||||
// The per-shard budget stays monotonic across object sizes.
|
||||
// PUT separately bounds inline body materialization.
|
||||
// The effective policy is monotonic across object sizes. This
|
||||
// table covers the boundaries that previously exposed the
|
||||
// fixed-shard read-ahead mismatch, including the 1 MiB case.
|
||||
assert!(!inline || previous, "inline decision must not re-enable at {object_size} bytes");
|
||||
previous = inline;
|
||||
}
|
||||
|
||||
assert!(
|
||||
config.should_inline(erasure.shard_file_size(1024 * 1024), data_shards, false),
|
||||
"1 MiB fits the per-shard budget for EC{data_shards}+{parity_shards}"
|
||||
!config.should_inline(erasure.shard_file_size(1024 * 1024), data_shards, false),
|
||||
"1 MiB must use the non-inline path for EC{data_shards}+{parity_shards}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inline_block_preserves_fixed_default_and_explicit_limits() {
|
||||
fn effective_inline_block_scales_default_budget_and_preserves_explicit_limit() {
|
||||
let config = Config::default();
|
||||
assert_eq!(config.inline_block(), 128 * 1024);
|
||||
assert!(!config.should_inline(0, 0, false));
|
||||
assert_eq!(config.effective_inline_block(8), 32 * 1024);
|
||||
assert_eq!(config.effective_inline_block(12), 21_846);
|
||||
assert_eq!(config.effective_inline_block(0), 0);
|
||||
|
||||
let explicit = lookup_config_for_pools_with_env(
|
||||
&KVS::new(),
|
||||
&[12],
|
||||
StorageClassEnvOverrides {
|
||||
inline_block: Some("32KiB".to_string()),
|
||||
inline_block: Some("128KiB".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.expect("explicit inline block should resolve");
|
||||
assert_eq!(explicit.inline_block(), 32 * 1024);
|
||||
assert_eq!(explicit.effective_inline_block(12), 128 * 1024);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -889,21 +905,6 @@ mod tests {
|
||||
assert!(decoded.should_inline(100 * 1024, 8, false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_scalar_inline_config_preserves_limit_without_explicit_marker() {
|
||||
let cfg = lookup_config_for_pools_with_env(&KVS::new(), &[12], no_env_overrides())
|
||||
.expect("default storage class should resolve");
|
||||
let mut legacy = serde_json::to_value(cfg).expect("serialize scalar config");
|
||||
legacy["inline_block"] = serde_json::json!(64 * 1024);
|
||||
legacy.as_object_mut().expect("config object").remove("inline_block_explicit");
|
||||
let decoded: Config = serde_json::from_value(legacy).expect("read historical scalar config");
|
||||
|
||||
assert!(decoded.should_inline(64 * 1024, 8, false));
|
||||
assert!(!decoded.should_inline(64 * 1024 + 1, 8, false));
|
||||
assert!(decoded.should_inline(8 * 1024, 8, true));
|
||||
assert!(!decoded.should_inline(8 * 1024 + 1, 8, true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lookup_config_reads_rrs_from_class_rrs_key() {
|
||||
// Regression: kvs.get(RRS) used RRS="REDUCED_REDUNDANCY" instead of
|
||||
|
||||
@@ -12378,7 +12378,6 @@ struct DecommissionCapacityLockOrderBarrierState {
|
||||
external_heal_target_lock_attempted: tokio::sync::Notify,
|
||||
target_gate_retry_entered: tokio::sync::Notify,
|
||||
target_gate_retry_entries: AtomicUsize,
|
||||
target_gate_retry_pause: std::sync::OnceLock<tokio::sync::Semaphore>,
|
||||
target_gate_inline_retry_entered: tokio::sync::Notify,
|
||||
target_gate_inline_retry_entries: AtomicUsize,
|
||||
target_gate_exact_reloads: AtomicUsize,
|
||||
@@ -12426,7 +12425,6 @@ impl DecommissionCapacityLockOrderBarrier {
|
||||
external_heal_target_lock_attempted: tokio::sync::Notify::new(),
|
||||
target_gate_retry_entered: tokio::sync::Notify::new(),
|
||||
target_gate_retry_entries: AtomicUsize::new(0),
|
||||
target_gate_retry_pause: std::sync::OnceLock::new(),
|
||||
target_gate_inline_retry_entered: tokio::sync::Notify::new(),
|
||||
target_gate_inline_retry_entries: AtomicUsize::new(0),
|
||||
target_gate_exact_reloads: AtomicUsize::new(0),
|
||||
@@ -12522,13 +12520,6 @@ impl DecommissionCapacityLockOrderBarrier {
|
||||
.expect("decommission entries should observe target gate contention");
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
pub(crate) fn pause_target_gate_retries(&self) -> &tokio::sync::Semaphore {
|
||||
self.state
|
||||
.target_gate_retry_pause
|
||||
.get_or_init(|| tokio::sync::Semaphore::new(0))
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
pub(crate) fn target_gate_inline_retries(&self) -> usize {
|
||||
self.state.target_gate_inline_retry_entries.load(Ordering::Acquire)
|
||||
@@ -12639,9 +12630,6 @@ impl DecommissionCapacityLockOrderBarrier {
|
||||
#[cfg(test)]
|
||||
impl Drop for DecommissionCapacityLockOrderBarrier {
|
||||
fn drop(&mut self) {
|
||||
if let Some(pause) = self.state.target_gate_retry_pause.get() {
|
||||
pause.close();
|
||||
}
|
||||
self.state.owner_release.notify_one();
|
||||
self.state.external_object_capacity_probe_release.notify_one();
|
||||
self.state.external_object_commit_phase_release.notify_one();
|
||||
@@ -12770,7 +12758,7 @@ async fn pause_decommission_cancel_target_timeout(store_id: uuid::Uuid) {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn observe_decommission_target_gate_retry(store_id: uuid::Uuid) {
|
||||
fn notify_decommission_target_gate_retry(store_id: uuid::Uuid) {
|
||||
let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
@@ -12781,11 +12769,6 @@ async fn observe_decommission_target_gate_retry(store_id: uuid::Uuid) {
|
||||
if let Some(barrier) = barrier {
|
||||
barrier.target_gate_retry_entries.fetch_add(1, Ordering::AcqRel);
|
||||
barrier.target_gate_retry_entered.notify_one();
|
||||
if let Some(pause) = barrier.target_gate_retry_pause.get()
|
||||
&& let Ok(permit) = pause.acquire().await
|
||||
{
|
||||
permit.forget();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17655,7 +17638,7 @@ impl ECStore {
|
||||
target_busy_attempt: usize,
|
||||
) -> Result<Option<DecommissionCapacityTargetPermit>> {
|
||||
#[cfg(test)]
|
||||
observe_decommission_target_gate_retry(self.id).await;
|
||||
notify_decommission_target_gate_retry(self.id);
|
||||
let mut wait_attempt = target_busy_attempt;
|
||||
let target_guard = loop {
|
||||
let retry_delay =
|
||||
|
||||
@@ -8360,16 +8360,13 @@ mod decommission_lock_order_tests {
|
||||
.await
|
||||
.expect("hold the target gate before starting both waiters");
|
||||
let retry_observer = DecommissionCapacityLockOrderBarrier::install(other_store.id, other_store.id);
|
||||
// Both workers must observe the original gate as busy, but slow disk I/O
|
||||
// in the holder or first waiter must not consume the second waiter's retry budget.
|
||||
let retry_pause = retry_observer.pause_target_gate_retries();
|
||||
|
||||
let mut workers = tokio::task::JoinSet::new();
|
||||
let mut workers = Vec::new();
|
||||
for object in objects {
|
||||
let worker_store = Arc::clone(&other_store);
|
||||
let worker_bucket = bucket.clone();
|
||||
let source_set = other_store.pools[0].get_disks_by_key(object);
|
||||
workers.spawn(async move {
|
||||
workers.push(tokio::spawn(async move {
|
||||
worker_store
|
||||
.decommission_entry_for_test(
|
||||
0,
|
||||
@@ -8381,7 +8378,7 @@ mod decommission_lock_order_tests {
|
||||
source_set,
|
||||
)
|
||||
.await
|
||||
});
|
||||
}));
|
||||
}
|
||||
retry_observer.wait_until_target_gate_retries(2).await;
|
||||
assert_eq!(
|
||||
@@ -8428,7 +8425,6 @@ mod decommission_lock_order_tests {
|
||||
);
|
||||
|
||||
drop(target_guard);
|
||||
retry_pause.add_permits(1);
|
||||
retry_observer.wait_until_owner_paused().await;
|
||||
assert_eq!(
|
||||
retry_observer.target_gate_exact_reloads(),
|
||||
@@ -8447,13 +8443,6 @@ mod decommission_lock_order_tests {
|
||||
.expect("a pinned waiter must not hold the alternate target gate");
|
||||
drop(alternate_probe);
|
||||
retry_observer.release_owner();
|
||||
tokio::time::timeout(Duration::from_secs(30), workers.join_next())
|
||||
.await
|
||||
.expect("the pinned waiter should finish before the other retries")
|
||||
.expect("the pinned waiter must exist")
|
||||
.expect("the pinned waiter task should not panic")
|
||||
.expect("the pinned waiter migration should succeed");
|
||||
retry_pause.add_permits(1);
|
||||
retry_observer.wait_until_owner_paused().await;
|
||||
assert_eq!(
|
||||
retry_observer.target_gate_exact_reloads(),
|
||||
@@ -8473,12 +8462,13 @@ mod decommission_lock_order_tests {
|
||||
drop(original_probe);
|
||||
retry_observer.release_owner();
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(30), workers.join_next())
|
||||
.await
|
||||
.expect("the reselected waiter should finish after permit transfer")
|
||||
.expect("the reselected waiter must exist")
|
||||
.expect("the reselected waiter task should not panic")
|
||||
.expect("the reselected waiter migration should succeed");
|
||||
for worker in workers {
|
||||
tokio::time::timeout(Duration::from_secs(30), worker)
|
||||
.await
|
||||
.expect("same-target waiter should finish after permit transfer")
|
||||
.expect("same-target waiter task should not panic")
|
||||
.expect("same-target waiter migration should succeed");
|
||||
}
|
||||
assert_eq!(
|
||||
retry_observer.target_gate_exact_reloads(),
|
||||
2,
|
||||
|
||||
@@ -842,7 +842,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for Sets {
|
||||
user_defined: dst_opts.user_defined.clone(),
|
||||
shard_integrity_write_mode: Some(src_info.shard_integrity_write_mode()),
|
||||
versioned: dst_opts.versioned,
|
||||
version_suspended: dst_opts.version_suspended,
|
||||
version_id: dst_opts.version_id.clone(),
|
||||
mod_time: dst_opts.mod_time,
|
||||
http_preconditions: dst_opts.http_preconditions.clone(),
|
||||
|
||||
@@ -180,9 +180,6 @@ pub(crate) const GET_METADATA_CACHE_DECISION_SKIP: &str = "skip";
|
||||
pub(crate) const GET_METADATA_CACHE_REASON_DATA_MOVEMENT: &str = "data_movement";
|
||||
pub(crate) const GET_METADATA_CACHE_REASON_DELETE_MARKER: &str = "delete_marker";
|
||||
pub(crate) const GET_METADATA_CACHE_REASON_DIST_ERASURE: &str = "dist_erasure";
|
||||
pub(crate) const GET_METADATA_CACHE_REASON_FENCE_LOCK_LOST: &str = "fence_lock_lost";
|
||||
pub(crate) const GET_METADATA_CACHE_REASON_FENCE_PROOF_MISMATCH: &str = "fence_proof_mismatch";
|
||||
pub(crate) const GET_METADATA_CACHE_REASON_FENCE_PROOF_UNAVAILABLE: &str = "fence_proof_unavailable";
|
||||
pub(crate) const GET_METADATA_CACHE_REASON_INCL_FREE_VERSIONS: &str = "incl_free_versions";
|
||||
pub(crate) const GET_METADATA_CACHE_REASON_INSUFFICIENT_CACHED_QUORUM: &str = "insufficient_cached_quorum";
|
||||
pub(crate) const GET_METADATA_CACHE_REASON_BODY_LAYOUT_REQUIRED: &str = "body_layout_required";
|
||||
|
||||
@@ -13,9 +13,11 @@
|
||||
// limitations under the License.
|
||||
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
fs::Metadata,
|
||||
path::{Path, PathBuf},
|
||||
time::Duration,
|
||||
sync::Mutex,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use tokio::{fs::File, io};
|
||||
|
||||
@@ -237,21 +239,72 @@ pub fn rename_std(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()
|
||||
std::fs::rename(from, to)
|
||||
}
|
||||
|
||||
// Only successful probes are cached. A missed bucket can be created at any time.
|
||||
static BUCKET_EXISTENCE_CACHE: std::sync::LazyLock<moka::sync::Cache<PathBuf, ()>> = std::sync::LazyLock::new(|| {
|
||||
moka::sync::Cache::builder()
|
||||
.max_capacity(4096)
|
||||
.time_to_live(Duration::from_secs(60))
|
||||
.build()
|
||||
});
|
||||
// Bucket existence cache - reduces statx syscalls for repeated bucket checks
|
||||
|
||||
/// Cache for bucket directory existence checks.
|
||||
struct BucketExistenceCache {
|
||||
cache: Mutex<HashMap<PathBuf, (Instant, bool)>>,
|
||||
ttl: Duration,
|
||||
}
|
||||
|
||||
impl BucketExistenceCache {
|
||||
fn new(ttl: Duration) -> Self {
|
||||
Self {
|
||||
cache: Mutex::new(HashMap::new()),
|
||||
ttl,
|
||||
}
|
||||
}
|
||||
|
||||
fn check_exists(&self, path: &Path) -> Option<bool> {
|
||||
let mut cache = self.cache.lock().ok()?;
|
||||
if let Some((timestamp, exists)) = cache.get(path) {
|
||||
if timestamp.elapsed() < self.ttl {
|
||||
return Some(*exists);
|
||||
}
|
||||
cache.remove(path);
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn record(&self, path: PathBuf, exists: bool) {
|
||||
if let Ok(mut cache) = self.cache.lock() {
|
||||
cache.insert(path, (Instant::now(), exists));
|
||||
}
|
||||
}
|
||||
|
||||
fn invalidate(&self, path: &Path) {
|
||||
if let Ok(mut cache) = self.cache.lock() {
|
||||
cache.remove(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static BUCKET_EXISTENCE_CACHE: std::sync::LazyLock<BucketExistenceCache> =
|
||||
std::sync::LazyLock::new(|| BucketExistenceCache::new(Duration::from_secs(60)));
|
||||
|
||||
/// Cached access check - reduces statx syscalls
|
||||
pub async fn cached_access(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
let path = path.as_ref();
|
||||
if BUCKET_EXISTENCE_CACHE.get(path).is_none() {
|
||||
access(path).await?;
|
||||
BUCKET_EXISTENCE_CACHE.insert(path.to_owned(), ());
|
||||
|
||||
if let Some(exists) = BUCKET_EXISTENCE_CACHE.check_exists(path) {
|
||||
if exists {
|
||||
return Ok(());
|
||||
}
|
||||
return Err(io::Error::new(io::ErrorKind::NotFound, "bucket not found (cached)"));
|
||||
}
|
||||
|
||||
let path_buf = path.to_path_buf();
|
||||
let result = access(&path_buf).await;
|
||||
|
||||
match &result {
|
||||
Ok(_) => BUCKET_EXISTENCE_CACHE.record(path_buf, true),
|
||||
Err(e) if e.kind() == io::ErrorKind::NotFound => {
|
||||
BUCKET_EXISTENCE_CACHE.record(path_buf, false);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
result?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -266,15 +319,6 @@ mod tests {
|
||||
use tempfile::TempDir;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
#[tokio::test]
|
||||
async fn bucket_creation_is_visible_after_a_cache_miss() {
|
||||
let root = TempDir::new().expect("bucket root");
|
||||
let path = root.path().join("bucket");
|
||||
assert_eq!(cached_access(&path).await.expect_err("not created yet").kind(), io::ErrorKind::NotFound);
|
||||
tokio::fs::create_dir(&path).await.expect("create bucket");
|
||||
cached_access(&path).await.expect("new bucket visible");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancelled_filesystem_jobs_retain_physical_resources() {
|
||||
use crate::disk::io_admission::{test_resource_units, with_optional_resource_unit};
|
||||
|
||||
+111
-1291
File diff suppressed because it is too large
Load Diff
@@ -231,26 +231,14 @@ fn rollback_metadata_std(destination_metadata: &Path, rollback_data_dir: Option<
|
||||
let Some(dst_parent) = destination_metadata.parent() else {
|
||||
return Err(std::io::Error::new(ErrorKind::InvalidInput, "missing object metadata parent"));
|
||||
};
|
||||
let rollback_parent = dst_parent.join(old_data_dir.to_string());
|
||||
let backup_path = rollback_parent.join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
std::fs::rename(&backup_path, destination_metadata)?;
|
||||
let _ = remove_file_if_exists(&backup_path);
|
||||
let _ = std::fs::remove_dir(rollback_parent);
|
||||
let backup_path = dst_parent.join(old_data_dir.to_string()).join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
std::fs::rename(backup_path, destination_metadata)?;
|
||||
} else {
|
||||
remove_file_if_exists(destination_metadata)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
pub(super) fn discard_staged_inline_rollback_backup_std(
|
||||
old_data_dir: Uuid,
|
||||
has_old_data_dir: Option<Uuid>,
|
||||
commit_guard: &os::RenameCommitGuard,
|
||||
) {
|
||||
os::discard_inline_rollback_backup_with_commit_guard_std(&old_data_dir.to_string(), has_old_data_dir.is_none(), commit_guard);
|
||||
}
|
||||
|
||||
fn rollback_inline_metadata_std(
|
||||
destination_metadata: &Path,
|
||||
rollback_data_dir: Option<Uuid>,
|
||||
@@ -389,77 +377,34 @@ pub(super) async fn lock_rename_commit_directories(
|
||||
result.map_err(to_file_error).map_err(DiskError::from)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[derive(Clone, Copy)]
|
||||
enum SourcePreparationMode {
|
||||
CreateParent,
|
||||
ExistingParent,
|
||||
FlatInlineTmp { defer_dst_preflight: bool },
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
enum PrepareCommitAndDestinationError {
|
||||
PreflightAccess(std::io::Error),
|
||||
Io(std::io::Error),
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn prepare_rename_commit_and_destination_metadata_std(
|
||||
mode: SourcePreparationMode,
|
||||
create_source_parent: bool,
|
||||
source_parent: &Path,
|
||||
destination_parent: &Path,
|
||||
destination_metadata: &Path,
|
||||
base_dir: &Path,
|
||||
publication_root: &os::PublicationRoot,
|
||||
) -> core::result::Result<(os::RenameCommitGuard, Option<os::ReadDestinationFile>), PrepareCommitAndDestinationError> {
|
||||
if matches!(mode, SourcePreparationMode::CreateParent) {
|
||||
std::fs::create_dir_all(source_parent).map_err(PrepareCommitAndDestinationError::Io)?;
|
||||
) -> std::io::Result<(os::RenameCommitGuard, Option<Bytes>)> {
|
||||
if create_source_parent {
|
||||
std::fs::create_dir_all(source_parent)?;
|
||||
}
|
||||
let defer_dst_preflight = matches!(
|
||||
mode,
|
||||
SourcePreparationMode::FlatInlineTmp {
|
||||
defer_dst_preflight: true
|
||||
}
|
||||
);
|
||||
let guard_res = if matches!(mode, SourcePreparationMode::FlatInlineTmp { .. }) {
|
||||
os::prepare_rename_commit_guard_flat_inline(source_parent, destination_parent, base_dir, publication_root)
|
||||
} else {
|
||||
os::prepare_rename_commit_guard(source_parent, destination_parent, base_dir, publication_root)
|
||||
};
|
||||
let guard = match guard_res {
|
||||
let guard = match os::prepare_rename_commit_guard(source_parent, destination_parent, base_dir, publication_root) {
|
||||
Ok(guard) => guard,
|
||||
Err(err) => {
|
||||
if err
|
||||
.get_ref()
|
||||
.and_then(|inner| inner.downcast_ref::<DiskError>())
|
||||
.is_some_and(|disk_err| matches!(disk_err, DiskError::InvalidPath))
|
||||
{
|
||||
return Err(PrepareCommitAndDestinationError::Io(err));
|
||||
}
|
||||
if defer_dst_preflight && let Err(access_err) = super::super::fs::access_std(base_dir) {
|
||||
return Err(PrepareCommitAndDestinationError::PreflightAccess(access_err));
|
||||
}
|
||||
return Err(PrepareCommitAndDestinationError::Io(match std::fs::symlink_metadata(base_dir) {
|
||||
return Err(match std::fs::symlink_metadata(base_dir) {
|
||||
Err(base_err) if base_err.kind() == ErrorKind::NotFound => base_err,
|
||||
_ => err,
|
||||
}));
|
||||
});
|
||||
}
|
||||
};
|
||||
if defer_dst_preflight
|
||||
&& !guard.has_prepared_destination()
|
||||
&& let Err(access_err) = super::super::fs::access_std(base_dir)
|
||||
{
|
||||
return Err(PrepareCommitAndDestinationError::PreflightAccess(access_err));
|
||||
}
|
||||
let data = os::read_destination_file_handle_with_commit_guard(destination_metadata, &guard)
|
||||
.map_err(PrepareCommitAndDestinationError::Io)?;
|
||||
Ok((guard, data))
|
||||
let data = os::read_destination_file_with_commit_guard(destination_metadata, &guard)?;
|
||||
Ok((guard, data.map(Bytes::from)))
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
enum FusedCommitAttempt {
|
||||
Committed(Box<RenameDataResp>),
|
||||
PreflightAccessErr(std::io::Error),
|
||||
CommitErr(DiskError),
|
||||
RestoreDataSource(os::RenameCommitGuard, DiskError),
|
||||
Fallback(os::RenameCommitGuard, Option<Bytes>, Box<FileInfo>),
|
||||
@@ -467,14 +412,6 @@ enum FusedCommitAttempt {
|
||||
ResumePanic(Box<dyn std::any::Any + Send + 'static>),
|
||||
}
|
||||
|
||||
/// File fdatasync grouping only owns staged shard files. Inline commits have
|
||||
/// no shard directory, so the switch must not evict them from the flat/fused
|
||||
/// metadata path where grouping can never run.
|
||||
#[cfg(any(target_os = "linux", test))]
|
||||
pub(super) fn file_fdatasync_grouping_requires_async_shard_sync(no_inline: bool) -> bool {
|
||||
no_inline && os::file_fdatasync_group_commit_enabled()
|
||||
}
|
||||
|
||||
async fn prepare_rename_commit_and_destination_metadata(
|
||||
create_source_parent: bool,
|
||||
source_parent: &Path,
|
||||
@@ -630,20 +567,7 @@ impl LocalDisk {
|
||||
// object commit is publishing into it. The peer's empty scan remains
|
||||
// optimistic; this lease establishes the local commit/delete order and
|
||||
// remains owned by any blocking syscall that outlives async cancellation.
|
||||
let dst_dir_group_commit = os::dst_dir_fsync_group_commit_enabled();
|
||||
let flat_inline_tmp = cfg!(target_os = "linux")
|
||||
&& (fi.data.is_some() || fi.size <= 0)
|
||||
&& src_volume == super::super::RUSTFS_META_TMP_BUCKET
|
||||
&& pool_meta_condition.is_none()
|
||||
&& !fi.is_healing()
|
||||
&& !dst_dir_group_commit
|
||||
&& !has_rename_data_test_failpoints(dst_path);
|
||||
let defer_dst_preflight = flat_inline_tmp && !skip_access_checks(dst_volume);
|
||||
let (dst_volume_dir, destination_object_path) = if defer_dst_preflight {
|
||||
self.io_get_bucket_and_object_open_path(dst_volume, dst_path)?
|
||||
} else {
|
||||
self.io_get_bucket_and_object_path(dst_volume, dst_path)?
|
||||
};
|
||||
let (dst_volume_dir, destination_object_path) = self.io_get_bucket_and_object_path(dst_volume, dst_path)?;
|
||||
let quota_fence_token =
|
||||
match rustfs_utils::http::metadata_compat::get_consistent_str(&fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX) {
|
||||
Some(value) => {
|
||||
@@ -792,8 +716,7 @@ impl LocalDisk {
|
||||
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
|
||||
}
|
||||
|
||||
if !defer_dst_preflight
|
||||
&& !skip_access_checks(dst_volume)
|
||||
if !skip_access_checks(dst_volume)
|
||||
&& let Err(e) = super::super::fs::access_std(&dst_volume_dir)
|
||||
{
|
||||
info!(
|
||||
@@ -819,6 +742,16 @@ impl LocalDisk {
|
||||
} else {
|
||||
STORAGE_FORMAT_FILE
|
||||
};
|
||||
let flat_inline_tmp = cfg!(target_os = "linux")
|
||||
&& !no_inline
|
||||
&& src_volume == super::super::RUSTFS_META_TMP_BUCKET
|
||||
&& !preserve_upload_metadata
|
||||
&& multipart_publication.is_none()
|
||||
&& pool_meta_condition.is_none()
|
||||
&& !fi_healing
|
||||
&& !os::dst_dir_fsync_group_commit_enabled()
|
||||
&& !os::file_fdatasync_group_commit_enabled()
|
||||
&& !has_rename_data_test_failpoints(dst_path);
|
||||
let src_file_path = match multipart_publication.as_ref() {
|
||||
Some(publication) => publication.canonical_head(),
|
||||
None if flat_inline_tmp => {
|
||||
@@ -900,13 +833,12 @@ impl LocalDisk {
|
||||
.ok_or_else(|| DiskError::other("missing object metadata parent"))?;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
let (rename_commit_guard, has_dst_buf) = if flat_inline_tmp
|
||||
|| (!preserve_upload_metadata
|
||||
&& multipart_publication.is_none()
|
||||
&& pool_meta_condition.is_none()
|
||||
&& !fi_healing
|
||||
&& !dst_dir_group_commit
|
||||
&& !file_fdatasync_grouping_requires_async_shard_sync(no_inline))
|
||||
let (rename_commit_guard, has_dst_buf) = if !preserve_upload_metadata
|
||||
&& multipart_publication.is_none()
|
||||
&& pool_meta_condition.is_none()
|
||||
&& !fi_healing
|
||||
&& !os::dst_dir_fsync_group_commit_enabled()
|
||||
&& !os::file_fdatasync_group_commit_enabled()
|
||||
{
|
||||
#[cfg(test)]
|
||||
if !no_inline {
|
||||
@@ -941,45 +873,25 @@ impl LocalDisk {
|
||||
let root_owned = self.root.clone();
|
||||
|
||||
let fused_attempt = move |is_cancelled: &dyn Fn() -> bool| -> std::io::Result<FusedCommitAttempt> {
|
||||
let source_mode = if flat_inline_tmp {
|
||||
SourcePreparationMode::FlatInlineTmp { defer_dst_preflight }
|
||||
} else if !no_inline {
|
||||
SourcePreparationMode::CreateParent
|
||||
} else {
|
||||
SourcePreparationMode::ExistingParent
|
||||
};
|
||||
let (guard, old_destination_file) = match prepare_rename_commit_and_destination_metadata_std(
|
||||
source_mode,
|
||||
let (guard, has_dst_buf) = prepare_rename_commit_and_destination_metadata_std(
|
||||
!no_inline && !flat_inline_tmp,
|
||||
&src_parent_owned,
|
||||
&dst_parent_owned,
|
||||
&dst_file_path_owned,
|
||||
&dst_volume_dir_owned,
|
||||
&publication_root_owned,
|
||||
) {
|
||||
Ok(res) => res,
|
||||
Err(PrepareCommitAndDestinationError::PreflightAccess(err)) => {
|
||||
return Ok(FusedCommitAttempt::PreflightAccessErr(err));
|
||||
}
|
||||
Err(PrepareCommitAndDestinationError::Io(err)) => return Err(err),
|
||||
};
|
||||
)?;
|
||||
// An existing xl.meta that replaces an existing version slot
|
||||
// requires a rollback directory; appending a new version keeps
|
||||
// an inline rollback backup for post-rename directory sync errors.
|
||||
if is_cancelled() {
|
||||
if flat_inline_tmp {
|
||||
guard.ensure_source_pinned()?;
|
||||
}
|
||||
return Ok(FusedCommitAttempt::Fallback(
|
||||
guard,
|
||||
old_destination_file.map(|read| read.bytes),
|
||||
Box::new(fi_moved),
|
||||
));
|
||||
return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, Box::new(fi_moved)));
|
||||
}
|
||||
let mut xlmeta = FileMeta::new();
|
||||
let mut dst_meta_unparsable = false;
|
||||
if let Some(ref old_file) = old_destination_file {
|
||||
if FileMeta::is_xl2_v1_format(&old_file.bytes)
|
||||
&& let Ok(nmeta) = FileMeta::load(&old_file.bytes)
|
||||
if let Some(ref dst_buf) = has_dst_buf {
|
||||
if FileMeta::is_xl2_v1_format(dst_buf)
|
||||
&& let Ok(nmeta) = FileMeta::load(dst_buf)
|
||||
{
|
||||
xlmeta = nmeta;
|
||||
} else {
|
||||
@@ -989,21 +901,14 @@ impl LocalDisk {
|
||||
let version_id = fi_moved.version_id.unwrap_or_default();
|
||||
let has_old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id));
|
||||
let rollback_data_dir = has_old_data_dir.or_else(|| {
|
||||
if old_destination_file.is_some() && xlmeta.find_version(Some(version_id)).is_ok() {
|
||||
if has_dst_buf.is_some() && xlmeta.find_version(Some(version_id)).is_ok() {
|
||||
Some(inline_metadata_rollback_dir(version_id, &xlmeta))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
});
|
||||
if rollback_data_dir.is_some() && has_rename_data_test_failpoints(&dst_path_str) {
|
||||
if flat_inline_tmp {
|
||||
guard.ensure_source_pinned()?;
|
||||
}
|
||||
return Ok(FusedCommitAttempt::Fallback(
|
||||
guard,
|
||||
old_destination_file.map(|read| read.bytes),
|
||||
Box::new(fi_moved),
|
||||
));
|
||||
return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, Box::new(fi_moved)));
|
||||
}
|
||||
#[cfg(test)]
|
||||
let probe_active = os::file_sync_probe::is_active();
|
||||
@@ -1015,17 +920,10 @@ impl LocalDisk {
|
||||
&& (probe_active || !super::is_presynced_tmp_data_dir(src_data_path))
|
||||
&& os::regular_files(src_data_path).is_ok_and(|files| files.len() >= os::PARALLEL_FILE_SYNC_THRESHOLD)
|
||||
{
|
||||
if flat_inline_tmp {
|
||||
guard.ensure_source_pinned()?;
|
||||
}
|
||||
return Ok(FusedCommitAttempt::Fallback(
|
||||
guard,
|
||||
old_destination_file.map(|read| read.bytes),
|
||||
Box::new(fi_moved),
|
||||
));
|
||||
return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, Box::new(fi_moved)));
|
||||
}
|
||||
let (old_current_size, old_current_source_checked, old_current_source) = observe_old_current(
|
||||
old_destination_file.is_some(),
|
||||
has_dst_buf.is_some(),
|
||||
dst_meta_unparsable,
|
||||
&xlmeta,
|
||||
&dst_volume_str,
|
||||
@@ -1038,9 +936,9 @@ impl LocalDisk {
|
||||
xlmeta.add_version(fi_moved)?;
|
||||
let version_signature = rename_data_versions_signature(&xlmeta);
|
||||
let new_dst_buf = xlmeta.marshal_msg().map_err(std::io::Error::other)?;
|
||||
let destination_was_absent = old_destination_file.is_none();
|
||||
let destination_was_absent = has_dst_buf.is_none();
|
||||
let mut skip_parent = dst_volume_dir_owned.clone();
|
||||
if old_destination_file.is_some() {
|
||||
if has_dst_buf.is_some() {
|
||||
skip_parent = dst_parent_owned.clone();
|
||||
}
|
||||
|
||||
@@ -1050,7 +948,7 @@ impl LocalDisk {
|
||||
os::create_prepared_rename_source_with_commit_guard(&src_file_path_owned, &dst_file_path_owned, &guard)?;
|
||||
let cleanup_flat_tmp = || {
|
||||
if flat_inline_tmp {
|
||||
os::unlink_flat_tmp_source_std(&src_file_path_owned, &src_parent_owned, &publication_root_owned);
|
||||
let _ = remove_file_if_exists(&src_file_path_owned);
|
||||
}
|
||||
};
|
||||
if let Err(err) = prepared_metadata_source.write_all(&new_dst_buf, false) {
|
||||
@@ -1063,14 +961,14 @@ impl LocalDisk {
|
||||
}
|
||||
|
||||
let mut staged_rollback_path = None;
|
||||
if rollback_data_dir.is_none()
|
||||
&& let Some(ref old_file) = old_destination_file
|
||||
&& cfg!(test)
|
||||
if (!no_inline || rollback_data_dir.is_none())
|
||||
&& let Some(ref old_metadata) = has_dst_buf
|
||||
&& (rollback_data_dir.is_some() || cfg!(test))
|
||||
{
|
||||
let backup_path = match create_local_inline_rollback_backup(
|
||||
&dst_file_path_owned,
|
||||
&src_file_path_owned,
|
||||
&old_file.bytes,
|
||||
old_metadata,
|
||||
sync_recovery_protocol,
|
||||
) {
|
||||
Ok(path) => path,
|
||||
@@ -1190,22 +1088,28 @@ impl LocalDisk {
|
||||
}
|
||||
return Ok(FusedCommitAttempt::RestoreDataSource(guard, DiskError::Unexpected));
|
||||
}
|
||||
if let (Some(old_data_dir), Some(old_file)) = (rollback_data_dir, old_destination_file.as_ref()) {
|
||||
if let (Some(old_data_dir), Some(dst_buf)) = (rollback_data_dir, has_dst_buf.as_ref()) {
|
||||
let backup_parent = dst_parent_owned.join(old_data_dir.to_string());
|
||||
if let Err(err) =
|
||||
os::mkdir_all_below_existing_base_std(&backup_parent, &skip_parent, &publication_root_owned)
|
||||
{
|
||||
return Ok(FusedCommitAttempt::RestoreDataSource(guard, to_file_error(err).into()));
|
||||
}
|
||||
let backup_path_guard = match guard.create_destination_directory_for_path_access(&backup_parent) {
|
||||
Ok(g) => g,
|
||||
Err(err) => return Ok(FusedCommitAttempt::RestoreDataSource(guard, to_file_error(err).into())),
|
||||
};
|
||||
let backup_path = backup_parent.join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
if let Err(err) = check_path_length(backup_path.to_string_lossy().as_ref()) {
|
||||
return Ok(FusedCommitAttempt::RestoreDataSource(guard, err));
|
||||
}
|
||||
#[cfg(test)]
|
||||
run_owned_file_before_open(&backup_path);
|
||||
if let Err(err) = os::stage_inline_rollback_backup_with_commit_guard_std(
|
||||
&dst_file_path_owned,
|
||||
&backup_parent,
|
||||
if let Err(err) = backup_path_guard.write_file_for_path_access(
|
||||
&backup_path,
|
||||
old_file,
|
||||
dst_buf.as_ref(),
|
||||
sync_recovery_protocol,
|
||||
sync_recovery_protocol,
|
||||
!should_fail_local_inline_rollback_hardlink(&dst_file_path_owned),
|
||||
&guard,
|
||||
) {
|
||||
return Ok(FusedCommitAttempt::RestoreDataSource(guard, to_file_error(err).into()));
|
||||
}
|
||||
@@ -1213,32 +1117,25 @@ impl LocalDisk {
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterBackupBeforeMetaCommit, &dst_path_str) {
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
|
||||
}
|
||||
} else if let (Some(old_data_dir), Some(old_file)) = (rollback_data_dir, old_destination_file.as_ref()) {
|
||||
} else if let (Some(old_data_dir), Some(staged_backup)) =
|
||||
(rollback_data_dir, staged_rollback_path.as_ref().map(StagedRollbackBackup::path))
|
||||
{
|
||||
let backup_parent = dst_parent_owned.join(old_data_dir.to_string());
|
||||
let backup_path = backup_parent.join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
if let Err(err) = check_path_length(backup_path.to_string_lossy().as_ref()) {
|
||||
drop(prepared_metadata_source);
|
||||
cleanup_flat_tmp();
|
||||
return Ok(FusedCommitAttempt::CommitErr(err));
|
||||
}
|
||||
#[cfg(test)]
|
||||
run_owned_file_before_open(&backup_path);
|
||||
if let Err(err) = os::stage_inline_rollback_backup_with_commit_guard_std(
|
||||
&dst_file_path_owned,
|
||||
&backup_parent,
|
||||
if let Err(err) = os::rename_all_with_precondition_and_parent_sync_std(
|
||||
staged_backup,
|
||||
&backup_path,
|
||||
old_file,
|
||||
&skip_parent,
|
||||
&publication_root_owned,
|
||||
None,
|
||||
sync_recovery_protocol,
|
||||
!should_fail_local_inline_rollback_hardlink(&dst_file_path_owned),
|
||||
&guard,
|
||||
) {
|
||||
drop(prepared_metadata_source);
|
||||
cleanup_flat_tmp();
|
||||
let _ = remove_file_if_exists(staged_backup);
|
||||
return Ok(FusedCommitAttempt::CommitErr(to_file_error(err).into()));
|
||||
}
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterBackupBeforeMetaCommit, &dst_path_str) {
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
|
||||
}
|
||||
staged_rollback_path = None;
|
||||
}
|
||||
|
||||
let rename_res = if should_fail_commit_rename(&dst_path_str) {
|
||||
@@ -1259,9 +1156,6 @@ impl LocalDisk {
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
if let Some(old_data_dir) = rollback_data_dir {
|
||||
discard_staged_inline_rollback_backup_std(old_data_dir, has_old_data_dir, &guard);
|
||||
}
|
||||
if no_inline {
|
||||
return Ok(FusedCommitAttempt::RestoreDataSource(guard, err));
|
||||
}
|
||||
@@ -1287,10 +1181,10 @@ impl LocalDisk {
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
|
||||
}
|
||||
|
||||
if durability.syncs_metadata_commit_directory(rollback_data_dir.is_some()) {
|
||||
if sync_commit_directories {
|
||||
if staged_rollback_path.is_none()
|
||||
&& rollback_data_dir.is_none()
|
||||
&& let Some(ref old_file) = old_destination_file
|
||||
&& let Some(ref old_metadata) = has_dst_buf
|
||||
&& (!no_inline || rollback_data_dir.is_none())
|
||||
{
|
||||
let fsync_res = (|| -> std::io::Result<()> {
|
||||
@@ -1301,7 +1195,7 @@ impl LocalDisk {
|
||||
})();
|
||||
if let Err(err) = fsync_res {
|
||||
let rollback_ok =
|
||||
write_local_inline_rollback_backup_from_bytes(&src_file_path_owned, &old_file.bytes, true)
|
||||
write_local_inline_rollback_backup_from_bytes(&src_file_path_owned, old_metadata, true)
|
||||
.and_then(|backup_path| {
|
||||
let backup = StagedRollbackBackup(backup_path);
|
||||
let res =
|
||||
@@ -1376,20 +1270,6 @@ impl LocalDisk {
|
||||
}
|
||||
return Ok(*resp);
|
||||
}
|
||||
Ok(FusedCommitAttempt::PreflightAccessErr(e)) => {
|
||||
info!(
|
||||
target: "rustfs_ecstore::disk::local",
|
||||
event = EVENT_DISK_LOCAL_ACCESS_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
|
||||
path = ?dst_volume_dir,
|
||||
operation = "rename_data_dst_access",
|
||||
error = %e,
|
||||
"Disk local access check failed"
|
||||
);
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
|
||||
}
|
||||
Ok(FusedCommitAttempt::RestoreDataSource(directory_guard, err)) => {
|
||||
let commit = RenameCommitContext {
|
||||
directory_guard,
|
||||
@@ -1426,7 +1306,7 @@ impl LocalDisk {
|
||||
&& !preserve_upload_metadata
|
||||
&& !publication.legacy
|
||||
&& !fi_healing
|
||||
&& !dst_dir_group_commit
|
||||
&& !os::dst_dir_fsync_group_commit_enabled()
|
||||
{
|
||||
enum MultipartFusedCommitOutcome {
|
||||
Committed(Box<RenameDataResp>),
|
||||
@@ -1456,19 +1336,15 @@ impl LocalDisk {
|
||||
true,
|
||||
move || -> std::io::Result<MultipartFusedCommitOutcome> {
|
||||
let (directory_guard_owned, has_dst_buf_owned) = match prepare_rename_commit_and_destination_metadata_std(
|
||||
SourcePreparationMode::ExistingParent,
|
||||
false,
|
||||
&src_parent_owned,
|
||||
&dst_parent_owned,
|
||||
&dst_file_path_owned,
|
||||
&dst_volume_dir_owned,
|
||||
&publication_root_owned,
|
||||
) {
|
||||
Ok((guard, old_file)) => (guard, old_file.map(|read| read.bytes)),
|
||||
Err(
|
||||
PrepareCommitAndDestinationError::PreflightAccess(err) | PrepareCommitAndDestinationError::Io(err),
|
||||
) => {
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into()));
|
||||
}
|
||||
Ok(res) => res,
|
||||
Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into())),
|
||||
};
|
||||
let mut xlmeta = FileMeta::new();
|
||||
let mut dst_meta_unparsable = false;
|
||||
@@ -2006,10 +1882,9 @@ impl LocalDisk {
|
||||
// durable. It is part of the metadata commit machinery, so relaxed
|
||||
// tiers leave it to the page cache like the xl.meta it mirrors.
|
||||
let backup_sync = durability.recovery_sync_mode();
|
||||
let recovery_backup = rollback_data_dir
|
||||
.zip(has_dst_buf.as_ref())
|
||||
.filter(|_| multipart_publication.as_ref().is_none_or(|publication| publication.legacy));
|
||||
if let Some((old_data_dir, dst_buf)) = recovery_backup {
|
||||
if let (Some(old_data_dir), Some(dst_buf)) = (rollback_data_dir, has_dst_buf.as_ref())
|
||||
&& multipart_publication.as_ref().is_none_or(|publication| publication.legacy)
|
||||
{
|
||||
let backup_parent = dst_file_parent.join(old_data_dir.to_string());
|
||||
#[cfg(not(windows))]
|
||||
if let Err(err) = os::make_dir_all(&backup_parent, &skip_parent).await {
|
||||
@@ -2123,7 +1998,7 @@ impl LocalDisk {
|
||||
// Persist the directory entries for both the data dir and xl.meta renames;
|
||||
// without this the commit itself can vanish on power loss. Relaxed tiers
|
||||
// accept that window (documented in docs/operations/durability-modes.md).
|
||||
if durability.syncs_metadata_commit_directory(recovery_backup.is_some())
|
||||
if durability.syncs_commit_directories()
|
||||
&& let Some(parent) = dst_file_path.parent()
|
||||
{
|
||||
let fsync_started = rustfs_io_metrics::put_stage_timer();
|
||||
@@ -2272,15 +2147,13 @@ impl LocalDisk {
|
||||
let dst_path_for_source = dst_path.to_string();
|
||||
let dst_volume_dir_for_commit = dst_volume_dir.clone();
|
||||
let publication_root_for_commit = self.publication_root.clone();
|
||||
let cleanup_src_parent = !flat_inline_tmp && multipart_publication.is_none() && !preserve_upload_metadata;
|
||||
let cleanup_src_parent = multipart_publication.is_none() && !preserve_upload_metadata;
|
||||
#[cfg(windows)]
|
||||
let source_parent = src_file_parent.to_path_buf();
|
||||
let rename_commit_guard_for_preparation = commit.directory_guard.clone();
|
||||
let sync_metadata_files = durability.syncs_metadata_files();
|
||||
let sync_commit_directories = durability.syncs_commit_directories();
|
||||
let sync_recovery_protocol = durability.syncs_recovery_protocol_state();
|
||||
let sync = durability.syncs_recovery_protocol_state();
|
||||
#[cfg(unix)]
|
||||
let receipt_identity = if sync_recovery_protocol
|
||||
let receipt_identity = if sync
|
||||
&& pool_meta_condition
|
||||
.as_ref()
|
||||
.is_some_and(|condition| condition.parent_raw_sha256.is_none())
|
||||
@@ -2308,7 +2181,7 @@ impl LocalDisk {
|
||||
}
|
||||
#[cfg(test)]
|
||||
run_inline_before_file_sync_admission(dst_path);
|
||||
let mut file_sync_admission = if sync_metadata_files || sync_commit_directories || sync_recovery_protocol {
|
||||
let mut file_sync_admission = if sync {
|
||||
let _admission = os::StageTimer::new(native_timing, "pool_meta_native_file_sync_admission");
|
||||
Some(
|
||||
os::acquire_file_sync_admission(self.file_sync_permits.clone())
|
||||
@@ -2376,10 +2249,14 @@ impl LocalDisk {
|
||||
xlmeta.add_version(fi)?;
|
||||
let version_signature = rename_data_versions_signature(&xlmeta);
|
||||
let new_buf = xlmeta.marshal_msg()?;
|
||||
// Inline payload shares xl.meta's file synchronization policy.
|
||||
// Write the staged xl.meta. Inline objects carry their data inside
|
||||
// xl.meta, so this is the durable preparation for the metadata commit:
|
||||
// relaxed tiers do no per-object fsync here at all (aligned
|
||||
// with MinIO's default), trading a documented power-loss
|
||||
// window for latency.
|
||||
{
|
||||
let _new_file = os::StageTimer::new(native_timing, "pool_meta_native_staged_write_sync");
|
||||
prepared_metadata_source.write_all(&new_buf, sync_metadata_files)?;
|
||||
prepared_metadata_source.write_all(&new_buf, sync)?;
|
||||
}
|
||||
run_inline_preparation_before_backup(&dst_path_for_failpoint);
|
||||
#[cfg(unix)]
|
||||
@@ -2410,10 +2287,10 @@ impl LocalDisk {
|
||||
if !backup.is_linked() {
|
||||
receipt_candidate = None;
|
||||
}
|
||||
if sync_recovery_protocol && !reuses_durability {
|
||||
if sync && !reuses_durability {
|
||||
let _backup_sync = os::StageTimer::new(native_timing, "pool_meta_native_backup_file_sync");
|
||||
backup.sync_data()?;
|
||||
} else if sync_recovery_protocol {
|
||||
} else if sync {
|
||||
rustfs_io_metrics::record_put_object_stage_count("pool_meta_native_backup_sync_reused", 1);
|
||||
}
|
||||
Some(backup)
|
||||
@@ -2427,7 +2304,7 @@ impl LocalDisk {
|
||||
let owned_backup = None;
|
||||
if let Some(ref old_metadata) = has_dst_buf
|
||||
&& owned_backup.is_none()
|
||||
&& (rollback_data_dir.is_some() || fi_healing || sync_recovery_protocol || cfg!(test))
|
||||
&& (rollback_data_dir.is_some() || fi_healing || sync || cfg!(test))
|
||||
{
|
||||
#[cfg(windows)]
|
||||
let backup_path = {
|
||||
@@ -2436,18 +2313,13 @@ impl LocalDisk {
|
||||
.parent()
|
||||
.ok_or_else(|| std::io::Error::new(ErrorKind::InvalidInput, "missing staging metadata parent"))?
|
||||
.join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
source_metadata_guard.write_file_for_path_access(
|
||||
&backup_path,
|
||||
old_metadata,
|
||||
sync_recovery_protocol,
|
||||
false,
|
||||
)?;
|
||||
source_metadata_guard.write_file_for_path_access(&backup_path, old_metadata, sync, false)?;
|
||||
backup_path
|
||||
};
|
||||
#[cfg(not(windows))]
|
||||
let backup_path = {
|
||||
let _backup = os::StageTimer::new(native_timing, "pool_meta_native_backup_create");
|
||||
create_local_inline_rollback_backup(&dst, &src, old_metadata, sync_recovery_protocol)?
|
||||
create_local_inline_rollback_backup(&dst, &src, old_metadata, sync)?
|
||||
};
|
||||
staged_rollback_path = Some(StagedRollbackBackup(backup_path));
|
||||
}
|
||||
@@ -2455,7 +2327,7 @@ impl LocalDisk {
|
||||
let destination_was_absent = has_dst_buf.is_none();
|
||||
if !is_cancelled()
|
||||
&& rollback_data_dir.is_none()
|
||||
&& (!sync_commit_directories || !os::dst_dir_fsync_group_commit_enabled())
|
||||
&& !os::dst_dir_fsync_group_commit_enabled()
|
||||
&& !has_inline_commit_test_failpoints(&dst_path_for_failpoint)
|
||||
{
|
||||
let rename_res = if should_fail_commit_rename(&dst_path_for_failpoint) {
|
||||
@@ -2472,10 +2344,6 @@ impl LocalDisk {
|
||||
)
|
||||
};
|
||||
if let Err(err) = rename_res {
|
||||
#[cfg(target_os = "linux")]
|
||||
if flat_inline_tmp {
|
||||
os::unlink_prepared_rename_source_with_commit_guard_std(&src, &rename_commit_guard_for_preparation);
|
||||
}
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
@@ -2489,7 +2357,7 @@ impl LocalDisk {
|
||||
return Ok(InlinePreparationOutcome::CommitErr(DiskError::from(err)));
|
||||
}
|
||||
|
||||
if sync_commit_directories
|
||||
if sync
|
||||
&& let Err(err) = fsync_inline_commit_std(
|
||||
&dst,
|
||||
&dst_volume_dir_for_commit,
|
||||
@@ -2591,10 +2459,6 @@ impl LocalDisk {
|
||||
}
|
||||
Ok(InlinePreparationOutcome::CommitErr(err)) | Err(err) => {
|
||||
drop(file_sync_admission.take());
|
||||
#[cfg(target_os = "linux")]
|
||||
if flat_inline_tmp {
|
||||
os::unlink_prepared_rename_source_with_commit_guard_std(&src_file_path, &commit.directory_guard);
|
||||
}
|
||||
for part_path in &invalidate_part_paths {
|
||||
self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
|
||||
}
|
||||
@@ -2688,7 +2552,7 @@ impl LocalDisk {
|
||||
}
|
||||
#[cfg(test)]
|
||||
run_rename_data_after_first_publication(&self.root, dst_volume, dst_path);
|
||||
if sync_commit_directories || sync_recovery_protocol {
|
||||
if sync {
|
||||
let _admission = os::StageTimer::new(native_timing, "pool_meta_native_file_sync_readmission");
|
||||
file_sync_admission = Some(
|
||||
os::acquire_file_sync_admission(self.file_sync_permits.clone())
|
||||
@@ -2737,10 +2601,6 @@ impl LocalDisk {
|
||||
};
|
||||
drop(head_publish);
|
||||
if let Err(err) = commit_result {
|
||||
#[cfg(target_os = "linux")]
|
||||
if flat_inline_tmp {
|
||||
os::unlink_prepared_rename_source_with_commit_guard_std(&src_file_path, &commit.directory_guard);
|
||||
}
|
||||
if let Some(backup_path) = local_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
@@ -2756,10 +2616,8 @@ impl LocalDisk {
|
||||
return Err(std::io::Error::other("test fail after metadata commit"));
|
||||
}
|
||||
|
||||
// The object directory also publishes the recovery backup's
|
||||
// directory entry, which file-sync must keep durable.
|
||||
if durability.syncs_metadata_commit_directory(rollback_data_dir.is_some())
|
||||
&& let Some(admission) = file_sync_admission.as_ref()
|
||||
// Persist the commit rename's directory entry across power loss.
|
||||
if let Some(admission) = file_sync_admission.as_ref()
|
||||
&& let Some(dst_parent) = dst_file_path.parent()
|
||||
{
|
||||
let _head_dir = os::StageTimer::new(native_timing, "pool_meta_native_head_dir_sync");
|
||||
@@ -2792,8 +2650,7 @@ impl LocalDisk {
|
||||
// not its own entry, so for a new inline object fsync the ancestor
|
||||
// chain up to and including the bucket. Overwrites already have a
|
||||
// durable object dir; the starts_with guard bounds the walk.
|
||||
if sync_commit_directories
|
||||
&& let Some(admission) = file_sync_admission.as_ref()
|
||||
if let Some(admission) = file_sync_admission.as_ref()
|
||||
&& destination_was_absent
|
||||
{
|
||||
let mut ancestor = dst_file_path.parent().and_then(|object_dir| object_dir.parent());
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,597 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Bounded emergency headroom for durable deletion. Healthy operations access
|
||||
//! only the shared owner's atomic state. Only allocation failures consume the
|
||||
//! reserve, without holding its mutex across transaction publication or quorum.
|
||||
|
||||
use std::{io, path::Path, sync::Arc};
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(super) const RESERVE_BYTES: u64 = 8 * 1024 * 1024;
|
||||
|
||||
pub(super) struct DeleteReserve {
|
||||
#[cfg(unix)]
|
||||
inner: unix::State,
|
||||
}
|
||||
|
||||
impl DeleteReserve {
|
||||
pub(super) fn acquire(root: &Path) -> io::Result<Arc<Self>> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
unix::acquire(root)
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
let _ = root;
|
||||
Ok(Arc::new(Self {}))
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn is_armed(&self) -> bool {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
self.inner.is_armed()
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
/// Populate the reserve on the disk's dedicated fsync lane.
|
||||
pub(super) fn initialize(&self) -> io::Result<()> {
|
||||
self.replenish()
|
||||
}
|
||||
|
||||
/// Refill only when sufficient free space remains for recovery. An armed
|
||||
/// owner returns immediately without opening files or probing capacity.
|
||||
pub(super) fn replenish(&self) -> io::Result<()> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
self.inner.replenish()
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Probe capacity during startup so a persisted reserve can make the disk
|
||||
/// writable before its normal writeability check.
|
||||
pub(super) fn release_if_low(&self) -> io::Result<()> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
self.inner.release_if_low()
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Retry an allocation-safe scratch operation once after ENOSPC. The first
|
||||
/// waiter releases the reserve; concurrent waiters serialize behind it and
|
||||
/// may use the same released capacity. The caller must ensure that a failed
|
||||
/// attempt leaves only disposable scratch or an idempotent primitive under
|
||||
/// the existing namespace owner. Never replay a transaction or a
|
||||
/// publication-plus-sync sequence with this helper.
|
||||
///
|
||||
/// The owner mutex covers release and the retry, excluding replenishment
|
||||
/// during this bounded operation. Later transaction phases and unrelated
|
||||
/// allocators can still compete for the released space.
|
||||
pub(super) fn retry_after_enospc<T>(&self, mut operation: impl FnMut() -> io::Result<T>) -> io::Result<T> {
|
||||
match operation() {
|
||||
#[cfg(unix)]
|
||||
Err(error) if error.kind() == io::ErrorKind::StorageFull => self.inner.retry_after_enospc(operation, error),
|
||||
result => result,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
mod unix {
|
||||
use super::*;
|
||||
use parking_lot::Mutex;
|
||||
use rustix::fs::{Mode, OFlags, mkdirat, open, openat};
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
fs::File,
|
||||
io::{Seek, SeekFrom, Write},
|
||||
os::unix::fs::MetadataExt,
|
||||
sync::{
|
||||
OnceLock, Weak,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
},
|
||||
};
|
||||
|
||||
const METADATA_DIR: &str = ".rustfs.sys";
|
||||
const RESERVE_NAME: &str = "delete-reserve";
|
||||
type RootIdentity = (u64, u64);
|
||||
type OwnerRegistry = Mutex<HashMap<RootIdentity, Weak<DeleteReserve>>>;
|
||||
static OWNERS: OnceLock<OwnerRegistry> = OnceLock::new();
|
||||
|
||||
pub(super) struct State {
|
||||
root: File,
|
||||
mutation: Mutex<()>,
|
||||
armed: AtomicBool,
|
||||
}
|
||||
|
||||
pub(super) fn acquire(path: &Path) -> io::Result<Arc<DeleteReserve>> {
|
||||
let root = File::from(open(
|
||||
path,
|
||||
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
|
||||
Mode::empty(),
|
||||
)?);
|
||||
let metadata = root.metadata()?;
|
||||
let identity = (metadata.dev(), metadata.ino());
|
||||
let mut registry = OWNERS.get_or_init(Mutex::default).lock();
|
||||
if let Some(owner) = registry.get(&identity).and_then(Weak::upgrade) {
|
||||
return Ok(owner);
|
||||
}
|
||||
registry.retain(|_, owner| owner.strong_count() != 0);
|
||||
let owner = Arc::new(DeleteReserve {
|
||||
inner: State {
|
||||
root,
|
||||
mutation: Mutex::new(()),
|
||||
armed: AtomicBool::new(false),
|
||||
},
|
||||
});
|
||||
registry.insert(identity, Arc::downgrade(&owner));
|
||||
Ok(owner)
|
||||
}
|
||||
|
||||
fn metadata_directory(root: &File, create: bool) -> io::Result<File> {
|
||||
if create {
|
||||
match mkdirat(root, METADATA_DIR, Mode::from_raw_mode(0o700)) {
|
||||
Ok(()) => root.sync_all()?,
|
||||
Err(rustix::io::Errno::EXIST) => {}
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
}
|
||||
Ok(File::from(openat(
|
||||
root,
|
||||
METADATA_DIR,
|
||||
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
|
||||
Mode::empty(),
|
||||
)?))
|
||||
}
|
||||
|
||||
fn reserve_file(parent: &File, create: bool) -> io::Result<File> {
|
||||
let flags = OFlags::RDWR | OFlags::NOFOLLOW | OFlags::NONBLOCK | OFlags::CLOEXEC;
|
||||
let file = File::from(openat(
|
||||
parent,
|
||||
RESERVE_NAME,
|
||||
if create { flags | OFlags::CREATE } else { flags },
|
||||
Mode::from_raw_mode(0o600),
|
||||
)?);
|
||||
let metadata = file.metadata()?;
|
||||
if !metadata.is_file() || metadata.nlink() != 1 || metadata.len() > RESERVE_BYTES {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid delete reserve file"));
|
||||
}
|
||||
Ok(file)
|
||||
}
|
||||
|
||||
fn absent_to_none<T>(result: io::Result<T>) -> io::Result<Option<T>> {
|
||||
match result {
|
||||
Ok(value) => Ok(Some(value)),
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None),
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
|
||||
impl State {
|
||||
pub(super) fn is_armed(&self) -> bool {
|
||||
self.armed.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub(super) fn replenish(&self) -> io::Result<()> {
|
||||
if self.is_armed() {
|
||||
return Ok(());
|
||||
}
|
||||
let _guard = self.mutation.lock();
|
||||
if self.is_armed() {
|
||||
return Ok(());
|
||||
}
|
||||
let existing = match absent_to_none(metadata_directory(&self.root, false))? {
|
||||
Some(parent) => absent_to_none(reserve_file(&parent, false))?.map(|file| (parent, file)),
|
||||
None => None,
|
||||
};
|
||||
if let Some((_, file)) = &existing {
|
||||
let metadata = file.metadata()?;
|
||||
if metadata.len() == RESERVE_BYTES && metadata.blocks().saturating_mul(512) >= RESERVE_BYTES {
|
||||
self.armed.store(true, Ordering::Release);
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
// A depleted reserve must not take newly released recovery space
|
||||
// away from a disk that is still nearly full.
|
||||
if available_bytes(&self.root)? < RESERVE_BYTES * 2 {
|
||||
return Ok(());
|
||||
}
|
||||
let (parent, mut file) = match existing {
|
||||
Some(existing) => existing,
|
||||
None => {
|
||||
let parent = metadata_directory(&self.root, true)?;
|
||||
let file = reserve_file(&parent, true)?;
|
||||
(parent, file)
|
||||
}
|
||||
};
|
||||
let metadata = file.metadata()?;
|
||||
// A length alone does not reserve blocks. Repair sparse files left
|
||||
// by interrupted allocation or older installations.
|
||||
if metadata.blocks().saturating_mul(512) < metadata.len() {
|
||||
file.set_len(0)?;
|
||||
}
|
||||
let length = file.seek(SeekFrom::End(0))?;
|
||||
let mut remaining = RESERVE_BYTES - length;
|
||||
let mut bytes = [0_u8; 64 * 1024];
|
||||
while remaining != 0 {
|
||||
let count = usize::try_from(remaining.min(64 * 1024)).map_err(io::Error::other)?;
|
||||
// Zero blocks can become holes on transparent-compression filesystems.
|
||||
rand::fill(&mut bytes[..count]);
|
||||
file.write_all(&bytes[..count])?;
|
||||
remaining -= u64::try_from(count).map_err(io::Error::other)?;
|
||||
}
|
||||
file.sync_all()?;
|
||||
if file.metadata()?.blocks().saturating_mul(512) < RESERVE_BYTES {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::Unsupported,
|
||||
"filesystem did not allocate delete reserve blocks",
|
||||
));
|
||||
}
|
||||
parent.sync_all()?;
|
||||
self.armed.store(true, Ordering::Release);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn release_locked(&self) -> io::Result<bool> {
|
||||
let Some(parent) = absent_to_none(metadata_directory(&self.root, false))? else {
|
||||
self.armed.store(false, Ordering::Release);
|
||||
return Ok(false);
|
||||
};
|
||||
let Some(file) = absent_to_none(reserve_file(&parent, false))? else {
|
||||
self.armed.store(false, Ordering::Release);
|
||||
return Ok(false);
|
||||
};
|
||||
let metadata = file.metadata()?;
|
||||
if metadata.len() == 0 {
|
||||
self.armed.store(false, Ordering::Release);
|
||||
return Ok(false);
|
||||
}
|
||||
file.set_len(0)?;
|
||||
self.armed.store(false, Ordering::Release);
|
||||
// The reserve contains no recovery authority. Truncation releases
|
||||
// blocks immediately and needs no durability barrier before retry.
|
||||
Ok(metadata.blocks() != 0)
|
||||
}
|
||||
|
||||
pub(super) fn release_if_low(&self) -> io::Result<()> {
|
||||
if available_bytes(&self.root)? < RESERVE_BYTES / 2 {
|
||||
let _guard = self.mutation.lock();
|
||||
self.release_locked()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn retry_after_enospc<T>(
|
||||
&self,
|
||||
mut operation: impl FnMut() -> io::Result<T>,
|
||||
original: io::Error,
|
||||
) -> io::Result<T> {
|
||||
let _guard = self.mutation.lock();
|
||||
match self.release_locked() {
|
||||
// A peer may have consumed the shared reserve while this
|
||||
// operation was failing. Waiting for the mutation lock and
|
||||
// replaying once lets it use the capacity that peer released.
|
||||
// The public contract only accepts disposable scratch or an
|
||||
// idempotent primitive, so the retry remains safe even when no
|
||||
// reserve was present. Invalid reserve state still fails closed.
|
||||
Ok(_) => operation(),
|
||||
Err(_) => Err(original),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn available_bytes(root: &File) -> io::Result<u64> {
|
||||
let stat = rustix::fs::fstatvfs(root)?;
|
||||
let block_size = if stat.f_frsize == 0 { stat.f_bsize } else { stat.f_frsize };
|
||||
let available = u128::from(stat.f_bavail) * u128::from(block_size);
|
||||
Ok(u64::try_from(available).unwrap_or(u64::MAX))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn reserve_path(root: &Path) -> std::path::PathBuf {
|
||||
root.join(METADATA_DIR).join(RESERVE_NAME)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_reserve_allocates_retries_once_and_replenishes_real_blocks() {
|
||||
let root = tempfile::tempdir().expect("create disk root");
|
||||
let owner = DeleteReserve::acquire(root.path()).expect("pin disk root");
|
||||
owner.initialize().expect("allocate reserve");
|
||||
let path = reserve_path(root.path());
|
||||
let initial = std::fs::metadata(&path).expect("reserve metadata");
|
||||
assert_eq!(initial.len(), RESERVE_BYTES);
|
||||
assert!(initial.blocks() * 512 >= RESERVE_BYTES);
|
||||
assert!(owner.is_armed());
|
||||
let mut calls = 0;
|
||||
let result = owner
|
||||
.retry_after_enospc(|| {
|
||||
calls += 1;
|
||||
if calls == 1 {
|
||||
Err(io::Error::new(io::ErrorKind::StorageFull, "scratch allocation"))
|
||||
} else {
|
||||
Ok(42)
|
||||
}
|
||||
})
|
||||
.expect("retry scratch allocation");
|
||||
assert_eq!(result, 42);
|
||||
assert_eq!(calls, 2);
|
||||
assert!(!owner.is_armed());
|
||||
assert_eq!(std::fs::metadata(&path).expect("released reserve").len(), 0);
|
||||
owner.replenish().expect("replenish reserve");
|
||||
assert!(owner.is_armed());
|
||||
assert_eq!(std::fs::metadata(&path).expect("replenished reserve").len(), RESERVE_BYTES);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_reserve_retries_once_without_local_reserve_capacity() {
|
||||
let root = tempfile::tempdir().expect("create disk root");
|
||||
let owner = DeleteReserve::acquire(root.path()).expect("pin disk root");
|
||||
let mut calls = 0;
|
||||
let error = owner
|
||||
.retry_after_enospc::<()>(|| {
|
||||
calls += 1;
|
||||
Err(io::Error::new(io::ErrorKind::StorageFull, format!("allocation {calls}")))
|
||||
})
|
||||
.expect_err("retry without a reserve still has no capacity");
|
||||
assert_eq!(calls, 2);
|
||||
assert_eq!(error.to_string(), "allocation 2");
|
||||
owner.initialize().expect("allocate reserve");
|
||||
let parent = metadata_directory(&owner.inner.root, false).expect("metadata directory");
|
||||
let reserve = reserve_file(&parent, false).expect("reserve file");
|
||||
reserve.set_len(0).expect("deplete reserve");
|
||||
calls = 0;
|
||||
assert!(
|
||||
owner
|
||||
.retry_after_enospc::<()>(|| {
|
||||
calls += 1;
|
||||
Err(io::Error::new(io::ErrorKind::StorageFull, "empty reserve"))
|
||||
})
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(calls, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_reserve_retry_ignores_other_errors_and_stays_bounded() {
|
||||
let root = tempfile::tempdir().expect("create disk root");
|
||||
let owner = DeleteReserve::acquire(root.path()).expect("pin disk root");
|
||||
owner.initialize().expect("allocate reserve");
|
||||
let mut calls = 0;
|
||||
let error = owner
|
||||
.retry_after_enospc::<()>(|| {
|
||||
calls += 1;
|
||||
Err(io::Error::new(io::ErrorKind::PermissionDenied, "permission failure"))
|
||||
})
|
||||
.expect_err("permission failures must propagate");
|
||||
assert_eq!(error.kind(), io::ErrorKind::PermissionDenied);
|
||||
assert_eq!(calls, 1);
|
||||
assert!(owner.is_armed());
|
||||
calls = 0;
|
||||
let error = owner
|
||||
.retry_after_enospc::<()>(|| {
|
||||
calls += 1;
|
||||
Err(io::Error::new(io::ErrorKind::StorageFull, format!("allocation {calls}")))
|
||||
})
|
||||
.expect_err("second allocation failure propagates");
|
||||
assert_eq!(calls, 2);
|
||||
assert_eq!(error.to_string(), "allocation 2");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_reserve_rejects_symlinks_and_hardlinks_without_hiding_operation_errors() {
|
||||
use std::os::unix::fs::symlink;
|
||||
let root = tempfile::tempdir().expect("create disk root");
|
||||
let owner = DeleteReserve::acquire(root.path()).expect("pin disk root");
|
||||
let external = tempfile::tempdir().expect("create external directory");
|
||||
let victim = external.path().join("victim");
|
||||
std::fs::write(&victim, b"preserve").expect("write victim");
|
||||
let metadata = root.path().join(METADATA_DIR);
|
||||
symlink(external.path(), &metadata).expect("symlink metadata directory");
|
||||
assert!(owner.replenish().is_err());
|
||||
std::fs::remove_file(&metadata).expect("remove directory link");
|
||||
std::fs::create_dir(&metadata).expect("create metadata directory");
|
||||
let reserve = reserve_path(root.path());
|
||||
symlink(&victim, &reserve).expect("symlink reserve");
|
||||
assert!(owner.replenish().is_err());
|
||||
let mut calls = 0;
|
||||
let error = owner
|
||||
.retry_after_enospc::<()>(|| {
|
||||
calls += 1;
|
||||
Err(io::Error::new(io::ErrorKind::StorageFull, "original error"))
|
||||
})
|
||||
.expect_err("invalid reserve cannot mask allocation failure");
|
||||
assert_eq!(calls, 1);
|
||||
assert_eq!(error.to_string(), "original error");
|
||||
std::fs::remove_file(&reserve).expect("remove reserve link");
|
||||
std::fs::hard_link(&victim, &reserve).expect("hardlink reserve");
|
||||
assert!(owner.replenish().is_err());
|
||||
assert!(owner.inner.release_locked().is_err());
|
||||
assert_eq!(std::fs::read(&victim).expect("read victim"), b"preserve");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_reserve_repairs_partial_and_sparse_files() {
|
||||
let root = tempfile::tempdir().expect("create disk root");
|
||||
let owner = DeleteReserve::acquire(root.path()).expect("pin disk root");
|
||||
let parent = metadata_directory(&owner.inner.root, true).expect("create metadata directory");
|
||||
let mut file = reserve_file(&parent, true).expect("create reserve file");
|
||||
file.write_all(b"partial allocation").expect("write partial reserve");
|
||||
owner.replenish().expect("finish partial allocation");
|
||||
let metadata = file.metadata().expect("partial reserve metadata");
|
||||
assert_eq!(metadata.len(), RESERVE_BYTES);
|
||||
assert!(metadata.blocks() * 512 >= RESERVE_BYTES);
|
||||
file.set_len(0).expect("truncate reserve");
|
||||
file.set_len(RESERVE_BYTES).expect("create sparse reserve");
|
||||
owner.inner.armed.store(false, Ordering::Release);
|
||||
assert!(file.metadata().expect("sparse reserve metadata").blocks() * 512 < RESERVE_BYTES);
|
||||
owner.replenish().expect("repair sparse allocation");
|
||||
let metadata = file.metadata().expect("repaired reserve metadata");
|
||||
assert_eq!(metadata.len(), RESERVE_BYTES);
|
||||
assert!(metadata.blocks() * 512 >= RESERVE_BYTES);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_reserve_shares_reopened_owner_and_serializes_recovery_with_replenishment() {
|
||||
let root = tempfile::tempdir().expect("create disk root");
|
||||
let first = DeleteReserve::acquire(root.path()).expect("open first root");
|
||||
let second = DeleteReserve::acquire(root.path()).expect("reopen root");
|
||||
assert!(Arc::ptr_eq(&first, &second));
|
||||
first.initialize().expect("allocate reserve");
|
||||
let (retry_entered_tx, retry_entered_rx) = std::sync::mpsc::channel();
|
||||
let (allow_retry_tx, allow_retry_rx) = std::sync::mpsc::channel();
|
||||
let (replenished_tx, replenished_rx) = std::sync::mpsc::channel();
|
||||
let recovery_owner = first.clone();
|
||||
let root_path = root.path().to_path_buf();
|
||||
std::thread::scope(|scope| {
|
||||
let recovery = scope.spawn(move || {
|
||||
let mut calls = 0;
|
||||
recovery_owner
|
||||
.retry_after_enospc(|| {
|
||||
calls += 1;
|
||||
if calls == 1 {
|
||||
return Err(io::Error::new(io::ErrorKind::StorageFull, "scratch allocation"));
|
||||
}
|
||||
assert!(recovery_owner.inner.mutation.try_lock().is_none(), "retry must retain the reserve mutex");
|
||||
retry_entered_tx.send(()).expect("notify retry");
|
||||
allow_retry_rx
|
||||
.recv_timeout(std::time::Duration::from_secs(10))
|
||||
.expect("allow retry completion");
|
||||
assert_eq!(std::fs::metadata(reserve_path(&root_path)).expect("depleted reserve").len(), 0);
|
||||
Ok(())
|
||||
})
|
||||
.expect("retry allocation");
|
||||
});
|
||||
retry_entered_rx
|
||||
.recv_timeout(std::time::Duration::from_secs(10))
|
||||
.expect("retry entered");
|
||||
let replenish = scope.spawn(move || {
|
||||
second.replenish().expect("concurrent replenishment");
|
||||
replenished_tx.send(()).expect("notify replenishment");
|
||||
});
|
||||
assert!(matches!(replenished_rx.try_recv(), Err(std::sync::mpsc::TryRecvError::Empty)));
|
||||
allow_retry_tx.send(()).expect("finish retry");
|
||||
recovery.join().expect("recovery worker");
|
||||
replenish.join().expect("replenishment worker");
|
||||
});
|
||||
assert!(first.is_armed());
|
||||
assert_eq!(std::fs::metadata(reserve_path(root.path())).expect("final reserve").len(), RESERVE_BYTES);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_reserve_gives_concurrent_enospc_waiters_one_retry() {
|
||||
let root = tempfile::tempdir().expect("create disk root");
|
||||
let first = DeleteReserve::acquire(root.path()).expect("open first owner");
|
||||
let second = DeleteReserve::acquire(root.path()).expect("open second owner");
|
||||
first.initialize().expect("allocate reserve");
|
||||
|
||||
let (first_retry_tx, first_retry_rx) = std::sync::mpsc::channel();
|
||||
let (allow_first_tx, allow_first_rx) = std::sync::mpsc::channel();
|
||||
let (second_failed_tx, second_failed_rx) = std::sync::mpsc::channel();
|
||||
let (second_done_tx, second_done_rx) = std::sync::mpsc::channel();
|
||||
|
||||
std::thread::scope(|scope| {
|
||||
let first_waiter = scope.spawn(move || {
|
||||
let mut calls = 0;
|
||||
first
|
||||
.retry_after_enospc(|| {
|
||||
calls += 1;
|
||||
if calls == 1 {
|
||||
return Err(io::Error::new(io::ErrorKind::StorageFull, "first ENOSPC"));
|
||||
}
|
||||
first_retry_tx.send(()).expect("notify first retry");
|
||||
allow_first_rx
|
||||
.recv_timeout(std::time::Duration::from_secs(10))
|
||||
.expect("allow first retry to finish");
|
||||
Ok(())
|
||||
})
|
||||
.expect("first waiter retries");
|
||||
assert_eq!(calls, 2);
|
||||
});
|
||||
|
||||
first_retry_rx
|
||||
.recv_timeout(std::time::Duration::from_secs(10))
|
||||
.expect("first waiter holds recovery lock");
|
||||
let second_waiter = scope.spawn(move || {
|
||||
let mut calls = 0;
|
||||
second
|
||||
.retry_after_enospc(|| {
|
||||
calls += 1;
|
||||
if calls == 1 {
|
||||
second_failed_tx.send(()).expect("notify second ENOSPC");
|
||||
return Err(io::Error::new(io::ErrorKind::StorageFull, "second ENOSPC"));
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.expect("second waiter retries after shared release");
|
||||
assert_eq!(calls, 2);
|
||||
second_done_tx.send(()).expect("notify second completion");
|
||||
});
|
||||
|
||||
second_failed_rx
|
||||
.recv_timeout(std::time::Duration::from_secs(10))
|
||||
.expect("second waiter observed ENOSPC");
|
||||
assert!(matches!(second_done_rx.try_recv(), Err(std::sync::mpsc::TryRecvError::Empty)));
|
||||
allow_first_tx.send(()).expect("finish first retry");
|
||||
first_waiter.join().expect("first waiter");
|
||||
second_done_rx
|
||||
.recv_timeout(std::time::Duration::from_secs(10))
|
||||
.expect("second waiter completed its bounded retry");
|
||||
second_waiter.join().expect("second waiter");
|
||||
});
|
||||
|
||||
assert_eq!(std::fs::metadata(reserve_path(root.path())).expect("depleted reserve").len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_reserve_rejects_oversize_files_and_fifos() {
|
||||
let root = tempfile::tempdir().expect("create disk root");
|
||||
let owner = DeleteReserve::acquire(root.path()).expect("pin disk root");
|
||||
let parent = metadata_directory(&owner.inner.root, true).expect("create metadata directory");
|
||||
let file = reserve_file(&parent, true).expect("create reserve file");
|
||||
file.set_len(RESERVE_BYTES + 1).expect("create oversized reserve");
|
||||
assert!(owner.replenish().is_err());
|
||||
assert!(owner.inner.release_locked().is_err());
|
||||
assert_eq!(file.metadata().expect("oversize metadata").len(), RESERVE_BYTES + 1);
|
||||
let path = reserve_path(root.path());
|
||||
std::fs::remove_file(&path).expect("remove oversized file");
|
||||
assert!(
|
||||
std::process::Command::new("mkfifo")
|
||||
.arg(&path)
|
||||
.status()
|
||||
.expect("create FIFO fixture")
|
||||
.success()
|
||||
);
|
||||
assert!(owner.replenish().is_err());
|
||||
assert!(owner.inner.release_locked().is_err());
|
||||
use std::os::unix::fs::FileTypeExt;
|
||||
assert!(std::fs::symlink_metadata(path).expect("FIFO metadata").file_type().is_fifo());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -17,27 +17,6 @@ use std::{
|
||||
};
|
||||
use uuid::Uuid;
|
||||
|
||||
fn verify_synced_target(
|
||||
dir: &Path,
|
||||
base: &Path,
|
||||
root: &os::PublicationRoot,
|
||||
length: u64,
|
||||
digest: [u8; 32],
|
||||
) -> std::io::Result<()> {
|
||||
// Target version histories are bounded by the prepared receipt, not the source-control record limit.
|
||||
let observed = os::observe_multipart_source_metadata(&dir.join(STORAGE_FORMAT_FILE), base, root, length).map_err(|err| {
|
||||
if err.kind() == std::io::ErrorKind::NotFound {
|
||||
IoError::other(DiskError::ConditionalDeleteConflict)
|
||||
} else {
|
||||
err
|
||||
}
|
||||
})?;
|
||||
if observed != (length, digest) {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
const STATE: &str = ".mpu-source-state";
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
@@ -2491,7 +2470,30 @@ impl NativeMultipartPublication {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
if target_already_synced {
|
||||
verify_synced_target(&target_dir, base, root, expected_length, expected_sha256)?;
|
||||
let pinned = os::pin_multipart_source_directory(&target_dir, base, root).map_err(|error| {
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
IoError::other(DiskError::ConditionalDeleteConflict)
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
let target = os::read_multipart_source_metadata_in_pinned(
|
||||
&pinned,
|
||||
&target_dir,
|
||||
std::ffi::OsStr::new(STORAGE_FORMAT_FILE),
|
||||
base,
|
||||
root,
|
||||
)
|
||||
.map_err(|error| {
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
IoError::other(DiskError::ConditionalDeleteConflict)
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
if target.len() as u64 != expected_length || raw_sha(&target) != expected_sha256 {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
} else {
|
||||
os::read_and_sync_multipart_target_metadata(
|
||||
&target_dir,
|
||||
@@ -2777,29 +2779,3 @@ impl LocalDisk {
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod target_receipt_tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn synced_target_uses_prepared_receipt_for_large_history() {
|
||||
let tmp = tempfile::tempdir().expect("temporary publication root");
|
||||
let base = tmp.path().canonicalize().expect("absolute root");
|
||||
let dir = base.join("target");
|
||||
std::fs::create_dir(&dir).expect("target directory");
|
||||
let root = os::PublicationRoot::new(&base).expect("publication root");
|
||||
let block = [0u8; 64 * 1024];
|
||||
let length = 16 * 1024 * 1024 + 1;
|
||||
std::fs::File::create(dir.join(STORAGE_FORMAT_FILE))
|
||||
.expect("target head")
|
||||
.set_len(length)
|
||||
.expect("sparse head");
|
||||
let mut hash = Sha256::new();
|
||||
for _ in 0..256 {
|
||||
hash.update(block);
|
||||
}
|
||||
hash.update([0]);
|
||||
verify_synced_target(&dir, &base, &root, length, hash.finalize().into()).expect("matching receipt");
|
||||
assert!(verify_synced_target(&dir, &base, &root, length - 1, [0; 32]).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user