mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-20 15:07:34 +00:00
Compare commits
163 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| aca434fdfc | |||
| 4c569451e1 | |||
| f83b964f2c | |||
| 117c9dc197 | |||
| 46f750c6c9 | |||
| 1f8970ba1b | |||
| 0c44fd3bb8 | |||
| 4cb724e64b | |||
| e8ee409f6b | |||
| 08e88f87eb | |||
| 0525b3dc04 | |||
| b8225fb52d | |||
| a1868cf6bb | |||
| a2acae3944 | |||
| 4713e21231 | |||
| 95fc8b7b01 | |||
| 2873de8de9 | |||
| 6cba2bd002 | |||
| 7c8e4c7711 | |||
| e8847f6f1d | |||
| ee7523bab6 | |||
| ca8769d7c2 | |||
| 83f3fbf2cd | |||
| f56477eb53 | |||
| 2f55ddb595 | |||
| 44e147e8eb | |||
| dd760dfa56 | |||
| 7be29ee088 | |||
| 70a0267d01 | |||
| 28ac9de62a | |||
| da6c626f2a | |||
| e2fee7cb2b | |||
| a115748cda | |||
| 1f7f5b8532 | |||
| 0721822cfa | |||
| 0b7fd7a367 | |||
| f3b0806547 | |||
| 1fbcacb2b6 | |||
| 7729e8c337 | |||
| a537b6ea98 | |||
| 3805f2970e | |||
| 62f0385769 | |||
| c312102bab | |||
| 35e6c8077d | |||
| 007c3b204b | |||
| 1c3fe4747e | |||
| 032566507a | |||
| 42ab123d69 | |||
| 22f6c668da | |||
| 9829661a98 | |||
| ee3ef73630 | |||
| 4c7cc8a943 | |||
| 42ee5b151b | |||
| 3cda284b4c | |||
| fb053dfb85 | |||
| 9707aa4f1f | |||
| 197e682488 | |||
| 0e0744cb49 | |||
| 20872bba63 | |||
| 30d8aee534 | |||
| 05e6fd2085 | |||
| 3f861328d8 | |||
| 210e1b214e | |||
| 55d931353f | |||
| 0e1e2e0003 | |||
| b3e60a8882 | |||
| e6a1f542a1 | |||
| 662781e45b | |||
| 7aceffc68c | |||
| e0b53cc8d9 | |||
| db370e3241 | |||
| ffe6bc86d3 | |||
| fc09148f08 | |||
| c9e1464a2a | |||
| 5d6358a0a5 | |||
| 0e3028d29e | |||
| dde2169c83 | |||
| 0b586c37e9 | |||
| 365014a385 | |||
| 7946b094f9 | |||
| 443e8afee1 | |||
| 782e3e5209 | |||
| cfffb1700e | |||
| cd62f3ba4f | |||
| 7bc419e368 | |||
| 7269243de7 | |||
| c4a7423f99 | |||
| 2db4ed244a | |||
| 93a61df1cf | |||
| feec0e9e4a | |||
| 38baeeb11d | |||
| 1af039c972 | |||
| a2fc7c13b4 | |||
| dbe01e5cb1 | |||
| ffc27fa6db | |||
| 7e8edab0d3 | |||
| 22057ad4c5 | |||
| 279a88ab56 | |||
| b4abe64ada | |||
| add363f084 | |||
| 1f2dbe36e5 | |||
| 67fbafa13e | |||
| 1000860cb8 | |||
| b783f404ad | |||
| 11a991c777 | |||
| 3f2b6f690a | |||
| 88b626e7a0 | |||
| 8cf6187ff4 | |||
| a0a0cee9eb | |||
| 80fff4b5c5 | |||
| d91dfcedd2 | |||
| 79c4fedf8c | |||
| 73c846e09c | |||
| 6938c467d0 | |||
| 5323df7049 | |||
| 55b6fbd260 | |||
| dd677cc121 | |||
| 0c2f327974 | |||
| df42cf1ce8 | |||
| d290d46b34 | |||
| 0d297ed8cd | |||
| b8dffa2f1d | |||
| 62b6f5f3fc | |||
| be0986115c | |||
| 1bd91c8dad | |||
| dcb22bd4d5 | |||
| fa17718444 | |||
| 99abffc57a | |||
| fc061c7930 | |||
| d86bf1360f | |||
| f6097d1df2 | |||
| 7e5d32eecf | |||
| 72da0e21c4 | |||
| 78a0cf321c | |||
| cbe38a120f | |||
| db0828aa1c | |||
| 3ca889b099 | |||
| e6112c526f | |||
| c5967686f1 | |||
| 859de565ad | |||
| 1f7cc593e7 | |||
| 1d5a50f094 | |||
| 53245be9f4 | |||
| c47432e353 | |||
| f622c1bce0 | |||
| a99d8f4f89 | |||
| 4d15b81623 | |||
| 836d30192f | |||
| 860c61ee6d | |||
| f600af4704 | |||
| 9e747e8475 | |||
| 96fc067765 | |||
| e809b4899e | |||
| 584f03f300 | |||
| d52d6fade7 | |||
| 3623d6db2d | |||
| 484f1254ca | |||
| c9c16efbb6 | |||
| f2a59073ca | |||
| c1af94d1d1 | |||
| f84bbfe28f | |||
| 378a2ff7f8 | |||
| b88b53aa6a |
@@ -31,14 +31,3 @@
|
||||
# the flag, so the two can no longer drift apart unnoticed.
|
||||
#
|
||||
# For CPU profiling, add `-C force-frame-pointers=yes` to that RUSTFLAGS value.
|
||||
|
||||
# Exact internode HTTP/2 PING ACK timeout accounting is also deliberately
|
||||
# default-off. Hyper exposes the exact timeout only through its unstable tracing
|
||||
# event, so diagnostic builds must opt in to both the feature and cfg:
|
||||
#
|
||||
# RUSTFLAGS="--cfg hyper_unstable_tracing" \
|
||||
# cargo build -p rustfs --features diagnostic-internode-http2-ping
|
||||
#
|
||||
# `crates/protos/build.rs` and `crates/obs/build.rs` fail fast if the feature and
|
||||
# cfg drift apart. Do not move this cfg into global rustflags: doing so would
|
||||
# recompile every dependency under an unstable, non-semver build contract.
|
||||
|
||||
@@ -1,10 +0,0 @@
|
||||
# Rulings for the gateway conformance lane (.github/workflows/gateway-acceptance.yml).
|
||||
#
|
||||
# One row per case that fails on the GATEWAY stack for a reviewed reason: `case`; `ledger`, the
|
||||
# rustfs/backlog#2684 entry that accepts the difference (R-0xx or K-n); a one-line `reason`; and
|
||||
# `failure`, a prefix of the failure the ruling accepts, as the runner's report.json renders it
|
||||
# (`[expect/status] at /expect/status: exchange #1, attempt 1: expected status 400, observed 403`).
|
||||
# scripts/check_conformance_rulings.py fails the lane on a failed case without a row, on a ruled
|
||||
# case whose failures do not start with its `failure`, and on a row whose case no longer fails or
|
||||
# no longer exists. The legacy stack is a reference run and takes no rulings: the `rustfs` profile
|
||||
# describes the gateway edge, not the legacy one.
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=9ccb03f4139e99ec11a63810523d504df39191b86e74b28a72dd9a075682293b
|
||||
sha256-linux=5ef052b03a03afbd00ebfcbbdcb754bbde414b7e6151aae3e2ee78024ce1d0b6
|
||||
sha256-darwin=8fef326a3fddd8a548e2cbcc6ea8159f831113595fb003eeebbd9aeb4ac23e48
|
||||
sha256-linux=8b4381578377704ac8b1b5156128b83f036ec4aedc9d863fff25df15112420fc
|
||||
|
||||
@@ -1 +1 @@
|
||||
sha256=fd5683273bba1eae3b4402a4e854a5d168ace845bd0e5fc801f8e336485a4394
|
||||
sha256=061d0a697967d2dc8f472ff885886e069b1cf4aeaa5f230e177b7236017c23b6
|
||||
|
||||
@@ -39,11 +39,7 @@ script-tests: ## Run shell script tests
|
||||
./scripts/test_entrypoint_directories.sh
|
||||
./scripts/test_internode_grpc_ab_bench.sh
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_internode_grpc_window.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_internode_rpc_timeout_boundary.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_internode_msgpack_peer_canary.py
|
||||
./scripts/test_internode_msgpack_peer_canary_runner.sh
|
||||
./scripts/test_object_batch_bench_enhanced.sh
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_inline_docker_lab.py
|
||||
./scripts/test_package_service_scripts.sh
|
||||
./scripts/test_hotpath_warp_ab_gate.sh
|
||||
./scripts/test_hotpath_warp_abba.sh
|
||||
@@ -103,8 +99,8 @@ script-tests: ## Run shell script tests
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_retain_validation_evidence.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_timing_report.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_junit_diff.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_conformance_rulings.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/compat_matrix_report.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_dual_stack.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_mint_stack_diff.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_runner_tools.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_api_coverage.py
|
||||
|
||||
+5
-15
@@ -99,11 +99,6 @@ threads-required = "num-test-threads"
|
||||
filter = '(package(rustfs-ecstore) & (test(/^core::pools_test::decommission_lock_order_tests::(active_decommission_migration_does_not_invert_capacity_and_delete_object(_tag)?_locks|decommission_moves_a_transitioned_version_expired_by_a_delete_all_rule_without_stalling)$/) | test(=services::integrity::tests::audit_migration_and_ack_recovery_preserve_source_and_reject_conflicts))) | (package(rustfs-heal) & test(=heal::mrf_queue::partial_write::tests::durable_retry_submission_error_keeps_checkpoint_and_uses_backoff))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# These eight-disk namespace-owner probes overlap durable cleanup with a lock contender. Reserve capacity so sibling disk fixtures cannot consume their lock and completion budgets.
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(rustfs-ecstore) & test(/^store::object::tests::(canceled_create_retains_usage_cleanup_namespace_owner|batch_delete_excludes_recovery_on_nonzero_pool|delete_excludes_recovery_with_(disjoint|identical|overlapping)_pool_lock_clients)$/)'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# These eight-disk decommission fixtures exercise concurrent retries,
|
||||
# free-version cleanup, and real capacity probes under production lock deadlines.
|
||||
# Reserve the run's capacity so unrelated IO cannot exhaust those budgets.
|
||||
@@ -141,9 +136,8 @@ setup = 'ecstore-large-stack'
|
||||
filter = '(package(rustfs) & test(/^(admin::handlers::replication::target_repair_tests::[a-z_]+|admin::handlers::plugin_attempt::v2_tests::delegation_v2_persists_and_vends_with_real_parent_intersection|admin::handlers::site_replication::tests::durable_iam_state_apply_iam_item_rejects_a_stale_recreate_after_a_replicated_delete|admin::service::federated_identity::tests::(binding_keeps_explicit_iam_mapping_strict|markerless_legacy_oidc_session_crosses_auth_and_iam_deletion_boundaries)|storage::scanner_dirty_journal::tests::(clear_callback_keeps_unknown_coverage_until_all_pending_work_is_confirmed|owned_journals_keep_peer_records_after_interleaved_writes_and_clear))$/)) | (package(rustfs-heal) & ((binary(heal_b5_versioned_regression_test) & test(/^serial_tests::(test_heal_unversioned_bucket_e2e|test_historical_null_recursive_heal_restores_physical_versions|test_recursive_deep_heal_reports_stale_delete_marker_repair)$/)) | (binary(usage_observation_read_repair_test) & test(=usage_observation_read_repair_survives_snapshot_cleanup)))) | (package(rustfs-test-utils) & test(=data_usage_snapshot_tests::data_usage_snapshot_recovery_wires_primary_backup_and_negative_cache))'
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
# The multipart refusal fixture composes real-store setup, abort, quota and legal-hold futures.
|
||||
[[profile.default.scripts]]
|
||||
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::complete::tests::completion_absent_binding_refusal_order_baseline|app::multipart_usecase::complete::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access_owned::tests::generation_guards::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
|
||||
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::complete::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
# These handler/store fixtures and the relocated-pool GET compose deep storage
|
||||
@@ -411,11 +405,6 @@ threads-required = "num-test-threads"
|
||||
filter = '(package(rustfs-ecstore) & (test(/^core::pools_test::decommission_lock_order_tests::(active_decommission_migration_does_not_invert_capacity_and_delete_object(_tag)?_locks|decommission_moves_a_transitioned_version_expired_by_a_delete_all_rule_without_stalling)$/) | test(=services::integrity::tests::audit_migration_and_ack_recovery_preserve_source_and_reject_conflicts))) | (package(rustfs-heal) & test(=heal::mrf_queue::partial_write::tests::durable_retry_submission_error_keeps_checkpoint_and_uses_backoff))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# These eight-disk namespace-owner probes overlap durable cleanup with a lock contender. Reserve capacity so sibling disk fixtures cannot consume their lock and completion budgets.
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(rustfs-ecstore) & test(/^store::object::tests::(canceled_create_retains_usage_cleanup_namespace_owner|batch_delete_excludes_recovery_on_nonzero_pool|delete_excludes_recovery_with_(disjoint|identical|overlapping)_pool_lock_clients)$/)'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# These eight-disk decommission fixtures exercise concurrent retries,
|
||||
# free-version cleanup, and real capacity probes under production lock deadlines.
|
||||
# Reserve the run's capacity so unrelated IO cannot exhaust those budgets.
|
||||
@@ -466,7 +455,7 @@ filter = '(package(rustfs) & test(/^(admin::handlers::replication::target_repair
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
[[profile.ci.scripts]]
|
||||
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::complete::tests::completion_absent_binding_refusal_order_baseline|app::multipart_usecase::complete::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access_owned::tests::generation_guards::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
|
||||
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::complete::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
[[profile.ci.scripts]]
|
||||
@@ -723,8 +712,9 @@ test-group = 'ecstore-serial-flaky'
|
||||
# fixture as the existing smoke modules.
|
||||
#
|
||||
# Dual-stack differential (rustfs/backlog#2740): `dual_stack_differential::`
|
||||
# starts two single-node servers on random ports with isolated temp
|
||||
# dirs, and runs one SDK script covering every S3 operation against both. Its
|
||||
# starts two single-node servers from the same binary, one per
|
||||
# RUSTFS_S3_STACK value, on random ports with isolated temp dirs, and runs one
|
||||
# SDK script covering every operation of `impl S3 for FS` against both. Its
|
||||
# comparator unit tests need no server. It meets the admission criteria
|
||||
# unchanged; the module clause admits a new differential test without another
|
||||
# filter edit.
|
||||
|
||||
@@ -9,9 +9,9 @@
|
||||
# Adding a target is one more [[targets]] table, for example:
|
||||
#
|
||||
# [[targets]]
|
||||
# name = "candidate-tuned"
|
||||
# name = "candidate-gateway"
|
||||
# binary = "candidate"
|
||||
# env = { RUSTFS_EXAMPLE_SETTING = "on" }
|
||||
# env = { RUSTFS_S3_STACK = "gateway" }
|
||||
|
||||
schema = 1
|
||||
|
||||
|
||||
@@ -114,14 +114,8 @@
|
||||
}
|
||||
],
|
||||
"upgrade_sources": {
|
||||
"basis": "The three previous-release sources pinned in .github/workflows/e2e-upgrade.yml are exercised today (source.version / source.asset / source.sha256). No other N-1/N-2 compatibility is claimed, and these digests are not inferred from tags.",
|
||||
"basis": "Exactly the two previous-release sources pinned in .github/workflows/e2e-upgrade.yml are exercised today (source.version / source.asset / source.sha256). No other N-1/N-2 compatibility is claimed, and these digests are not inferred from tags.",
|
||||
"sources": [
|
||||
{
|
||||
"version": "1.0.5",
|
||||
"role": "gateway compatibility baseline",
|
||||
"artifact_name": "rustfs-linux-x86_64-gnu-v1.0.5.zip",
|
||||
"artifact_sha256": "5addae2f74756bf80104d0975e6347cda843bc2daffdfcd1f0cb80118b95cbd0"
|
||||
},
|
||||
{
|
||||
"version": "1.0.1",
|
||||
"role": "N-1",
|
||||
@@ -136,6 +130,6 @@
|
||||
}
|
||||
],
|
||||
"rollback_supported": false,
|
||||
"rollback_basis": "The release upgrade lane checks bucket-config rollback. The hosted compatibility workflow also checks in-flight multipart rollback to 1.0.5. Full data-plane rollback is not certified and is never a default claim."
|
||||
"rollback_basis": "Only the bucket-config-rollback case (rollback_to_previous_release_reads_current_bucket_metadata) is exercised. Full data-plane rollback is not certified and is never a default claim."
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,6 +28,8 @@ rustfs/src/app/storage_api.rs
|
||||
rustfs/src/admin/router.rs
|
||||
rustfs/src/app/metadata_route.rs
|
||||
rustfs/src/server/http.rs
|
||||
rustfs/src/server/layer.rs
|
||||
rustfs/src/server/hybrid.rs
|
||||
rustfs/src/protocols/client.rs
|
||||
|
||||
# e2e oracle: the test target that verifies S3 behaviour against s3s
|
||||
|
||||
@@ -13,6 +13,6 @@
|
||||
# s3_error_lines lines holding s3_error!, same scope, minus those the script
|
||||
# header's rule clears (rustfs_s3_types's macro); fails closed
|
||||
# ecstore_files Rust files referencing s3s paths under crates/ecstore/src
|
||||
files=33
|
||||
s3_error_lines=89
|
||||
files=63
|
||||
s3_error_lines=209
|
||||
ecstore_files=0
|
||||
|
||||
@@ -22,7 +22,7 @@ rustfs-connect-symbol-catalog 6 0
|
||||
rustfs-credentials 24 0
|
||||
rustfs-crypto 67 0
|
||||
rustfs-data-usage 58 0
|
||||
rustfs-ecstore 7858 3
|
||||
rustfs-ecstore 7858 2
|
||||
rustfs-extension-schema 23 0
|
||||
rustfs-filemeta 361 0
|
||||
rustfs-gateway-adapter 74 0
|
||||
|
||||
@@ -34,10 +34,6 @@ services:
|
||||
- RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT:-5}
|
||||
- RUSTFS_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_LOCK_ACQUIRE_TIMEOUT:-5}
|
||||
- RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
|
||||
- RUSTFS_DURABILITY_MODE=${RUSTFS_DURABILITY_MODE:-strict}
|
||||
- RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE=${RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS:-0}
|
||||
# Internode gRPC optimization knobs (grpc-optimization P0-P3). Defaults match the binary
|
||||
# defaults, so leaving these unset is a no-op; the A/B driver exports them to toggle a stage.
|
||||
- RUSTFS_INTERNODE_RPC_TCP_NODELAY=${RUSTFS_INTERNODE_RPC_TCP_NODELAY:-true}
|
||||
@@ -47,8 +43,6 @@ services:
|
||||
- RUSTFS_INTERNODE_CHANNEL_ISOLATION=${RUSTFS_INTERNODE_CHANNEL_ISOLATION:-false}
|
||||
- RUSTFS_INTERNODE_BULK_CHANNELS=${RUSTFS_INTERNODE_BULK_CHANNELS:-2}
|
||||
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY:-false}
|
||||
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED:-false}
|
||||
- RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED=${RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED:-false}
|
||||
- RUSTFS_INTERNODE_PREWARM=${RUSTFS_INTERNODE_PREWARM:-false}
|
||||
- RUSTFS_INTERNODE_OFFLINE_BYPASS=${RUSTFS_INTERNODE_OFFLINE_BYPASS:-false}
|
||||
- RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS=${RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS:-5}
|
||||
@@ -62,7 +56,7 @@ services:
|
||||
- node1_data_2:/data/rustfs2
|
||||
- node1_data_3:/data/rustfs3
|
||||
ports:
|
||||
- "${RUSTFS_NODE1_PORT:-9000}:9000"
|
||||
- "9000:9000"
|
||||
networks:
|
||||
- rustfs-cluster-net
|
||||
|
||||
@@ -87,10 +81,6 @@ services:
|
||||
- RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT:-5}
|
||||
- RUSTFS_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_LOCK_ACQUIRE_TIMEOUT:-5}
|
||||
- RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
|
||||
- RUSTFS_DURABILITY_MODE=${RUSTFS_DURABILITY_MODE:-strict}
|
||||
- RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE=${RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS:-0}
|
||||
# Internode gRPC optimization knobs (grpc-optimization P0-P3). Defaults match the binary
|
||||
# defaults, so leaving these unset is a no-op; the A/B driver exports them to toggle a stage.
|
||||
- RUSTFS_INTERNODE_RPC_TCP_NODELAY=${RUSTFS_INTERNODE_RPC_TCP_NODELAY:-true}
|
||||
@@ -100,8 +90,6 @@ services:
|
||||
- RUSTFS_INTERNODE_CHANNEL_ISOLATION=${RUSTFS_INTERNODE_CHANNEL_ISOLATION:-false}
|
||||
- RUSTFS_INTERNODE_BULK_CHANNELS=${RUSTFS_INTERNODE_BULK_CHANNELS:-2}
|
||||
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY:-false}
|
||||
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED:-false}
|
||||
- RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED=${RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED:-false}
|
||||
- RUSTFS_INTERNODE_PREWARM=${RUSTFS_INTERNODE_PREWARM:-false}
|
||||
- RUSTFS_INTERNODE_OFFLINE_BYPASS=${RUSTFS_INTERNODE_OFFLINE_BYPASS:-false}
|
||||
- RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS=${RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS:-5}
|
||||
@@ -115,7 +103,7 @@ services:
|
||||
- node2_data_2:/data/rustfs2
|
||||
- node2_data_3:/data/rustfs3
|
||||
ports:
|
||||
- "${RUSTFS_NODE2_PORT:-9001}:9000"
|
||||
- "9001:9000"
|
||||
networks:
|
||||
- rustfs-cluster-net
|
||||
|
||||
@@ -140,10 +128,6 @@ services:
|
||||
- RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT:-5}
|
||||
- RUSTFS_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_LOCK_ACQUIRE_TIMEOUT:-5}
|
||||
- RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
|
||||
- RUSTFS_DURABILITY_MODE=${RUSTFS_DURABILITY_MODE:-strict}
|
||||
- RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE=${RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS:-0}
|
||||
# Internode gRPC optimization knobs (grpc-optimization P0-P3). Defaults match the binary
|
||||
# defaults, so leaving these unset is a no-op; the A/B driver exports them to toggle a stage.
|
||||
- RUSTFS_INTERNODE_RPC_TCP_NODELAY=${RUSTFS_INTERNODE_RPC_TCP_NODELAY:-true}
|
||||
@@ -153,8 +137,6 @@ services:
|
||||
- RUSTFS_INTERNODE_CHANNEL_ISOLATION=${RUSTFS_INTERNODE_CHANNEL_ISOLATION:-false}
|
||||
- RUSTFS_INTERNODE_BULK_CHANNELS=${RUSTFS_INTERNODE_BULK_CHANNELS:-2}
|
||||
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY:-false}
|
||||
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED:-false}
|
||||
- RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED=${RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED:-false}
|
||||
- RUSTFS_INTERNODE_PREWARM=${RUSTFS_INTERNODE_PREWARM:-false}
|
||||
- RUSTFS_INTERNODE_OFFLINE_BYPASS=${RUSTFS_INTERNODE_OFFLINE_BYPASS:-false}
|
||||
- RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS=${RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS:-5}
|
||||
@@ -168,7 +150,7 @@ services:
|
||||
- node3_data_2:/data/rustfs2
|
||||
- node3_data_3:/data/rustfs3
|
||||
ports:
|
||||
- "${RUSTFS_NODE3_PORT:-9002}:9000"
|
||||
- "9002:9000"
|
||||
networks:
|
||||
- rustfs-cluster-net
|
||||
|
||||
@@ -193,10 +175,6 @@ services:
|
||||
- RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT:-5}
|
||||
- RUSTFS_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_LOCK_ACQUIRE_TIMEOUT:-5}
|
||||
- RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
|
||||
- RUSTFS_DURABILITY_MODE=${RUSTFS_DURABILITY_MODE:-strict}
|
||||
- RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE=${RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS:-0}
|
||||
# Internode gRPC optimization knobs (grpc-optimization P0-P3). Defaults match the binary
|
||||
# defaults, so leaving these unset is a no-op; the A/B driver exports them to toggle a stage.
|
||||
- RUSTFS_INTERNODE_RPC_TCP_NODELAY=${RUSTFS_INTERNODE_RPC_TCP_NODELAY:-true}
|
||||
@@ -206,8 +184,6 @@ services:
|
||||
- RUSTFS_INTERNODE_CHANNEL_ISOLATION=${RUSTFS_INTERNODE_CHANNEL_ISOLATION:-false}
|
||||
- RUSTFS_INTERNODE_BULK_CHANNELS=${RUSTFS_INTERNODE_BULK_CHANNELS:-2}
|
||||
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY:-false}
|
||||
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED:-false}
|
||||
- RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED=${RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED:-false}
|
||||
- RUSTFS_INTERNODE_PREWARM=${RUSTFS_INTERNODE_PREWARM:-false}
|
||||
- RUSTFS_INTERNODE_OFFLINE_BYPASS=${RUSTFS_INTERNODE_OFFLINE_BYPASS:-false}
|
||||
- RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS=${RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS:-5}
|
||||
@@ -221,7 +197,7 @@ services:
|
||||
- node4_data_2:/data/rustfs2
|
||||
- node4_data_3:/data/rustfs3
|
||||
ports:
|
||||
- "${RUSTFS_NODE4_PORT:-9003}:9000"
|
||||
- "9003:9000"
|
||||
networks:
|
||||
- rustfs-cluster-net
|
||||
|
||||
|
||||
@@ -1,58 +0,0 @@
|
||||
# syntax=docker/dockerfile:1.6
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Server-only Linux validation image. The build context supplies a Git bundle.
|
||||
FROM rust:1.100.0-trixie AS build-tools
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates clang lld libssl-dev pkg-config protobuf-compiler python3 python3-boto3 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
ENV RUSTUP_TOOLCHAIN=1.100.0 \
|
||||
CARGO_BUILD_JOBS=4 \
|
||||
CARGO_NET_GIT_FETCH_WITH_CLI=true \
|
||||
CARGO_TARGET_DIR=/work/rustfs/target \
|
||||
RUSTFLAGS="-C link-arg=-fuse-ld=lld"
|
||||
|
||||
FROM build-tools AS build
|
||||
ARG SOURCE_REVISION
|
||||
COPY repository.bundle /tmp/repository.bundle
|
||||
COPY inspect_geometry.rs /tmp/inspect_geometry.rs
|
||||
RUN git clone /tmp/repository.bundle /work/rustfs \
|
||||
&& git -C /work/rustfs checkout --detach "$SOURCE_REVISION" \
|
||||
&& test "$(git -C /work/rustfs rev-parse HEAD)" = "$SOURCE_REVISION"
|
||||
WORKDIR /work/rustfs
|
||||
RUN --mount=type=cache,id=rustfs-inline-registry,target=/usr/local/cargo/registry \
|
||||
--mount=type=cache,id=rustfs-inline-git,target=/usr/local/cargo/git \
|
||||
--mount=type=cache,id=rustfs-inline-linux-target,target=/work/rustfs/target \
|
||||
cargo build --locked --release --bin rustfs \
|
||||
&& install -Dm755 target/release/rustfs /out/rustfs
|
||||
|
||||
# Build the diagnostic tool after preserving the clean-source server binary.
|
||||
RUN cp /tmp/inspect_geometry.rs crates/filemeta/examples/inline_lab_inspect.rs
|
||||
RUN --mount=type=cache,id=rustfs-inline-registry,target=/usr/local/cargo/registry \
|
||||
--mount=type=cache,id=rustfs-inline-git,target=/usr/local/cargo/git \
|
||||
--mount=type=cache,id=rustfs-inline-linux-target,target=/work/rustfs/target \
|
||||
cargo build --locked --release -p rustfs-filemeta --example inline_lab_inspect \
|
||||
&& install -Dm755 target/release/examples/inline_lab_inspect /out/inline_lab_inspect
|
||||
|
||||
FROM debian:trixie-slim AS runtime
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates curl libssl3t64 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
ARG SOURCE_REVISION
|
||||
LABEL org.opencontainers.image.revision="$SOURCE_REVISION" \
|
||||
rustfs.validation="inline"
|
||||
COPY --from=build /out/rustfs /usr/local/bin/rustfs
|
||||
COPY --from=build /out/inline_lab_inspect /usr/local/bin/inline_lab_inspect
|
||||
ENTRYPOINT ["/usr/local/bin/rustfs"]
|
||||
@@ -1,5 +0,0 @@
|
||||
FROM debian:trixie-slim
|
||||
COPY mc /usr/bin/mc
|
||||
RUN chmod 0755 /usr/bin/mc && /usr/bin/mc --version
|
||||
LABEL rustfs.validation="inline"
|
||||
ENTRYPOINT ["/usr/bin/mc"]
|
||||
@@ -1,133 +0,0 @@
|
||||
# Inline geometry Linux lab
|
||||
|
||||
**Use this when** validating inline placement, block geometry, Range reads and reader rollback in a task-owned local Docker cluster.
|
||||
|
||||
The adapter reuses the repository's four-node compose topology with 16 logical Docker volumes and EC12+4. It binds one S3 port to loopback, keeps strict durability and checks running binary hashes and persisted object geometry. Docker volumes share the host storage substrate. Performance observations describe this local virtualized lab.
|
||||
|
||||
## Build a frozen server image
|
||||
|
||||
Use the current stable toolchain pinned in the lab Dockerfile. Supply a Git bundle and the metadata inspector source as the build context. The server is compiled from a clean checkout before the diagnostic example is added.
|
||||
|
||||
```bash
|
||||
set -euo pipefail
|
||||
LAB_BUILD_DIR="$(mktemp -d)"
|
||||
LAB_REVISION="$(git rev-parse HEAD)"
|
||||
git bundle create "$LAB_BUILD_DIR/repository.bundle" HEAD
|
||||
cp crates/filemeta/examples/inline_lab_inspect.rs "$LAB_BUILD_DIR/inspect_geometry.rs"
|
||||
docker --context "$LAB_CONTEXT" build --target build-tools \
|
||||
-t rustfs-inline-build-tools:1.100 \
|
||||
-f .docker/inline-lab/Dockerfile .docker/inline-lab
|
||||
docker --context "$LAB_CONTEXT" build \
|
||||
--build-arg SOURCE_REVISION="$LAB_REVISION" \
|
||||
-t rustfs-inline-validation:local \
|
||||
-f .docker/inline-lab/Dockerfile "$LAB_BUILD_DIR"
|
||||
LAB_EXPORT_ID="$(docker --context "$LAB_CONTEXT" create \
|
||||
--label rustfs.validation=inline rustfs-inline-validation:local)"
|
||||
docker --context "$LAB_CONTEXT" cp \
|
||||
"$LAB_EXPORT_ID:/usr/local/bin/rustfs" "$LAB_BUILD_DIR/rustfs-linux"
|
||||
docker --context "$LAB_CONTEXT" rm "$LAB_EXPORT_ID"
|
||||
```
|
||||
|
||||
## Configure and probe
|
||||
|
||||
Select a new compose project and a private state directory. The adapter generates test credentials there. Existing resources require matching ownership labels. Configure checks the image's source revision and pins immutable image IDs. Each operation takes an exclusive state lock. Deployment waits for `/health/ready` on every node, covering storage, IAM, lock quorum and peer dependencies.
|
||||
|
||||
```bash
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" configure \
|
||||
--context "$LAB_CONTEXT" --project "$LAB_PROJECT" \
|
||||
--image rustfs-inline-validation:local --revision "$LAB_REVISION" \
|
||||
--binary "$LAB_BUILD_DIR/rustfs-linux" --port 19080
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" deploy \
|
||||
--bucket inline-geometry-probe --shard-bytes 0
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" probe \
|
||||
--bucket inline-geometry-probe
|
||||
```
|
||||
|
||||
The probe verifies complete bytes, HEAD sizes, Range contents, EC12+4 and the actual persisted block. It records inline markers for 4 KiB, 1 MiB, 1 MiB + 1 byte, 1.25 MiB, 1,572,856 bytes, 1,572,857 bytes and 1.5 MiB objects in `probe.json`. The server-only image includes the existing filemeta decoder through a small geometry inspector. Console assets and the administration CLI are outside this lab image.
|
||||
|
||||
## ABBA deployment hook
|
||||
|
||||
Use `inline_docker_lab.py ... deploy` in the existing `run_hotpath_warp_abba.sh` external deploy hook. Export `RUSTFS_ERASURE_SHARD_BYTES` according to `HOTPATH_ABBA_PHASE`. The adapter uses the supplied leg, phase, bucket, binary and evidence-file context, resets only owned lab volumes, checks running configuration and binary hashes, creates the fresh bucket and writes the runner's deployment evidence.
|
||||
|
||||
Use the same frozen binary for both sides of a geometry experiment. Fix the inline budget and encoding mode, then compare zero with one candidate shard value. Include the CPU, memory and image identities from every node in each result. The state `config.json` supplies the generated test credentials for the load driver. Set `DOCKER_CONTEXT` to the selected context when the driver captures Docker resource snapshots.
|
||||
|
||||
For content compatibility across an image or geometry change, configure the next selected image and artifact using the same state, then deploy with `--keep-data` and an existing bucket. This retains owned volumes. Compare existing objects and version IDs with the manifest prepared before the change.
|
||||
|
||||
## Cleanup
|
||||
|
||||
```bash
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" down --purge
|
||||
```
|
||||
|
||||
Cleanup checks ownership for containers, volumes and networks. A configuration cannot adopt another project's resources. Raw measurements, state, Git bundles and build outputs stay in task artifact directories.
|
||||
|
||||
Unit checks run with `scripts/python_bin.sh scripts/test_inline_docker_lab.py` and are included in `make script-tests`.
|
||||
|
||||
## Linux content-probe client
|
||||
|
||||
The PUT after-probe requires a working `mc`. For a macOS client affected by cgo initialization failures, compile the same frozen public mc module for Linux with CGO disabled, build it with `Dockerfile.mc`, and use `scripts/inline_docker_mc.sh` as the process-local `mc` command. Record the module version, checksum and resulting image ID with the run.
|
||||
|
||||
The adapter rewrites only the lab loopback alias endpoint to the private `node1` endpoint. Alias configuration stays in the probe's own `MC_CONFIG_DIR`, and `cat` streams raw bytes. The existing HEAD, GET and hash checks continue to execute.
|
||||
|
||||
```bash
|
||||
export RUSTFS_INLINE_LAB_STATE="$LAB_STATE"
|
||||
export RUSTFS_INLINE_MC_IMAGE="$LAB_MC_IMAGE"
|
||||
mkdir -p "$LAB_TOOLS"
|
||||
ln -s "$PWD/scripts/inline_docker_mc.sh" "$LAB_TOOLS/mc"
|
||||
export PATH="$LAB_TOOLS:$PATH"
|
||||
```
|
||||
|
||||
## Compare transport and placement independently
|
||||
|
||||
`deploy --rpc-mode json-compat` explicitly clears both MessagePack-only flags on every node. `deploy --rpc-mode msgpack-only` enables both flags for the selected homogeneous lab fleet. Deployment evidence records the mode and checks both running flags. The default is `json-compat`. The ABBA hook may select the same mode through `RUSTFS_INLINE_LAB_RPC_MODE`.
|
||||
|
||||
First compare the base and bounded-multi-block binaries at `--shard-bytes 0` in JSON compatibility mode. Then repeat with MessagePack-only on both sides. A same-binary transport ABBA isolates the encoding effect. Keep the 128 KiB budget, CPU sets, strict durability, load and object set fixed. Check actual markers and block sizes at the boundary sizes before timing. Retain baseline-drift and candidate gates, error logs and raw request distributions.
|
||||
|
||||
For a rolling downgrade, redeploy the current binary with `--keep-data --rpc-mode json-compat` before introducing an older node. Check old and new reads, version IDs and batch deletes against the persisted manifest. A running MessagePack-only flag pair establishes the lab configuration. Compatibility with a particular old reader requires the explicit reader test.
|
||||
|
||||
## Preserve a compatibility manifest
|
||||
|
||||
Run the seed once in a fresh lab bucket. It records full byte oracles and the observed version inventory, including any additional version matching a known body, and leaves a 5 MiB multipart part ready for the next reader. Each bucket has its own private `compat-<bucket>.json` manifest. Keep the same state and volumes across image changes. A fresh deployment or cleanup that resets volumes moves existing compatibility manifests into `compat-history`. Retained-data deployments preserve the active manifests.
|
||||
|
||||
```bash
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" compat-seed --bucket inline-compat
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" compat-check --bucket inline-compat
|
||||
# Configure the next frozen image and binary, then retain the data.
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" deploy \
|
||||
--bucket inline-compat --shard-bytes 0 --rpc-mode json-compat --keep-data
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" compat-check --bucket inline-compat
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" compat-finish-mpu --bucket inline-compat
|
||||
```
|
||||
|
||||
The check compares every persisted version ID, HEAD length, complete SHA256, cross-block Range and per-version geometry from `xl.meta`. The seed includes zero-length objects, null versions, the 192 KiB versioned admission boundary and the accumulated 128 KiB shard boundary. Seed a second bucket on the new writer before the downgrade, then check both manifests using the selected old reader. Complete an old upload on the new binary and a new upload on the old binary. After returning to the current fleet, `compat-delete --bucket <bucket>` deletes the recorded versions in one S3 batch and checks the exact remaining inventory and bytes.
|
||||
|
||||
Successful multipart-completion and batch-delete receipts are saved before subsequent content and geometry checks. If either check fails, rerun the same command with the same state directory to resume verification from the saved completion state. After a seed check fails with a saved manifest, use `compat-check` to resume verification.
|
||||
|
||||
For mixed-version testing, retain JSON compatibility mode throughout the rolling replacement. Verify each node's immutable image and binary, then perform manifest checks and new writes through both coordinator versions using `--coordinator node1` and `--coordinator node4`. The manifest records the actual writer coordinator revision from the running image. Stop one owned node to check read availability at EC12+4, restore it and check the manifests again. Record the exact oldest tested source revision, toolchain and build profile with the evidence. The version boundary is established by these executed cells.
|
||||
|
||||
An owned lab can replace just its fourth node through a compose image override. Select the frozen old image first and verify its revision label. Keep `compose.json` in JSON compatibility mode for this cell.
|
||||
|
||||
```bash
|
||||
set -euo pipefail
|
||||
OLD_IMAGE_ID="$(docker --context "$LAB_CONTEXT" image inspect "$OLD_IMAGE" --format '{{.Id}}')"
|
||||
test "$(docker --context "$LAB_CONTEXT" image inspect "$OLD_IMAGE_ID" --format '{{index .Config.Labels "org.opencontainers.image.revision"}}')" = "$OLD_REVISION"
|
||||
INLINE_OLD_IMAGE_ID="$OLD_IMAGE_ID" scripts/python_bin.sh - <<'PYTHON' > "$LAB_STATE/mixed-image.json"
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
override = {"services": {"node4": {"image": os.environ["INLINE_OLD_IMAGE_ID"]}}}
|
||||
json.dump(override, sys.stdout)
|
||||
PYTHON
|
||||
docker --context "$LAB_CONTEXT" compose --project-name "$LAB_PROJECT" \
|
||||
--file "$LAB_STATE/compose.json" --file "$LAB_STATE/mixed-image.json" \
|
||||
up --detach --no-deps --no-build --pull never node4
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" compat-check \
|
||||
--bucket inline-compat --coordinator node4
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" compat-seed \
|
||||
--bucket inline-mixed-writer --coordinator node4
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" compat-check \
|
||||
--bucket inline-mixed-writer --coordinator node1
|
||||
```
|
||||
|
||||
Inspect all four running image IDs and binary hashes before recording the mixed-fleet result. Redeploy the configured current image with `--keep-data --rpc-mode json-compat` to restore a homogeneous fleet. The same manifest checks apply after stopping and restarting one owned node through the base compose file.
|
||||
@@ -34,10 +34,6 @@ If you want the Kafka-backed HA Tempo path, use `docker-compose-example-for-rust
|
||||
- **High Performance**: Optimized configurations for batching, compression, and memory management.
|
||||
- **Standardized Protocols**: Built entirely on OpenTelemetry standards.
|
||||
|
||||
## HTTP Request Latency Percentiles
|
||||
|
||||
The main RustFS dashboard shows request latency percentiles in milliseconds. The HTTP server request, response-body chunk, and response-body stream histograms export bucket boundaries in seconds. During a rolling upgrade or rollback that changes the bucket layout, compute percentiles from nodes that share one layout and from a query window entirely on that layout. The bundled dashboard and recording rules use a five-minute window. The mean from `_sum` divided by `_count` remains usable across the change.
|
||||
|
||||
## GET Performance Optimization Dashboards
|
||||
|
||||
Three pre-built Grafana dashboards are included for monitoring RustFS GET performance optimization rollout:
|
||||
|
||||
@@ -34,10 +34,6 @@
|
||||
- **高性能**: 针对批处理、压缩和内存管理进行了优化配置。
|
||||
- **标准化协议**: 完全基于 OpenTelemetry 标准构建。
|
||||
|
||||
## HTTP 请求延迟分位数
|
||||
|
||||
RustFS 主仪表盘以毫秒显示请求延迟分位数。HTTP 服务的请求耗时、响应体分块延迟和响应体流传输耗时直方图以秒为单位导出桶边界。滚动升级或回滚导致桶边界变化时,只对采用同一套桶边界的节点和时间窗口计算分位数。自带仪表盘和记录规则使用五分钟窗口。变更期间仍可用 `_sum` 除以 `_count` 计算平均值。
|
||||
|
||||
## GET 性能优化仪表盘
|
||||
|
||||
包含三个预构建的 Grafana 仪表盘,用于监控 RustFS GET 性能优化发布:
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
# Copyright 2026 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Both-stack leg
|
||||
description: Prove which S3 stack one e2e leg's servers selected and upload the leg for the summary (call with if always)
|
||||
|
||||
inputs:
|
||||
profile:
|
||||
description: The nextest profile the leg ran
|
||||
required: true
|
||||
stack:
|
||||
description: The stack the leg requested, legacy or gateway
|
||||
required: true
|
||||
logs:
|
||||
description: Directory or file holding the servers' output
|
||||
required: true
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Prove which stack the servers selected
|
||||
shell: bash
|
||||
env:
|
||||
PROFILE: ${{ inputs.profile }}
|
||||
STACK: ${{ inputs.stack }}
|
||||
LOGS: ${{ inputs.logs }}
|
||||
run: |
|
||||
# Outside the workspace: e2e_binary.py refuses a run whose source tree changed.
|
||||
mkdir -p "$RUNNER_TEMP/leg"
|
||||
junit="target/nextest/${PROFILE}/junit.xml"
|
||||
if [ -s "$junit" ]; then cp "$junit" "$RUNNER_TEMP/leg/junit.xml"; fi
|
||||
# A gateway leg stopped by its preflight has no output; the preflight's stack.json stays.
|
||||
if [ -e "$LOGS" ]; then python3 scripts/dual_stack.py stack --expect "$STACK" --out "$RUNNER_TEMP/leg/stack.json" "$LOGS"; fi
|
||||
|
||||
- name: Upload the leg
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: dual-stack-leg-${{ inputs.profile }}-${{ inputs.stack }}
|
||||
overwrite: true
|
||||
path: ${{ runner.temp }}/leg
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
@@ -0,0 +1,56 @@
|
||||
# Copyright 2026 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Both-stack summary
|
||||
description: Diff the legacy and gateway legs of this run (scripts/junit_diff.py) and upload one dated summary
|
||||
|
||||
inputs:
|
||||
profiles:
|
||||
description: Comma-separated nextest profiles whose two legs this run uploaded
|
||||
required: true
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Download the legs
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
pattern: dual-stack-leg-*
|
||||
path: legs
|
||||
|
||||
- name: Diff every profile and write the summary
|
||||
id: summary
|
||||
shell: bash
|
||||
env:
|
||||
PROFILES: ${{ inputs.profiles }}
|
||||
run: |
|
||||
# The shell runs with -e; the exit status is reported after the summary is written.
|
||||
set +e -u -o pipefail
|
||||
date="$(date -u +%F)"
|
||||
echo "date=${date}" >> "$GITHUB_OUTPUT"
|
||||
mkdir -p legs
|
||||
python3 scripts/dual_stack.py summary --profiles "$PROFILES" --legs legs --date "$date" --out-dir dual-stack
|
||||
status=$?
|
||||
if [ -f dual-stack/summary.md ]; then cat dual-stack/summary.md >> "$GITHUB_STEP_SUMMARY"; fi
|
||||
exit "$status"
|
||||
|
||||
- name: Upload the dated summary
|
||||
if: always() && steps.summary.outputs.date != ''
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: dual-stack-summary-${{ steps.summary.outputs.date }}
|
||||
overwrite: true
|
||||
path: dual-stack
|
||||
if-no-files-found: error
|
||||
retention-days: 30
|
||||
@@ -296,34 +296,6 @@
|
||||
"origin": "baseline",
|
||||
"alert_contract": "scheduled-validation"
|
||||
},
|
||||
{
|
||||
"workflow": ".github/workflows/rdma-feature-check.yml",
|
||||
"name": "S3-over-RDMA feature check",
|
||||
"roles": [
|
||||
"pr",
|
||||
"periodic",
|
||||
"manual"
|
||||
],
|
||||
"events": [
|
||||
"pull_request",
|
||||
"schedule",
|
||||
"workflow_dispatch"
|
||||
],
|
||||
"cadence": "weekly",
|
||||
"timezone": "UTC",
|
||||
"schedule": [
|
||||
"37 5 * * 1"
|
||||
],
|
||||
"owner": "@overtrue",
|
||||
"policy": "active",
|
||||
"monitor": "workflow",
|
||||
"max_age_hours": 192,
|
||||
"result_owner": "rdma-feature-check.yml",
|
||||
"disposition": "compiles the nixl feature and runs rustfs-nixl unit tests on Linux; the feature is off in every other build",
|
||||
"failure_owner": "workflow",
|
||||
"origin": "added",
|
||||
"alert_contract": "scheduled-validation"
|
||||
},
|
||||
{
|
||||
"workflow": ".github/workflows/rustfs-validation-sweep.yml",
|
||||
"name": "RustFS Validation Sweep",
|
||||
@@ -814,7 +786,7 @@
|
||||
"monitor": "workflow",
|
||||
"max_age_hours": 36,
|
||||
"result_owner": "dual-stack-nightly.yml",
|
||||
"disposition": "nightly run of the two e2e profiles only ci.yml runs, on the gateway stack, with the JUnit reports retained for 30 days",
|
||||
"disposition": "both-stack JUnit diff of the two profiles only ci.yml runs; the gateway legs are report-only until T2.10",
|
||||
"failure_owner": "watchdog",
|
||||
"origin": "added"
|
||||
},
|
||||
@@ -1005,31 +977,6 @@
|
||||
"origin": "baseline",
|
||||
"alert_contract": "scheduled-validation"
|
||||
},
|
||||
{
|
||||
"workflow": ".github/workflows/gateway-acceptance.yml",
|
||||
"name": "Gateway Acceptance",
|
||||
"roles": [
|
||||
"periodic",
|
||||
"manual"
|
||||
],
|
||||
"events": [
|
||||
"schedule",
|
||||
"workflow_dispatch"
|
||||
],
|
||||
"cadence": "nightly",
|
||||
"timezone": "UTC",
|
||||
"schedule": [
|
||||
"29 3 * * *"
|
||||
],
|
||||
"owner": "@overtrue",
|
||||
"policy": "active",
|
||||
"monitor": "workflow",
|
||||
"max_age_hours": 36,
|
||||
"result_owner": "gateway-acceptance.yml",
|
||||
"disposition": "gateway conformance corpus (rustfs profile) and client matrix against the candidate on both S3 stacks",
|
||||
"failure_owner": "watchdog",
|
||||
"origin": "added"
|
||||
},
|
||||
{
|
||||
"workflow": ".github/workflows/gateway-integration-sync.yml",
|
||||
"name": "Gateway Integration Sync",
|
||||
|
||||
+175
-13
@@ -92,6 +92,7 @@ jobs:
|
||||
connect: ${{ steps.scope.outputs.connect || 'true' }}
|
||||
packages: ${{ steps.scope.outputs.packages }}
|
||||
full_validation: ${{ steps.scope.outputs.full_validation || 'true' }}
|
||||
gateway_smoke: ${{ steps.scope.outputs.gateway_smoke || 'true' }}
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
@@ -118,7 +119,7 @@ jobs:
|
||||
echo "Base CI policy unavailable; running the full matrix."
|
||||
fi
|
||||
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=8$' "$GITHUB_OUTPUT"; then
|
||||
printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
|
||||
printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'gateway_smoke=true' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
|
||||
echo "Base policy does not cover this workflow contract; retaining the full matrix."
|
||||
fi
|
||||
- name: Retain verification plan
|
||||
@@ -675,6 +676,54 @@ jobs:
|
||||
retention-days: 3
|
||||
if-no-files-found: warn
|
||||
|
||||
test-and-lint-gateway-stack:
|
||||
name: Test and Lint (gateway-stack)
|
||||
# The gateway S3 stack (rustfs/backlog#2734) sits behind the rustfs
|
||||
# `gateway-stack` feature, which no other pull-request lane lints or tests.
|
||||
# Unlike the dormant feature lanes above, this one runs for every code
|
||||
# change: each operation slice puts its handler tests and its refusal-matrix
|
||||
# row behind the feature, and a lane that ran only on demand would let them
|
||||
# rot behind a green pull request.
|
||||
if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
# Self-hosted for build scale and cache ownership: a workspace-scale Cargo
|
||||
# build that reads the ci-dev cache written on this image.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: ci-dev
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
# A feature that does not compile fails here; nothing below is skipped
|
||||
# into a pass.
|
||||
- name: Run clippy with gateway-stack
|
||||
run: cargo clippy --locked -p rustfs --all-targets --features gateway-stack -- -D warnings
|
||||
|
||||
# The gateway stack's own tests: the assembly, the operation table and
|
||||
# its completeness, the shared handler sequence, the stack selection, and
|
||||
# the embedded server driven through the whole middleware stack. The
|
||||
# filter names modules, never operations.
|
||||
- name: Run tests with gateway-stack
|
||||
env:
|
||||
# The same bounded link concurrency as the other feature lanes.
|
||||
CARGO_BUILD_JOBS: "2"
|
||||
run: |
|
||||
cargo nextest run --profile ci --locked -p rustfs --features gateway-stack --no-tests fail \
|
||||
--status-level all --final-status-level all \
|
||||
-E 'test(/gateway_stack|gateway_handlers|server::stack::/) | binary(embedded_gateway_stack_test)'
|
||||
|
||||
build-rustfs-debug-binary:
|
||||
name: Build RustFS Debug Binary
|
||||
if: contains(fromJSON('["full", "e2e"]'), needs.classify-changes.outputs.mode) && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
@@ -937,6 +986,8 @@ jobs:
|
||||
# Retain the producer image and fixture paths until the runtime pilot passes.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
RUSTFS_S3_STACK: legacy
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -1057,6 +1108,125 @@ jobs:
|
||||
path: ${{ runner.temp }}/rustfs-e2e-*/rustfs.log
|
||||
retention-days: 3
|
||||
|
||||
# The same e2e-smoke suite against the same debug binary with the gateway
|
||||
# S3 stack selected (rustfs/backlog#2734). Report-only: it is outside the
|
||||
# Test and Lint aggregate and becomes required at T2.10. The test-binary
|
||||
# download, identity, membership, run and completeness steps must stay
|
||||
# identical to e2e-tests; scripts/ci_gate.py enforces that.
|
||||
#
|
||||
# When it runs: every push to main or gateway/integration, merge group,
|
||||
# nightly schedule and manual dispatch, and every pull request that changes a
|
||||
# gateway-stack input (GATEWAY_SMOKE_PATHS and the rustfs-gateway pins in
|
||||
# scripts/ci_gate.py, published by Select CI scope as gateway_smoke).
|
||||
# Why it is skipped on every other pull request: until T2.10 the gateway
|
||||
# stack does not serve the whole smoke suite, so the lane fails on any
|
||||
# revision. A pull request that touches none of its inputs can neither cause
|
||||
# nor fix that failure, and a red check there says nothing about the change.
|
||||
# Skipped is the honest state; a failure must never be rendered green, so no
|
||||
# step here may carry continue-on-error. A selection that is missing or
|
||||
# unreadable runs the lane. At T2.10, when the lane becomes required, it
|
||||
# must run on every pull request again: delete the selection, do not extend
|
||||
# it.
|
||||
e2e-tests-gateway:
|
||||
name: e2e-smoke (gateway, report-only until T2.10)
|
||||
needs: [ classify-changes, build-rustfs-debug-binary ]
|
||||
if: github.event_name != 'pull_request' || needs.classify-changes.outputs.gateway_smoke != 'false'
|
||||
runs-on: sm-standard-2
|
||||
timeout-minutes: 30
|
||||
continue-on-error: true
|
||||
env:
|
||||
RUSTFS_S3_STACK: gateway
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install verification runtime
|
||||
uses: ./.github/actions/test-runtime
|
||||
|
||||
- name: Download E2E test binaries
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: e2e-test-binaries
|
||||
path: ${{ runner.temp }}
|
||||
|
||||
- name: Verify E2E archive identity
|
||||
run: python3 scripts/ci_test_artifact.py verify --configuration e2e --archive "$RUNNER_TEMP/e2e-tests.tar.zst"
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install awscurl
|
||||
run: |
|
||||
python3 -m pip install --user --upgrade pip "awscurl==0.44"
|
||||
echo "AWSCURL_PATH=$HOME/.local/bin/awscurl" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Verify awscurl
|
||||
run: test -x "$AWSCURL_PATH"
|
||||
|
||||
# Download after the cache restore so the freshly built binary from the
|
||||
# build job always wins over anything restored into target/debug.
|
||||
- name: Download debug binary
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: rustfs-debug-binary
|
||||
path: target/debug
|
||||
|
||||
- name: Make binary executable
|
||||
run: chmod +x ./target/debug/rustfs
|
||||
|
||||
# Build the e2e test graph once. The archive is reused by the smoke
|
||||
# selection guard, security exact-count check, and run below, avoiding a
|
||||
# second compile of the same e2e_test target on cold runners (backlog#1645).
|
||||
- name: Verify e2e smoke test membership
|
||||
env:
|
||||
NEXTEST_ARCHIVE: ${{ runner.temp }}/e2e-tests.tar.zst
|
||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-smoke-list.json
|
||||
run: |
|
||||
cargo nextest list --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" --workspace-remap "$GITHUB_WORKSPACE" --extract-to "$GITHUB_WORKSPACE" --extract-overwrite --message-format json > "${NEXTEST_LISTING}"
|
||||
python3 ./scripts/check_test_wiring.py --check-profile e2e-smoke "${NEXTEST_LISTING}"
|
||||
./scripts/check_security_smoke_count.sh check "${NEXTEST_LISTING}"
|
||||
|
||||
# PR smoke subset of the in-repo e2e suite (backlog#1149 ci-4). The
|
||||
# profile.e2e-smoke default-filter in .config/nextest.toml is the single
|
||||
# wiring mechanism for e2e tests in CI — extend that filter instead of
|
||||
# adding new e2e jobs here. Each test spawns its own rustfs server on a
|
||||
# random port and reuses the downloaded debug binary above.
|
||||
- name: Run e2e smoke suite
|
||||
env:
|
||||
NEXTEST_ARCHIVE: ${{ runner.temp }}/e2e-tests.tar.zst
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-smoke-logs
|
||||
SAMPLER_INTERVAL_SECS: "15"
|
||||
run: |
|
||||
./scripts/ci/resource_sampler.sh start e2e-smoke
|
||||
trap './scripts/ci/resource_sampler.sh stop' EXIT
|
||||
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-smoke --cargo-metadata target/nextest/cargo-metadata.json --binaries-metadata target/nextest/binaries-metadata.json --workspace-remap "$GITHUB_WORKSPACE" --target-dir-remap "$GITHUB_WORKSPACE/target" \
|
||||
--status-level all --final-status-level all --failure-output final
|
||||
|
||||
- name: Verify e2e smoke execution completeness
|
||||
run: python3 scripts/ci_test_artifact.py evidence "$RUNNER_TEMP/rustfs-e2e-smoke-list.json" target/nextest/e2e-smoke/junit.xml
|
||||
|
||||
- name: Upload e2e smoke diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-smoke-gateway-diagnostics-${{ github.run_number }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
${{ runner.temp }}/rustfs-e2e-smoke-logs/
|
||||
${{ runner.temp }}/rustfs-e2e-smoke-list.json
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Upload e2e smoke JUnit report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-smoke-gateway-junit-${{ github.run_number }}-${{ github.run_attempt }}
|
||||
path: target/nextest/e2e-smoke/junit.xml
|
||||
if-no-files-found: warn
|
||||
|
||||
e2e-full:
|
||||
name: End-to-End Tests (full merge gate)
|
||||
# Merge gate only (backlog#1149 ci-5): the never-automated user-visible
|
||||
@@ -1091,12 +1261,9 @@ jobs:
|
||||
- name: Verify E2E archive identity
|
||||
run: python3 scripts/ci_test_artifact.py verify --configuration e2e --archive "$RUNNER_TEMP/e2e-tests.tar.zst"
|
||||
|
||||
- name: Install network fault-injection tools and the persistence test clock
|
||||
- name: Install network fault-injection tools
|
||||
run: |
|
||||
sudo apt-get install -y iptables libfaketime
|
||||
clock_library="$(dpkg-query -L libfaketime | grep -E '/libfaketime[.]so[.]1$')"
|
||||
test -f "$clock_library"
|
||||
echo "RUSTFS_PERSISTENCE_CLOCK_LIBRARY=$clock_library" >> "$GITHUB_ENV"
|
||||
sudo apt-get install -y iptables
|
||||
sudo -n iptables --version
|
||||
# The endpoint-blackhole heal scenario needs CAP_NET_ADMIN. Containerised
|
||||
# runners can run iptables but not touch the rule set; the test then logs
|
||||
@@ -1203,14 +1370,8 @@ jobs:
|
||||
# default-filter in .config/nextest.toml is the single wiring mechanism —
|
||||
# extend that filter, never add ad-hoc e2e jobs here. Reuses the downloaded
|
||||
# debug binary; each test spawns its own rustfs server on a random port.
|
||||
- name: Download pinned release for persistence byte parity
|
||||
id: persistence-reference
|
||||
uses: ./.github/actions/reference-binary
|
||||
|
||||
- name: Run e2e full suite
|
||||
env:
|
||||
RUSTFS_PERSISTENCE_REFERENCE_BINARY: ${{ steps.persistence-reference.outputs.path }}
|
||||
RUSTFS_PERSISTENCE_XML_DIR: ${{ runner.temp }}/rustfs-persistence-xml
|
||||
CARGO_BIN_EXE_rustfs: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs
|
||||
RUSTFS_E2E_STARTUP_CAS_BINARY: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs
|
||||
RUSTFS_E2E_STARTUP_CAS_BUILD_MANIFEST: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs.e2e-startup-cas-build.json
|
||||
@@ -1229,7 +1390,6 @@ jobs:
|
||||
path: |
|
||||
target/nextest/e2e-full/junit.xml
|
||||
${{ runner.temp }}/rustfs-e2e-full-list.json
|
||||
${{ runner.temp }}/rustfs-persistence-xml
|
||||
retention-days: 7
|
||||
|
||||
- name: Upload startup CAS evidence
|
||||
@@ -1452,6 +1612,7 @@ jobs:
|
||||
- fips-provider
|
||||
- test-and-lint-rio-v2
|
||||
- test-and-lint-protocols
|
||||
- test-and-lint-gateway-stack
|
||||
- build-rustfs-debug-binary
|
||||
- uring-integration
|
||||
- e2e-tests
|
||||
@@ -1487,6 +1648,7 @@ jobs:
|
||||
- core-contract
|
||||
- test-and-lint-rio-v2
|
||||
- test-and-lint-protocols
|
||||
- test-and-lint-gateway-stack
|
||||
- build-rustfs-debug-binary
|
||||
- build-rustfs-debug-binary-rio-v2
|
||||
- uring-integration
|
||||
|
||||
@@ -12,18 +12,21 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Nightly run of the two e2e profiles only ci.yml runs, e2e-smoke and e2e-full
|
||||
# (rustfs/backlog#2772, T3.1), on the gateway stack, which is the only S3 stack.
|
||||
# The workflow keeps its file and registry name from the period when it diffed two
|
||||
# stacks. Each leg uploads its JUnit report and server logs for 30 days; the one
|
||||
# comparison against the last legacy record (the `dual-stack-leg-<profile>-legacy`
|
||||
# artifacts of the final both-stack run) is made by hand with scripts/junit_diff.py.
|
||||
# Nightly both-stack diff of the two e2e profiles only ci.yml runs, e2e-smoke
|
||||
# and e2e-full (rustfs/backlog#2772, T3.1). The other profiles carry a `stack`
|
||||
# dimension in their own workflows, where the existing lane is the legacy leg.
|
||||
#
|
||||
# Left out: `dual_stack_differential::` (servers with an explicit
|
||||
# One build serves every leg. One preflight starts that binary with
|
||||
# RUSTFS_S3_STACK=gateway (scripts/dual_stack.py preflight); without a proven
|
||||
# gateway start the gateway legs do not run and the summary says `skipped:
|
||||
# gateway stack refuses startup: <why>`. Which stack a leg ran is read from the
|
||||
# servers' own start-up line. The gateway legs are report-only until T2.10.
|
||||
#
|
||||
# Left out of both stacks: `dual_stack_differential::` (servers with an explicit
|
||||
# stack), `fake_s3_target::`, the upgrade write-readiness cases and one deadline
|
||||
# case (no server), and `distributed_startup_regression_test::` (needs ci.yml's
|
||||
# hooks-binary manifest). e2e-full uses ci.yml's pinned mc and Vault binaries
|
||||
# and network fault-injection tools.
|
||||
# hooks-binary manifest). e2e-full runs without the mc, Vault and iptables
|
||||
# tools ci.yml installs: their cases fail on both stacks.
|
||||
|
||||
name: dual-stack-nightly
|
||||
|
||||
@@ -47,11 +50,13 @@ defaults:
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Build once
|
||||
name: Build once and probe the gateway stack
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
# The runner and cache of ci.yml's build-rustfs-debug-binary.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 60
|
||||
outputs:
|
||||
stacks: ${{ steps.preflight.outputs.stacks }}
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
@@ -81,15 +86,47 @@ jobs:
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
# Without a proven gateway start there is no gateway leg to run.
|
||||
- name: Gateway preflight
|
||||
id: preflight
|
||||
run: |
|
||||
if python3 scripts/dual_stack.py preflight --binary target/debug/rustfs --out "$RUNNER_TEMP/preflight/stack.json"; then
|
||||
echo 'stacks=["legacy","gateway"]' >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo 'stacks=["legacy"]' >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# The evidence stands in for a gateway leg that is skipped; a leg that runs overwrites it.
|
||||
- name: Upload the preflight evidence for e2e-smoke
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: dual-stack-leg-e2e-smoke-gateway
|
||||
path: ${{ runner.temp }}/preflight
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
|
||||
- name: Upload the preflight evidence for e2e-full
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: dual-stack-leg-e2e-full-gateway
|
||||
path: ${{ runner.temp }}/preflight
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
|
||||
leg:
|
||||
name: ${{ matrix.profile }}
|
||||
name: ${{ matrix.profile }} (${{ matrix.stack }}${{ matrix.stack == 'gateway' && ', report-only until T2.10' || '' }})
|
||||
needs: [ build ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 120
|
||||
# Report-only until T2.10: the gateway stack cannot serve the suites yet.
|
||||
continue-on-error: ${{ matrix.stack == 'gateway' }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
profile: [ e2e-smoke, e2e-full ]
|
||||
stack: ${{ fromJSON(needs.build.outputs.stacks) }}
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
@@ -123,57 +160,10 @@ jobs:
|
||||
echo "AWSCURL_PATH=$HOME/.local/bin/awscurl" >> "$GITHUB_ENV"
|
||||
test -x "$HOME/.local/bin/awscurl"
|
||||
|
||||
- name: Install network fault-injection tools
|
||||
if: matrix.profile == 'e2e-full'
|
||||
run: |
|
||||
sudo apt-get install -y iptables
|
||||
sudo -n iptables --version
|
||||
# The endpoint-blackhole heal scenario needs CAP_NET_ADMIN. Containerised
|
||||
# runners can run iptables but not touch the rule set; the test then logs
|
||||
# a skip instead of failing, so surface that here where it is visible.
|
||||
if ! sudo -n iptables -w 5 -S OUTPUT >/dev/null 2>&1; then
|
||||
echo "::warning::iptables cannot read the OUTPUT chain on this runner (no CAP_NET_ADMIN); the endpoint-blackhole heal scenario will be skipped"
|
||||
fi
|
||||
|
||||
- name: Install mc
|
||||
if: matrix.profile == 'e2e-full'
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
MC_VERSION: RELEASE.2025-08-13T08-35-41Z
|
||||
MC_SHA256: 01f866e9c5f9b87c2b09116fa5d7c06695b106242d829a8bb32990c00312e891
|
||||
run: |
|
||||
if ! command -v gh >/dev/null 2>&1; then
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y gh
|
||||
fi
|
||||
MC_BINARY="mc.linux-amd64.${MC_VERSION}"
|
||||
python3 .github/actions/setup/download_release_asset.py --repository minio/mc --tag "$MC_VERSION" --asset "$MC_BINARY" --output "$RUNNER_TEMP/mc" --sha256 "$MC_SHA256"
|
||||
echo "${MC_SHA256} $RUNNER_TEMP/mc" | sha256sum --check --status
|
||||
chmod +x "$RUNNER_TEMP/mc"
|
||||
echo "$RUNNER_TEMP" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Verify mc
|
||||
if: matrix.profile == 'e2e-full'
|
||||
run: mc --version
|
||||
|
||||
- name: Install Vault
|
||||
if: matrix.profile == 'e2e-full'
|
||||
run: |
|
||||
VAULT_VERSION="1.17.6"
|
||||
VAULT_ARCHIVE="vault_${VAULT_VERSION}_linux_amd64.zip"
|
||||
curl -fsSLo "$RUNNER_TEMP/$VAULT_ARCHIVE" "https://releases.hashicorp.com/vault/${VAULT_VERSION}/${VAULT_ARCHIVE}"
|
||||
echo "0cddc1fbbb88583b5ba5b845f9f8fae47c6fb39a6d48cd543c6ba6fd3ac1a669 $RUNNER_TEMP/$VAULT_ARCHIVE" | sha256sum --check --status
|
||||
unzip -q "$RUNNER_TEMP/$VAULT_ARCHIVE" -d "$RUNNER_TEMP/vault-bin"
|
||||
echo "RUSTFS_TEST_VAULT_BIN=$RUNNER_TEMP/vault-bin/vault" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Verify Vault
|
||||
if: matrix.profile == 'e2e-full'
|
||||
run: |
|
||||
"$RUSTFS_TEST_VAULT_BIN" version
|
||||
|
||||
- name: Run the profile
|
||||
- name: Run the profile on the ${{ matrix.stack }} stack
|
||||
env:
|
||||
PROFILE: ${{ matrix.profile }}
|
||||
RUSTFS_S3_STACK: ${{ matrix.stack }}
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/server-logs
|
||||
run: |
|
||||
chmod +x target/debug/rustfs
|
||||
@@ -182,14 +172,23 @@ jobs:
|
||||
filter="$filter & not test(=list_objects_v2_pagination_test::tests::test_list_objects_v2_deadline_rejects_unresponsive_peer)"
|
||||
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile "$PROFILE" -p e2e_test -E "$filter"
|
||||
|
||||
- name: Upload the leg
|
||||
- name: Prove the stack and upload the leg
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
uses: ./.github/actions/dual-stack-leg
|
||||
with: { profile: "${{ matrix.profile }}", stack: "${{ matrix.stack }}", logs: "${{ runner.temp }}/server-logs" }
|
||||
|
||||
summary:
|
||||
name: Both-stack diff summary
|
||||
needs: [ build, leg ]
|
||||
if: github.repository == 'rustfs/rustfs' && always()
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
name: e2e-nightly-${{ matrix.profile }}
|
||||
overwrite: true
|
||||
path: |
|
||||
target/nextest/${{ matrix.profile }}/junit.xml
|
||||
${{ runner.temp }}/server-logs
|
||||
if-no-files-found: warn
|
||||
retention-days: 30
|
||||
persist-credentials: false
|
||||
|
||||
- uses: ./.github/actions/dual-stack-summary
|
||||
with:
|
||||
profiles: e2e-smoke,e2e-full
|
||||
|
||||
@@ -53,11 +53,14 @@ concurrency:
|
||||
jobs:
|
||||
distributed:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Distributed 4-node 4-disk e2e
|
||||
name: Distributed 4-node 4-disk e2e (${{ matrix.stack }}${{ matrix.stack == 'gateway' && ', report-only until T2.10' || '' }})
|
||||
# GitHub-hosted VM: loop and tmpfs mounts work here. sm-standard-4 is an
|
||||
# ARC pod and rejects both (`mount -o loop` ENOENT, tmpfs "read-only").
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 180
|
||||
# The gateway leg is report-only until T2.10.
|
||||
continue-on-error: ${{ matrix.stack == 'gateway' }}
|
||||
strategy: { fail-fast: false, matrix: { stack: [legacy, gateway] } }
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
NO_PROXY: 127.0.0.1,localhost
|
||||
@@ -129,6 +132,11 @@ jobs:
|
||||
run: |
|
||||
python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks
|
||||
|
||||
# A gateway binary that refuses to start skips the suite; the summary says why.
|
||||
- name: Gateway preflight
|
||||
if: matrix.stack == 'gateway'
|
||||
run: python3 scripts/dual_stack.py preflight --binary target/debug/rustfs --out "$RUNNER_TEMP/leg/stack.json"
|
||||
|
||||
- name: Verify distributed e2e membership
|
||||
env:
|
||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-distributed-list.json
|
||||
@@ -140,6 +148,7 @@ jobs:
|
||||
env:
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-distributed-logs
|
||||
FILTER: ${{ inputs.filter }}
|
||||
RUSTFS_S3_STACK: ${{ matrix.stack }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
rm -f target/nextest/e2e-distributed/junit.xml
|
||||
@@ -150,11 +159,19 @@ jobs:
|
||||
fi
|
||||
test -s target/nextest/e2e-distributed/junit.xml
|
||||
|
||||
- name: Prove the stack and upload the leg
|
||||
if: always()
|
||||
uses: ./.github/actions/dual-stack-leg
|
||||
with:
|
||||
profile: e2e-distributed
|
||||
stack: ${{ matrix.stack }}
|
||||
logs: ${{ runner.temp }}/rustfs-e2e-distributed-logs
|
||||
|
||||
- name: Upload distributed e2e diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-distributed-${{ github.run_number }}
|
||||
name: e2e-distributed-${{ github.run_number }}-${{ matrix.stack }}
|
||||
path: |
|
||||
target/nextest/e2e-distributed/junit.xml
|
||||
${{ runner.temp }}/rustfs-e2e-distributed-list.json
|
||||
@@ -174,6 +191,20 @@ jobs:
|
||||
fi
|
||||
done
|
||||
|
||||
dual-stack-summary:
|
||||
name: Both-stack diff summary
|
||||
needs: [distributed]
|
||||
if: github.repository == 'rustfs/rustfs' && always()
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: ./.github/actions/dual-stack-summary
|
||||
with:
|
||||
profiles: e2e-distributed
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [distributed]
|
||||
|
||||
@@ -54,15 +54,17 @@ concurrency:
|
||||
jobs:
|
||||
repl-nightly:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Replication e2e (nightly)
|
||||
name: Replication e2e (nightly, ${{ matrix.stack }}${{ matrix.stack == 'gateway' && ', report-only until T2.10' || '' }})
|
||||
# dual-node tests spawn two full rustfs servers each; the extra headroom
|
||||
# over the per-PR sm-standard-2 e2e job keeps the parallel process fan-out
|
||||
# from starving the runner. Mirrors the ILM serial lane's runner class.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 45
|
||||
# The gateway leg is report-only until T2.10.
|
||||
continue-on-error: ${{ matrix.stack == 'gateway' }}
|
||||
strategy: { fail-fast: false, matrix: { stack: [legacy, gateway] } }
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
RUSTFS_BUILD_FEATURES: ""
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -94,7 +96,12 @@ jobs:
|
||||
# Build once and carry its source/binary identity into the test invocation.
|
||||
- name: Build rustfs binary
|
||||
run: |
|
||||
python3 scripts/e2e_binary.py build --bins --features "$RUSTFS_BUILD_FEATURES"
|
||||
python3 scripts/e2e_binary.py build --bins
|
||||
|
||||
# A gateway binary that refuses to start skips the suite; the summary says why.
|
||||
- name: Gateway preflight
|
||||
if: matrix.stack == 'gateway'
|
||||
run: python3 scripts/dual_stack.py preflight --binary target/debug/rustfs --out "$RUNNER_TEMP/leg/stack.json"
|
||||
|
||||
- name: Verify replication e2e membership
|
||||
env:
|
||||
@@ -106,13 +113,22 @@ jobs:
|
||||
- name: Run replication e2e nightly suite
|
||||
env:
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-repl-nightly-logs
|
||||
run: python3 scripts/e2e_binary.py run --features "$RUSTFS_BUILD_FEATURES" -- cargo nextest run --profile e2e-repl-nightly -p e2e_test
|
||||
RUSTFS_S3_STACK: ${{ matrix.stack }}
|
||||
run: python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-repl-nightly -p e2e_test
|
||||
|
||||
- name: Prove the stack and upload the leg
|
||||
if: always()
|
||||
uses: ./.github/actions/dual-stack-leg
|
||||
with:
|
||||
profile: e2e-repl-nightly
|
||||
stack: ${{ matrix.stack }}
|
||||
logs: ${{ runner.temp }}/rustfs-e2e-repl-nightly-logs
|
||||
|
||||
- name: Upload nextest junit report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-replication-nightly-junit-${{ github.run_number }}
|
||||
name: e2e-replication-nightly-junit-${{ github.run_number }}-${{ matrix.stack }}
|
||||
path: |
|
||||
target/nextest/e2e-repl-nightly/junit.xml
|
||||
${{ runner.temp }}/rustfs-e2e-repl-nightly-list.json
|
||||
@@ -122,9 +138,12 @@ jobs:
|
||||
|
||||
cluster-nightly:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Cluster fault e2e (nightly)
|
||||
name: Cluster fault e2e (nightly, ${{ matrix.stack }}${{ matrix.stack == 'gateway' && ', report-only until T2.10' || '' }})
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
# The gateway leg is report-only until T2.10.
|
||||
continue-on-error: ${{ matrix.stack == 'gateway' }}
|
||||
strategy: { fail-fast: false, matrix: { stack: [legacy, gateway] } }
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
@@ -145,6 +164,11 @@ jobs:
|
||||
run: |
|
||||
python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks
|
||||
|
||||
# A gateway binary that refuses to start skips the suite; the summary says why.
|
||||
- name: Gateway preflight
|
||||
if: matrix.stack == 'gateway'
|
||||
run: python3 scripts/dual_stack.py preflight --binary target/debug/rustfs --out "$RUNNER_TEMP/leg/stack.json"
|
||||
|
||||
- name: Verify cluster fault e2e membership
|
||||
env:
|
||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-nightly-list.json
|
||||
@@ -156,13 +180,22 @@ jobs:
|
||||
env:
|
||||
CARGO_BIN_EXE_rustfs: ${{ github.workspace }}/target/debug/rustfs
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-nightly-logs
|
||||
RUSTFS_S3_STACK: ${{ matrix.stack }}
|
||||
run: python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-nightly -p e2e_test
|
||||
|
||||
- name: Prove the stack and upload the leg
|
||||
if: always()
|
||||
uses: ./.github/actions/dual-stack-leg
|
||||
with:
|
||||
profile: e2e-nightly
|
||||
stack: ${{ matrix.stack }}
|
||||
logs: ${{ runner.temp }}/rustfs-e2e-nightly-logs
|
||||
|
||||
- name: Upload cluster fault diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-cluster-nightly-${{ github.run_number }}
|
||||
name: e2e-cluster-nightly-${{ github.run_number }}-${{ matrix.stack }}
|
||||
path: |
|
||||
target/nextest/e2e-nightly/junit.xml
|
||||
${{ runner.temp }}/rustfs-e2e-nightly-list.json
|
||||
@@ -172,9 +205,12 @@ jobs:
|
||||
|
||||
protocols-nightly:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Protocol e2e (nightly)
|
||||
name: Protocol e2e (nightly, ${{ matrix.stack }}${{ matrix.stack == 'gateway' && ', report-only until T2.10' || '' }})
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
# The gateway leg is report-only until T2.10.
|
||||
continue-on-error: ${{ matrix.stack == 'gateway' }}
|
||||
strategy: { fail-fast: false, matrix: { stack: [legacy, gateway] } }
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
RUSTFS_BUILD_FEATURES: ftps,webdav,sftp
|
||||
@@ -201,6 +237,11 @@ jobs:
|
||||
- name: Build protocol server
|
||||
run: python3 scripts/e2e_binary.py build --features "$RUSTFS_BUILD_FEATURES"
|
||||
|
||||
# A gateway binary that refuses to start skips the suite; the summary says why.
|
||||
- name: Gateway preflight
|
||||
if: matrix.stack == 'gateway'
|
||||
run: python3 scripts/dual_stack.py preflight --binary target/debug/rustfs --out "$RUNNER_TEMP/leg/stack.json"
|
||||
|
||||
# The suite owns fixed protocol ports and serializes its internal cases.
|
||||
- name: Verify protocol e2e membership
|
||||
env:
|
||||
@@ -212,16 +253,28 @@ jobs:
|
||||
- name: Run protocol e2e nightly suite
|
||||
env:
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-protocol-e2e-logs
|
||||
RUSTFS_S3_STACK: ${{ matrix.stack }}
|
||||
# These servers set no log level of their own; the stack proof reads their start-up line.
|
||||
RUST_LOG: rustfs=info
|
||||
# The servers inherit nextest's stdout (--no-capture): keep it where the stack proof reads it.
|
||||
run: |
|
||||
set -o pipefail
|
||||
mkdir -p "$RUSTFS_E2E_LOG_DIR"
|
||||
python3 scripts/e2e_binary.py run --features "$RUSTFS_BUILD_FEATURES" -- cargo nextest run -j 1 --profile e2e-protocols -p e2e_test --no-capture 2>&1 | tee "$RUSTFS_E2E_LOG_DIR/nextest.log"
|
||||
|
||||
- name: Prove the stack and upload the leg
|
||||
if: always()
|
||||
uses: ./.github/actions/dual-stack-leg
|
||||
with:
|
||||
profile: e2e-protocols
|
||||
stack: ${{ matrix.stack }}
|
||||
logs: ${{ runner.temp }}/rustfs-protocol-e2e-logs
|
||||
|
||||
- name: Upload protocol diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-protocol-nightly-${{ github.run_number }}
|
||||
name: e2e-protocol-nightly-${{ github.run_number }}-${{ matrix.stack }}
|
||||
path: |
|
||||
target/nextest/e2e-protocols/junit.xml
|
||||
${{ runner.temp }}/rustfs-e2e-protocols-list.json
|
||||
@@ -229,6 +282,20 @@ jobs:
|
||||
retention-days: 7
|
||||
if-no-files-found: warn
|
||||
|
||||
dual-stack-summary:
|
||||
name: Both-stack diff summary
|
||||
needs: [repl-nightly, cluster-nightly, protocols-nightly]
|
||||
if: github.repository == 'rustfs/rustfs' && always()
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: ./.github/actions/dual-stack-summary
|
||||
with:
|
||||
profiles: e2e-repl-nightly,e2e-nightly,e2e-protocols
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [repl-nightly, cluster-nightly, protocols-nightly]
|
||||
|
||||
@@ -90,16 +90,11 @@ jobs:
|
||||
- version: 1.0.1
|
||||
asset: rustfs-linux-x86_64-gnu-v1.0.1.zip
|
||||
sha256: 9742e183cb0dfb0d48fa909797af572fe47b7f827569036baa4156e0d3958ee5
|
||||
- version: 1.0.5
|
||||
asset: rustfs-linux-x86_64-gnu-v1.0.5.zip
|
||||
sha256: 5addae2f74756bf80104d0975e6347cda843bc2daffdfcd1f0cb80118b95cbd0
|
||||
revision: 34e6449ed90e3d3a106f6655342e0b3127f33082
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
UPGRADE_SOURCE_VERSION: ${{ matrix.source.version }}
|
||||
UPGRADE_SOURCE_ASSET: ${{ matrix.source.asset }}
|
||||
UPGRADE_SOURCE_SHA256: ${{ matrix.source.sha256 }}
|
||||
UPGRADE_SOURCE_REVISION: ${{ matrix.source.revision }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -137,13 +132,6 @@ jobs:
|
||||
unzip -q "$archive" -d "$SOURCE_DIR"
|
||||
chmod +x "$SOURCE_DIR/rustfs"
|
||||
test -x "$SOURCE_DIR/rustfs"
|
||||
if [[ -n "$UPGRADE_SOURCE_REVISION" ]]; then
|
||||
mkdir -p "$RUNNER_TEMP/rustfs-upgrade-logs"
|
||||
version_log="$RUNNER_TEMP/rustfs-upgrade-logs/source-version.txt"
|
||||
"$SOURCE_DIR/rustfs" --version | tee "$version_log"
|
||||
grep -Fx "rustfs $UPGRADE_SOURCE_VERSION" "$version_log"
|
||||
grep -Eq "^git commit[[:space:]]*:[[:space:]]*$UPGRADE_SOURCE_REVISION$" "$version_log"
|
||||
fi
|
||||
echo "RUSTFS_UPGRADE_SOURCE_BINARY=$SOURCE_DIR/rustfs" >> "$GITHUB_ENV"
|
||||
echo "RUSTFS_E2E_LOG_DIR=$RUNNER_TEMP/rustfs-upgrade-logs" >> "$GITHUB_ENV"
|
||||
|
||||
@@ -163,14 +151,12 @@ jobs:
|
||||
multipart-layout-upgrade:direct_upgrade_from_previous_release_preserves_multipart_layouts
|
||||
multipart-layout-baseline:previous_release_baseline_replicates_multipart_layouts
|
||||
)
|
||||
case "$UPGRADE_SOURCE_VERSION" in
|
||||
1.0.0-rc.5)
|
||||
cases+=(odm-config-rollback:rc5_rollback_requires_restoring_odm_configuration)
|
||||
cases+=(checkpoint-upgrade:rc5_checkpoint_upgrade_rewinds_signed_progress_without_losing_objects) ;;
|
||||
1.0.1) cases+=(legacy-delete-reservation:stable_upgrade_deletes_foreign_reservations_without_losing_live_objects) ;;
|
||||
1.0.5) ;; # The bucket rollback case also completes an in-flight multipart upload.
|
||||
*) echo "::error::Unknown upgrade source $UPGRADE_SOURCE_VERSION"; exit 1 ;;
|
||||
esac
|
||||
if [[ "$UPGRADE_SOURCE_VERSION" == "1.0.0-rc.5" ]]; then
|
||||
cases+=(odm-config-rollback:rc5_rollback_requires_restoring_odm_configuration)
|
||||
cases+=(checkpoint-upgrade:rc5_checkpoint_upgrade_rewinds_signed_progress_without_losing_objects)
|
||||
else
|
||||
cases+=(legacy-delete-reservation:stable_upgrade_deletes_foreign_reservations_without_losing_live_objects)
|
||||
fi
|
||||
status=0
|
||||
for entry in "${cases[@]}"; do
|
||||
artifact="${entry%%:*}"
|
||||
@@ -192,7 +178,7 @@ jobs:
|
||||
exit "$status"
|
||||
|
||||
- name: Run mixed-version MPU recovery
|
||||
if: ${{ !cancelled() && steps.verify_source.outcome == 'success' && matrix.source.version != '1.0.5' }}
|
||||
if: ${{ !cancelled() && steps.verify_source.outcome == 'success' }}
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
NO_PROXY: 127.0.0.1,localhost,::1
|
||||
@@ -213,24 +199,6 @@ jobs:
|
||||
"$mpu_test" -- --exact --nocapture --test-threads=1 \
|
||||
2>&1 | tee "$RUSTFS_E2E_LOG_DIR/test.log"
|
||||
|
||||
- name: Verify gateway baseline G09 evidence
|
||||
if: ${{ success() && matrix.source.version == '1.0.5' }}
|
||||
shell: bash
|
||||
run: |
|
||||
for field in mixed_version_reader_evidence mixed_version_writer_evidence rollback_payload_evidence; do
|
||||
directory=mixed-version-upgrade
|
||||
if [[ "$field" == rollback_payload_evidence ]]; then directory=bucket-config-rollback; fi
|
||||
jq -e --arg field "$field" --arg current "$GITHUB_SHA" --arg previous "$UPGRADE_SOURCE_REVISION" \
|
||||
'.gate == "G09" and .field == $field and .current_revision == $current and .previous_revision == $previous
|
||||
and ($field != "rollback_payload_evidence" or .rollback_payload_replayed == true)' \
|
||||
"$RUNNER_TEMP/rustfs-upgrade-g09-evidence/$directory/G09-$field.json"
|
||||
done
|
||||
jq -e --arg current "$GITHUB_SHA" --arg previous "$UPGRADE_SOURCE_REVISION" \
|
||||
'.current_revision == $current and .previous_revision == $previous
|
||||
and .part_sizes == [5242880, 5242880, 4096] and .bytes == (.part_sizes | add)
|
||||
and (.rollback_read_sha256 | test("^[a-f0-9]{64}$")) and .rollback_read_sha256 == .roll_forward_read_sha256' \
|
||||
"$RUNNER_TEMP/rustfs-upgrade-g09-evidence/bucket-config-rollback/inflight-multipart-rollback.json"
|
||||
|
||||
- name: Upload scanner/heal G09 evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
|
||||
@@ -1,336 +0,0 @@
|
||||
# Copyright 2026 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Gateway black-box acceptance lanes (rustfs/backlog#2775 T3.4, #2776 T3.5).
|
||||
#
|
||||
# Runs two suites of the rustfs/gateway repository against the candidate binary,
|
||||
# one server:
|
||||
#
|
||||
# - the conformance corpus, `--profile rustfs`, judged by
|
||||
# scripts/check_conformance_rulings.py against .config/conformance-rulings.toml
|
||||
# (red: an unruled failure, a ruled case failing differently, a stale ruling,
|
||||
# a skip on the server's own answer);
|
||||
# - the client matrix, ci/compat/run_matrix.sh --external, judged by
|
||||
# scripts/compat_matrix_report.py (no `fail`, no `sut-unregistered`, every
|
||||
# client ran; a client that could not be installed is reported as skipped).
|
||||
#
|
||||
# The suites are checked out at the rustfs/gateway rev Cargo.toml pins, so the
|
||||
# suite and the framework under test never drift apart. The verdict is computed
|
||||
# from what the run left on disk (startup record, runner exit code, liveness
|
||||
# after the corpus, report), never from the exit status of an earlier step.
|
||||
#
|
||||
# Not a pull-request gate. Scheduled failures are owned by the scheduled-validation
|
||||
# watchdog (.github/scheduled-validations.json).
|
||||
|
||||
name: Gateway Acceptance
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "29 3 * * *"
|
||||
workflow_dispatch:
|
||||
|
||||
# Matches ci.yml: these variables are hashed into the rust-cache key, so the
|
||||
# candidate build reads the same ci-dev cache as build-rustfs-debug-binary.
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
RUST_BACKTRACE: 1
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
plan:
|
||||
name: Read the gateway pin
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
outputs:
|
||||
gateway-rev: ${{ steps.pin.outputs.rev }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Read the rustfs/gateway rev from Cargo.toml
|
||||
id: pin
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 - <<'PY' >> "$GITHUB_OUTPUT"
|
||||
import re, sys, tomllib
|
||||
dependencies = tomllib.load(open("Cargo.toml", "rb"))["workspace"]["dependencies"]
|
||||
revs = {spec.get("rev") for spec in dependencies.values()
|
||||
if isinstance(spec, dict) and spec.get("git") == "https://github.com/rustfs/gateway.git"}
|
||||
if len(revs) != 1 or not re.fullmatch(r"[0-9a-f]{40}", str(next(iter(revs)))):
|
||||
sys.exit(f"Cargo.toml must pin rustfs/gateway at exactly one full rev, found {sorted(map(str, revs))}")
|
||||
print(f"rev={revs.pop()}")
|
||||
PY
|
||||
cat "$GITHUB_OUTPUT"
|
||||
|
||||
build-candidate:
|
||||
name: Build candidate binary
|
||||
# The same debug build as ci.yml's build-rustfs-debug-binary, reading the
|
||||
# ci-dev cache written on sm-standard-4.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: ci-dev
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
- name: Build debug binary
|
||||
run: python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks
|
||||
|
||||
- name: Upload candidate binary
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: gateway-acceptance-candidate-binary
|
||||
overwrite: true
|
||||
path: |
|
||||
target/debug/rustfs
|
||||
target/debug/rustfs.e2e.json
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
build-gateway-tools:
|
||||
name: Build gateway conformance runner and compat-sut
|
||||
needs: [ plan ]
|
||||
# The gateway workspace builds with its own rust-toolchain.toml, which rustup
|
||||
# installs on first use; nothing here names a toolchain.
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: Checkout rustfs/gateway at the pinned rev
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
repository: rustfs/gateway
|
||||
ref: ${{ needs.plan.outputs.gateway-rev }}
|
||||
path: gateway
|
||||
persist-credentials: false
|
||||
|
||||
- name: Build
|
||||
working-directory: gateway
|
||||
run: |
|
||||
cargo build --locked --release -p rustfs-gateway-conformance -p rustfs-gateway-compat-sut \
|
||||
--bin rustfs-gateway-conformance --bin compat-sut
|
||||
|
||||
- name: Upload tools
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: gateway-acceptance-tools
|
||||
overwrite: true
|
||||
path: |
|
||||
gateway/target/release/rustfs-gateway-conformance
|
||||
gateway/target/release/compat-sut
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
acceptance:
|
||||
name: Acceptance
|
||||
needs: [ plan, build-candidate, build-gateway-tools ]
|
||||
# GitHub-hosted like the s3-tests targets of reference-parity.yml: the job
|
||||
# only runs prebuilt binaries and the pinned clients, none of which needs Docker.
|
||||
runs-on: ubuntu-latest
|
||||
# About 3x the 15 minutes the job took in run 43325101234 (client install about
|
||||
# 8, matrix about 3, corpus about 3), so a hung server costs 45 runner minutes, not more.
|
||||
timeout-minutes: 45
|
||||
env:
|
||||
ENDPOINT: http://127.0.0.1:9000
|
||||
# The conformance suite signs with these fixed fixture credentials (the AWS
|
||||
# documentation example pair), so the candidate is started with them and the
|
||||
# client matrix is handed the same pair.
|
||||
ACCESS_KEY: AKIAIOSFODNN7EXAMPLE
|
||||
SECRET_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Checkout rustfs/gateway at the pinned rev
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
repository: rustfs/gateway
|
||||
ref: ${{ needs.plan.outputs.gateway-rev }}
|
||||
path: gateway
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download candidate binary
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: gateway-acceptance-candidate-binary
|
||||
path: target/debug
|
||||
|
||||
- name: Download gateway tools
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: gateway-acceptance-tools
|
||||
path: tools
|
||||
|
||||
# Records `ready`, or what the process said when it exited, in lane/startup.txt.
|
||||
- name: Start candidate
|
||||
run: |
|
||||
set -uo pipefail
|
||||
chmod +x target/debug/rustfs tools/*
|
||||
mkdir -p lane "$RUNNER_TEMP"/data/rustfs{0..3}
|
||||
target/debug/rustfs --version | tee lane/candidate-version.txt
|
||||
# The four directories share the runner's one disk; the bypass is the CI case the
|
||||
# physical-disk guard sanctions (as in mint.yml).
|
||||
RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true RUST_LOG=info target/debug/rustfs --address 127.0.0.1:9000 \
|
||||
--access-key "$ACCESS_KEY" --secret-key "$SECRET_KEY" "$RUNNER_TEMP"/data/rustfs{0..3} \
|
||||
> lane/rustfs.log 2>&1 &
|
||||
pid=$!
|
||||
echo "$pid" > lane/rustfs.pid
|
||||
# Liveness first: a ready answer counts only while this process is still running.
|
||||
for _ in $(seq 120); do
|
||||
if ! kill -0 "$pid" 2>/dev/null; then
|
||||
status=0
|
||||
wait "$pid" || status=$?
|
||||
echo "did not start: exit ${status}: $(grep -v '^[[:space:]]*$' lane/rustfs.log | tail -n 1)" > lane/startup.txt
|
||||
cat lane/startup.txt
|
||||
exit 1
|
||||
fi
|
||||
if curl -sf "$ENDPOINT/health/ready" >/dev/null; then
|
||||
echo ready > lane/startup.txt
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "did not start: /health/ready did not answer within 120 seconds" > lane/startup.txt
|
||||
exit 1
|
||||
|
||||
# The exit code is data for the judge step, not this step's result.
|
||||
- name: Run gateway conformance (rustfs profile)
|
||||
run: |
|
||||
set +e
|
||||
tools/rustfs-gateway-conformance run --root gateway/conformance --profile rustfs \
|
||||
--endpoint "$ENDPOINT" --allow-external-fixtures \
|
||||
--json lane/report.json > lane/conformance.log 2>&1
|
||||
echo "$?" > lane/runner-exit.txt
|
||||
if kill -0 "$(cat lane/rustfs.pid)" 2>/dev/null; then echo alive; else echo exited; fi > lane/alive-after-conformance.txt
|
||||
tail -n 20 lane/conformance.log
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
# Toolchains for the matrix's SDK driver programs, as the gateway's own
|
||||
# client-matrix.yml sets them up; the client versions come from
|
||||
# gateway/compat/versions.toml.
|
||||
- uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
|
||||
with:
|
||||
go-version: "1.26"
|
||||
cache: false
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
with:
|
||||
node-version: "22"
|
||||
- uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9 # v4.3.1
|
||||
with:
|
||||
dotnet-version: "8.0.x"
|
||||
- uses: actions/setup-java@cf277c60eb25467037889841efdb72551f06f6c3 # v4.9.1
|
||||
with:
|
||||
distribution: temurin
|
||||
java-version: "21"
|
||||
- name: Install Maven
|
||||
env:
|
||||
MAVEN_VERSION: "3.9.16"
|
||||
MAVEN_SHA512: "831a8591fe20c8243b1dbe7d71e3244f31d1665b0804b2e825e38cbbe5ce0cafb8338851f90780735568773e0a6cd07bbec107cda0b896b008b861075358b6f6"
|
||||
run: |
|
||||
set -euo pipefail
|
||||
archive="$RUNNER_TEMP/apache-maven.tar.gz"
|
||||
curl -sSfL -o "$archive" "https://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz"
|
||||
echo "${MAVEN_SHA512} ${archive}" | sha512sum -c -
|
||||
tar -xzf "$archive" -C "$RUNNER_TEMP"
|
||||
echo "$RUNNER_TEMP/apache-maven-${MAVEN_VERSION}/bin" >> "$GITHUB_PATH"
|
||||
|
||||
# One client at a time, so a client that cannot be installed is reported
|
||||
# as skipped with its reason instead of stopping the others.
|
||||
- name: Install pinned clients
|
||||
working-directory: gateway
|
||||
run: |
|
||||
set -uo pipefail
|
||||
python3 -m pip install --disable-pip-version-check "pyyaml==6.0.2"
|
||||
installed=()
|
||||
for client in $(python3 -c 'import tomllib; print(" ".join(sorted(tomllib.load(open("compat/versions.toml", "rb"))["clients"])))'); do
|
||||
if bash ci/compat/install_clients.sh --clients "$client" > "../lane/install-${client}.log" 2>&1; then
|
||||
installed+=("$client")
|
||||
else
|
||||
echo "${client}: skipped: $(tail -n 1 "../lane/install-${client}.log")" >> ../lane/skipped-clients.txt
|
||||
fi
|
||||
done
|
||||
echo "COMPAT_CLIENTS=$(IFS=,; echo "${installed[*]}")" >> "$GITHUB_ENV"
|
||||
cat ../lane/skipped-clients.txt 2>/dev/null || echo "every pinned client installed"
|
||||
|
||||
# The exit code is data for the judge step, not this step's result.
|
||||
- name: Run client matrix (external)
|
||||
working-directory: gateway
|
||||
env:
|
||||
GATEWAY_COMPAT_SUT_BIN: ${{ github.workspace }}/tools/compat-sut
|
||||
GATEWAY_COMPAT_RUN_DIR: ${{ github.workspace }}/lane/compat
|
||||
GATEWAY_COMPAT_MATRIX_OUT: ${{ github.workspace }}/lane/compat/matrix.json
|
||||
run: |
|
||||
set +e
|
||||
export GATEWAY_COMPAT_ACCESS_KEY="$ACCESS_KEY" GATEWAY_COMPAT_SECRET_KEY="$SECRET_KEY"
|
||||
PATH="$(go env GOPATH)/bin:$PATH"
|
||||
if [[ -f ../lane/skipped-clients.txt ]]; then
|
||||
bash ci/compat/run_matrix.sh --external "$ENDPOINT" --product rustfs --clients "$COMPAT_CLIENTS"
|
||||
else
|
||||
bash ci/compat/run_matrix.sh --external "$ENDPOINT" --product rustfs
|
||||
fi
|
||||
echo "$?" > ../lane/matrix-exit.txt
|
||||
|
||||
- name: Judge and summarize
|
||||
if: always()
|
||||
run: |
|
||||
set -uo pipefail
|
||||
status=0
|
||||
python3 scripts/check_conformance_rulings.py --run-dir lane | tee -a "$GITHUB_STEP_SUMMARY" || status=1
|
||||
python3 scripts/compat_matrix_report.py check --run-dir lane | tee -a "$GITHUB_STEP_SUMMARY" || status=1
|
||||
exit "$status"
|
||||
|
||||
- name: Upload lane results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: gateway-acceptance-lane
|
||||
overwrite: true
|
||||
path: |
|
||||
lane/*.txt
|
||||
lane/*.log
|
||||
lane/report.json
|
||||
lane/compat/matrix.json
|
||||
lane/compat/probe.jsonl
|
||||
lane/compat/sut.log
|
||||
lane/compat/results/**
|
||||
if-no-files-found: error
|
||||
retention-days: 14
|
||||
@@ -44,6 +44,13 @@
|
||||
# well inside the 120-minute budget and buys deterministic infra. The
|
||||
# docker-capable self-hosted `dind-sm-standard-2` label was the alternative but
|
||||
# has fewer cores and reintroduces fleet-state risk for no reliability gain.
|
||||
#
|
||||
# Stack dimension (rustfs/backlog#2774, T3.3): one image, run with RUSTFS_S3_STACK=legacy and
|
||||
# =gateway. `compare` (scripts/mint_stack_diff.py) holds the gate that every case legacy passes
|
||||
# passes on gateway with no new failure. The gateway leg and `compare` are report-only until
|
||||
# T2.10. A gateway server that refuses to start stops its leg at "Wait for RustFS ready", before
|
||||
# mint runs; the summary says `skipped: gateway stack refuses startup: <why>`. The stack is read
|
||||
# from the server's own start-up line (scripts/dual_stack.py stack).
|
||||
|
||||
name: mint
|
||||
|
||||
@@ -138,8 +145,16 @@ jobs:
|
||||
retention-days: 1
|
||||
|
||||
mint:
|
||||
name: mint
|
||||
name: mint (${{ matrix.stack }}${{ matrix.stack == 'gateway' && ', report-only until T2.10' || '' }})
|
||||
needs: [image]
|
||||
# Report-only until T2.10: the gateway stack cannot serve the suites yet.
|
||||
continue-on-error: ${{ matrix.stack == 'gateway' }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
stack: [legacy, gateway]
|
||||
env:
|
||||
STACK: ${{ matrix.stack }}
|
||||
# GitHub-hosted: reliably provides a running Docker daemon + buildx +
|
||||
# python3. See the header note (ci-2) for why the self-hosted sm-standard-4
|
||||
# label was abandoned (no docker.sock on the pod its only run landed on).
|
||||
@@ -179,6 +194,7 @@ jobs:
|
||||
-e RUSTFS_SECRET_KEY="${S3_SECRET_KEY}" \
|
||||
-e RUSTFS_VOLUMES="/data/rustfs{0...3}" \
|
||||
-e RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true \
|
||||
-e RUSTFS_S3_STACK="${STACK}" \
|
||||
-e RUST_LOG=rustfs=info \
|
||||
-v /tmp/rustfs-mint:/data \
|
||||
rustfs-ci
|
||||
@@ -275,13 +291,47 @@ jobs:
|
||||
mkdir -p artifacts/rustfs-mint
|
||||
docker logs rustfs-mint > artifacts/rustfs-mint/rustfs.log 2>&1 || true
|
||||
|
||||
- name: Prove which stack the server selected
|
||||
if: always()
|
||||
run: python3 scripts/dual_stack.py stack --expect "${STACK}" --out artifacts/stack.json artifacts/rustfs-mint
|
||||
|
||||
- name: Upload artifacts
|
||||
if: always() && env.ACT != 'true'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: mint
|
||||
name: mint-${{ matrix.stack }}
|
||||
path: artifacts/**
|
||||
|
||||
compare:
|
||||
name: Compare stacks (report-only until T2.10)
|
||||
needs: [mint]
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download both legs
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
pattern: mint-{legacy,gateway}
|
||||
path: legs
|
||||
|
||||
# A leg that never uploaded leaves no directory: create both so that the
|
||||
# comparison reports "no result" for it instead of failing to find it.
|
||||
- name: Compare the stacks
|
||||
run: |
|
||||
# The shell runs with -e; the exit status is reported after the summary is written.
|
||||
set +e -u -o pipefail
|
||||
mkdir -p legs/mint-legacy legs/mint-gateway
|
||||
python3 scripts/mint_stack_diff.py legs/mint-legacy legs/mint-gateway | tee report.txt
|
||||
status="${PIPESTATUS[0]}"
|
||||
{ echo '## Mint: gateway versus legacy'; echo; echo '```'; cat report.txt; echo '```'; } >> "$GITHUB_STEP_SUMMARY"
|
||||
exit "$status"
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [image, mint]
|
||||
|
||||
@@ -68,11 +68,14 @@ env:
|
||||
|
||||
jobs:
|
||||
minio-source:
|
||||
name: MinIO source (read-through, list-through, backfill)
|
||||
name: MinIO source (read-through, list-through, backfill, ${{ matrix.stack }}${{ matrix.stack == 'gateway' && ', report-only until T2.10' || '' }})
|
||||
# Keep this scheduled interoperability lane scoped to the upstream repo.
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 90
|
||||
# The gateway leg is report-only until T2.10.
|
||||
continue-on-error: ${{ matrix.stack == 'gateway' }}
|
||||
strategy: { fail-fast: false, matrix: { stack: [legacy, gateway] } }
|
||||
env:
|
||||
NO_PROXY: 127.0.0.1,localhost
|
||||
# Fixed credentials of the container this job starts and throws away;
|
||||
@@ -137,6 +140,11 @@ jobs:
|
||||
- name: Build the RustFS binary under test
|
||||
run: python3 scripts/e2e_binary.py build --bins
|
||||
|
||||
# A gateway binary that refuses to start skips the suite; the summary says why.
|
||||
- name: Gateway preflight
|
||||
if: matrix.stack == 'gateway'
|
||||
run: python3 scripts/dual_stack.py preflight --binary target/debug/rustfs --out "$RUNNER_TEMP/leg/stack.json"
|
||||
|
||||
# The lane selects tests by module, so a rename would quietly shrink it.
|
||||
# The committed digest in .config/e2e-odm-interop-selection.txt fails
|
||||
# closed on that.
|
||||
@@ -147,9 +155,18 @@ jobs:
|
||||
|
||||
- name: Run the interop cases against MinIO
|
||||
env:
|
||||
RUSTFS_S3_STACK: ${{ matrix.stack }}
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-logs
|
||||
run: python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-odm-interop -p e2e_test --no-tests=fail
|
||||
|
||||
- name: Prove the stack and upload the leg
|
||||
if: always()
|
||||
uses: ./.github/actions/dual-stack-leg
|
||||
with:
|
||||
profile: e2e-odm-interop
|
||||
stack: ${{ matrix.stack }}
|
||||
logs: ${{ runner.temp }}/rustfs-e2e-logs
|
||||
|
||||
- name: Build the MinIO interop report
|
||||
if: always()
|
||||
uses: ./.github/actions/odm-interop-report
|
||||
@@ -173,7 +190,7 @@ jobs:
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: odm-interop-minio-${{ github.run_number }}-${{ github.run_attempt }}
|
||||
name: odm-interop-minio-${{ github.run_number }}-${{ github.run_attempt }}-${{ matrix.stack }}
|
||||
path: |
|
||||
artifacts/odm-interop/minio
|
||||
target/nextest/e2e-odm-interop/junit.xml
|
||||
@@ -299,6 +316,20 @@ jobs:
|
||||
target/nextest/e2e-odm-interop/junit.xml
|
||||
retention-days: 14
|
||||
|
||||
dual-stack-summary:
|
||||
name: Both-stack diff summary
|
||||
needs: [minio-source]
|
||||
if: github.repository == 'rustfs/rustfs' && always()
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: ./.github/actions/dual-stack-summary
|
||||
with:
|
||||
profiles: e2e-odm-interop
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [minio-source, cloud-source]
|
||||
|
||||
@@ -1,91 +0,0 @@
|
||||
name: S3-over-RDMA feature check
|
||||
|
||||
# The `nixl` and `cuobject` features are off in every other build, so the code
|
||||
# behind them is compiled nowhere else. This job compiles the `nixl` half on
|
||||
# Linux and runs rustfs-nixl's unit tests; nixl-sys builds through its stub
|
||||
# fallback, so no NIXL SDK is needed.
|
||||
# The cuObject half needs the NVIDIA cuObject SDK and is not compiled here.
|
||||
# Not a required check and not part of the `Test and Lint` aggregate.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- "crates/nixl/**"
|
||||
- "crates/cuobject/**"
|
||||
- "rustfs/src/storage/rpc/nixl*"
|
||||
- "rustfs/src/app/object/cuobject.rs"
|
||||
- "rustfs/src/app/object/get.rs"
|
||||
- "rustfs/src/app/object/put.rs"
|
||||
- "rustfs/src/app/multipart_usecase/upload_part.rs"
|
||||
- "Cargo.toml"
|
||||
- "rustfs/Cargo.toml"
|
||||
- ".github/workflows/rdma-feature-check.yml"
|
||||
schedule:
|
||||
- cron: "37 5 * * 1"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: rdma-feature-check-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
nixl-check:
|
||||
name: cargo check and unit tests (nixl)
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: ci-feat-nixl
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
- name: Install only missing native build tools
|
||||
run: |
|
||||
packages=()
|
||||
command -v g++ >/dev/null || packages+=(g++)
|
||||
ldconfig -p | grep -q 'libclang' || packages+=(libclang-dev)
|
||||
if (( ${#packages[@]} )); then
|
||||
sudo apt-get update
|
||||
sudo apt-get install --no-install-recommends -y "${packages[@]}"
|
||||
fi
|
||||
|
||||
- name: Check the nixl feature build
|
||||
run: cargo check --locked -p rustfs --features nixl
|
||||
|
||||
# These unit tests use a fake sysfs and plain host memory; none needs an
|
||||
# RDMA device or the NIXL libraries.
|
||||
- name: Run the rustfs-nixl unit tests
|
||||
run: cargo test --locked -p rustfs-nixl --features sys
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [nixl-check]
|
||||
# Only scheduled runs open or update the tracking issue; pull-request and
|
||||
# manual runs stay quiet.
|
||||
if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Open or update failure-tracking issue
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}
|
||||
@@ -32,9 +32,9 @@
|
||||
#
|
||||
# The `differential` job adds the 73-operation dual-stack differential
|
||||
# (crates/e2e_test/src/dual_stack_differential, rustfs/backlog#2740) against the
|
||||
# same pinned reference: side A is the released binary (RUSTFS_E2E_REFERENCE_BINARY),
|
||||
# side B is the candidate built above.
|
||||
# It reuses the candidate build artifact, never builds the candidate
|
||||
# same pinned reference: side A is the released binary (RUSTFS_E2E_REFERENCE_BINARY,
|
||||
# started without RUSTFS_S3_STACK), side B is the candidate built above on its
|
||||
# legacy stack. It reuses the candidate build artifact, never builds the candidate
|
||||
# twice, and prints reference_side, version_a and version_b from the report the test
|
||||
# wrote; it fails when side A was not the reference or both sides report the same
|
||||
# version. Every (op, field, a, b) difference goes to the job summary. Report-only
|
||||
@@ -310,7 +310,7 @@ jobs:
|
||||
chmod +x target/debug/rustfs
|
||||
test -x "$REFERENCE_PATH"
|
||||
mkdir -p "$RUSTFS_E2E_LOG_DIR"
|
||||
# Side A: the released binary. Side B: the candidate.
|
||||
# Side A: the released binary. Side B: the candidate on its legacy stack.
|
||||
export RUSTFS_E2E_REFERENCE_BINARY="$REFERENCE_PATH"
|
||||
test_name='dual_stack_differential::tests::both_stacks_answer_every_operation_identically'
|
||||
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- \
|
||||
|
||||
@@ -13,8 +13,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Deployment
|
||||
|
||||
- **The S3 protocol stack is now rustfs/gateway**, and it is the only S3 stack. The legacy s3s-based stack is removed, and so are the `RUSTFS_S3_STACK` environment variable and the `gateway-stack` cargo feature. There is no on-disk format change. `RUSTFS_S3_STACK` no longer selects anything: a server started with it set, to any value, refuses to start and prints `RUSTFS_S3_STACK is no longer supported; the gateway stack is the only S3 stack since <version>`, where `<version>` is the release's version. Remove the variable from service units, container environments, Helm values and orchestration files before upgrading. To roll back, follow the [S3 stack rollback procedure](docs/operations/s3-stack-rollback.md).
|
||||
|
||||
- Before upgrading, verify that the configured root access key is at least 3 bytes and the root secret key is at least 8 bytes. RustFS now rejects shorter values at startup and identifies the setting to update. Version 1.0.5 accepted these settings at startup and returned HTTP 500 when root requested STS credentials. Coordinate root credential changes across all nodes and clients. See [root credential rotation considerations](docs/operations/two-factor-auth.md#root-credentials-cannot-be-changed-at-runtime).
|
||||
|
||||
- Existing volumes created by older root-running container images must be writable by the non-root runtime identity before upgrading. The default container UID/GID is `10001:10001`; stop the affected instances, verify host mount paths, and for ordinary bind mounts run `sudo chown -R 10001:10001 /actual/data /actual/logs` on each RustFS data/log mount. Account for ACLs, NFS, Kubernetes, and rootless UID mappings. Image-build ownership changes do not migrate mounted data. See the Docker Quick Start in [README.md](README.md).
|
||||
@@ -46,8 +44,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Vault KMS backends no longer fail when the server answers reads from a replica**: a Vault or OpenBao node that serves reads locally (an OpenBao standby, a Vault Enterprise performance standby) can return the state from before a write for a short while after acknowledging it. Against such a node the KV2 backend failed every key creation with `404 Key not found` while leaving the key stored without its description and tags, failed operations issued right after a create or update with `Key not found` or `Concurrent modification`, and the Transit backend failed a disable followed quickly by an enable. Key creation on the KV2 backend is now a single create-only write that carries the description and tags. Both backends now re-read a key record when the answer is older than a write the same RustFS node made in the previous five seconds, pause before re-reading after a lost check-and-set write and allow five attempts instead of three, and retry a Transit key delete that is refused because the deletion switch is not visible yet. A different RustFS node reading the same replica can still observe the earlier state until the replica catches up; see [OpenBao as the Vault KMS backend](docs/operations/kms-openbao-compatibility.md).
|
||||
|
||||
- **KMS keys could not be described by ARN through the path route**: `GET /rustfs/admin/v3/kms/keys/{key_id}` read the path segment verbatim, so a key id that has to be percent-encoded to fit in one segment — an AWS key ARN, with its `:` and `/` — was looked up, authorized and audited under its encoded spelling and answered `404`. The segment is now percent-decoded first; a segment that is not valid UTF-8 after decoding is refused with `400 InvalidArgument`. The `describe-key?keyId=` form was unaffected.
|
||||
|
||||
- **SSE-KMS objects written through the AWS KMS backend could not be read**: `GetObject`, `CopyObject` and every other read of an `aws:kms` object failed with `400 InvalidRequest` ("SSE-S3 requires RUSTFS_SSE_S3_MASTER_KEY to be set ...") on a deployment using `RUSTFS_KMS_BACKEND=aws`, although the write and `HeadObject` succeeded. The AWS backend stored the `GenerateDataKey` blob as it came from AWS, the read path took those unrecognized bytes for a locally wrapped key, and AWS was never asked to decrypt. The backend now stores the blob inside the same data-key envelope the other backends use, and a blob stored without one is routed to AWS when the AWS backend is active. Objects written by 1.0.4 and 1.0.5 are intact and become readable after the upgrade, with no rewrite. Setting `RUSTFS_SSE_S3_MASTER_KEY` was never a workaround. An object written by this release is not readable by an older one; the same was already true of every object the older release wrote itself.
|
||||
|
||||
Generated
+437
-777
File diff suppressed because it is too large
Load Diff
+37
-48
@@ -23,8 +23,6 @@ members = [
|
||||
"crates/connect-symbol-catalog", # Release-only CPU profile symbol catalog generator
|
||||
"crates/credentials", # Credential management system
|
||||
"crates/crypto", # Cryptography and security features
|
||||
"crates/cuobject", # NVIDIA cuObject S3-over-RDMA staging and transfers (optional)
|
||||
"crates/nixl", # NIXL/UCX S3-over-RDMA server side (optional)
|
||||
"crates/ecstore", # Erasure coding storage implementation
|
||||
"crates/e2e_test", # End-to-end test suite
|
||||
"crates/filemeta", # File metadata management
|
||||
@@ -71,7 +69,6 @@ members = [
|
||||
"crates/zip", # ZIP file handling and compression
|
||||
"crates/zstd-compat", # Fixed codec for dependencies requiring zstd 0.13
|
||||
]
|
||||
exclude = ["vendor/hyper"]
|
||||
resolver = "3"
|
||||
|
||||
[workspace.package]
|
||||
@@ -81,6 +78,7 @@ repository = "https://github.com/rustfs/rustfs"
|
||||
rust-version = "1.99.0"
|
||||
version = "1.0.5"
|
||||
homepage = "https://rustfs.com"
|
||||
description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
|
||||
keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
|
||||
categories = ["web-programming", "development-tools", "filesystem", "network-programming"]
|
||||
authors = ["RustFS Team"]
|
||||
@@ -95,6 +93,7 @@ redundant_clone = "warn"
|
||||
|
||||
[workspace.dependencies]
|
||||
# RustFS Internal Crates
|
||||
rustfs = { path = "./rustfs", version = "1.0.5" }
|
||||
rustfs-gateway-adapter = { path = "crates/gateway-adapter", version = "1.0.5" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.5" }
|
||||
rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.5" }
|
||||
@@ -107,8 +106,6 @@ rustfs-config = { path = "./crates/config", version = "1.0.5" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.5" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.5" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.5" }
|
||||
rustfs-cuobject = { path = "crates/cuobject", version = "1.0.5" }
|
||||
rustfs-nixl = { path = "crates/nixl", version = "1.0.5" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.5" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.5" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.5" }
|
||||
@@ -152,16 +149,14 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.5" }
|
||||
# engine crates move onto from s3s (rustfs/backlog#2745). The one pin of the
|
||||
# gateway in this workspace; no crate may enable a `compat-s3s*` feature except
|
||||
# the s3s-facing edge of the rustfs binary.
|
||||
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "60dcd2a55713dbcc9dbd6e165922e1a0df4473d7", version = "0.42.0", default-features = false }
|
||||
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "60dcd2a55713dbcc9dbd6e165922e1a0df4473d7", version = "0.2.0", default-features = false }
|
||||
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "7915d06950f88611d883f6ab3d95efc90c65dec0", version = "0.40.0", default-features = false }
|
||||
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "7915d06950f88611d883f6ab3d95efc90c65dec0", version = "0.2.0", default-features = false }
|
||||
# The gateway framework itself, consumed for its extension traits by the ring-2
|
||||
# adapter. `default-features = false` keeps the `server` feature, and with it the
|
||||
# server crate, out of the build.
|
||||
rustfs-gateway = { git = "https://github.com/rustfs/gateway.git", rev = "60dcd2a55713dbcc9dbd6e165922e1a0df4473d7", version = "0.66.0", default-features = false }
|
||||
rustfs-gateway = { git = "https://github.com/rustfs/gateway.git", rev = "7915d06950f88611d883f6ab3d95efc90c65dec0", version = "0.62.0", default-features = false }
|
||||
# Test-only: builds the `EffectiveHost` a host-resolution test queries with.
|
||||
rustfs-gateway-http = { git = "https://github.com/rustfs/gateway.git", rev = "60dcd2a55713dbcc9dbd6e165922e1a0df4473d7", default-features = false }
|
||||
rustfs-gateway-core = { git = "https://github.com/rustfs/gateway.git", rev = "60dcd2a55713dbcc9dbd6e165922e1a0df4473d7", version = "0.63.0", default-features = false }
|
||||
rustfs-gateway-dialect-rustfs-admin = { git = "https://github.com/rustfs/gateway.git", rev = "60dcd2a55713dbcc9dbd6e165922e1a0df4473d7", version = "0.13.0", default-features = false }
|
||||
rustfs-gateway-http = { git = "https://github.com/rustfs/gateway.git", rev = "7915d06950f88611d883f6ab3d95efc90c65dec0", default-features = false }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-channel = "2.5.0"
|
||||
@@ -169,13 +164,13 @@ async_zip = { default-features = false, version = "0.0.19" }
|
||||
mysql_async = { default-features = false, version = "0.37.1" }
|
||||
async-compression = { version = "0.4.52" }
|
||||
async-trait = "0.1.92"
|
||||
async-nats = { version = "0.53.0", default-features = false }
|
||||
async-nats = { version = "0.52.0", default-features = false }
|
||||
axum = "0.8.9"
|
||||
futures = "0.3.34"
|
||||
futures-core = "0.3.34"
|
||||
futures-lite = "2.6.1"
|
||||
futures-util = "0.3.34"
|
||||
pollster = "1.0.2"
|
||||
pollster = "1.0.1"
|
||||
pulsar = { default-features = false, version = "=6.9.0" }
|
||||
lapin = { default-features = false, version = "4.12.3" }
|
||||
hyper = { version = "1.12.0" }
|
||||
@@ -189,7 +184,6 @@ memchr = "2.8.3"
|
||||
minlz = "1.2.3"
|
||||
reqwest = "0.13.5"
|
||||
rustfs-kafka-async = { version = "1.4.1" }
|
||||
slab = { version = "0.4.12" }
|
||||
socket2 = { version = "0.6.5" }
|
||||
tokio = { version = "1.54.1" }
|
||||
tokio-rustls = { default-features = false, version = "0.26.6" }
|
||||
@@ -210,19 +204,18 @@ quinn = { version = "0.11.12", default-features = false, features = [
|
||||
] }
|
||||
|
||||
# Serialization and Data Formats
|
||||
apache-avro = { version = "0.23.0", features = ["snappy", "zstandard"] }
|
||||
bytes = { version = "1.12.2" }
|
||||
apache-avro = { version = "0.22.0", features = ["snappy", "zstandard"] }
|
||||
bytes = { version = "1.12.1" }
|
||||
csv-core = "0.1.13"
|
||||
bytesize = "2.7.0"
|
||||
byteorder = "1.5.0"
|
||||
flatbuffers = "25.12.19"
|
||||
form_urlencoded = "1.2.2"
|
||||
prost = "0.14.6"
|
||||
prost = "0.14.5"
|
||||
quick-xml = "0.42.0"
|
||||
rmp = { version = "0.8.15" }
|
||||
rmp-serde = { version = "1.3.1" }
|
||||
serde = { version = "1.0.229" }
|
||||
serde-value = "0.7.0"
|
||||
serde_ignored = { version = "0.1" }
|
||||
serde_with = { version = "3", default-features = false, features = ["macros", "std"] }
|
||||
serde_json = { version = "1.0.154" }
|
||||
@@ -240,11 +233,11 @@ blake2 = "0.11.0"
|
||||
chacha20poly1305 = { version = "0.11.0" }
|
||||
crc-fast = "1.10.0"
|
||||
hmac = { version = "0.13.0" }
|
||||
jsonwebtoken = { version = "12.0.0" }
|
||||
jsonwebtoken = { version = "11.1.0" }
|
||||
openidconnect = { default-features = false, version = "4.1.0" }
|
||||
pbkdf2 = "0.13.0"
|
||||
p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
|
||||
rsa = { version = "0.10.0", features = ["pkcs1v15"] }
|
||||
rsa = { version = "=0.10.0-rc.18" }
|
||||
rustls = { default-features = false, version = "0.23.46" }
|
||||
rustls-native-certs = "0.8"
|
||||
rustls-pki-types = "1.16.0"
|
||||
@@ -252,13 +245,13 @@ x509-parser = "0.18.1"
|
||||
sha1 = "0.11.0"
|
||||
sha2 = "0.11.1"
|
||||
subtle = "2.6"
|
||||
zeroize = { version = "1.9.2" }
|
||||
zeroize = { version = "1.9.1" }
|
||||
proptest = "1"
|
||||
|
||||
# Time and Date
|
||||
chrono = { version = "0.4.45" }
|
||||
humantime = "2.4.0"
|
||||
jiff = { version = "1.0.1" }
|
||||
jiff = { version = "0.2.43" }
|
||||
time = { version = "0.3.55" }
|
||||
|
||||
# Database
|
||||
@@ -275,18 +268,18 @@ tar-codec = "0.0.16"
|
||||
tar-framing = "0.0.16"
|
||||
atoi = "3.1.1"
|
||||
atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.13.3" }
|
||||
aws-config = { version = "1.13.2" }
|
||||
aws-credential-types = { version = "1.3.1" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.133.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.160.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.127.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.131.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.159.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.126.0" }
|
||||
aws-smithy-async = { version = "1.3.1" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.5.3" }
|
||||
aws-smithy-runtime-api = { version = "1.22.0" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.5.2" }
|
||||
aws-smithy-runtime-api = { version = "1.21.0" }
|
||||
aws-smithy-types = { version = "1.9.4" }
|
||||
base64-simd = "0.8.0"
|
||||
brotli = "9.1.3"
|
||||
clap = { version = "4.6.8" }
|
||||
brotli = "9.1.1"
|
||||
clap = { version = "4.6.7" }
|
||||
const-str = { version = "1.1.0" }
|
||||
convert_case = "0.12.0"
|
||||
criterion = { version = "0.8" }
|
||||
@@ -294,7 +287,7 @@ crossbeam-queue = "0.3.14"
|
||||
crossbeam-channel = "0.5.17"
|
||||
crossbeam-deque = "0.8.8"
|
||||
crossbeam-utils = "0.8.23"
|
||||
datafusion = { default-features = false, version = "55.3.0" }
|
||||
datafusion = { default-features = false, version = "55.2.0" }
|
||||
derive_builder = "0.20.2"
|
||||
ed25519-dalek = "3.0.0"
|
||||
enumset = "1.1.15"
|
||||
@@ -313,14 +306,12 @@ hostname = "0.4.2"
|
||||
ipnetwork = { version = "0.21.1" }
|
||||
lazy_static = "1.5.1"
|
||||
libc = "0.2.191"
|
||||
cuobject = { version = "0.1.0", default-features = false }
|
||||
libsystemd = "0.7.2"
|
||||
local-ip-address = "0.6.13"
|
||||
log = "0.4"
|
||||
memmap2 = "0.9.12"
|
||||
nixl-sys = { git = "https://github.com/houseme/nixl", rev = "cc636c83f13d692e85b67e55aed092413d16fb41" }
|
||||
lz4 = "1.28.1"
|
||||
lz4_flex = { version = "0.14.2", default-features = false, features = ["std", "frame"] }
|
||||
lz4_flex = { version = "=0.14.0", default-features = false, features = ["std", "frame"] }
|
||||
matchit = "0.9.2"
|
||||
md-5 = "0.11.0"
|
||||
mime_guess = "2.0.5"
|
||||
@@ -339,14 +330,14 @@ percent-encoding = "2.3.2"
|
||||
qrcode-rs = { version = "2.2.0", default-features = false, features = ["std", "svg"] }
|
||||
pin-project-lite = "0.2.17"
|
||||
pretty_assertions = "1.4.1"
|
||||
rand = { version = "0.10.4" }
|
||||
rand = { version = "0.10.3" }
|
||||
ratelimit = "2.0.0"
|
||||
rayon = "1.12.0"
|
||||
rustfs-erasure-codec = { version = "9.1.0" }
|
||||
rustfs-erasure-codec = { version = "9.0.0" }
|
||||
reed-solomon-simd = "3.1.0"
|
||||
regex = { version = "1.13.2" }
|
||||
regex = { version = "1.13.1" }
|
||||
rumqttc = { package = "rumqttc-next", version = "0.34.0" }
|
||||
redis = { version = "1.8.2" }
|
||||
redis = { version = "1.8.1" }
|
||||
rustify = { version = "0.7", default-features = false }
|
||||
rustix = { version = "1.1.6" }
|
||||
rust-embed = { version = "8.14.0" }
|
||||
@@ -366,7 +357,7 @@ snap = "1.1.2"
|
||||
starshard = { version = "2.4.1" }
|
||||
strum = { version = "0.28.0" }
|
||||
sync_wrapper = "1.0.2"
|
||||
sysinfo = "0.40.4"
|
||||
sysinfo = "0.40.3"
|
||||
temp-env = "0.3.6"
|
||||
tempfile = "3.27.0"
|
||||
test-case = "3.4.1"
|
||||
@@ -380,15 +371,15 @@ tracing-subscriber = { version = "0.3.23" }
|
||||
transform-stream = "0.3.1"
|
||||
url = "2.5.8"
|
||||
urlencoding = "2.1.3"
|
||||
uuid = { version = "1.29.3" }
|
||||
uuid = { version = "1.29.2" }
|
||||
vaultrs = { version = "0.8.0" }
|
||||
tar = "0.4.46"
|
||||
walkdir = "2.5.0"
|
||||
winapi-util = "0.1.11"
|
||||
windows = { version = "0.62.2" }
|
||||
windows-sys = "0.61.2"
|
||||
xxhash-rust = { version = "0.8.21" }
|
||||
zip = "9.7.0"
|
||||
xxhash-rust = { version = "0.8.19" }
|
||||
zip = "9.5.1"
|
||||
zstd = "0.14.4"
|
||||
|
||||
# Observability and Metrics
|
||||
@@ -405,11 +396,11 @@ opentelemetry-stdout = { version = "0.33.1" }
|
||||
pyroscope = { version = "2.1.5" }
|
||||
|
||||
# FTP and SFTP
|
||||
libunftp = { version = "0.25.0" }
|
||||
libunftp = { version = "0.24.0" }
|
||||
unftp-core = "0.1.1"
|
||||
suppaftp = { version = "13.0.0" }
|
||||
suppaftp = { version = "12.2.1" }
|
||||
rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
russh = { version = "0.65.2" }
|
||||
russh = { version = "0.65.1" }
|
||||
russh-sftp = "3.0.1"
|
||||
|
||||
# WebDAV
|
||||
@@ -418,7 +409,7 @@ dav-server = "0.12.0"
|
||||
# Performance Analysis and Memory Profiling
|
||||
rustfs-mimalloc = { version = "0.7.4" }
|
||||
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
|
||||
hotpath = { version = "0.32.1", default-features = false }
|
||||
hotpath = { version = "0.31.2", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.49" }
|
||||
|
||||
@@ -458,6 +449,4 @@ strip = "none"
|
||||
|
||||
# Pin upstream fixes until compatible crates.io releases are available.
|
||||
[patch.crates-io]
|
||||
# Remove after Hyper releases HTTP/2 Expect support (hyperium/hyper#4217).
|
||||
hyper = { path = "vendor/hyper" }
|
||||
zstd = { path = "crates/zstd-compat" }
|
||||
|
||||
@@ -170,17 +170,10 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
||||
;; \
|
||||
esac
|
||||
|
||||
# `RC_RELEASE=disabled` is reserved for local server-only validation when the
|
||||
# GitHub release asset is unavailable. The placeholder fails if invoked, so a
|
||||
# production image that needs the CLI must keep the default release download.
|
||||
# TARGETARCH is only provided by BuildKit. Fall back to TARGETPLATFORM (for
|
||||
# example set by docker-compose) and then to the build host, like the cargo
|
||||
# step above, so every path that builds this file passes a real architecture.
|
||||
RUN set -eu; \
|
||||
if [ "${RC_RELEASE}" = "disabled" ]; then \
|
||||
install -m 0755 /bin/false /usr/local/bin/rc; \
|
||||
exit 0; \
|
||||
fi; \
|
||||
rc_arch="${TARGETARCH:-}"; \
|
||||
if [ -z "${rc_arch}" ]; then \
|
||||
case "${TARGETPLATFORM:-}" in \
|
||||
|
||||
@@ -311,10 +311,6 @@ x env use rustfs
|
||||
rustfs --help
|
||||
```
|
||||
|
||||
### 7\. Docker Swarm with the SwarmCLI chart (Option 7)
|
||||
|
||||
Follow the [SwarmCLI chart guide](https://docs.rustfs.com/en/installation/cloud-native/swarmcli-chart) to run RustFS on Docker Swarm, as a single-node service or a distributed, erasure-coded cluster with one peer per node, with the open-source [SwarmCLI `rustfs` chart](https://github.com/Eldara-Tech/swarmcli-charts/tree/main/charts/rustfs). The chart keeps the root credentials in Docker secrets and refuses the default `rustfsadmin` keys, upgrades and rolls back as versioned releases, and can serve the S3 API over HTTPS through Traefik. Its Console stays off until you enable it. The chart and [SwarmCLI](https://github.com/Eldara-Tech/swarmcli) are maintained by Eldara Tech, not by the RustFS project.
|
||||
|
||||
---
|
||||
|
||||
### Accessing RustFS
|
||||
|
||||
@@ -239,10 +239,6 @@ x env use rustfs
|
||||
rustfs --help
|
||||
```
|
||||
|
||||
### 7\. 使用 SwarmCLI Chart 部署到 Docker Swarm (Option 7)
|
||||
|
||||
请按照 [SwarmCLI Chart 指南](https://docs.rustfs.com/zh/installation/cloud-native/swarmcli-chart),使用开源的 [SwarmCLI `rustfs` Chart](https://github.com/Eldara-Tech/swarmcli-charts/tree/main/charts/rustfs) 在 Docker Swarm 上运行 RustFS,可部署为单节点服务,或每个节点一个对等节点、采用纠删码的分布式集群。该 Chart 将 root 凭证保存在 Docker Secret 中并拒绝默认的 `rustfsadmin` 密钥,以带版本的发布进行升级和回滚,并可通过 Traefik 以 HTTPS 提供 S3 API。控制台默认关闭,需要时再启用。该 Chart 和 [SwarmCLI](https://github.com/Eldara-Tech/swarmcli) 由 Eldara Tech 维护,并非 RustFS 项目的一部分。
|
||||
|
||||
---
|
||||
|
||||
### 访问 RustFS
|
||||
|
||||
@@ -30,11 +30,12 @@ workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
hotpath = []
|
||||
hotpath-alloc = ["hotpath"]
|
||||
hotpath-cpu = ["hotpath"]
|
||||
hotpath = ["hotpath/hotpath"]
|
||||
hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"]
|
||||
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
crc-fast = { workspace = true }
|
||||
http = { workspace = true }
|
||||
|
||||
@@ -30,12 +30,13 @@ workspace = true
|
||||
[features]
|
||||
default = []
|
||||
test-util = []
|
||||
hotpath = []
|
||||
hotpath-alloc = ["hotpath"]
|
||||
hotpath-cpu = ["hotpath"]
|
||||
hotpath = ["hotpath/hotpath", "hotpath/tokio"]
|
||||
hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"]
|
||||
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"]
|
||||
|
||||
[dependencies]
|
||||
arc-swap.workspace = true
|
||||
hotpath.workspace = true
|
||||
tokio = { workspace = true, features = ["fs", "rt-multi-thread"] }
|
||||
tonic = { workspace = true, features = ["gzip", "deflate"] }
|
||||
uuid = { workspace = true, features = ["v4", "fast-rng", "macro-diagnostics"] }
|
||||
|
||||
@@ -23,80 +23,7 @@ pub static GLOBAL_LOCAL_NODE_NAME: LazyLock<RwLock<String>> = LazyLock::new(|| R
|
||||
pub static GLOBAL_RUSTFS_HOST: LazyLock<RwLock<String>> = LazyLock::new(|| RwLock::new("".to_string()));
|
||||
pub static GLOBAL_RUSTFS_PORT: LazyLock<RwLock<String>> = LazyLock::new(|| RwLock::new("9000".to_string()));
|
||||
pub static GLOBAL_RUSTFS_ADDR: LazyLock<RwLock<String>> = LazyLock::new(|| RwLock::new("".to_string()));
|
||||
static NEXT_GRPC_CHANNEL_ID: AtomicU64 = AtomicU64::new(1);
|
||||
static NEXT_GRPC_CONNECTION_GENERATION: AtomicU64 = AtomicU64::new(1);
|
||||
|
||||
#[derive(Debug)]
|
||||
struct GrpcConnectionStateInner {
|
||||
channel_id: u64,
|
||||
generation: AtomicU64,
|
||||
}
|
||||
|
||||
/// Lock-free identity and physical-connection generation shared by every clone of one gRPC channel.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct GrpcConnectionState(Arc<GrpcConnectionStateInner>);
|
||||
|
||||
/// A point-in-time identity for the physical connection currently used by a gRPC channel.
|
||||
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
|
||||
pub struct GrpcConnectionSnapshot {
|
||||
pub channel_id: u64,
|
||||
pub generation: u64,
|
||||
}
|
||||
|
||||
impl GrpcConnectionState {
|
||||
pub fn new() -> Self {
|
||||
Self(Arc::new(GrpcConnectionStateInner {
|
||||
channel_id: next_nonzero_id(&NEXT_GRPC_CHANNEL_ID),
|
||||
generation: AtomicU64::new(0),
|
||||
}))
|
||||
}
|
||||
|
||||
/// Publish a successfully opened physical transport before tonic makes it request-ready.
|
||||
pub fn record_connected(&self) -> GrpcConnectionSnapshot {
|
||||
let generation = next_nonzero_id(&NEXT_GRPC_CONNECTION_GENERATION);
|
||||
self.0.generation.store(generation, Ordering::Release);
|
||||
GrpcConnectionSnapshot {
|
||||
channel_id: self.0.channel_id,
|
||||
generation,
|
||||
}
|
||||
}
|
||||
|
||||
/// Return `None` until the first physical transport has connected successfully.
|
||||
pub fn snapshot(&self) -> Option<GrpcConnectionSnapshot> {
|
||||
let generation = self.0.generation.load(Ordering::Acquire);
|
||||
(generation != 0).then_some(GrpcConnectionSnapshot {
|
||||
channel_id: self.0.channel_id,
|
||||
generation,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for GrpcConnectionState {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
fn next_nonzero_id(counter: &AtomicU64) -> u64 {
|
||||
loop {
|
||||
let id = counter.fetch_add(1, Ordering::Relaxed);
|
||||
if id != 0 {
|
||||
return id;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub static GLOBAL_CONN_MAP: LazyLock<RwLock<HashMap<String, Channel>>> = LazyLock::new(|| RwLock::new(HashMap::new()));
|
||||
|
||||
#[derive(Clone)]
|
||||
struct TrackedGrpcConnection {
|
||||
channel: Channel,
|
||||
connection_state: GrpcConnectionState,
|
||||
}
|
||||
|
||||
static GLOBAL_TRACKED_CONN_MAP: LazyLock<RwLock<HashMap<String, TrackedGrpcConnection>>> =
|
||||
LazyLock::new(|| RwLock::new(HashMap::new()));
|
||||
static GLOBAL_CONN_CACHE_WRITER: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
|
||||
pub static GLOBAL_ROOT_CERT: LazyLock<RwLock<Option<Vec<u8>>>> = LazyLock::new(|| RwLock::new(None));
|
||||
pub static GLOBAL_MTLS_IDENTITY: LazyLock<RwLock<Option<MtlsIdentityPem>>> = LazyLock::new(|| RwLock::new(None));
|
||||
pub static GLOBAL_OUTBOUND_TLS_GENERATION: LazyLock<AtomicU64> = LazyLock::new(|| AtomicU64::new(0));
|
||||
@@ -304,16 +231,6 @@ pub async fn evict_connection(addr: &str) {
|
||||
|
||||
/// Evict a stale/dead connection from the global connection cache with an explicit log level.
|
||||
pub async fn evict_connection_with_log_level(addr: &str, log_level: ConnectionEvictionLogLevel) {
|
||||
let _ = remove_connection_with_log_level(addr, log_level).await;
|
||||
}
|
||||
|
||||
/// Remove a cached connection and report whether it existed. Transport owners use this to
|
||||
/// attach a rebuild reason only to real cache transitions.
|
||||
pub async fn remove_connection_with_log_level(addr: &str, log_level: ConnectionEvictionLogLevel) -> bool {
|
||||
// Writes are rare and serialized so the optimized tracked cache and public compatibility map
|
||||
// cannot publish different channel clones for one key.
|
||||
let _writer = GLOBAL_CONN_CACHE_WRITER.lock().await;
|
||||
GLOBAL_TRACKED_CONN_MAP.write().await.remove(addr);
|
||||
let removed = GLOBAL_CONN_MAP.write().await.remove(addr);
|
||||
if removed.is_some() {
|
||||
match log_level {
|
||||
@@ -337,7 +254,6 @@ pub async fn remove_connection_with_log_level(addr: &str, log_level: ConnectionE
|
||||
}
|
||||
}
|
||||
}
|
||||
removed.is_some()
|
||||
}
|
||||
|
||||
/// Get a cached gRPC connection for the given address.
|
||||
@@ -345,37 +261,8 @@ pub async fn cached_connection(addr: &str) -> Option<Channel> {
|
||||
GLOBAL_CONN_MAP.read().await.get(addr).cloned()
|
||||
}
|
||||
|
||||
/// Get a cached channel together with its shared physical-connection state, when tracked.
|
||||
pub async fn cached_connection_with_state(addr: &str) -> Option<(Channel, Option<GrpcConnectionState>)> {
|
||||
if let Some(cached) = GLOBAL_TRACKED_CONN_MAP.read().await.get(addr).cloned() {
|
||||
return Some((cached.channel, Some(cached.connection_state)));
|
||||
}
|
||||
// A miss may race a low-frequency publish/evict. Serialize only this fallback, recheck the
|
||||
// tracked map, then consult legacy entries injected through the compatibility API.
|
||||
let _writer = GLOBAL_CONN_CACHE_WRITER.lock().await;
|
||||
if let Some(cached) = GLOBAL_TRACKED_CONN_MAP.read().await.get(addr).cloned() {
|
||||
return Some((cached.channel, Some(cached.connection_state)));
|
||||
}
|
||||
cached_connection(addr).await.map(|channel| (channel, None))
|
||||
}
|
||||
|
||||
/// Cache a gRPC connection for the given address.
|
||||
pub async fn cache_connection(addr: String, channel: Channel) {
|
||||
let _writer = GLOBAL_CONN_CACHE_WRITER.lock().await;
|
||||
GLOBAL_TRACKED_CONN_MAP.write().await.remove(&addr);
|
||||
GLOBAL_CONN_MAP.write().await.insert(addr, channel);
|
||||
}
|
||||
|
||||
/// Cache a gRPC channel and its physical-connection state as one coherent entry.
|
||||
pub async fn cache_connection_with_state(addr: String, channel: Channel, connection_state: GrpcConnectionState) {
|
||||
let _writer = GLOBAL_CONN_CACHE_WRITER.lock().await;
|
||||
GLOBAL_TRACKED_CONN_MAP.write().await.insert(
|
||||
addr.clone(),
|
||||
TrackedGrpcConnection {
|
||||
channel: channel.clone(),
|
||||
connection_state,
|
||||
},
|
||||
);
|
||||
GLOBAL_CONN_MAP.write().await.insert(addr, channel);
|
||||
}
|
||||
|
||||
@@ -392,8 +279,6 @@ pub async fn has_cached_connection(addr: &str) -> bool {
|
||||
|
||||
/// Clear all cached connections. Useful for full cluster reset/recovery.
|
||||
pub async fn clear_all_connections() {
|
||||
let _writer = GLOBAL_CONN_CACHE_WRITER.lock().await;
|
||||
GLOBAL_TRACKED_CONN_MAP.write().await.clear();
|
||||
let mut map = GLOBAL_CONN_MAP.write().await;
|
||||
let count = map.len();
|
||||
map.clear();
|
||||
@@ -478,46 +363,4 @@ mod tls_snapshot_tests {
|
||||
assert!(!debug.contains("PRIVATE-KEY-MARKER"));
|
||||
assert!(debug.contains("[REDACTED]"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn conditional_connection_removal_preserves_legacy_evict_api() {
|
||||
let key = "http://conditional-eviction-test.invalid";
|
||||
cache_connection(
|
||||
key.to_owned(),
|
||||
tonic::transport::Endpoint::from_static("http://127.0.0.1:1").connect_lazy(),
|
||||
)
|
||||
.await;
|
||||
assert!(remove_connection_with_log_level(key, ConnectionEvictionLogLevel::Debug).await);
|
||||
assert!(!remove_connection_with_log_level(key, ConnectionEvictionLogLevel::Debug).await);
|
||||
let _: () = evict_connection_with_log_level(key, ConnectionEvictionLogLevel::Debug).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cached_connection_keeps_channel_and_transport_state_coherent() {
|
||||
let key = "http://tracked-connection-test.invalid";
|
||||
let channel = tonic::transport::Endpoint::from_static("http://127.0.0.1:1").connect_lazy();
|
||||
let state = GrpcConnectionState::new();
|
||||
assert!(state.snapshot().is_none());
|
||||
let first = state.record_connected();
|
||||
cache_connection_with_state(key.to_owned(), channel, state.clone()).await;
|
||||
|
||||
let (_, cached_state) = cached_connection_with_state(key).await.expect("tracked cache entry");
|
||||
let cached_state = cached_state.expect("tracked cache state");
|
||||
assert_eq!(cached_state.snapshot(), Some(first));
|
||||
let second = state.record_connected();
|
||||
assert_eq!(cached_state.snapshot(), Some(second));
|
||||
assert_eq!(first.channel_id, second.channel_id);
|
||||
assert_ne!(first.generation, second.generation);
|
||||
|
||||
cache_connection(
|
||||
key.to_owned(),
|
||||
tonic::transport::Endpoint::from_static("http://127.0.0.1:2").connect_lazy(),
|
||||
)
|
||||
.await;
|
||||
let (_, replacement_state) = cached_connection_with_state(key).await.expect("legacy replacement entry");
|
||||
assert!(replacement_state.is_none(), "legacy replacement must remove stale tracked state");
|
||||
|
||||
assert!(remove_connection_with_log_level(key, ConnectionEvictionLogLevel::Debug).await);
|
||||
assert!(cached_connection_with_state(key).await.is_none());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,6 +25,7 @@ keywords = ["configuration", "settings", "management", "rustfs", "Minio"]
|
||||
categories = ["web-programming", "development-tools", "config"]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
const-str = { workspace = true, optional = true, features = ["std", "proc"] }
|
||||
serde = { workspace = true, optional = true, features = ["derive"] }
|
||||
serde_json = { workspace = true, optional = true, features = ["raw_value"] }
|
||||
@@ -34,9 +35,9 @@ workspace = true
|
||||
|
||||
[features]
|
||||
default = ["constants"]
|
||||
hotpath = []
|
||||
hotpath-alloc = ["hotpath"]
|
||||
hotpath-cpu = ["hotpath"]
|
||||
hotpath = ["hotpath/hotpath"]
|
||||
hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"]
|
||||
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"]
|
||||
audit = ["dep:const-str", "constants"]
|
||||
constants = ["dep:const-str"]
|
||||
notify = ["dep:const-str", "constants"]
|
||||
|
||||
@@ -121,11 +121,6 @@ when restoring `/rustfs/console` for a custom OEM build.
|
||||
- `RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS` (canonical)
|
||||
- `RUSTFS_SCANNER_CYCLE_MAX_OBJECTS` (canonical)
|
||||
- `RUSTFS_SCANNER_CYCLE_MAX_DIRECTORIES` (canonical)
|
||||
- `RUSTFS_SCANNER_BUCKET_QUANTUM_DURATION_SECS` (canonical)
|
||||
- `RUSTFS_SCANNER_BUCKET_QUANTUM_OBJECTS` (canonical)
|
||||
- `RUSTFS_SCANNER_BUCKET_QUANTUM_DIRECTORIES` (canonical)
|
||||
- `RUSTFS_SCANNER_BUCKET_QUANTUM_ENTRIES` (canonical)
|
||||
- `RUSTFS_SCANNER_BUCKET_QUANTUM_NAME_BYTES` (canonical)
|
||||
- `RUSTFS_SCANNER_MAX_CONCURRENT_SET_SCANS` (canonical)
|
||||
- `RUSTFS_SCANNER_MAX_CONCURRENT_DISK_SCANS` (canonical)
|
||||
- `RUSTFS_SCANNER_WALKERS_PER_DISK` (canonical)
|
||||
@@ -135,12 +130,6 @@ Scanner cycle budget controls:
|
||||
- When `RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS` is unset, the finite default is 1800 seconds (30 minutes), matching the scanner benchmark guidance.
|
||||
- An explicit `0` preserves the compatibility behavior of an unbounded runtime budget. Object and directory budgets likewise remain unbounded when explicitly set to `0`.
|
||||
|
||||
Scanner bucket quantum controls:
|
||||
|
||||
- `RUSTFS_SCANNER_BUCKET_QUANTUM_DURATION_SECS`, `RUSTFS_SCANNER_BUCKET_QUANTUM_OBJECTS`, `RUSTFS_SCANNER_BUCKET_QUANTUM_DIRECTORIES`, `RUSTFS_SCANNER_BUCKET_QUANTUM_ENTRIES`, and `RUSTFS_SCANNER_BUCKET_QUANTUM_NAME_BYTES` cap one bucket admission. Their persistent keys are `scanner.bucket_quantum_duration`, `scanner.bucket_quantum_objects`, `scanner.bucket_quantum_directories`, `scanner.bucket_quantum_entries`, and `scanner.bucket_quantum_name_bytes`; duration is in whole seconds and the remaining keys are work counts or bytes.
|
||||
- All five defaults are `0` for controlled rollout. A positive value enables that dimension; `0` disables that dimension, and all five zero values disable the bucket quantum. Parent cycle limits remain in force. Invalid unsigned integers and durations outside the platform timer range are rejected; environment overrides take precedence but cannot hide invalid persisted values. Admin scanner status reports each effective value and source under `runtime_config.bucket_quantum_*`, using `bucket_quantum_duration_seconds` for duration.
|
||||
- Disabled defaults are an initial rollout state, not completion of usage-convergence acceptance. Production defaults require matched-topology A/B measurements and the remote progress/fairness checks in [Scanner runtime controls](../../docs/operations/scanner-runtime-controls.md#bucket-quantum-rollout-and-acceptance). These controls do not certify partial usage or acknowledge pending dirty work.
|
||||
|
||||
Scanner recovery retry controls:
|
||||
|
||||
- `RUSTFS_SCANNER_RECOVERY_BACKOFF_CAP_SECS` caps the wait between automatic startup recovery retries. The persistent admin key is `scanner.recovery_backoff_cap`; the environment takes precedence over the persistent value, and the unset default remains 600 seconds (10 minutes). Values must be positive integer seconds within the platform timer range; `0`, malformed values, and overflow are rejected. A valid environment override does not hide an invalid persistent value from `admin config set` validation.
|
||||
@@ -302,8 +291,6 @@ Drive timeout profile preset:
|
||||
|
||||
RustFS production guidance remains direct-attached local POSIX filesystems. Network-mounted filesystems (for example `nfs`, `cifs`, `smb2`, and `fuse.*`) are treated as unsupported by this startup guard. On Windows, endpoints on network drives (UNC shares such as `\\server\share` and mapped drive letters) are treated the same way, even though Windows reports the server's filesystem (usually `NTFS`) for them.
|
||||
|
||||
A local POSIX filesystem bind-mounted into a gVisor sandbox with directfs is a supported storage path. gVisor presents that bind mount as `9p`/`V9FS`, so the host filesystem type is not visible; the startup guard recognizes mountinfo options `trans=fd` and `directfs` and does not treat the mount as unsupported. Genuine 9p, virtio, and NFS shares remain unsupported.
|
||||
|
||||
## 📄 License
|
||||
|
||||
This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.
|
||||
|
||||
@@ -35,8 +35,8 @@ pub const DEFAULT_HEALTH_OBJECT_PROGRESS_TIMEOUT_MS: u64 = 30_000;
|
||||
/// Additional time beyond the configured object-lock acquisition deadline.
|
||||
pub const HEALTH_OBJECT_PROGRESS_LOCK_MARGIN_MS: u64 = 5_000;
|
||||
|
||||
/// End-to-end budget for cluster health readiness probes (milliseconds).
|
||||
/// Includes cache/singleflight waiting, storage observations, and lock checks.
|
||||
/// Timeout for cluster health readiness collectors (milliseconds).
|
||||
/// This bounds expensive storage and lock quorum checks used by cluster probes.
|
||||
pub const ENV_HEALTH_CLUSTER_TIMEOUT_MS: &str = "RUSTFS_HEALTH_CLUSTER_TIMEOUT_MS";
|
||||
pub const DEFAULT_HEALTH_CLUSTER_TIMEOUT_MS: u64 = 2000;
|
||||
|
||||
|
||||
@@ -121,17 +121,9 @@ pub const DEFAULT_INTERNODE_RPC_MSGPACK_ONLY: bool = false;
|
||||
pub const ENV_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED: &str = "RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED";
|
||||
pub const DEFAULT_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED: bool = false;
|
||||
|
||||
/// Permit current per-peer capability proof to replace fleet-wide confirmation.
|
||||
///
|
||||
/// This remains ineffective unless [`ENV_INTERNODE_RPC_MSGPACK_ONLY`] is also enabled. Unknown,
|
||||
/// unsupported, revoked, or stale connection generations keep dual-writing JSON.
|
||||
pub const ENV_INTERNODE_RPC_MSGPACK_PEER_CAPABILITY: &str = "RUSTFS_INTERNODE_RPC_MSGPACK_PEER_CAPABILITY";
|
||||
pub const DEFAULT_INTERNODE_RPC_MSGPACK_PEER_CAPABILITY: bool = false;
|
||||
|
||||
// Compile-time invariants: dual-write by default so the base build is byte-for-byte legacy behavior.
|
||||
const _: () = assert!(!DEFAULT_INTERNODE_RPC_MSGPACK_ONLY);
|
||||
const _: () = assert!(!DEFAULT_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED);
|
||||
const _: () = assert!(!DEFAULT_INTERNODE_RPC_MSGPACK_PEER_CAPABILITY);
|
||||
|
||||
/// Require target-bound v2 signatures on every internode gRPC request, rejecting the legacy
|
||||
/// constant-target fallback instead of accepting it (<https://github.com/rustfs/backlog/issues/1327>).
|
||||
@@ -302,14 +294,6 @@ pub const DEFAULT_INTERNODE_BULK_CONCURRENCY_LIMIT: usize = 64;
|
||||
pub const ENV_INTERNODE_BULK_BUFFER_SIZE: &str = "RUSTFS_INTERNODE_BULK_BUFFER_SIZE";
|
||||
pub const DEFAULT_INTERNODE_BULK_BUFFER_SIZE: usize = 1024;
|
||||
|
||||
/// Per-peer limit held through complete bulk-unary response decoding and validation. Zero keeps
|
||||
/// the compatibility behavior with no additional admission gate.
|
||||
pub const ENV_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT: &str = "RUSTFS_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT";
|
||||
pub const DEFAULT_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT: usize = 0;
|
||||
pub const ENV_INTERNODE_BULK_BODY_WAIT_TIMEOUT_MS: &str = "RUSTFS_INTERNODE_BULK_BODY_WAIT_TIMEOUT_MS";
|
||||
pub const DEFAULT_INTERNODE_BULK_BODY_WAIT_TIMEOUT_MS: u64 = 1_000;
|
||||
const _: () = assert!(DEFAULT_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT == 0);
|
||||
|
||||
/// Profile selector for conservative internode HTTP data-plane client tuning.
|
||||
pub const ENV_INTERNODE_HTTP_TUNING_PROFILE: &str = "RUSTFS_INTERNODE_HTTP_TUNING_PROFILE";
|
||||
pub const DEFAULT_INTERNODE_HTTP_TUNING_PROFILE: &str = "legacy";
|
||||
@@ -349,130 +333,8 @@ pub const DEFAULT_INTERNODE_DATA_TRANSPORT: &str = "tcp-http";
|
||||
/// Legacy alias for "tcp-http". Both values select the TCP/HTTP transport backend.
|
||||
pub const INTERNODE_DATA_TRANSPORT_TCP: &str = "tcp";
|
||||
|
||||
/// Internode reads whose data moves over NIXL/UCX (InfiniBand rails or RoCE by
|
||||
/// LID/GID, no IP needed); requests, writes and walks stay on TCP/HTTP. Requires
|
||||
/// a binary built with the `nixl` feature and `RUSTFS_NIXL_ENABLE`; until the
|
||||
/// NIXL agent is up, and for any read it cannot serve, it uses TCP/HTTP.
|
||||
pub const INTERNODE_DATA_TRANSPORT_NIXL: &str = "nixl";
|
||||
|
||||
/// Known internode transport backend names accepted by the config parser.
|
||||
pub const KNOWN_INTERNODE_DATA_TRANSPORT_BACKENDS: &[&str] = &[
|
||||
DEFAULT_INTERNODE_DATA_TRANSPORT,
|
||||
INTERNODE_DATA_TRANSPORT_TCP,
|
||||
INTERNODE_DATA_TRANSPORT_NIXL,
|
||||
];
|
||||
|
||||
/// Check the configured RDMA devices at startup and leave out any that is down,
|
||||
/// slower than its peers, or loses its link during a short watch.
|
||||
pub const ENV_RUSTFS_NIXL_DEVICE_CHECK: &str = "RUSTFS_NIXL_DEVICE_CHECK";
|
||||
pub const DEFAULT_RUSTFS_NIXL_DEVICE_CHECK: bool = true;
|
||||
/// Leave out a device whose link dropped this many times more since boot than
|
||||
/// the quietest device of its link layer; 0 skips this test.
|
||||
pub const ENV_RUSTFS_NIXL_DEVICE_MAX_LINK_DROPS: &str = "RUSTFS_NIXL_DEVICE_MAX_LINK_DROPS";
|
||||
pub const DEFAULT_RUSTFS_NIXL_DEVICE_MAX_LINK_DROPS: u64 = 16;
|
||||
/// How long the startup device check watches the link counters; 0 skips the watch.
|
||||
pub const ENV_RUSTFS_NIXL_DEVICE_FLAP_WINDOW_MS: &str = "RUSTFS_NIXL_DEVICE_FLAP_WINDOW_MS";
|
||||
pub const DEFAULT_RUSTFS_NIXL_DEVICE_FLAP_WINDOW_MS: u64 = 3_000;
|
||||
|
||||
/// NIXL internode reads in flight per remote shard stream.
|
||||
pub const ENV_RUSTFS_NIXL_INTERNODE_DEPTH: &str = "RUSTFS_NIXL_INTERNODE_DEPTH";
|
||||
pub const DEFAULT_RUSTFS_NIXL_INTERNODE_DEPTH: usize = 4;
|
||||
|
||||
/// Whether S3 GETs and PUTs carrying the NIXL headers are served over NIXL/UCX
|
||||
/// S3-over-RDMA (requires a build with the `nixl` feature and libnixl). The
|
||||
/// pipeline settings below (`RUSTFS_CUOBJECT_GET_FANOUT*`, `_INFLIGHT_WRITES`,
|
||||
/// `_SHARD_ARENA_BYTES`) apply to this transport as well.
|
||||
pub const ENV_RUSTFS_NIXL_ENABLE: &str = "RUSTFS_NIXL_ENABLE";
|
||||
pub const DEFAULT_RUSTFS_NIXL_ENABLE: bool = false;
|
||||
/// This node's NIXL agent name; defaults to `rustfs-<first local endpoint host>`.
|
||||
pub const ENV_RUSTFS_NIXL_AGENT_NAME: &str = "RUSTFS_NIXL_AGENT_NAME";
|
||||
/// Comma-separated UCX device names (`mlx5_9,mlx5_10`; NIXL adds the port); empty lets UCX use every device.
|
||||
pub const ENV_RUSTFS_NIXL_UCX_DEVICES: &str = "RUSTFS_NIXL_UCX_DEVICES";
|
||||
/// UCX progress workers of the agent.
|
||||
pub const ENV_RUSTFS_NIXL_NUM_WORKERS: &str = "RUSTFS_NIXL_NUM_WORKERS";
|
||||
pub const DEFAULT_RUSTFS_NIXL_NUM_WORKERS: u32 = 1;
|
||||
/// Staging chunk size per transfer descriptor to a client.
|
||||
pub const ENV_RUSTFS_NIXL_CHUNK_BYTES: &str = "RUSTFS_NIXL_CHUNK_BYTES";
|
||||
pub const DEFAULT_RUSTFS_NIXL_CHUNK_BYTES: usize = 16 * 1024 * 1024;
|
||||
/// Registered staging memory of the agent.
|
||||
pub const ENV_RUSTFS_NIXL_POOL_BYTES: &str = "RUSTFS_NIXL_POOL_BYTES";
|
||||
pub const DEFAULT_RUSTFS_NIXL_POOL_BYTES: usize = 1024 * 1024 * 1024;
|
||||
/// Transfers in flight per node at most.
|
||||
pub const ENV_RUSTFS_NIXL_MAX_INFLIGHT: &str = "RUSTFS_NIXL_MAX_INFLIGHT";
|
||||
pub const DEFAULT_RUSTFS_NIXL_MAX_INFLIGHT: usize = 256;
|
||||
/// A transfer that has not completed after this many seconds fails the request.
|
||||
pub const ENV_RUSTFS_NIXL_TRANSFER_TIMEOUT_SECS: &str = "RUSTFS_NIXL_TRANSFER_TIMEOUT_SECS";
|
||||
pub const DEFAULT_RUSTFS_NIXL_TRANSFER_TIMEOUT_SECS: u64 = 60;
|
||||
|
||||
/// Whether S3 GETs carrying an `x-amz-rdma-token` are served over cuObject S3-over-RDMA
|
||||
/// (requires a build with the `cuobject` feature and libcuobjserver).
|
||||
pub const ENV_RUSTFS_CUOBJECT_ENABLE: &str = "RUSTFS_CUOBJECT_ENABLE";
|
||||
pub const DEFAULT_RUSTFS_CUOBJECT_ENABLE: bool = false;
|
||||
|
||||
/// Comma-separated RDMA-capable IP addresses that get a cuObject server each.
|
||||
/// Defaults to the hosts of this node's own endpoints.
|
||||
pub const ENV_RUSTFS_CUOBJECT_ADDRS: &str = "RUSTFS_CUOBJECT_ADDRS";
|
||||
|
||||
/// UDP/RDMA control port of the cuObject servers.
|
||||
pub const ENV_RUSTFS_CUOBJECT_PORT: &str = "RUSTFS_CUOBJECT_PORT";
|
||||
pub const DEFAULT_RUSTFS_CUOBJECT_PORT: u16 = 18515;
|
||||
|
||||
/// Staging chunk size per RDMA write to a client.
|
||||
pub const ENV_RUSTFS_CUOBJECT_CHUNK_BYTES: &str = "RUSTFS_CUOBJECT_CHUNK_BYTES";
|
||||
pub const DEFAULT_RUSTFS_CUOBJECT_CHUNK_BYTES: usize = 16 * 1024 * 1024;
|
||||
|
||||
/// Registered staging memory per cuObject server.
|
||||
pub const ENV_RUSTFS_CUOBJECT_POOL_BYTES: &str = "RUSTFS_CUOBJECT_POOL_BYTES";
|
||||
pub const DEFAULT_RUSTFS_CUOBJECT_POOL_BYTES: usize = 1024 * 1024 * 1024;
|
||||
|
||||
/// Dynamically connected initiators per cuObject server (concurrent client connections).
|
||||
/// Staging chunks (RDMA writes) kept in flight per GET, so decoding never waits
|
||||
/// for the write of the previous chunk; each costs one chunk of the pool.
|
||||
pub const ENV_RUSTFS_CUOBJECT_INFLIGHT_WRITES: &str = "RUSTFS_CUOBJECT_INFLIGHT_WRITES";
|
||||
pub const DEFAULT_RUSTFS_CUOBJECT_INFLIGHT_WRITES: usize = 3;
|
||||
|
||||
/// Staging chunks (RDMA reads from the client buffer) kept in flight per
|
||||
/// RDMA PUT body, so ingest never waits on one chunk's round trip at a time;
|
||||
/// each costs one chunk of the pool.
|
||||
pub const ENV_RUSTFS_CUOBJECT_PUT_READ_AHEAD: &str = "RUSTFS_CUOBJECT_PUT_READ_AHEAD";
|
||||
pub const DEFAULT_RUSTFS_CUOBJECT_PUT_READ_AHEAD: usize = 4;
|
||||
|
||||
/// Concurrent erasure-block-aligned segments one cuObject GET is split into
|
||||
/// on the server (each segment decodes and RDMA-writes its own stripes), so a
|
||||
/// single stream keeps several stripes in flight. 1 disables the fan-out.
|
||||
pub const ENV_RUSTFS_CUOBJECT_GET_FANOUT: &str = "RUSTFS_CUOBJECT_GET_FANOUT";
|
||||
pub const DEFAULT_RUSTFS_CUOBJECT_GET_FANOUT: usize = 8;
|
||||
|
||||
/// Smallest segment worth its own producer (rounded up to whole erasure
|
||||
/// blocks): each segment pays a reader setup per part it touches, so short
|
||||
/// requests stay sequential.
|
||||
pub const ENV_RUSTFS_CUOBJECT_GET_FANOUT_MIN_BYTES: &str = "RUSTFS_CUOBJECT_GET_FANOUT_MIN_BYTES";
|
||||
pub const DEFAULT_RUSTFS_CUOBJECT_GET_FANOUT_MIN_BYTES: u64 = 192 * 1024 * 1024;
|
||||
|
||||
/// Segment producers a node runs across all its cuObject GETs: each new GET
|
||||
/// gets a fair share (budget / open GETs, at least 1, at most the fan-out)
|
||||
/// and never waits for one, so many-stream load degrades to sequential GETs
|
||||
/// instead of a reader-setup storm.
|
||||
pub const ENV_RUSTFS_CUOBJECT_GET_FANOUT_BUDGET: &str = "RUSTFS_CUOBJECT_GET_FANOUT_BUDGET";
|
||||
pub const DEFAULT_RUSTFS_CUOBJECT_GET_FANOUT_BUDGET: usize = 16;
|
||||
|
||||
/// Registered host memory the erasure decoder reads shards into so a
|
||||
/// cuObject GET can RDMA-write data shards straight from those buffers
|
||||
/// (zero-copy emit); 0 keeps the staging copy. Blocks are
|
||||
/// `RUSTFS_ERASURE_SHARD_BYTES` (4 MiB when unset); objects with larger
|
||||
/// shards fall back to heap buffers and the staging copy.
|
||||
pub const ENV_RUSTFS_CUOBJECT_SHARD_ARENA_BYTES: &str = "RUSTFS_CUOBJECT_SHARD_ARENA_BYTES";
|
||||
pub const DEFAULT_RUSTFS_CUOBJECT_SHARD_ARENA_BYTES: usize = 0;
|
||||
|
||||
pub const ENV_RUSTFS_CUOBJECT_NUM_DCIS: &str = "RUSTFS_CUOBJECT_NUM_DCIS";
|
||||
pub const DEFAULT_RUSTFS_CUOBJECT_NUM_DCIS: u32 = 128;
|
||||
|
||||
/// Complete cuObject transfers through the library's asynchronous mode: the
|
||||
/// write is submitted with a handle and the task waits on the channel's
|
||||
/// completion descriptor instead of holding a blocking thread for the whole
|
||||
/// transfer. `false` keeps one blocking library call per transfer.
|
||||
pub const ENV_RUSTFS_CUOBJECT_ASYNC_TRANSFERS: &str = "RUSTFS_CUOBJECT_ASYNC_TRANSFERS";
|
||||
pub const DEFAULT_RUSTFS_CUOBJECT_ASYNC_TRANSFERS: bool = true;
|
||||
pub const KNOWN_INTERNODE_DATA_TRANSPORT_BACKENDS: &[&str] = &[DEFAULT_INTERNODE_DATA_TRANSPORT, INTERNODE_DATA_TRANSPORT_TCP];
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
@@ -517,12 +379,8 @@ mod tests {
|
||||
fn internode_channel_isolation_defaults_and_env_names() {
|
||||
// The isolation-off default is asserted at compile time next to the constant definition.
|
||||
assert_eq!(DEFAULT_INTERNODE_BULK_CHANNELS, 2);
|
||||
assert_eq!(DEFAULT_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT, 0);
|
||||
assert_eq!(DEFAULT_INTERNODE_BULK_BODY_WAIT_TIMEOUT_MS, 1_000);
|
||||
assert_eq!(ENV_INTERNODE_CHANNEL_ISOLATION, "RUSTFS_INTERNODE_CHANNEL_ISOLATION");
|
||||
assert_eq!(ENV_INTERNODE_BULK_CHANNELS, "RUSTFS_INTERNODE_BULK_CHANNELS");
|
||||
assert_eq!(ENV_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT, "RUSTFS_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT");
|
||||
assert_eq!(ENV_INTERNODE_BULK_BODY_WAIT_TIMEOUT_MS, "RUSTFS_INTERNODE_BULK_BODY_WAIT_TIMEOUT_MS");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -533,7 +391,6 @@ mod tests {
|
||||
ENV_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED,
|
||||
"RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED"
|
||||
);
|
||||
assert_eq!(ENV_INTERNODE_RPC_MSGPACK_PEER_CAPABILITY, "RUSTFS_INTERNODE_RPC_MSGPACK_PEER_CAPABILITY");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -608,6 +465,6 @@ mod tests {
|
||||
assert_eq!(ENV_RUSTFS_INTERNODE_DATA_TRANSPORT, "RUSTFS_INTERNODE_DATA_TRANSPORT");
|
||||
assert_eq!(DEFAULT_INTERNODE_DATA_TRANSPORT, "tcp-http");
|
||||
assert_eq!(INTERNODE_DATA_TRANSPORT_TCP, "tcp");
|
||||
assert_eq!(KNOWN_INTERNODE_DATA_TRANSPORT_BACKENDS, &["tcp-http", "tcp", "nixl"]);
|
||||
assert_eq!(KNOWN_INTERNODE_DATA_TRANSPORT_BACKENDS, &["tcp-http", "tcp"]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -152,33 +152,6 @@ pub const DEFAULT_SHARD_INTEGRITY_WRITE: bool = false;
|
||||
pub const ENV_SHARD_INTEGRITY_FLEET_CONFIRMED: &str = "RUSTFS_SHARD_INTEGRITY_FLEET_CONFIRMED";
|
||||
pub const DEFAULT_SHARD_INTEGRITY_FLEET_CONFIRMED: bool = false;
|
||||
|
||||
/// Request parsing switch for the per-object erasure-code block-size hint.
|
||||
/// The default keeps the feature inert during rolling upgrades.
|
||||
pub const ENV_EC_BLOCK_SIZE_HINT_ENABLE: &str = "RUSTFS_EC_BLOCK_SIZE_HINT_ENABLE";
|
||||
pub const DEFAULT_EC_BLOCK_SIZE_HINT_ENABLE: bool = false;
|
||||
|
||||
/// Operator confirmation that every writer and reader in the fleet understands
|
||||
/// the persisted erasure block-size geometry.
|
||||
pub const ENV_EC_BLOCK_SIZE_HINT_FLEET_CONFIRMED: &str = "RUSTFS_EC_BLOCK_SIZE_HINT_FLEET_CONFIRMED";
|
||||
pub const DEFAULT_EC_BLOCK_SIZE_HINT_FLEET_CONFIRMED: bool = false;
|
||||
|
||||
/// Write new EC shards with a bounded bitrot frame size so Range GETs do not
|
||||
/// inherit the full persisted layout block as their physical read quantum.
|
||||
/// The marker is only emitted after the fleet confirmation gate is enabled;
|
||||
/// readers without this feature continue to consume legacy objects unchanged.
|
||||
pub const ENV_EC_READ_QUANTUM_ENABLE: &str = "RUSTFS_EC_READ_QUANTUM_ENABLE";
|
||||
pub const DEFAULT_EC_READ_QUANTUM_ENABLE: bool = false;
|
||||
|
||||
/// Operator confirmation that every reader and writer understands the
|
||||
/// persisted EC read-quantum marker and its per-frame bitrot layout.
|
||||
pub const ENV_EC_READ_QUANTUM_FLEET_CONFIRMED: &str = "RUSTFS_EC_READ_QUANTUM_FLEET_CONFIRMED";
|
||||
pub const DEFAULT_EC_READ_QUANTUM_FLEET_CONFIRMED: bool = false;
|
||||
|
||||
const _: () = assert!(!DEFAULT_EC_BLOCK_SIZE_HINT_ENABLE);
|
||||
const _: () = assert!(!DEFAULT_EC_BLOCK_SIZE_HINT_FLEET_CONFIRMED);
|
||||
const _: () = assert!(!DEFAULT_EC_READ_QUANTUM_ENABLE);
|
||||
const _: () = assert!(!DEFAULT_EC_READ_QUANTUM_FLEET_CONFIRMED);
|
||||
|
||||
const _: () = assert!(!DEFAULT_SHARD_INTEGRITY_WRITE);
|
||||
const _: () = assert!(!DEFAULT_SHARD_INTEGRITY_FLEET_CONFIRMED);
|
||||
|
||||
|
||||
@@ -48,7 +48,7 @@ pub const ENV_RUNTIME_DIAL9_S3_PREFIX: &str = "RUSTFS_RUNTIME_DIAL9_S3_PREFIX";
|
||||
pub const DEFAULT_WORKER_THREADS: usize = 16;
|
||||
pub const DEFAULT_MAX_BLOCKING_THREADS: usize = 1024;
|
||||
pub const DEFAULT_THREAD_PRINT_ENABLED: bool = false;
|
||||
pub const DEFAULT_THREAD_STACK_SIZE: usize = 2 * MI_B; // 2 MiB
|
||||
pub const DEFAULT_THREAD_STACK_SIZE: usize = MI_B; // 1 MiB
|
||||
pub const DEFAULT_THREAD_KEEP_ALIVE: u64 = 60; // seconds
|
||||
pub const DEFAULT_GLOBAL_QUEUE_INTERVAL: u32 = 31;
|
||||
pub const DEFAULT_THREAD_NAME: &str = "rustfs-worker";
|
||||
|
||||
@@ -44,21 +44,6 @@ pub const SCANNER_CYCLE_MAX_OBJECTS: &str = "cycle_max_objects";
|
||||
/// Scanner config key capping directories entered by one cycle.
|
||||
pub const SCANNER_CYCLE_MAX_DIRECTORIES: &str = "cycle_max_directories";
|
||||
|
||||
/// Per-bucket admission limit in seconds; zero disables this dimension.
|
||||
pub const SCANNER_BUCKET_QUANTUM_DURATION: &str = "bucket_quantum_duration";
|
||||
|
||||
/// Per-bucket admission limit in objects; zero disables this dimension.
|
||||
pub const SCANNER_BUCKET_QUANTUM_OBJECTS: &str = "bucket_quantum_objects";
|
||||
|
||||
/// Per-bucket admission limit in directories; zero disables this dimension.
|
||||
pub const SCANNER_BUCKET_QUANTUM_DIRECTORIES: &str = "bucket_quantum_directories";
|
||||
|
||||
/// Per-bucket admission limit in raw entries; zero disables this dimension.
|
||||
pub const SCANNER_BUCKET_QUANTUM_ENTRIES: &str = "bucket_quantum_entries";
|
||||
|
||||
/// Per-bucket admission limit in raw entry-name bytes; zero disables this dimension.
|
||||
pub const SCANNER_BUCKET_QUANTUM_NAME_BYTES: &str = "bucket_quantum_name_bytes";
|
||||
|
||||
/// Scanner config key setting the periodic bitrot scan cycle in seconds.
|
||||
pub const SCANNER_BITROT_CYCLE: &str = "bitrot_cycle";
|
||||
|
||||
@@ -108,11 +93,6 @@ pub const SCANNER_KEYS: &[&str] = &[
|
||||
SCANNER_CYCLE_MAX_DURATION,
|
||||
SCANNER_CYCLE_MAX_OBJECTS,
|
||||
SCANNER_CYCLE_MAX_DIRECTORIES,
|
||||
SCANNER_BUCKET_QUANTUM_DURATION,
|
||||
SCANNER_BUCKET_QUANTUM_OBJECTS,
|
||||
SCANNER_BUCKET_QUANTUM_DIRECTORIES,
|
||||
SCANNER_BUCKET_QUANTUM_ENTRIES,
|
||||
SCANNER_BUCKET_QUANTUM_NAME_BYTES,
|
||||
SCANNER_BITROT_CYCLE,
|
||||
SCANNER_IDLE_MODE,
|
||||
SCANNER_CACHE_SAVE_TIMEOUT,
|
||||
@@ -161,36 +141,6 @@ pub const ENV_SCANNER_CYCLE_MAX_OBJECTS: &str = "RUSTFS_SCANNER_CYCLE_MAX_OBJECT
|
||||
/// - Example: `export RUSTFS_SCANNER_CYCLE_MAX_DIRECTORIES=100000`
|
||||
pub const ENV_SCANNER_CYCLE_MAX_DIRECTORIES: &str = "RUSTFS_SCANNER_CYCLE_MAX_DIRECTORIES";
|
||||
|
||||
/// Per-bucket admission limit in seconds; zero disables this dimension.
|
||||
pub const ENV_SCANNER_BUCKET_QUANTUM_DURATION_SECS: &str = "RUSTFS_SCANNER_BUCKET_QUANTUM_DURATION_SECS";
|
||||
|
||||
/// Disabled until controlled-fleet acceptance selects production limits.
|
||||
pub const DEFAULT_SCANNER_BUCKET_QUANTUM_DURATION_SECS: u64 = 0;
|
||||
|
||||
/// Per-bucket admission limit in objects; zero disables this dimension.
|
||||
pub const ENV_SCANNER_BUCKET_QUANTUM_OBJECTS: &str = "RUSTFS_SCANNER_BUCKET_QUANTUM_OBJECTS";
|
||||
|
||||
/// Disabled until controlled-fleet acceptance selects production limits.
|
||||
pub const DEFAULT_SCANNER_BUCKET_QUANTUM_OBJECTS: u64 = 0;
|
||||
|
||||
/// Per-bucket admission limit in directories; zero disables this dimension.
|
||||
pub const ENV_SCANNER_BUCKET_QUANTUM_DIRECTORIES: &str = "RUSTFS_SCANNER_BUCKET_QUANTUM_DIRECTORIES";
|
||||
|
||||
/// Disabled until controlled-fleet acceptance selects production limits.
|
||||
pub const DEFAULT_SCANNER_BUCKET_QUANTUM_DIRECTORIES: u64 = 0;
|
||||
|
||||
/// Per-bucket admission limit in raw entries; zero disables this dimension.
|
||||
pub const ENV_SCANNER_BUCKET_QUANTUM_ENTRIES: &str = "RUSTFS_SCANNER_BUCKET_QUANTUM_ENTRIES";
|
||||
|
||||
/// Disabled until controlled-fleet acceptance selects production limits.
|
||||
pub const DEFAULT_SCANNER_BUCKET_QUANTUM_ENTRIES: u64 = 0;
|
||||
|
||||
/// Per-bucket admission limit in raw entry-name bytes; zero disables this dimension.
|
||||
pub const ENV_SCANNER_BUCKET_QUANTUM_NAME_BYTES: &str = "RUSTFS_SCANNER_BUCKET_QUANTUM_NAME_BYTES";
|
||||
|
||||
/// Disabled until controlled-fleet acceptance selects production limits.
|
||||
pub const DEFAULT_SCANNER_BUCKET_QUANTUM_NAME_BYTES: u64 = 0;
|
||||
|
||||
/// Environment variable that selects the scanner speed preset.
|
||||
/// Valid values: `fastest`, `fast`, `default`, `slow`, `slowest`.
|
||||
/// Controls the sleep factor, maximum sleep duration, and cycle interval.
|
||||
|
||||
@@ -26,11 +26,12 @@ categories = ["web-programming", "development-tools", "data-structures", "securi
|
||||
|
||||
[features]
|
||||
default = []
|
||||
hotpath = []
|
||||
hotpath-alloc = ["hotpath"]
|
||||
hotpath-cpu = ["hotpath"]
|
||||
hotpath = ["hotpath/hotpath"]
|
||||
hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"]
|
||||
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
base64-simd = { workspace = true }
|
||||
hmac = { workspace = true }
|
||||
rand = { workspace = true, features = ["serde"] }
|
||||
|
||||
@@ -29,6 +29,7 @@ documentation = "https://docs.rs/rustfs-crypto/latest/rustfs_crypto/"
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
aes-gcm = { workspace = true, optional = true, features = ["rand_core"] }
|
||||
argon2 = { workspace = true, optional = true }
|
||||
chacha20poly1305 = { workspace = true, optional = true }
|
||||
@@ -36,7 +37,7 @@ jsonwebtoken = { workspace = true, features = ["aws_lc_rs"] }
|
||||
base64-simd = { workspace = true }
|
||||
pbkdf2 = { workspace = true, optional = true }
|
||||
rand = { workspace = true, features = ["serde"] }
|
||||
rsa = { workspace = true }
|
||||
rsa = { workspace = true, features = ["sha2"] }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
sha2 = { workspace = true, optional = true }
|
||||
thiserror.workspace = true
|
||||
@@ -49,9 +50,9 @@ time = { workspace = true, features = ["parsing", "formatting", "macros", "serde
|
||||
|
||||
[features]
|
||||
default = ["crypto", "fips"]
|
||||
hotpath = []
|
||||
hotpath-alloc = ["hotpath"]
|
||||
hotpath-cpu = ["hotpath"]
|
||||
hotpath = ["hotpath/hotpath"]
|
||||
hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"]
|
||||
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"]
|
||||
fips = []
|
||||
crypto = [
|
||||
"dep:aes-gcm",
|
||||
|
||||
@@ -18,6 +18,9 @@ use crate::Error;
|
||||
use crate::jwt::Claims;
|
||||
|
||||
pub fn decode(token: &str, token_secret: &[u8]) -> Result<TokenData<Claims>, Error> {
|
||||
let key = DecodingKey::from_secret(token_secret)?;
|
||||
Ok(jsonwebtoken::decode(token, &key, &Validation::new(Algorithm::HS512))?)
|
||||
Ok(jsonwebtoken::decode(
|
||||
token,
|
||||
&DecodingKey::from_secret(token_secret),
|
||||
&Validation::new(Algorithm::HS512),
|
||||
)?)
|
||||
}
|
||||
|
||||
@@ -18,6 +18,9 @@ use crate::Error;
|
||||
use crate::jwt::Claims;
|
||||
|
||||
pub fn encode(token_secret: &[u8], claims: &Claims) -> Result<String, Error> {
|
||||
let key = EncodingKey::from_secret(token_secret)?;
|
||||
Ok(jsonwebtoken::encode(&Header::new(Algorithm::HS512), claims, &key)?)
|
||||
Ok(jsonwebtoken::encode(
|
||||
&Header::new(Algorithm::HS512),
|
||||
claims,
|
||||
&EncodingKey::from_secret(token_secret),
|
||||
)?)
|
||||
}
|
||||
|
||||
@@ -13,9 +13,9 @@
|
||||
// limitations under the License.
|
||||
|
||||
use rsa::{
|
||||
Pkcs1v15Encrypt, PrivateKey, PublicKey,
|
||||
Pkcs1v15Encrypt, RsaPrivateKey, RsaPublicKey,
|
||||
pkcs8::{DecodePrivateKey, DecodePublicKey},
|
||||
pss::{Signature, SigningKey, VerifyingKey},
|
||||
pss::{BlindedSigningKey, Signature, VerifyingKey},
|
||||
sha2::Sha256,
|
||||
signature::{RandomizedSigner, Verifier},
|
||||
traits::PublicKeyParts,
|
||||
@@ -37,7 +37,7 @@ pub struct Token {
|
||||
pub fn gencode(token: &Token, key: &str) -> Result<String> {
|
||||
let data = serde_json::to_vec(token)?;
|
||||
let mut rng = rand::rng();
|
||||
let public_key = PublicKey::from_public_key_pem(key).map_err(Error::other)?;
|
||||
let public_key = RsaPublicKey::from_public_key_pem(key).map_err(Error::other)?;
|
||||
let encrypted_data = public_key.encrypt(&mut rng, Pkcs1v15Encrypt, &data).map_err(Error::other)?;
|
||||
Ok(base64_simd::URL_SAFE_NO_PAD.encode_to_string(&encrypted_data))
|
||||
}
|
||||
@@ -51,7 +51,7 @@ pub fn parse(token: &str, key: &str) -> Result<Token> {
|
||||
let encrypted_data = base64_simd::URL_SAFE_NO_PAD
|
||||
.decode_to_vec(token.as_bytes())
|
||||
.map_err(Error::other)?;
|
||||
let private_key = PrivateKey::from_pkcs8_pem(key).map_err(Error::other)?;
|
||||
let private_key = RsaPrivateKey::from_pkcs8_pem(key).map_err(Error::other)?;
|
||||
let decrypted_data = private_key.decrypt(Pkcs1v15Encrypt, &encrypted_data).map_err(Error::other)?;
|
||||
serde_json::from_slice(&decrypted_data).map_err(Error::other)
|
||||
}
|
||||
@@ -63,8 +63,8 @@ pub fn parse(token: &str, key: &str) -> Result<Token> {
|
||||
pub fn sign_license_token(token: &Token, private_key_pem: &str) -> Result<String> {
|
||||
let payload = serde_json::to_vec(token)?;
|
||||
let mut rng = rand::rng();
|
||||
let private_key = PrivateKey::from_pkcs8_pem(private_key_pem).map_err(Error::other)?;
|
||||
let signing_key = SigningKey::<Sha256>::new(private_key);
|
||||
let private_key = RsaPrivateKey::from_pkcs8_pem(private_key_pem).map_err(Error::other)?;
|
||||
let signing_key = BlindedSigningKey::<Sha256>::new(private_key);
|
||||
let signature: Signature = signing_key.try_sign_with_rng(&mut rng, &payload).map_err(Error::other)?;
|
||||
let signature: Box<[u8]> = signature.into();
|
||||
|
||||
@@ -80,7 +80,7 @@ pub fn parse_signed_license_token(token: &str, public_key_pem: &str) -> Result<T
|
||||
let signed_payload = base64_simd::URL_SAFE_NO_PAD
|
||||
.decode_to_vec(token.as_bytes())
|
||||
.map_err(Error::other)?;
|
||||
let public_key = PublicKey::from_public_key_pem(public_key_pem).map_err(Error::other)?;
|
||||
let public_key = RsaPublicKey::from_public_key_pem(public_key_pem).map_err(Error::other)?;
|
||||
let signature_len = public_key.size();
|
||||
|
||||
if signed_payload.len() <= signature_len {
|
||||
@@ -103,7 +103,7 @@ pub fn parse_license_with_public_key(license: &str, public_key: &str) -> Result<
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rsa::{
|
||||
PrivateKey,
|
||||
RsaPrivateKey,
|
||||
pkcs8::{EncodePrivateKey, EncodePublicKey, LineEnding},
|
||||
};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
@@ -112,8 +112,8 @@ mod tests {
|
||||
fn test_sign_license_token_and_parse_signed_license_token() {
|
||||
let mut rng = rand::rng();
|
||||
let bits = 2048;
|
||||
let private_key = PrivateKey::new(&mut rng, bits).expect("Failed to generate private key");
|
||||
let public_key = PublicKey::from(&private_key);
|
||||
let private_key = RsaPrivateKey::new(&mut rng, bits).expect("Failed to generate private key");
|
||||
let public_key = RsaPublicKey::from(&private_key);
|
||||
|
||||
let private_key_pem = private_key
|
||||
.to_pkcs8_pem(LineEnding::LF)
|
||||
@@ -144,8 +144,8 @@ mod tests {
|
||||
fn test_legacy_gencode_and_parse_roundtrip() {
|
||||
let mut rng = rand::rng();
|
||||
let bits = 2048;
|
||||
let private_key = PrivateKey::new(&mut rng, bits).expect("Failed to generate private key");
|
||||
let public_key = PublicKey::from(&private_key);
|
||||
let private_key = RsaPrivateKey::new(&mut rng, bits).expect("Failed to generate private key");
|
||||
let public_key = RsaPublicKey::from(&private_key);
|
||||
|
||||
let private_key_pem = private_key
|
||||
.to_pkcs8_pem(LineEnding::LF)
|
||||
@@ -173,8 +173,8 @@ mod tests {
|
||||
#[test]
|
||||
fn test_parse_signed_license_token_rejects_tampered_payload() {
|
||||
let mut rng = rand::rng();
|
||||
let private_key = PrivateKey::new(&mut rng, 2048).expect("Failed to generate private key");
|
||||
let public_key = PublicKey::from(&private_key);
|
||||
let private_key = RsaPrivateKey::new(&mut rng, 2048).expect("Failed to generate private key");
|
||||
let public_key = RsaPublicKey::from(&private_key);
|
||||
let private_key_pem = private_key
|
||||
.to_pkcs8_pem(LineEnding::LF)
|
||||
.expect("failed to encode private key pem");
|
||||
@@ -206,8 +206,8 @@ mod tests {
|
||||
#[test]
|
||||
fn test_parse_signed_license_token_rejects_invalid_token() {
|
||||
let mut rng = rand::rng();
|
||||
let private_key = PrivateKey::new(&mut rng, 2048).expect("Failed to generate private key");
|
||||
let public_key = PublicKey::from(&private_key);
|
||||
let private_key = RsaPrivateKey::new(&mut rng, 2048).expect("Failed to generate private key");
|
||||
let public_key = RsaPublicKey::from(&private_key);
|
||||
let public_key_pem = public_key
|
||||
.to_public_key_pem(LineEnding::LF)
|
||||
.expect("failed to encode public key pem");
|
||||
|
||||
@@ -1,26 +0,0 @@
|
||||
[package]
|
||||
name = "rustfs-cuobject"
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
rust-version.workspace = true
|
||||
version.workspace = true
|
||||
homepage.workspace = true
|
||||
description = "RustFS staging pool and transfers for NVIDIA cuObject S3-over-RDMA (GPUDirect object loading)"
|
||||
keywords = ["rdma", "cuobject", "gpudirect", "s3"]
|
||||
categories = ["network-programming"]
|
||||
documentation = "https://docs.rs/rustfs-cuobject/latest/rustfs_cuobject/"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
bytes = { workspace = true, optional = true }
|
||||
cuobject = { workspace = true, default-features = false, optional = true }
|
||||
parking_lot = { workspace = true, optional = true }
|
||||
thiserror = { workspace = true, optional = true }
|
||||
tokio = { workspace = true, features = ["net", "rt-multi-thread", "sync", "time"], optional = true }
|
||||
|
||||
[features]
|
||||
# Compiles the cuObject server bindings (needs the NVIDIA cuObject SDK).
|
||||
server = ["dep:bytes", "dep:cuobject", "cuobject/server", "dep:parking_lot", "dep:thiserror", "dep:tokio"]
|
||||
@@ -1,775 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Server side of NVIDIA cuObject S3-over-RDMA.
|
||||
//!
|
||||
//! A client that supports the protocol (NVIDIA cuObject, NIXL's object plugin,
|
||||
//! …) sends an ordinary S3 GET with an `x-amz-rdma-token` header describing a
|
||||
//! registered client buffer (host or GPU memory). Instead of streaming the
|
||||
//! body over HTTP, the server RDMA-writes the object bytes into that buffer
|
||||
//! through NVIDIA's `libcuobjserver` and answers with an empty body plus
|
||||
//! `x-amz-rdma-reply` and `x-amz-rdma-bytes-transferred`.
|
||||
//!
|
||||
//! The library bindings come from the [`cuobject`] crate. This crate owns
|
||||
//! RustFS's pool of registered, page-aligned host staging buffers: the regular
|
||||
//! read path fills a staging chunk and
|
||||
//! [`CuObjectServer::write_to_client_owned`] pushes it.
|
||||
//!
|
||||
//! Everything below needs the `server` feature, which compiles the
|
||||
//! NVIDIA cuObject server bindings; without it the crate is empty, so a
|
||||
//! workspace build does not need the cuObject SDK.
|
||||
#![cfg(feature = "server")]
|
||||
|
||||
use bytes::Bytes;
|
||||
use cuobject::{Channel, Completion, RegisteredBuffer, Server, ServerOptions, SgBuffer, SgEntry, Source};
|
||||
use std::collections::VecDeque;
|
||||
use std::net::IpAddr;
|
||||
use std::num::NonZeroUsize;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use thiserror::Error;
|
||||
use tokio::io::Interest;
|
||||
use tokio::io::unix::AsyncFd;
|
||||
use tokio::sync::Semaphore;
|
||||
|
||||
/// Header carrying the client's RDMA descriptor on a request.
|
||||
pub const RDMA_TOKEN_HEADER: &str = cuobject::TOKEN_HEADER;
|
||||
/// Header carrying the transfer status on the response (HTTP-style code).
|
||||
pub const RDMA_REPLY_HEADER: &str = cuobject::REPLY_HEADER;
|
||||
/// Header carrying the payload byte count moved over RDMA.
|
||||
pub const RDMA_BYTES_TRANSFERRED_HEADER: &str = cuobject::BYTES_TRANSFERRED_HEADER;
|
||||
/// Reply code for a declined transfer (the client then errors or falls back).
|
||||
pub const RDMA_REPLY_NOT_IMPLEMENTED: u16 = cuobject::REPLY_NOT_IMPLEMENTED;
|
||||
/// Largest single `handleGetObject` the library accepts.
|
||||
pub const MAX_RDMA_OP_BYTES: usize = cuobject::MAX_TRANSFER_BYTES;
|
||||
/// Default RDMA control port of the cuObject server.
|
||||
pub const DEFAULT_PORT: u16 = cuobject::DEFAULT_PORT;
|
||||
/// Most ranges one scatter-gather transfer may carry (the library's `max_sge`).
|
||||
pub const MAX_SG_RANGES: usize = cuobject::DEFAULT_MAX_SG_ENTRIES;
|
||||
|
||||
/// Version of the cuObject server headers the bindings were built against, as
|
||||
/// (major, minor).
|
||||
pub fn library_version() -> (u32, u32) {
|
||||
cuobject::header_version()
|
||||
}
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum CuObjectError {
|
||||
#[error("cuobject {op} failed: {source}")]
|
||||
Os {
|
||||
op: &'static str,
|
||||
#[source]
|
||||
source: std::io::Error,
|
||||
},
|
||||
#[error("cuobject rdma token is malformed: {0}")]
|
||||
Token(String),
|
||||
#[error("cuobject rdma token buffer ({buffer} bytes) is smaller than the response ({needed} bytes)")]
|
||||
TokenTooSmall { buffer: u64, needed: u64 },
|
||||
#[error("cuobject rejected the request: {0}")]
|
||||
Invalid(&'static str),
|
||||
#[error("cuobject transfer failed with verbs status {verbs_status}: {source}")]
|
||||
Transfer {
|
||||
verbs_status: i32,
|
||||
#[source]
|
||||
source: std::io::Error,
|
||||
},
|
||||
#[error("cuobject transfer moved {moved} of {expected} bytes")]
|
||||
ShortTransfer { moved: u64, expected: u64 },
|
||||
#[error("cuobject staging pool exhausted")]
|
||||
PoolExhausted,
|
||||
#[error("cuobject library offers no completion descriptor for channel {0}")]
|
||||
AsyncUnavailable(u16),
|
||||
#[error("cuobject read source failed: {0}")]
|
||||
Source(#[source] std::io::Error),
|
||||
}
|
||||
|
||||
impl From<cuobject::Error> for CuObjectError {
|
||||
fn from(error: cuobject::Error) -> Self {
|
||||
match error {
|
||||
cuobject::Error::Library { op, source } => Self::Os { op, source },
|
||||
cuobject::Error::Transfer { verbs_status, source } => Self::Transfer { verbs_status, source },
|
||||
cuobject::Error::InvalidArgument(reason) => Self::Invalid(reason),
|
||||
cuobject::Error::InvalidToken(reason) => Self::Token(reason.to_owned()),
|
||||
cuobject::Error::TokenTooSmall { buffer, needed } => Self::TokenTooSmall { buffer, needed },
|
||||
cuobject::Error::AsyncUnavailable(channel) => Self::AsyncUnavailable(channel),
|
||||
other => Self::Os {
|
||||
op: "cuobject",
|
||||
source: std::io::Error::other(other),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Parsed `x-amz-rdma-token`.
|
||||
///
|
||||
/// Wire format (cuObject RDMA descriptor protocol): colon-separated hex
|
||||
/// fields `addr:size:rkey:lid:dctn:gid_present:gid`; only the leading
|
||||
/// address and size are interpreted here, the library consumes the rest.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RdmaToken {
|
||||
inner: cuobject::RdmaToken,
|
||||
/// Start of the client buffer (a copy of the parsed field).
|
||||
pub remote_addr: u64,
|
||||
/// Length of the client buffer (a copy of the parsed field).
|
||||
pub buffer_len: u64,
|
||||
}
|
||||
|
||||
impl RdmaToken {
|
||||
pub fn parse(raw: &str) -> Result<Self, CuObjectError> {
|
||||
let inner = cuobject::RdmaToken::parse(raw)?;
|
||||
Ok(Self {
|
||||
remote_addr: inner.remote_addr(),
|
||||
buffer_len: inner.buffer_len(),
|
||||
inner,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn as_str(&self) -> &str {
|
||||
self.inner.as_str()
|
||||
}
|
||||
}
|
||||
|
||||
/// Tuning for one server instance.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct CuObjectConfig {
|
||||
pub port: u16,
|
||||
pub num_dcis: i32,
|
||||
pub cq_depth: u32,
|
||||
/// Size of one staging buffer; one RDMA write moves at most this much.
|
||||
pub chunk_bytes: usize,
|
||||
/// Total registered staging memory per server instance.
|
||||
pub pool_bytes: usize,
|
||||
/// Complete transfers through the library's asynchronous mode (submit,
|
||||
/// then wait on the channel's completion descriptor) instead of one
|
||||
/// blocking library call per transfer.
|
||||
pub async_transfers: bool,
|
||||
}
|
||||
|
||||
impl Default for CuObjectConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
port: DEFAULT_PORT,
|
||||
num_dcis: 128,
|
||||
cq_depth: 640,
|
||||
chunk_bytes: 64 * 1024 * 1024,
|
||||
pool_bytes: 1024 * 1024 * 1024,
|
||||
async_transfers: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct Staging {
|
||||
buffer: RegisteredBuffer<Arc<Server>>,
|
||||
free: parking_lot::Mutex<VecDeque<usize>>,
|
||||
permits: Semaphore,
|
||||
chunk_bytes: usize,
|
||||
}
|
||||
|
||||
/// One cuObject server bound to the RDMA device that owns an IP address.
|
||||
pub struct CuObjectServer {
|
||||
server: Arc<Server>,
|
||||
staging: Staging,
|
||||
addr: IpAddr,
|
||||
port: u16,
|
||||
/// One permit per library channel (DC initiator): a transfer waits here
|
||||
/// instead of failing with `EBUSY` when every channel is in use.
|
||||
channels: Arc<Semaphore>,
|
||||
async_transfers: bool,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for CuObjectServer {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("CuObjectServer")
|
||||
.field("addr", &self.addr)
|
||||
.field("port", &self.port)
|
||||
.field("chunk_bytes", &self.staging.chunk_bytes)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
/// Several byte ranges of one [`RegisteredArena`], sent in order as one
|
||||
/// RDMA operation (at most [`MAX_SG_RANGES`]).
|
||||
pub struct ArenaRanges {
|
||||
pub arena: Arc<RegisteredArena>,
|
||||
/// `(local_offset, len)` pairs, in stream order.
|
||||
pub ranges: Vec<(usize, usize)>,
|
||||
}
|
||||
|
||||
impl ArenaRanges {
|
||||
pub fn total_len(&self) -> usize {
|
||||
self.ranges.iter().map(|(_, len)| *len).sum()
|
||||
}
|
||||
}
|
||||
|
||||
/// A byte range of a [`RegisteredArena`] to send from.
|
||||
pub struct ArenaRange {
|
||||
pub arena: Arc<RegisteredArena>,
|
||||
pub local_offset: usize,
|
||||
pub len: usize,
|
||||
}
|
||||
|
||||
/// Host memory registered with one server, owned by the caller (see
|
||||
/// [`CuObjectServer::register_arena`]).
|
||||
pub struct RegisteredArena {
|
||||
buffer: RegisteredBuffer<Arc<Server>>,
|
||||
server: Arc<CuObjectServer>,
|
||||
id: u64,
|
||||
}
|
||||
|
||||
impl RegisteredArena {
|
||||
/// Length of the registered memory.
|
||||
pub fn len(&self) -> usize {
|
||||
self.buffer.len()
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.buffer.is_empty()
|
||||
}
|
||||
|
||||
/// Process-unique id of this registration.
|
||||
pub fn id(&self) -> u64 {
|
||||
self.id
|
||||
}
|
||||
|
||||
/// The server this memory is registered with.
|
||||
pub fn server(&self) -> &Arc<CuObjectServer> {
|
||||
&self.server
|
||||
}
|
||||
|
||||
/// Raw pointer to the start of the registered memory. The memory lives as
|
||||
/// long as this arena; callers hand out disjoint ranges of it and never
|
||||
/// touch a range while a transfer sends it.
|
||||
pub fn as_mut_ptr(&self) -> *mut u8 {
|
||||
self.buffer.as_mut_ptr()
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for RegisteredArena {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("RegisteredArena")
|
||||
.field("id", &self.id)
|
||||
.field("len", &self.len())
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
/// A reserved staging chunk; returned to the pool on drop.
|
||||
///
|
||||
/// The chunk's index is unique while it is reserved, so its range of the
|
||||
/// staging buffer has one owner. Transfers take the chunk by value, so no
|
||||
/// Rust borrow of the range overlaps an RDMA operation on it.
|
||||
pub struct StagingChunk {
|
||||
server: Arc<CuObjectServer>,
|
||||
index: usize,
|
||||
len: usize,
|
||||
}
|
||||
|
||||
impl StagingChunk {
|
||||
pub fn capacity(&self) -> usize {
|
||||
self.server.staging.chunk_bytes
|
||||
}
|
||||
|
||||
/// Bytes currently staged.
|
||||
pub fn len(&self) -> usize {
|
||||
self.len
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.len == 0
|
||||
}
|
||||
|
||||
/// Copies `data` at `self.len()` and advances the staged length.
|
||||
pub fn append(&mut self, data: &[u8]) -> Result<(), CuObjectError> {
|
||||
if self.len + data.len() > self.capacity() {
|
||||
return Err(CuObjectError::Invalid("staging append exceeds chunk"));
|
||||
}
|
||||
self.spare_mut()?[..data.len()].copy_from_slice(data);
|
||||
self.len += data.len();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn clear(&mut self) {
|
||||
self.len = 0;
|
||||
}
|
||||
|
||||
pub fn is_full(&self) -> bool {
|
||||
self.len >= self.capacity()
|
||||
}
|
||||
|
||||
/// The bytes staged so far.
|
||||
#[allow(unsafe_code)]
|
||||
pub fn staged(&self) -> Result<&[u8], CuObjectError> {
|
||||
// SAFETY: the range lies in this chunk (see the type docs); `&self`
|
||||
// excludes `spare_mut`, and transfers own the chunk they move.
|
||||
Ok(unsafe { self.server.staging.buffer.slice(self.start(), self.len)? })
|
||||
}
|
||||
|
||||
/// The unfilled remainder of the chunk, for a reader to fill directly.
|
||||
#[allow(unsafe_code)]
|
||||
pub fn spare_mut(&mut self) -> Result<&mut [u8], CuObjectError> {
|
||||
let len = self.capacity() - self.len;
|
||||
// SAFETY: the range lies in this chunk (see the type docs) and
|
||||
// `&mut self` excludes every other borrow of it.
|
||||
Ok(unsafe { self.server.staging.buffer.slice_mut(self.start() + self.len, len)? })
|
||||
}
|
||||
|
||||
/// Marks `n` more bytes of the spare area as staged.
|
||||
pub fn advance(&mut self, n: usize) -> Result<(), CuObjectError> {
|
||||
if self.len + n > self.capacity() {
|
||||
return Err(CuObjectError::Invalid("staging advance exceeds chunk"));
|
||||
}
|
||||
self.len += n;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn start(&self) -> usize {
|
||||
self.index * self.capacity()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for StagingChunk {
|
||||
fn drop(&mut self) {
|
||||
self.server.staging.free.lock().push_back(self.index);
|
||||
self.server.staging.permits.add_permits(1);
|
||||
}
|
||||
}
|
||||
|
||||
/// The local side of one GET, owned by the transfer until the DMA finished.
|
||||
enum Local {
|
||||
/// A range of the staging buffer, inside a chunk the transfer also owns.
|
||||
Staging { offset: usize, len: usize },
|
||||
/// A range of a caller-owned arena.
|
||||
Arena {
|
||||
arena: Arc<RegisteredArena>,
|
||||
offset: usize,
|
||||
len: usize,
|
||||
},
|
||||
/// Several ranges of a caller-owned arena, sent as one operation.
|
||||
ArenaList {
|
||||
arena: Arc<RegisteredArena>,
|
||||
entries: Vec<SgEntry>,
|
||||
},
|
||||
}
|
||||
|
||||
impl Local {
|
||||
fn len(&self) -> usize {
|
||||
match self {
|
||||
Self::Staging { len, .. } | Self::Arena { len, .. } => *len,
|
||||
Self::ArenaList { entries, .. } => entries.iter().map(|entry| entry.size).sum(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Why an asynchronous GET did not complete.
|
||||
enum AsyncFailure {
|
||||
/// The library has no completion descriptor; nothing was submitted.
|
||||
Unavailable,
|
||||
/// The GET failed, or was never submitted; nothing is in flight.
|
||||
Failed(CuObjectError),
|
||||
/// Waiting failed after submission: the NIC may still read the memory.
|
||||
InFlight(CuObjectError),
|
||||
}
|
||||
|
||||
impl From<cuobject::Error> for AsyncFailure {
|
||||
fn from(error: cuobject::Error) -> Self {
|
||||
Self::Failed(error.into())
|
||||
}
|
||||
}
|
||||
|
||||
/// The handle of the one GET in flight on an asynchronous channel.
|
||||
const ASYNC_HANDLE: NonZeroUsize = NonZeroUsize::MIN;
|
||||
|
||||
impl CuObjectServer {
|
||||
/// Creates the server and its registered staging pool.
|
||||
pub fn new(addr: IpAddr, config: &CuObjectConfig) -> Result<Arc<Self>, CuObjectError> {
|
||||
if config.chunk_bytes == 0 || config.chunk_bytes > MAX_RDMA_OP_BYTES || config.pool_bytes < config.chunk_bytes {
|
||||
return Err(CuObjectError::Invalid("invalid staging configuration"));
|
||||
}
|
||||
let mut options = ServerOptions::default();
|
||||
// A non-positive count keeps the library default.
|
||||
options.num_dcis = u32::try_from(config.num_dcis).unwrap_or(0);
|
||||
options.cq_depth = config.cq_depth;
|
||||
let server = Arc::new(Server::new(addr, config.port, &options)?);
|
||||
let chunks = config.pool_bytes / config.chunk_bytes;
|
||||
let buffer = RegisteredBuffer::new(Arc::clone(&server), chunks * config.chunk_bytes)?;
|
||||
Ok(Arc::new(Self {
|
||||
channels: Arc::new(Semaphore::new(usize::try_from(config.num_dcis).unwrap_or(1).max(1))),
|
||||
async_transfers: config.async_transfers,
|
||||
server,
|
||||
staging: Staging {
|
||||
buffer,
|
||||
free: parking_lot::Mutex::new((0..chunks).collect()),
|
||||
permits: Semaphore::new(chunks),
|
||||
chunk_bytes: config.chunk_bytes,
|
||||
},
|
||||
addr,
|
||||
port: config.port,
|
||||
}))
|
||||
}
|
||||
|
||||
pub fn addr(&self) -> IpAddr {
|
||||
self.addr
|
||||
}
|
||||
|
||||
pub fn port(&self) -> u16 {
|
||||
self.port
|
||||
}
|
||||
|
||||
pub fn chunk_bytes(&self) -> usize {
|
||||
self.staging.chunk_bytes
|
||||
}
|
||||
|
||||
/// Reserves one staging chunk, waiting up to `timeout` for a free one.
|
||||
pub async fn reserve(self: &Arc<Self>, timeout: Duration) -> Result<StagingChunk, CuObjectError> {
|
||||
let permit = tokio::time::timeout(timeout, self.staging.permits.acquire())
|
||||
.await
|
||||
.map_err(|_| CuObjectError::PoolExhausted)?
|
||||
.map_err(|_| CuObjectError::PoolExhausted)?;
|
||||
permit.forget();
|
||||
let index = self.staging.free.lock().pop_front().ok_or(CuObjectError::PoolExhausted)?;
|
||||
Ok(StagingChunk {
|
||||
server: Arc::clone(self),
|
||||
index,
|
||||
len: 0,
|
||||
})
|
||||
}
|
||||
|
||||
/// Reserves one staging chunk without waiting; `None` when the pool is empty.
|
||||
pub fn try_reserve(self: &Arc<Self>) -> Option<StagingChunk> {
|
||||
let permit = self.staging.permits.try_acquire().ok()?;
|
||||
permit.forget();
|
||||
let index = self.staging.free.lock().pop_front()?;
|
||||
Some(StagingChunk {
|
||||
server: Arc::clone(self),
|
||||
index,
|
||||
len: 0,
|
||||
})
|
||||
}
|
||||
|
||||
/// Staging chunks currently free in the pool (a hint; another reservation may win).
|
||||
pub fn available_chunks(&self) -> usize {
|
||||
self.staging.permits.available_permits()
|
||||
}
|
||||
|
||||
/// Registers `len` bytes of zero-filled, page-aligned host memory with this
|
||||
/// server for callers that manage the memory themselves (a shard arena the
|
||||
/// decoder reads into and the server RDMA-writes from without staging).
|
||||
pub fn register_arena(self: &Arc<Self>, len: usize) -> Result<Arc<RegisteredArena>, CuObjectError> {
|
||||
let buffer = RegisteredBuffer::new(Arc::clone(&self.server), len)?;
|
||||
static NEXT_ID: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);
|
||||
Ok(Arc::new(RegisteredArena {
|
||||
buffer,
|
||||
server: Arc::clone(self),
|
||||
id: NEXT_ID.fetch_add(1, std::sync::atomic::Ordering::Relaxed),
|
||||
}))
|
||||
}
|
||||
|
||||
/// RDMA-writes the staged bytes of `chunk` to `token.remote_addr +
|
||||
/// remote_offset`. The transfer owns the chunk until the DMA finished, so
|
||||
/// cancelling the awaiting future can never release the staging slot under
|
||||
/// an in-flight write. Returns the chunk with the byte count for reuse.
|
||||
pub async fn write_to_client_owned(
|
||||
self: &Arc<Self>,
|
||||
key: &str,
|
||||
token: &RdmaToken,
|
||||
remote_offset: u64,
|
||||
chunk: StagingChunk,
|
||||
) -> Result<(u64, StagingChunk), CuObjectError> {
|
||||
if chunk.is_empty() {
|
||||
return Ok((0, chunk));
|
||||
}
|
||||
let local = Local::Staging {
|
||||
offset: chunk.start(),
|
||||
len: chunk.len(),
|
||||
};
|
||||
self.get_owned(key, token, remote_offset, local, chunk).await
|
||||
}
|
||||
|
||||
/// RDMA-writes `source.len` bytes at `source.local_offset` of
|
||||
/// `source.arena` to `token.remote_addr + remote_offset`, keeping `keep`
|
||||
/// alive until the DMA finished (the caller passes whatever owns the arena
|
||||
/// bytes). The arena must have been registered with this server.
|
||||
pub async fn write_to_client_from_arena_owned<T: Send + 'static>(
|
||||
self: &Arc<Self>,
|
||||
key: &str,
|
||||
token: &RdmaToken,
|
||||
remote_offset: u64,
|
||||
source: ArenaRange,
|
||||
keep: T,
|
||||
) -> Result<(u64, T), CuObjectError> {
|
||||
if source.len == 0 {
|
||||
return Ok((0, keep));
|
||||
}
|
||||
let local = Local::Arena {
|
||||
arena: source.arena,
|
||||
offset: source.local_offset,
|
||||
len: source.len,
|
||||
};
|
||||
self.get_owned(key, token, remote_offset, local, keep).await
|
||||
}
|
||||
|
||||
/// RDMA-writes all of `source.ranges` (in order, as one operation) to
|
||||
/// `token.remote_addr + remote_offset`, keeping `keep` alive until the DMA
|
||||
/// finished. The arena must have been registered with this server.
|
||||
pub async fn write_to_client_sg_owned<T: Send + 'static>(
|
||||
self: &Arc<Self>,
|
||||
key: &str,
|
||||
token: &RdmaToken,
|
||||
remote_offset: u64,
|
||||
source: ArenaRanges,
|
||||
keep: T,
|
||||
) -> Result<(u64, T), CuObjectError> {
|
||||
if source.ranges.len() > MAX_SG_RANGES {
|
||||
return Err(CuObjectError::Invalid("too many scatter-gather ranges"));
|
||||
}
|
||||
if source.total_len() == 0 {
|
||||
return Ok((0, keep));
|
||||
}
|
||||
let entries = source
|
||||
.ranges
|
||||
.iter()
|
||||
.map(|&(offset, size)| SgEntry {
|
||||
offset: offset as u64,
|
||||
size,
|
||||
})
|
||||
.collect();
|
||||
let local = Local::ArenaList {
|
||||
arena: source.arena,
|
||||
entries,
|
||||
};
|
||||
self.get_owned(key, token, remote_offset, local, keep).await
|
||||
}
|
||||
|
||||
/// RDMA-reads `len` bytes from `token.remote_addr + remote_offset` into the
|
||||
/// (empty) `chunk`, owning it for the whole DMA like
|
||||
/// [`Self::write_to_client_owned`]. On success the chunk holds `len` staged
|
||||
/// bytes.
|
||||
pub async fn read_from_client_owned(
|
||||
self: &Arc<Self>,
|
||||
key: &str,
|
||||
token: &RdmaToken,
|
||||
remote_offset: u64,
|
||||
len: usize,
|
||||
mut chunk: StagingChunk,
|
||||
) -> Result<StagingChunk, CuObjectError> {
|
||||
chunk.clear();
|
||||
if len == 0 {
|
||||
return Ok(chunk);
|
||||
}
|
||||
if len > chunk.capacity() {
|
||||
return Err(CuObjectError::Invalid("read exceeds the staging chunk"));
|
||||
}
|
||||
let server = Arc::clone(self);
|
||||
let key = key.to_owned();
|
||||
let token = token.inner.clone();
|
||||
let channel_permit = self.channel_permit().await?;
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let _channel_permit = channel_permit;
|
||||
let mut channel = Channel::lease(Arc::clone(&server.server))?;
|
||||
let moved = channel.put(&key, &token, remote_offset, &server.staging.buffer, chunk.start(), len)?;
|
||||
check_moved(moved as u64, len as u64)?;
|
||||
chunk.advance(len)?;
|
||||
Ok(chunk)
|
||||
})
|
||||
.await
|
||||
.map_err(|error| join_error("read task", error))?
|
||||
}
|
||||
|
||||
async fn channel_permit(&self) -> Result<tokio::sync::OwnedSemaphorePermit, CuObjectError> {
|
||||
Arc::clone(&self.channels)
|
||||
.acquire_owned()
|
||||
.await
|
||||
.map_err(|_| CuObjectError::Invalid("server closed"))
|
||||
}
|
||||
|
||||
/// RDMA-writes `local` to the client and hands `keep` back once the DMA
|
||||
/// finished. The transfer runs in a task of its own that owns `local`,
|
||||
/// `keep` and the channel, so dropping this future never releases memory
|
||||
/// the NIC may still read.
|
||||
async fn get_owned<T: Send + 'static>(
|
||||
self: &Arc<Self>,
|
||||
key: &str,
|
||||
token: &RdmaToken,
|
||||
remote_offset: u64,
|
||||
local: Local,
|
||||
keep: T,
|
||||
) -> Result<(u64, T), CuObjectError> {
|
||||
let expected = local.len() as u64;
|
||||
let server = Arc::clone(self);
|
||||
let key = key.to_owned();
|
||||
let token = token.inner.clone();
|
||||
let channel_permit = self.channel_permit().await?;
|
||||
tokio::spawn(async move {
|
||||
if server.async_transfers {
|
||||
match server.get_async(&key, &token, remote_offset, &local).await {
|
||||
Ok(()) => {
|
||||
drop(channel_permit);
|
||||
return Ok((expected, keep));
|
||||
}
|
||||
Err(AsyncFailure::Unavailable) => {}
|
||||
Err(AsyncFailure::Failed(error)) => return Err(error),
|
||||
Err(AsyncFailure::InFlight(error)) => {
|
||||
// The NIC may still read this memory: never release it.
|
||||
std::mem::forget(local);
|
||||
std::mem::forget(keep);
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let _channel_permit = channel_permit;
|
||||
let mut channel = Channel::lease(Arc::clone(&server.server))?;
|
||||
let mut list = None;
|
||||
let source = server.source(&local, &mut list)?;
|
||||
let moved = channel.get(&key, &token, remote_offset, source)?;
|
||||
check_moved(moved as u64, expected)?;
|
||||
// `keep` (the chunk or the arena owner) is released only here.
|
||||
Ok((expected, keep))
|
||||
})
|
||||
.await
|
||||
.map_err(|error| join_error("write task", error))?
|
||||
})
|
||||
.await
|
||||
.map_err(|error| join_error("write task", error))?
|
||||
}
|
||||
|
||||
/// Moves `local` with the library's asynchronous mode: the GET is
|
||||
/// submitted on a channel of its own and the task waits on that channel's
|
||||
/// completion descriptor instead of holding a blocking thread.
|
||||
// SAFETY: the allowance covers only this function; its two unsafe calls
|
||||
// each state the invariant they rely on.
|
||||
#[allow(unsafe_code)]
|
||||
async fn get_async(
|
||||
&self,
|
||||
key: &str,
|
||||
token: &cuobject::RdmaToken,
|
||||
remote_offset: u64,
|
||||
local: &Local,
|
||||
) -> Result<(), AsyncFailure> {
|
||||
let mut channel = Channel::lease(Arc::clone(&self.server))?;
|
||||
let fd = match channel.completion_fd() {
|
||||
Ok(fd) => fd,
|
||||
Err(cuobject::Error::AsyncUnavailable(_)) => return Err(AsyncFailure::Unavailable),
|
||||
Err(error) => return Err(error.into()),
|
||||
};
|
||||
// SAFETY: the library owns the descriptor and keeps it open while the
|
||||
// server lives; `fd` holds the server, and nothing here closes or
|
||||
// replaces the descriptor while it is registered.
|
||||
let completions = unsafe { AsyncFd::register_with_interest(fd, Interest::READABLE) }.map_err(|error| {
|
||||
AsyncFailure::Failed(CuObjectError::Os {
|
||||
op: "completion fd",
|
||||
source: error.into_parts().1,
|
||||
})
|
||||
})?;
|
||||
let mut list = None;
|
||||
let source = self.source(local, &mut list).map_err(AsyncFailure::Failed)?;
|
||||
// SAFETY: the memory behind `source` belongs to `local`, owned by the
|
||||
// calling task together with the transfer's `keep`; staging chunks and
|
||||
// arena ranges given to a transfer are not borrowed by anyone else.
|
||||
// This function returns only after the completion carrying the handle
|
||||
// was drained, except through `InFlight`, where the channel and the
|
||||
// list are leaked here and the caller leaks `local` and `keep`.
|
||||
unsafe { channel.get_async(key, token, remote_offset, source, ASYNC_HANDLE) }?;
|
||||
let mut events = [Completion::default(); 8];
|
||||
loop {
|
||||
let mut ready = match completions.readable().await {
|
||||
Ok(ready) => ready,
|
||||
Err(source) => {
|
||||
std::mem::forget(list);
|
||||
std::mem::forget(channel);
|
||||
return Err(AsyncFailure::InFlight(CuObjectError::Os {
|
||||
op: "completion wait",
|
||||
source,
|
||||
}));
|
||||
}
|
||||
};
|
||||
loop {
|
||||
let landed = match channel.process_completions(&mut events) {
|
||||
Ok(landed) => landed,
|
||||
Err(error) => {
|
||||
std::mem::forget(list);
|
||||
std::mem::forget(channel);
|
||||
return Err(AsyncFailure::InFlight(error.into()));
|
||||
}
|
||||
};
|
||||
if let Some(done) = events[..landed].iter().find(|event| event.handle() == ASYNC_HANDLE.get()) {
|
||||
// A completion carries no byte count: success moved the whole range.
|
||||
return done
|
||||
.result()
|
||||
.map_err(|source| AsyncFailure::Failed(CuObjectError::Transfer { verbs_status: 0, source }));
|
||||
}
|
||||
if landed < events.len() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
ready.clear_ready();
|
||||
}
|
||||
}
|
||||
|
||||
/// The library view of `local`; a scatter-gather list is built into `list`.
|
||||
fn source<'a: 'l, 'l>(
|
||||
&'a self,
|
||||
local: &'a Local,
|
||||
list: &'l mut Option<SgBuffer<'a, Arc<Server>>>,
|
||||
) -> Result<Source<'l, Arc<Server>>, CuObjectError> {
|
||||
Ok(match local {
|
||||
Local::Staging { offset, len } => Source::Range {
|
||||
buffer: &self.staging.buffer,
|
||||
offset: *offset,
|
||||
len: *len,
|
||||
},
|
||||
Local::Arena { arena, offset, len } => Source::Range {
|
||||
buffer: &arena.buffer,
|
||||
offset: *offset,
|
||||
len: *len,
|
||||
},
|
||||
Local::ArenaList { arena, entries } => Source::List(list.insert(arena.buffer.sg_buffer(entries)?)),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn check_moved(moved: u64, expected: u64) -> Result<u64, CuObjectError> {
|
||||
if moved != expected {
|
||||
return Err(CuObjectError::ShortTransfer { moved, expected });
|
||||
}
|
||||
Ok(moved)
|
||||
}
|
||||
|
||||
fn join_error(op: &'static str, error: tokio::task::JoinError) -> CuObjectError {
|
||||
CuObjectError::Os {
|
||||
op,
|
||||
source: std::io::Error::other(error),
|
||||
}
|
||||
}
|
||||
|
||||
/// Copies owned bytes into staging; convenience for callers holding `Bytes`.
|
||||
pub fn fill_chunk(chunk: &mut StagingChunk, data: &Bytes) -> Result<(), CuObjectError> {
|
||||
chunk.append(data)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn token_parses_leading_fields_and_rejects_garbage() {
|
||||
let token =
|
||||
RdmaToken::parse("0000f52eddc00000:00100000:001824d2:0000:002873:1:00000000000000000000ffff0a0d5410").expect("token");
|
||||
assert_eq!(token.remote_addr, 0x0000_f52e_ddc0_0000);
|
||||
assert_eq!(token.buffer_len, 0x0010_0000);
|
||||
assert!(matches!(RdmaToken::parse("nothex:00100000:1:2:3:4:5"), Err(CuObjectError::Token(_))));
|
||||
assert!(RdmaToken::parse("0000f52eddc00000:00000000:1:2:3:4:5").is_err());
|
||||
assert!(RdmaToken::parse("0000f52eddc00000:00100000").is_err());
|
||||
}
|
||||
}
|
||||
@@ -39,8 +39,5 @@ serde = { workspace = true, features = ["derive"] }
|
||||
rmp-serde = { workspace = true }
|
||||
rustfs-filemeta = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
serde_json.workspace = true
|
||||
|
||||
[lib]
|
||||
doctest = false
|
||||
|
||||
@@ -66,12 +66,6 @@ pub const LEGACY_DATA_USAGE_OBJECT_NAME: &str = ".usage.json";
|
||||
/// stale tier names from growing persisted per-tier maps without bound.
|
||||
pub const UNKNOWN_TIER: &str = "UNKNOWN_TIER";
|
||||
|
||||
/// Whether a remote tier name can be represented in scanner accounting.
|
||||
/// Local storage classes are checked separately by the configuration owner.
|
||||
pub fn is_scannable_tier_name(name: &str) -> bool {
|
||||
!name.is_empty() && name.len() <= 256 && !name.bytes().any(|byte| byte.is_ascii_control()) && name != UNKNOWN_TIER
|
||||
}
|
||||
|
||||
/// Returns true when `existing_last_update` is ahead of `now` by more than
|
||||
/// [`USAGE_LAST_UPDATE_FUTURE_TOLERANCE`], i.e. the persisted timestamp cannot be
|
||||
/// trusted for staleness comparisons and a fresh snapshot save must be allowed.
|
||||
@@ -480,12 +474,6 @@ pub struct BucketTargetUsageInfo {
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct BucketUsageInfo {
|
||||
pub size: u64,
|
||||
/// Provenance of this value. Missing and unknown states remain unproven.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub usage_value_state: Option<crate::BucketUsageValueState>,
|
||||
/// Certification of this exact value, independent of aggregation time.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub usage_certification: Option<crate::BucketUsageCertification>,
|
||||
/// Bucket identity certified by every set contributing to this snapshot.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub bucket_incarnation: Option<String>,
|
||||
@@ -592,18 +580,6 @@ pub struct DataUsageInfo {
|
||||
/// which last-known-good generation contributed to the view.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub usage_snapshot_set_states: Vec<DataUsageSnapshotSetState>,
|
||||
/// Complete mixed observations for admin display only. Legacy bucket maps
|
||||
/// and quota consumers continue to use the canonical values above.
|
||||
#[serde(
|
||||
default,
|
||||
skip_serializing_if = "HashMap::is_empty",
|
||||
deserialize_with = "crate::usage_observation::deserialize_bucket_observations"
|
||||
)]
|
||||
pub usage_snapshot_bucket_observations: HashMap<String, crate::DataUsageBucketObservation>,
|
||||
/// Response-only marker: a projected admin view must never be persisted or
|
||||
/// admitted as a quota/cache baseline, including after serialization.
|
||||
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
|
||||
pub usage_snapshot_admin_projection: bool,
|
||||
/// Buckets whose dirty-usage generation moved while this complete
|
||||
/// snapshot was being walked. Their `buckets_usage` values are the walk
|
||||
/// result, which the cycle could not certify as current; the next cycle
|
||||
@@ -733,8 +709,7 @@ pub fn observed_data_usage_is_newer(observed: &DataUsageInfo, authoritative: &Da
|
||||
/// This applies the structural, exact-baseline, and generation checks of
|
||||
/// [`observed_data_usage_is_newer`] without retaining the baseline's bucket maps.
|
||||
pub fn observed_data_usage_is_newer_than_identity(observed: &DataUsageInfo, authoritative: DataUsageSnapshotIdentity) -> bool {
|
||||
!observed.usage_snapshot_admin_projection
|
||||
&& observed.usage_snapshot_converged == Some(false)
|
||||
observed.usage_snapshot_converged == Some(false)
|
||||
&& (observed.is_complete_bucket_usage_snapshot() || observed.is_valid_partial_snapshot())
|
||||
&& observed.usage_snapshot_authoritative_baseline == Some(authoritative)
|
||||
&& data_usage_identity_is_newer(observed.snapshot_identity(), authoritative)
|
||||
@@ -2049,8 +2024,7 @@ impl DataUsageInfo {
|
||||
|
||||
/// Whether this snapshot authoritatively covers every reported bucket.
|
||||
pub fn is_complete_bucket_usage_snapshot(&self) -> bool {
|
||||
!self.usage_snapshot_admin_projection
|
||||
&& !self.usage_snapshot_bootstrap_pending
|
||||
!self.usage_snapshot_bootstrap_pending
|
||||
&& self.usage_snapshot_complete
|
||||
&& self.last_update.is_some()
|
||||
&& u64::try_from(self.buckets_usage.len()).ok() == Some(self.buckets_count)
|
||||
@@ -2060,8 +2034,7 @@ impl DataUsageInfo {
|
||||
/// admin display. Partial data is accepted only with unique set states,
|
||||
/// a plan digest for every state, and at least one usable generation.
|
||||
pub fn is_valid_partial_snapshot(&self) -> bool {
|
||||
if self.usage_snapshot_admin_projection
|
||||
|| self.usage_snapshot_bootstrap_pending
|
||||
if self.usage_snapshot_bootstrap_pending
|
||||
|| !self.usage_snapshot_partial
|
||||
|| self.usage_snapshot_converged != Some(false)
|
||||
|| self.last_update.is_none()
|
||||
@@ -2099,11 +2072,9 @@ impl DataUsageInfo {
|
||||
Ok(name) => name,
|
||||
Err(_) => return,
|
||||
};
|
||||
self.usage_snapshot_bucket_observations.remove(&bucket_name);
|
||||
|
||||
// Update bucket statistics
|
||||
if let Some(bucket_usage) = self.buckets_usage.get_mut(&bucket_name) {
|
||||
bucket_usage.mark_observed();
|
||||
bucket_usage.size += meta_object.size as u64;
|
||||
bucket_usage.objects_count += 1;
|
||||
bucket_usage.versions_count += 1; // Simplified: assume 1 version per object
|
||||
@@ -2156,7 +2127,6 @@ impl DataUsageInfo {
|
||||
Ok(name) => name,
|
||||
Err(_) => return,
|
||||
};
|
||||
self.usage_snapshot_bucket_observations.remove(&bucket_name);
|
||||
|
||||
// Calculate accurate statistics from all versions
|
||||
let mut total_size = 0u64;
|
||||
@@ -2185,7 +2155,6 @@ impl DataUsageInfo {
|
||||
|
||||
// Update bucket statistics
|
||||
if let Some(bucket_usage) = self.buckets_usage.get_mut(&bucket_name) {
|
||||
bucket_usage.mark_observed();
|
||||
bucket_usage.size += total_size;
|
||||
bucket_usage.objects_count += 1;
|
||||
bucket_usage.versions_count += versions_count;
|
||||
@@ -2366,9 +2335,6 @@ impl BucketUsageInfo {
|
||||
/// Add size summary to this bucket usage
|
||||
/// Merge another BucketUsageInfo into this one
|
||||
pub fn merge(&mut self, other: &BucketUsageInfo) {
|
||||
// Adding contributions produces a new value, not either input's
|
||||
// whole-bucket certificate. The scanner may certify the final sum.
|
||||
self.mark_observed();
|
||||
if self.bucket_incarnation != other.bucket_incarnation {
|
||||
self.bucket_incarnation = None;
|
||||
}
|
||||
|
||||
@@ -13,7 +13,5 @@
|
||||
// limitations under the License.
|
||||
|
||||
pub mod data_usage;
|
||||
pub mod usage_observation;
|
||||
|
||||
pub use data_usage::*;
|
||||
pub use usage_observation::*;
|
||||
|
||||
@@ -1,265 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
//! Diagnostic provenance and response-only projections of complete bucket observations.
|
||||
|
||||
use crate::{BucketUsageInfo, DataUsageInfo, DataUsageSnapshotIdentity};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{collections::HashMap, time::SystemTime};
|
||||
|
||||
pub const DATA_USAGE_BUCKET_OBSERVATION_VERSION: u16 = 1;
|
||||
|
||||
/// Provenance of the returned bucket value, independent of quota authority.
|
||||
#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum BucketUsageValueState {
|
||||
Certified,
|
||||
Carried,
|
||||
Observed,
|
||||
#[serde(other)]
|
||||
Unknown,
|
||||
}
|
||||
|
||||
/// A certification of this exact bucket value, never its publication timestamp.
|
||||
#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)]
|
||||
pub struct BucketUsageCertification {
|
||||
pub epoch: u64,
|
||||
pub cycle: u64,
|
||||
pub certified_at: SystemTime,
|
||||
}
|
||||
|
||||
/// Immutable provenance of one complete traversal contribution.
|
||||
///
|
||||
/// The scanner validates its storage-owned sweep identity before constructing
|
||||
/// this public projection. Equal start/request plans are allowed when a fresh
|
||||
/// set contributes alongside another set's mixed traversal.
|
||||
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
|
||||
pub struct DataUsageBucketObservationSource {
|
||||
pub pool_index: u64,
|
||||
pub set_index: u64,
|
||||
pub bucket_incarnation: String,
|
||||
pub scanner_epoch: u64,
|
||||
pub scanner_cycle: u64,
|
||||
pub scan_plan_digest: [u8; 32],
|
||||
pub started_plan: [u8; 32],
|
||||
pub requested_plan: [u8; 32],
|
||||
pub completed_at: SystemTime,
|
||||
}
|
||||
|
||||
/// An additive companion; legacy bucket maps never contain this mixed value.
|
||||
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
|
||||
pub struct DataUsageBucketObservation {
|
||||
pub version: u16,
|
||||
pub usage: BucketUsageInfo,
|
||||
/// Exact enclosing candidate identity. Rewrapping preserves source times.
|
||||
pub snapshot_identity: DataUsageSnapshotIdentity,
|
||||
/// Oldest contribution time, so a newer set cannot hide an older value.
|
||||
pub completed_at: SystemTime,
|
||||
/// Strictly ordered, complete coverage of the enclosing live set inventory.
|
||||
pub sources: Vec<DataUsageBucketObservationSource>,
|
||||
}
|
||||
|
||||
/// Companion evolution must not make an otherwise usable canonical snapshot
|
||||
/// unreadable. Unsupported or malformed diagnostic entries never grant proof.
|
||||
pub(crate) fn deserialize_bucket_observations<'de, D>(
|
||||
deserializer: D,
|
||||
) -> Result<HashMap<String, DataUsageBucketObservation>, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
#[derive(Deserialize)]
|
||||
#[serde(untagged)]
|
||||
enum Entry {
|
||||
Observation(Box<DataUsageBucketObservation>),
|
||||
Ignored(serde::de::IgnoredAny),
|
||||
}
|
||||
Ok(HashMap::<String, Entry>::deserialize(deserializer)?
|
||||
.into_iter()
|
||||
.filter_map(|(bucket, entry)| match entry {
|
||||
Entry::Observation(observation) if observation.version == DATA_USAGE_BUCKET_OBSERVATION_VERSION => {
|
||||
Some((bucket, *observation))
|
||||
}
|
||||
Entry::Observation(_) | Entry::Ignored(_) => None,
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
impl BucketUsageInfo {
|
||||
/// Attach a certificate only after the caller proves complete bucket coverage.
|
||||
pub fn certify(&mut self, certification: BucketUsageCertification) {
|
||||
self.usage_value_state = Some(BucketUsageValueState::Certified);
|
||||
self.usage_certification = Some(certification);
|
||||
}
|
||||
|
||||
/// Preserve the certificate when carrying the exact prior bucket value.
|
||||
pub fn mark_carried(&mut self) {
|
||||
self.normalize_usage_provenance();
|
||||
self.usage_value_state = Some(BucketUsageValueState::Carried);
|
||||
}
|
||||
|
||||
/// The value changed without a complete bucket certificate.
|
||||
pub fn mark_observed(&mut self) {
|
||||
self.usage_value_state = Some(BucketUsageValueState::Observed);
|
||||
self.usage_certification = None;
|
||||
}
|
||||
|
||||
/// Future or legacy provenance cannot authorize a retained certificate.
|
||||
pub fn normalize_usage_provenance(&mut self) {
|
||||
if !matches!(
|
||||
self.usage_value_state,
|
||||
Some(BucketUsageValueState::Certified | BucketUsageValueState::Carried)
|
||||
) {
|
||||
self.usage_certification = None;
|
||||
}
|
||||
if self.usage_value_state == Some(BucketUsageValueState::Certified) && self.usage_certification.is_none() {
|
||||
self.usage_value_state = Some(BucketUsageValueState::Unknown);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl DataUsageBucketObservation {
|
||||
/// Verify the public provenance against the enclosing candidate and current
|
||||
/// storage-owned incarnation. This never grants publication or ACK authority.
|
||||
pub fn matches_snapshot(&self, snapshot: &DataUsageInfo, incarnation: &str) -> bool {
|
||||
let Some(root_generation) = snapshot.scanner_epoch.zip(snapshot.scanner_cycle) else {
|
||||
return false;
|
||||
};
|
||||
if self.version != DATA_USAGE_BUCKET_OBSERVATION_VERSION
|
||||
|| snapshot.usage_snapshot_admin_projection
|
||||
|| self.snapshot_identity != snapshot.snapshot_identity()
|
||||
|| incarnation.is_empty()
|
||||
|| self.usage.bucket_incarnation.as_deref() != Some(incarnation)
|
||||
|| self.sources.is_empty()
|
||||
|| self.usage.usage_certification.is_some()
|
||||
|| self.usage.usage_value_state != Some(BucketUsageValueState::Observed)
|
||||
|| self.sources.iter().map(|source| source.completed_at).min() != Some(self.completed_at)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let mut previous = None;
|
||||
let mut plan = None;
|
||||
for state in &snapshot.usage_snapshot_set_states {
|
||||
let key = (state.pool_index, state.set_index);
|
||||
if previous.is_some_and(|previous| previous >= key)
|
||||
|| state.scan_plan_digest.is_none()
|
||||
|| plan.is_some_and(|plan| Some(plan) != state.scan_plan_digest)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
previous = Some(key);
|
||||
plan = state.scan_plan_digest;
|
||||
}
|
||||
let mut sources = self.sources.iter();
|
||||
for state in snapshot.usage_snapshot_set_states.iter().filter(|state| !state.tombstone) {
|
||||
let Some(source) = sources.next() else {
|
||||
return false;
|
||||
};
|
||||
let Some(set_generation) = state.scanner_epoch.zip(state.scanner_cycle) else {
|
||||
return false;
|
||||
};
|
||||
if (source.pool_index, source.set_index) != (state.pool_index, state.set_index)
|
||||
|| source.bucket_incarnation != incarnation
|
||||
|| Some(source.scan_plan_digest) != state.scan_plan_digest
|
||||
|| (source.scanner_epoch, source.scanner_cycle) > set_generation
|
||||
|| set_generation > root_generation
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
sources.next().is_none()
|
||||
}
|
||||
}
|
||||
|
||||
impl DataUsageInfo {
|
||||
/// Overlay validated companions on an owned admin response only.
|
||||
///
|
||||
/// `incarnations` must come from current storage metadata, not the companion.
|
||||
/// The projection is marked explicitly so persistence and quota boundaries
|
||||
/// can reject accidental reuse. Invalid companions leave canonical values
|
||||
/// unchanged. Counter overflow rejects the whole projection.
|
||||
pub fn project_bucket_observations_for_admin(&mut self, incarnations: &HashMap<String, String>) -> usize {
|
||||
if self.usage_snapshot_admin_projection
|
||||
|| (!self.is_complete_bucket_usage_snapshot() && !self.is_valid_partial_snapshot())
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
let observations =
|
||||
self.usage_snapshot_bucket_observations
|
||||
.iter()
|
||||
.filter(|(bucket, observation)| {
|
||||
if self.usage_snapshot_complete
|
||||
&& (!self.usage_snapshot_pending_buckets.iter().any(|pending| pending == *bucket)
|
||||
|| u64::try_from(self.usage_snapshot_pending_buckets.len()).ok()
|
||||
!= Some(self.usage_snapshot_pending_bucket_count))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if self
|
||||
.buckets_usage
|
||||
.get(*bucket)
|
||||
.and_then(|usage| usage.usage_certification)
|
||||
.is_some_and(|certificate| {
|
||||
observation.sources.iter().all(|source| {
|
||||
(source.scanner_epoch, source.scanner_cycle) <= (certificate.epoch, certificate.cycle)
|
||||
})
|
||||
})
|
||||
{
|
||||
return false;
|
||||
}
|
||||
incarnations.get(*bucket).is_some_and(|incarnation| {
|
||||
self.buckets_usage
|
||||
.get(*bucket)
|
||||
.and_then(|usage| usage.bucket_incarnation.as_ref())
|
||||
.is_none_or(|canonical| canonical == incarnation)
|
||||
&& observation.matches_snapshot(self, incarnation)
|
||||
})
|
||||
})
|
||||
.collect::<HashMap<_, _>>();
|
||||
if observations.is_empty() {
|
||||
return 0;
|
||||
}
|
||||
let mut values = self
|
||||
.buckets_usage
|
||||
.iter()
|
||||
.filter(|(bucket, _)| !observations.contains_key(bucket))
|
||||
.map(|(_, usage)| usage)
|
||||
.chain(observations.values().map(|observation| &observation.usage));
|
||||
let totals = values.try_fold([0_u64; 4], |total, usage| {
|
||||
Some([
|
||||
total[0].checked_add(usage.size)?,
|
||||
total[1].checked_add(usage.objects_count)?,
|
||||
total[2].checked_add(usage.versions_count)?,
|
||||
total[3].checked_add(usage.delete_markers_count)?,
|
||||
])
|
||||
});
|
||||
let Some([size, objects, versions, markers]) = totals else {
|
||||
return 0;
|
||||
};
|
||||
let projected = observations.len();
|
||||
for (bucket, observation) in observations {
|
||||
self.buckets_usage.insert(bucket.clone(), observation.usage.clone());
|
||||
}
|
||||
self.bucket_sizes = self
|
||||
.buckets_usage
|
||||
.iter()
|
||||
.map(|(bucket, usage)| (bucket.clone(), usage.size))
|
||||
.collect();
|
||||
self.buckets_count = u64::try_from(self.buckets_usage.len()).unwrap_or(u64::MAX);
|
||||
self.objects_total_size = size;
|
||||
self.objects_total_count = objects;
|
||||
self.versions_total_count = versions;
|
||||
self.delete_markers_total_count = markers;
|
||||
self.replication_info.clear();
|
||||
self.tier_stats = None;
|
||||
self.unknown_tier_stats = None;
|
||||
self.usage_snapshot_complete = false;
|
||||
self.usage_snapshot_partial = true;
|
||||
self.usage_snapshot_converged = Some(false);
|
||||
self.usage_snapshot_observational = true;
|
||||
self.usage_snapshot_admin_projection = true;
|
||||
projected
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -1,323 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
use crate::DataUsageSnapshotSetState;
|
||||
use std::time::Duration;
|
||||
|
||||
const INCARNATION: &str = "11111111-1111-4111-8111-111111111111";
|
||||
|
||||
fn time(seconds: u64) -> SystemTime {
|
||||
SystemTime::UNIX_EPOCH + Duration::from_secs(seconds)
|
||||
}
|
||||
|
||||
fn usage(size: u64) -> BucketUsageInfo {
|
||||
BucketUsageInfo {
|
||||
size,
|
||||
objects_count: size,
|
||||
versions_count: size,
|
||||
bucket_incarnation: Some(INCARNATION.to_string()),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn snapshot() -> DataUsageInfo {
|
||||
let mut main = usage(10);
|
||||
main.certify(BucketUsageCertification {
|
||||
epoch: 2,
|
||||
cycle: 3,
|
||||
certified_at: time(30),
|
||||
});
|
||||
main.mark_carried();
|
||||
let mut snapshot = DataUsageInfo {
|
||||
last_update: Some(time(90)),
|
||||
scanner_epoch: Some(2),
|
||||
scanner_cycle: Some(9),
|
||||
buckets_count: 1,
|
||||
buckets_usage: HashMap::from([("bucket".to_string(), main)]),
|
||||
bucket_sizes: HashMap::from([("bucket".to_string(), 10)]),
|
||||
usage_snapshot_complete: true,
|
||||
usage_snapshot_converged: Some(false),
|
||||
usage_snapshot_pending_buckets: vec!["bucket".to_string()],
|
||||
usage_snapshot_pending_bucket_count: 1,
|
||||
usage_snapshot_set_states: (0..2)
|
||||
.map(|set_index| DataUsageSnapshotSetState {
|
||||
pool_index: 0,
|
||||
set_index,
|
||||
scanner_epoch: Some(2),
|
||||
scanner_cycle: Some(9),
|
||||
scan_plan_digest: Some([1; 32]),
|
||||
complete: true,
|
||||
..Default::default()
|
||||
})
|
||||
.collect(),
|
||||
..Default::default()
|
||||
};
|
||||
snapshot.calculate_totals();
|
||||
let mut observed = usage(25);
|
||||
observed.mark_observed();
|
||||
let observation = DataUsageBucketObservation {
|
||||
version: DATA_USAGE_BUCKET_OBSERVATION_VERSION,
|
||||
usage: observed,
|
||||
snapshot_identity: snapshot.snapshot_identity(),
|
||||
completed_at: time(40),
|
||||
sources: (0..2)
|
||||
.map(|set_index| DataUsageBucketObservationSource {
|
||||
pool_index: 0,
|
||||
set_index,
|
||||
bucket_incarnation: INCARNATION.to_string(),
|
||||
scanner_epoch: 2,
|
||||
scanner_cycle: 4 + set_index,
|
||||
scan_plan_digest: [1; 32],
|
||||
started_plan: [2; 32],
|
||||
requested_plan: [3; 32],
|
||||
completed_at: time(40 + set_index),
|
||||
})
|
||||
.collect(),
|
||||
};
|
||||
snapshot
|
||||
.usage_snapshot_bucket_observations
|
||||
.insert("bucket".to_string(), observation);
|
||||
snapshot
|
||||
}
|
||||
|
||||
fn incarnations() -> HashMap<String, String> {
|
||||
HashMap::from([("bucket".to_string(), INCARNATION.to_string())])
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provenance_is_additive_and_unknown_states_do_not_keep_certificates() {
|
||||
let legacy = usage(10);
|
||||
let bytes = serde_json::to_vec(&legacy).expect("encode a legacy bucket without provenance");
|
||||
let text = std::str::from_utf8(&bytes).expect("JSON is UTF-8");
|
||||
assert!(!text.contains("usage_value_state"));
|
||||
assert!(!text.contains("usage_certification"));
|
||||
let decoded: BucketUsageInfo = serde_json::from_slice(&bytes).expect("read legacy bucket");
|
||||
assert_eq!(decoded.usage_value_state, None);
|
||||
assert_eq!(decoded.usage_certification, None);
|
||||
|
||||
let mut wire = serde_json::to_value(legacy).expect("encode future bucket fixture");
|
||||
wire["usage_value_state"] = serde_json::json!("future_state");
|
||||
wire["usage_certification"] =
|
||||
serde_json::json!({"epoch":2,"cycle":3,"certified_at":{"secs_since_epoch":30,"nanos_since_epoch":0}});
|
||||
let mut future: BucketUsageInfo = serde_json::from_value(wire).expect("unknown diagnostic enum remains readable");
|
||||
future.normalize_usage_provenance();
|
||||
assert_eq!(future.usage_value_state, Some(BucketUsageValueState::Unknown));
|
||||
assert_eq!(future.usage_certification, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn carrying_preserves_time_but_adding_a_source_invalidates_the_certificate() {
|
||||
let mut bucket = snapshot().buckets_usage.remove("bucket").expect("canonical bucket");
|
||||
bucket.mark_carried();
|
||||
assert_eq!(bucket.usage_certification.map(|certification| certification.certified_at), Some(time(30)));
|
||||
bucket.merge(&usage(1));
|
||||
assert_eq!(bucket.size, 11);
|
||||
assert_eq!(bucket.usage_value_state, Some(BucketUsageValueState::Observed));
|
||||
assert_eq!(bucket.usage_certification, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn direct_object_accumulation_does_not_reuse_the_old_certificate() {
|
||||
let mut simple = snapshot();
|
||||
simple.add_object("bucket/object", &rustfs_filemeta::MetaObject::default());
|
||||
assert_eq!(simple.buckets_usage["bucket"].usage_value_state, Some(BucketUsageValueState::Observed));
|
||||
assert_eq!(simple.buckets_usage["bucket"].usage_certification, None);
|
||||
assert!(simple.usage_snapshot_bucket_observations.is_empty());
|
||||
|
||||
let mut versioned = snapshot();
|
||||
versioned.add_object_from_file_meta("bucket/object", &rustfs_filemeta::FileMeta::default());
|
||||
assert_eq!(versioned.buckets_usage["bucket"].usage_value_state, Some(BucketUsageValueState::Observed));
|
||||
assert_eq!(versioned.buckets_usage["bucket"].usage_certification, None);
|
||||
assert!(versioned.usage_snapshot_bucket_observations.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn future_or_malformed_companions_do_not_break_the_canonical_reader() {
|
||||
let canonical = snapshot();
|
||||
for entry in [
|
||||
serde_json::json!({"version": 2, "future_payload": {"new": true}}),
|
||||
serde_json::json!({"version": 1, "usage": "broken"}),
|
||||
] {
|
||||
let mut wire = serde_json::to_value(&canonical).expect("canonical JSON");
|
||||
wire["usage_snapshot_bucket_observations"]["bucket"] = entry;
|
||||
let decoded: DataUsageInfo = serde_json::from_value(wire).expect("diagnostics cannot break canonical data");
|
||||
assert_eq!(decoded.buckets_usage, canonical.buckets_usage);
|
||||
assert!(decoded.usage_snapshot_bucket_observations.is_empty());
|
||||
assert!(decoded.is_complete_bucket_usage_snapshot());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn complete_companion_projects_only_the_admin_copy_and_keeps_measurement_times() {
|
||||
let canonical = snapshot();
|
||||
let canonical_bytes = serde_json::to_vec(&canonical).expect("encode canonical candidate");
|
||||
let mut response = canonical.clone();
|
||||
assert_eq!(response.project_bucket_observations_for_admin(&incarnations()), 1);
|
||||
assert_eq!(response.buckets_usage["bucket"].size, 25);
|
||||
assert_eq!(response.buckets_usage["bucket"].usage_value_state, Some(BucketUsageValueState::Observed));
|
||||
assert_eq!(response.buckets_usage["bucket"].usage_certification, None);
|
||||
assert_eq!(response.objects_total_size, 25);
|
||||
assert_eq!(response.bucket_sizes["bucket"], 25);
|
||||
assert_eq!(response.last_update, canonical.last_update);
|
||||
assert_eq!(response.usage_snapshot_bucket_observations["bucket"].completed_at, time(40));
|
||||
assert!(response.usage_snapshot_admin_projection);
|
||||
assert!(response.usage_snapshot_partial);
|
||||
assert!(response.usage_snapshot_observational);
|
||||
assert_eq!(response.usage_snapshot_converged, Some(false));
|
||||
assert!(!response.is_complete_bucket_usage_snapshot());
|
||||
assert!(
|
||||
!response.is_valid_partial_snapshot(),
|
||||
"response-only values cannot be a storage candidate"
|
||||
);
|
||||
assert_eq!(serde_json::to_vec(&canonical).expect("reencode original"), canonical_bytes);
|
||||
assert_eq!(canonical.buckets_usage["bucket"].size, 10);
|
||||
assert_eq!(
|
||||
canonical.buckets_usage["bucket"]
|
||||
.usage_certification
|
||||
.map(|certificate| certificate.certified_at),
|
||||
Some(time(30))
|
||||
);
|
||||
assert_eq!(response.project_bucket_observations_for_admin(&incarnations()), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_companions_preserve_the_canonical_bucket() {
|
||||
for invalid in 0..11 {
|
||||
let mut candidate = snapshot();
|
||||
let observation = candidate
|
||||
.usage_snapshot_bucket_observations
|
||||
.get_mut("bucket")
|
||||
.expect("companion");
|
||||
match invalid {
|
||||
0 => observation.version += 1,
|
||||
1 => observation.snapshot_identity.scanner_cycle = Some(8),
|
||||
2 => {
|
||||
observation.sources.pop();
|
||||
}
|
||||
3 => observation.sources[1].set_index = 0,
|
||||
4 => observation.sources[1].scanner_cycle = 10,
|
||||
5 => observation.sources[1].scan_plan_digest = [9; 32],
|
||||
6 => observation.sources[1].bucket_incarnation = "other".to_string(),
|
||||
7 => observation.completed_at = time(41),
|
||||
8 => observation.usage.certify(BucketUsageCertification {
|
||||
epoch: 2,
|
||||
cycle: 9,
|
||||
certified_at: time(90),
|
||||
}),
|
||||
9 => candidate.usage_snapshot_set_states[1].scanner_cycle = None,
|
||||
10 => {
|
||||
candidate
|
||||
.buckets_usage
|
||||
.get_mut("bucket")
|
||||
.expect("canonical")
|
||||
.bucket_incarnation = Some("recreated".to_string())
|
||||
}
|
||||
_ => unreachable!("bounded fixture variants"),
|
||||
}
|
||||
let before = candidate.clone();
|
||||
assert_eq!(
|
||||
candidate.project_bucket_observations_for_admin(&incarnations()),
|
||||
0,
|
||||
"invalid case {invalid}"
|
||||
);
|
||||
assert_eq!(candidate, before, "invalid case {invalid} changed canonical data");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_fresh_set_can_contribute_without_refreshing_an_older_mixed_set() {
|
||||
let mut candidate = snapshot();
|
||||
let observation = candidate
|
||||
.usage_snapshot_bucket_observations
|
||||
.get_mut("bucket")
|
||||
.expect("companion");
|
||||
observation.sources[1].started_plan = observation.sources[1].requested_plan;
|
||||
observation.sources[1].completed_at = time(90);
|
||||
assert_eq!(candidate.project_bucket_observations_for_admin(&incarnations()), 1);
|
||||
assert_eq!(candidate.usage_snapshot_bucket_observations["bucket"].completed_at, time(40));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn companion_only_partial_keeps_legacy_maps_empty_until_admin_projection() {
|
||||
let mut candidate = snapshot();
|
||||
candidate.buckets_usage.clear();
|
||||
candidate.bucket_sizes.clear();
|
||||
candidate.buckets_count = 0;
|
||||
candidate.calculate_totals();
|
||||
candidate.usage_snapshot_complete = false;
|
||||
candidate.usage_snapshot_partial = true;
|
||||
candidate.usage_snapshot_observational = true;
|
||||
for state in &mut candidate.usage_snapshot_set_states {
|
||||
state.complete = false;
|
||||
}
|
||||
assert!(candidate.is_valid_partial_snapshot());
|
||||
let wire = serde_json::to_value(&candidate).expect("encode companion-only observation");
|
||||
assert_eq!(wire["buckets_usage"], serde_json::json!({}));
|
||||
assert_eq!(wire["bucket_sizes"], serde_json::json!({}));
|
||||
assert_eq!(wire["objects_total_size"], 0);
|
||||
assert_eq!(candidate.project_bucket_observations_for_admin(&incarnations()), 1);
|
||||
assert_eq!(candidate.objects_total_size, 25);
|
||||
assert!(candidate.usage_snapshot_partial);
|
||||
assert!(!candidate.usage_snapshot_complete);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn projection_rejects_counter_overflow_without_partial_mutation() {
|
||||
let mut candidate = snapshot();
|
||||
candidate.buckets_usage.insert("cold".to_string(), usage(u64::MAX));
|
||||
candidate.buckets_count = 2;
|
||||
let before = candidate.clone();
|
||||
assert_eq!(candidate.project_bucket_observations_for_admin(&incarnations()), 0);
|
||||
assert_eq!(candidate, before);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_newer_canonical_certificate_and_missing_pending_prevent_old_overlay() {
|
||||
for newer_certificate in [false, true] {
|
||||
let mut candidate = snapshot();
|
||||
if newer_certificate {
|
||||
candidate
|
||||
.buckets_usage
|
||||
.get_mut("bucket")
|
||||
.expect("canonical")
|
||||
.certify(BucketUsageCertification {
|
||||
epoch: 2,
|
||||
cycle: 9,
|
||||
certified_at: time(90),
|
||||
});
|
||||
} else {
|
||||
candidate.usage_snapshot_pending_buckets.clear();
|
||||
candidate.usage_snapshot_pending_bucket_count = 0;
|
||||
}
|
||||
assert_eq!(candidate.project_bucket_observations_for_admin(&incarnations()), 0);
|
||||
assert_eq!(candidate.buckets_usage["bucket"].size, 10);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provenance_and_companions_change_candidate_bytes_and_round_trip_as_maps() {
|
||||
let canonical = snapshot();
|
||||
let original = serde_json::to_vec(&canonical).expect("encode candidate");
|
||||
let mut changed = canonical.clone();
|
||||
changed
|
||||
.buckets_usage
|
||||
.get_mut("bucket")
|
||||
.expect("bucket")
|
||||
.usage_certification
|
||||
.as_mut()
|
||||
.expect("certificate")
|
||||
.certified_at = time(31);
|
||||
assert_ne!(serde_json::to_vec(&changed).expect("encode changed certificate"), original);
|
||||
let mut changed = canonical.clone();
|
||||
changed
|
||||
.usage_snapshot_bucket_observations
|
||||
.get_mut("bucket")
|
||||
.expect("observation")
|
||||
.sources[0]
|
||||
.started_plan = [8; 32];
|
||||
assert_ne!(serde_json::to_vec(&changed).expect("encode changed source"), original);
|
||||
let named = rmp_serde::to_vec_named(&canonical).expect("encode named map");
|
||||
let restored: DataUsageInfo = rmp_serde::from_slice(&named).expect("read named map with additive fields");
|
||||
assert_eq!(restored, canonical);
|
||||
}
|
||||
+11
-13
@@ -98,8 +98,11 @@ rustfs-madmin.workspace = true
|
||||
rustfs-filemeta.workspace = true
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
|
||||
aws-sdk-sts = { workspace = true, default-features = false, features = ["default-https-client", "rt-tokio"] }
|
||||
aws-config = { workspace = true }
|
||||
aws-smithy-http-client = { workspace = true, default-features = false, features = ["rustls-aws-lc"] }
|
||||
aws-smithy-types.workspace = true
|
||||
async-compression = { workspace = true, features = ["tokio", "bzip2", "lz4", "xz"] }
|
||||
async-trait = { workspace = true }
|
||||
flate2.workspace = true
|
||||
http.workspace = true
|
||||
@@ -123,32 +126,27 @@ urlencoding.workspace = true
|
||||
walkdir.workspace = true
|
||||
base64-simd = { workspace = true }
|
||||
rand = { workspace = true, features = ["serde"] }
|
||||
jiff = { workspace = true, features = ["serde"] }
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
hex-simd = { workspace = true }
|
||||
md-5 = { workspace = true }
|
||||
opentelemetry-proto = { workspace = true }
|
||||
prost.workspace = true
|
||||
sha2 = { workspace = true }
|
||||
rustfs-s3-types = { workspace = true }
|
||||
rustfs-tokio-tar = { workspace = true }
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
zstd.workspace = true
|
||||
time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }
|
||||
suppaftp = { workspace = true, features = ["rustls-aws-lc-rs", "tokio-rustls-aws-lc-rs"] }
|
||||
rcgen.workspace = true
|
||||
local-ip-address.workspace = true
|
||||
anyhow.workspace = true
|
||||
rustls = { workspace = true, default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
|
||||
russh = { workspace = true, features = ["serde"] }
|
||||
russh-sftp = { workspace = true }
|
||||
zip.workspace = true
|
||||
clap = { workspace = true, features = ["derive", "env"] }
|
||||
|
||||
[dev-dependencies]
|
||||
serde-value.workspace = true
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
async-compression = { workspace = true, features = ["tokio", "bzip2", "lz4", "xz"] }
|
||||
aws-config = { workspace = true }
|
||||
aws-sdk-sts = { workspace = true, default-features = false, features = ["default-https-client", "rt-tokio"] }
|
||||
opentelemetry-proto = { workspace = true }
|
||||
russh = { workspace = true, features = ["serde"] }
|
||||
russh-sftp = { workspace = true }
|
||||
suppaftp = { workspace = true, features = ["rustls-aws-lc-rs", "tokio-rustls-aws-lc-rs"] }
|
||||
rustfs-tokio-tar = { workspace = true }
|
||||
zip.workspace = true
|
||||
same-file.workspace = true
|
||||
aws-smithy-http-client = { workspace = true, default-features = false, features = ["test-util"] }
|
||||
rustfs-heal.workspace = true
|
||||
|
||||
@@ -29,7 +29,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
|
||||
| **distributed 4×4** | [`src/distributed/`](src/distributed) | Nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
|
||||
| **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
|
||||
| **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
|
||||
| **dual-stack differential** | `dual_stack_differential` | Two single-node servers answer one SDK script for all 73 S3 operations and every differing `(op, field, a, b)` is reported. Both sides run the binary under test on its only (gateway) stack unless `RUSTFS_E2E_REFERENCE_BINARY` names a released `rustfs` binary, which then runs side A. A missing, non-file or non-executable path fails the run; it never skips or falls back to the same-binary mode. `RUSTFS_E2E_STACK_B_REGION` starts side B in another region to inject a difference. The report in `<target>/dual-stack/<run>.json` records `reference_side` and each side's `--version` output |
|
||||
| **dual-stack differential** | `dual_stack_differential` | Two single-node servers answer one SDK script for all 73 S3 operations and every differing `(op, field, a, b)` is reported. Side A is the binary under test on `RUSTFS_S3_STACK=legacy`; side B runs `RUSTFS_E2E_STACK_B`. Set `RUSTFS_E2E_REFERENCE_BINARY` to a released `rustfs` binary to make side A that binary instead (started without `RUSTFS_S3_STACK`, which older releases do not know). A missing, non-file or non-executable path fails the run; it never skips or falls back to the same-binary mode. The report in `<target>/dual-stack/<run>.json` records `reference_side` and each side's `--version` output |
|
||||
|
||||
The external-tool `storage_metric_ownership_test` validates the OTLP/Collector/Prometheus path, including a rolling upgrade and node failures. See the [storage metrics guide](../../docs/operations/storage-metrics.md) for its required binaries and focused command.
|
||||
|
||||
@@ -211,8 +211,6 @@ the wiring source of truth. Committed test-ID digests under
|
||||
|
||||
**Endpoint blackhole scenario skipped** — `heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_after_target_endpoint_blackhole` installs a loopback `iptables` DROP rule and therefore needs `CAP_NET_ADMIN` (root or passwordless `sudo -n iptables`). A host where `iptables` is missing or cannot read the OUTPUT chain (typical inside an unprivileged container, where the nf_tables backend reports "Permission denied" even under `sudo`) logs a `heal_interruption_skipped` warning and returns without exercising heal. Set `RUSTFS_E2E_REQUIRE_NET_FAULT_INJECTION=1` on lanes that do provision the capability so a broken runner fails instead of skipping.
|
||||
|
||||
The full suite compares persisted bucket configurations and object records against the release pinned in `.config/reference-binary.toml`. Set `RUSTFS_PERSISTENCE_REFERENCE_BINARY` to the verified executable and `RUSTFS_PERSISTENCE_CLOCK_LIBRARY` to the absolute path of libfaketime (`libfaketime.so.1` on Linux or `libfaketime.1.dylib` on macOS). To retain the two sets of 13 XML documents for the gateway golden corpus, set `RUSTFS_PERSISTENCE_XML_DIR` to an output directory. CI includes them in the full suite's JUnit artifact.
|
||||
|
||||
**Reproduce a CI failure locally** — run the exact profile/lane:
|
||||
|
||||
```bash
|
||||
|
||||
@@ -21,8 +21,8 @@ use crate::common::{
|
||||
};
|
||||
use http::header::{AUTHORIZATION, HOST};
|
||||
use reqwest::StatusCode;
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::{constants::UNSIGNED_PAYLOAD, sign_v4};
|
||||
use s3s::Body;
|
||||
use serde_json::Value;
|
||||
use std::{error::Error, time::Duration};
|
||||
|
||||
|
||||
@@ -146,11 +146,7 @@ async fn create_service_account_for(
|
||||
Some(serde_json::json!({ "targetUser": parent }).to_string()),
|
||||
)
|
||||
.await?;
|
||||
service_account_keys(&response)
|
||||
}
|
||||
|
||||
fn service_account_keys(response: &str) -> Result<(String, String), Box<dyn Error + Send + Sync>> {
|
||||
let response: serde_json::Value = serde_json::from_str(response)?;
|
||||
let response: serde_json::Value = serde_json::from_str(&response)?;
|
||||
let access_key = response["credentials"]["accessKey"]
|
||||
.as_str()
|
||||
.ok_or("service account response should contain credentials.accessKey")?
|
||||
@@ -348,134 +344,6 @@ async fn test_update_service_account_enforces_owner_and_parent_scope() -> TestRe
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_delegated_service_account_creation_through_console_sessions() -> TestResult {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_without_cleanup(vec![]).await?;
|
||||
let bucket = "delegated-backups";
|
||||
let target = "backup-user";
|
||||
let operator = "backup-operator";
|
||||
let operator_secret = "operatorSecret123";
|
||||
let wildcard_admin = "backup-admin";
|
||||
let wildcard_secret = "adminSecret123";
|
||||
let s3 = env.create_s3_client();
|
||||
s3.create_bucket().bucket(bucket).send().await?;
|
||||
for (name, secret) in [
|
||||
(target, "backupSecret123"),
|
||||
(operator, operator_secret),
|
||||
(wildcard_admin, wildcard_secret),
|
||||
("other-user", "otherUserSecret123"),
|
||||
] {
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::PUT,
|
||||
&format!("/rustfs/admin/v3/add-user?accessKey={name}"),
|
||||
Some(serde_json::json!({"secretKey":secret,"status":"enabled"}).to_string()),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
let delegation = serde_json::json!({
|
||||
"Version":"2012-10-17",
|
||||
"Statement":[
|
||||
{"Effect":"Allow","Action":"admin:CreateServiceAccount"},
|
||||
{"Effect":"Allow","Action":"sts:AssumeRole","Resource":"arn:aws:s3:::*"},
|
||||
{"Effect":"Allow","Action":"admin:CreateServiceAccountForUser","Condition":{"StringEquals":{"svc:TargetUser":target}}}
|
||||
]
|
||||
});
|
||||
let wildcard = serde_json::json!({
|
||||
"Version":"2012-10-17",
|
||||
"Statement":[
|
||||
{"Effect":"Allow","Action":"admin:*"},
|
||||
{"Effect":"Allow","Action":"sts:AssumeRole","Resource":"arn:aws:s3:::*"}
|
||||
]
|
||||
});
|
||||
for (user, name, policy) in [
|
||||
(wildcard_admin, "backup-admin-wildcard", wildcard.to_string()),
|
||||
(target, "backup-writer", bucket_rw_policy(bucket)),
|
||||
(operator, "backup-key-manager", delegation.to_string()),
|
||||
] {
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::PUT,
|
||||
&format!("/rustfs/admin/v3/add-canned-policy?name={name}"),
|
||||
Some(policy),
|
||||
)
|
||||
.await?;
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::POST,
|
||||
"/rustfs/admin/v3/idp/builtin/policy/attach",
|
||||
Some(serde_json::json!({"policies":[name],"user":user}).to_string()),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
for (name, secret) in [
|
||||
(operator, operator_secret),
|
||||
(wildcard_admin, wildcard_secret),
|
||||
(env.access_key.as_str(), env.secret_key.as_str()),
|
||||
] {
|
||||
let session = build_test_sts_client(&env.url, name, secret, None, "delegated-create")
|
||||
.assume_role()
|
||||
.role_arn("arn:aws:iam::123456789012:role/console")
|
||||
.role_session_name("console")
|
||||
.send()
|
||||
.await?;
|
||||
let caller = session.credentials().ok_or("missing console session")?;
|
||||
let create = |user: &str| {
|
||||
admin_request_with_session_token(
|
||||
&env.url,
|
||||
http::Method::PUT,
|
||||
"/rustfs/admin/v3/add-service-account",
|
||||
Some(serde_json::json!({"targetUser":user}).to_string()),
|
||||
caller.access_key_id(),
|
||||
caller.secret_access_key(),
|
||||
Some(caller.session_token()),
|
||||
)
|
||||
};
|
||||
let (status, response) = create(target).await?;
|
||||
assert_eq!(status, StatusCode::OK, "create for target as {name}");
|
||||
let (key, secret) = service_account_keys(&response)?;
|
||||
let client = build_s3_client(&env.url, &key, &secret);
|
||||
wait_for_s3_put(&client, bucket, "backup", Duration::from_secs(10)).await?;
|
||||
let denied = client
|
||||
.create_bucket()
|
||||
.bucket("outside-backup-policy")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("target cannot create unrelated buckets");
|
||||
assert_eq!(denied.as_service_error().and_then(|e| e.code()), Some("AccessDenied"));
|
||||
if name != env.access_key {
|
||||
let (status, _) = create(&env.access_key).await?;
|
||||
assert_eq!(status, StatusCode::FORBIDDEN, "delegated targets exclude root");
|
||||
}
|
||||
if name == operator {
|
||||
let (status, _) = create("other-user").await?;
|
||||
assert_eq!(status, StatusCode::FORBIDDEN, "outside delegated target");
|
||||
}
|
||||
}
|
||||
let restricted = build_test_sts_client(&env.url, &env.access_key, &env.secret_key, None, "restricted-root")
|
||||
.assume_role()
|
||||
.role_arn("arn:aws:iam::123456789012:role/console")
|
||||
.role_session_name("restricted-console")
|
||||
.policy(r#"{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"admin:CreateServiceAccount"}]}"#)
|
||||
.send()
|
||||
.await?;
|
||||
let caller = restricted.credentials().ok_or("missing restricted root session")?;
|
||||
let (status, _) = admin_request_with_session_token(
|
||||
&env.url,
|
||||
http::Method::PUT,
|
||||
"/rustfs/admin/v3/add-service-account",
|
||||
Some(serde_json::json!({"targetUser":target}).to_string()),
|
||||
caller.access_key_id(),
|
||||
caller.secret_access_key(),
|
||||
Some(caller.session_token()),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(status, StatusCode::FORBIDDEN, "restricted root session requires delegated permission");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Full user -> policy -> service-account lifecycle, proving each management
|
||||
/// call takes effect on the data plane, not just that the endpoint answers 200.
|
||||
#[tokio::test]
|
||||
|
||||
@@ -34,15 +34,12 @@
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::common::{
|
||||
AdminTransport, RustFSTestEnvironment, admin_attach_user_policy_via, admin_create_user, admin_request_with_session_token,
|
||||
init_logging, local_http_client, parse_assume_role_credentials,
|
||||
};
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use hmac::{Hmac, KeyInit as _, Mac};
|
||||
use http::header::HOST;
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use sha1::Sha1;
|
||||
use std::error::Error;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
@@ -399,54 +396,6 @@ mod tests {
|
||||
let (status, _) = signed_request(&env.url, http::Method::GET, ACCOUNT_INFO_PATH, None, user_ak, old_sk).await?;
|
||||
assert!(status.is_client_error(), "the old secret must stop working, status: {status}");
|
||||
|
||||
env.stop_server();
|
||||
drop(env);
|
||||
console_session_rotates_the_iam_password_and_revokes_the_old_session().await
|
||||
}
|
||||
|
||||
async fn console_session_rotates_the_iam_password_and_revokes_the_old_session() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let user = "console-password-user";
|
||||
let old_password = uuid::Uuid::new_v4().to_string();
|
||||
let new_password = uuid::Uuid::new_v4().to_string();
|
||||
admin_create_user(&env, user, &old_password).await?;
|
||||
admin_attach_user_policy_via(AdminTransport::Signed, &env.url, &env.access_key, &env.secret_key, "readonly", user)
|
||||
.await?;
|
||||
|
||||
let (status, response) = assume_role(&env.url, user, &old_password, None).await?;
|
||||
assert_eq!(status, reqwest::StatusCode::OK, "the current password must sign in");
|
||||
let (session_key, session_secret, token) = parse_assume_role_credentials(&response)?;
|
||||
let session_request = |method, path, body| {
|
||||
admin_request_with_session_token(&env.url, method, path, body, &session_key, &session_secret, Some(&token))
|
||||
};
|
||||
let password_request = |current: &str| {
|
||||
Some(serde_json::json!({ "current_secret_key": current, "new_secret_key": new_password }).to_string())
|
||||
};
|
||||
|
||||
let (status, body) =
|
||||
session_request(http::Method::POST, ACCOUNT_PASSWORD_PATH, password_request(&session_secret)).await?;
|
||||
assert_eq!(
|
||||
status,
|
||||
reqwest::StatusCode::BAD_REQUEST,
|
||||
"the session secret must not be accepted as the IAM password"
|
||||
);
|
||||
assert!(body.contains("InvalidRequest") && body.contains("the current secret key is incorrect"));
|
||||
|
||||
let (status, body) = session_request(http::Method::POST, ACCOUNT_PASSWORD_PATH, password_request(&old_password)).await?;
|
||||
assert_eq!(status, reqwest::StatusCode::OK, "the same session must accept the current IAM password");
|
||||
assert_eq!(serde_json::from_str::<serde_json::Value>(&body)?["sessions_revoked"], 1);
|
||||
|
||||
let (status, _) = session_request(http::Method::GET, ACCOUNT_INFO_PATH, None).await?;
|
||||
assert_eq!(status, reqwest::StatusCode::FORBIDDEN, "the pre-rotation session must be revoked");
|
||||
let (status, _) = assume_role(&env.url, user, &old_password, None).await?;
|
||||
assert_eq!(status, reqwest::StatusCode::FORBIDDEN, "the old password must no longer sign in");
|
||||
let (status, response) = assume_role(&env.url, user, &new_password, None).await?;
|
||||
assert_eq!(status, reqwest::StatusCode::OK, "the new password must sign in");
|
||||
parse_assume_role_credentials(&response)?;
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -15,9 +15,9 @@
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use http::header::HOST;
|
||||
use reqwest::StatusCode;
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use serde::Deserialize;
|
||||
use std::error::Error;
|
||||
|
||||
|
||||
@@ -17,9 +17,9 @@ use aws_sdk_s3::primitives::ByteStream;
|
||||
use http::header::HOST;
|
||||
use reqwest::StatusCode;
|
||||
use rustfs_madmin::{ITEM_OFFLINE, InfoMessage, StorageInfo};
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use serde::Deserialize;
|
||||
use std::error::Error;
|
||||
use tokio::time::{Duration, sleep, timeout};
|
||||
|
||||
@@ -21,9 +21,9 @@ mod tests {
|
||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use reqwest::StatusCode;
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::{pre_sign_v4, sign_v4};
|
||||
use s3s::Body;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::error::Error;
|
||||
use std::io::{Cursor, Write};
|
||||
|
||||
@@ -137,40 +137,6 @@ async fn test_bucket_policy_authenticated_user() -> Result<(), Box<dyn std::erro
|
||||
.await
|
||||
.map_err(|e| format!("DeleteObject failed: {}", e))?;
|
||||
|
||||
for (allowed, may_read) in [("s3:GetBucketPolicy", true), ("s3:PutBucketPolicy", false)] {
|
||||
let policy = serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Principal": { "AWS": [user_access] },
|
||||
"Action": allowed,
|
||||
"Resource": format!("arn:aws:s3:::{bucket_name}")
|
||||
}]
|
||||
})
|
||||
.to_string();
|
||||
admin_client
|
||||
.put_bucket_policy()
|
||||
.bucket(bucket_name)
|
||||
.policy(&policy)
|
||||
.send()
|
||||
.await?;
|
||||
let read = user_client.get_bucket_policy().bucket(bucket_name).send().await;
|
||||
let write = user_client
|
||||
.put_bucket_policy()
|
||||
.bucket(bucket_name)
|
||||
.policy(&policy)
|
||||
.send()
|
||||
.await;
|
||||
if may_read {
|
||||
read?;
|
||||
let denied = write.expect_err("GetBucketPolicy must not grant PutBucketPolicy");
|
||||
assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
|
||||
} else {
|
||||
let denied = read.expect_err("PutBucketPolicy must not grant GetBucketPolicy");
|
||||
assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
|
||||
write?;
|
||||
}
|
||||
}
|
||||
info!("Test Passed!");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -37,9 +37,9 @@
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, local_http_client};
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::error::Error;
|
||||
@@ -170,15 +170,6 @@ impl DiskFaultHarness {
|
||||
/// Start (or restart after [`Self::kill_server`]) the RustFS server with
|
||||
/// all volume directories and the same address.
|
||||
pub async fn start_server(&mut self) -> ChaosResult<()> {
|
||||
self.start_server_with_readiness(false).await
|
||||
}
|
||||
|
||||
/// Start a process once its node read-readiness probe passes.
|
||||
pub async fn start_server_at_read_quorum(&mut self) -> ChaosResult<()> {
|
||||
self.start_server_with_readiness(true).await
|
||||
}
|
||||
|
||||
async fn start_server_with_readiness(&mut self, read_quorum_only: bool) -> ChaosResult<()> {
|
||||
// The common helper always appends `env.temp_dir` as the final storage
|
||||
// path: point it at the last disk for startup and pass the remaining
|
||||
// disks explicitly, then restore it to the root so Drop cleans up
|
||||
@@ -190,13 +181,7 @@ impl DiskFaultHarness {
|
||||
let arg_refs: Vec<&str> = disk_args.iter().map(String::as_str).collect();
|
||||
let env_refs: Vec<(&str, &str)> = self.extra_env.iter().map(|(k, v)| (k.as_str(), v.as_str())).collect();
|
||||
|
||||
let result = if read_quorum_only {
|
||||
self.env
|
||||
.start_rustfs_server_with_env_at_read_quorum(arg_refs, &env_refs)
|
||||
.await
|
||||
} else {
|
||||
self.env.start_rustfs_server_with_env(arg_refs, &env_refs).await
|
||||
};
|
||||
let result = self.env.start_rustfs_server_with_env(arg_refs, &env_refs).await;
|
||||
self.env.temp_dir = self.root.to_string_lossy().to_string();
|
||||
result
|
||||
}
|
||||
|
||||
@@ -29,9 +29,9 @@ use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use reqwest::Client as HttpClient;
|
||||
use reqwest::StatusCode;
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use serde_json;
|
||||
use std::fs as stdfs;
|
||||
use std::io::{self, ErrorKind};
|
||||
@@ -722,7 +722,7 @@ impl RustFSTestEnvironment {
|
||||
cleanup_existing: bool,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let binary_path = rustfs_binary_path();
|
||||
self.start_rustfs_server_inner_with_binary(&binary_path, extra_args, extra_env, cleanup_existing, false)
|
||||
self.start_rustfs_server_inner_with_binary(&binary_path, extra_args, extra_env, cleanup_existing)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -732,7 +732,6 @@ impl RustFSTestEnvironment {
|
||||
extra_args: Vec<&str>,
|
||||
extra_env: &[(&str, &str)],
|
||||
cleanup_existing: bool,
|
||||
read_quorum_only: bool,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
if cleanup_existing {
|
||||
self.cleanup_existing_processes().await?;
|
||||
@@ -757,7 +756,7 @@ impl RustFSTestEnvironment {
|
||||
self.process = Some(process);
|
||||
|
||||
// Wait for server to be ready
|
||||
self.wait_for_server_ready_with_probe(read_quorum_only).await?;
|
||||
self.wait_for_server_ready().await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -771,7 +770,7 @@ impl RustFSTestEnvironment {
|
||||
extra_args: Vec<&str>,
|
||||
extra_env: &[(&str, &str)],
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, true, false)
|
||||
self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, true)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -783,7 +782,7 @@ impl RustFSTestEnvironment {
|
||||
extra_args: Vec<&str>,
|
||||
extra_env: &[(&str, &str)],
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, false, false)
|
||||
self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, false)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -801,18 +800,6 @@ impl RustFSTestEnvironment {
|
||||
self.start_rustfs_server_inner(extra_args, extra_env, true).await
|
||||
}
|
||||
|
||||
/// Start a degraded read-serving process using the node readiness probe.
|
||||
/// ListBuckets retains its namespace write-quorum requirement.
|
||||
pub async fn start_rustfs_server_with_env_at_read_quorum(
|
||||
&mut self,
|
||||
extra_args: Vec<&str>,
|
||||
extra_env: &[(&str, &str)],
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let binary_path = rustfs_binary_path();
|
||||
self.start_rustfs_server_inner_with_binary(&binary_path, extra_args, extra_env, true, true)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Start RustFS server without cleaning up other running RustFS processes.
|
||||
///
|
||||
/// This is useful for tests that need multiple independent RustFS instances
|
||||
@@ -841,16 +828,8 @@ impl RustFSTestEnvironment {
|
||||
/// Fails immediately if the spawned server process has already exited
|
||||
/// (e.g. a port bind failure) instead of polling out the full timeout.
|
||||
pub async fn wait_for_server_ready(&mut self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
self.wait_for_server_ready_with_probe(false).await
|
||||
}
|
||||
|
||||
async fn wait_for_server_ready_with_probe(
|
||||
&mut self,
|
||||
read_quorum_only: bool,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
info!("Waiting for RustFS server to be ready on {}", self.address);
|
||||
let client = self.create_s3_client();
|
||||
let health_client = HttpClient::builder().no_proxy().timeout(Duration::from_secs(2)).build()?;
|
||||
|
||||
for i in 0..60 {
|
||||
if let Some(process) = self.process.as_mut()
|
||||
@@ -859,16 +838,7 @@ impl RustFSTestEnvironment {
|
||||
return Err(format!("RustFS server process exited before becoming ready: {status}").into());
|
||||
}
|
||||
|
||||
let ready = if read_quorum_only {
|
||||
health_client
|
||||
.get(format!("{}/minio/health/ready", self.url))
|
||||
.send()
|
||||
.await
|
||||
.is_ok_and(|response| response.status().is_success())
|
||||
} else {
|
||||
TcpStream::connect(&self.address).await.is_ok() && client.list_buckets().send().await.is_ok()
|
||||
};
|
||||
if ready {
|
||||
if TcpStream::connect(&self.address).await.is_ok() && client.list_buckets().send().await.is_ok() {
|
||||
info!("✅ RustFS server is ready after {} attempts", i + 1);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
@@ -13,8 +13,8 @@
|
||||
// limitations under the License.
|
||||
|
||||
//! Over-the-wire smoke net for the embedded console listener (backlog#1154
|
||||
//! peri-4). The console is a second HTTP listener for the control-plane and
|
||||
//! unauthenticated console routes under `/rustfs/console`;
|
||||
//! peri-4). The console is a second HTTP listener that serves the full S3/admin
|
||||
//! surface plus the unauthenticated console routes under `/rustfs/console`;
|
||||
//! before this test its behaviour was only covered by in-process router tests
|
||||
//! (`rustfs/src/admin/console.rs`), so a misconfiguration that exposed
|
||||
//! credentials through the public console endpoints or fail-opened the
|
||||
@@ -26,13 +26,13 @@
|
||||
//! * `/rustfs/console/license` exposes only the coarse `licensed` flag.
|
||||
//! * the console SPA prefix dispatches to the static handler, never to the
|
||||
//! S3 API (no S3 error XML), whether or not console assets are embedded.
|
||||
//! * unauthenticated admin API requests are denied with 403 AccessDenied.
|
||||
//! * S3 root/object requests on the console listener return an empty 404,
|
||||
//! including authenticated requests, before S3 dispatch.
|
||||
//! * unauthenticated requests to the admin API and the S3 root on the console
|
||||
//! listener are denied (403 AccessDenied) — the extra listener does not
|
||||
//! fail-open the protected surface.
|
||||
//! * a listener with the console disabled (the main S3 port here) does not
|
||||
//! serve the unauthenticated console endpoints at all.
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client, signed_request_with_client};
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use std::error::Error;
|
||||
use tokio::time::{Duration, sleep};
|
||||
|
||||
@@ -166,39 +166,13 @@ async fn test_console_over_the_wire_smoke() -> TestResult {
|
||||
let admin_body = admin_response.text().await?;
|
||||
assert!(admin_body.contains("AccessDenied"), "admin denial must be AccessDenied: {admin_body}");
|
||||
|
||||
let signed_data_root = signed_request_with_client(
|
||||
&client,
|
||||
http::Method::GET,
|
||||
&format!("{}/", env.url),
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
let s3_root_response = client.get(format!("{console_base}/")).send().await?;
|
||||
assert_eq!(
|
||||
signed_data_root.status(),
|
||||
reqwest::StatusCode::OK,
|
||||
"the same credentials must reach S3 on the data listener"
|
||||
s3_root_response.status(),
|
||||
reqwest::StatusCode::FORBIDDEN,
|
||||
"unauthenticated S3 root on the console listener must be denied"
|
||||
);
|
||||
|
||||
for path in ["/", "/console-object-bucket/key"] {
|
||||
let url = format!("{console_base}{path}");
|
||||
for signed in [false, true] {
|
||||
let response = if signed {
|
||||
signed_request_with_client(&client, http::Method::GET, &url, &env.access_key, &env.secret_key, None, None).await?
|
||||
} else {
|
||||
client.get(&url).send().await?
|
||||
};
|
||||
assert_eq!(
|
||||
response.status(),
|
||||
reqwest::StatusCode::NOT_FOUND,
|
||||
"S3 route {path}, signed={signed} must be isolated from the console listener"
|
||||
);
|
||||
assert!(response.bytes().await?.is_empty(), "isolated S3 routes must not reach a handler");
|
||||
}
|
||||
}
|
||||
|
||||
// --- console disabled on a listener means no console surface at all ------
|
||||
// The main S3 listener of this same process runs with the console disabled,
|
||||
// so its console paths must not answer with the unauthenticated console
|
||||
|
||||
@@ -44,9 +44,9 @@ mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use http::header::HOST;
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use std::error::Error;
|
||||
|
||||
/// Signed raw `PUT` copy request with an explicit copy-source conditional
|
||||
|
||||
@@ -21,9 +21,9 @@ mod tests {
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
||||
use http::header::HOST;
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use std::error::Error;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpStream;
|
||||
|
||||
@@ -26,7 +26,7 @@ use aws_sdk_s3::types::{
|
||||
DefaultRetention, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHoldStatus, ObjectLockRetentionMode,
|
||||
ObjectLockRule,
|
||||
};
|
||||
use jiff::{SignedDuration, Timestamp};
|
||||
use chrono::{Duration as ChronoDuration, Utc};
|
||||
|
||||
fn delete_denied(error: &SdkError<DeleteObjectError>, context: &str) -> TestResult {
|
||||
let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
|
||||
@@ -66,7 +66,7 @@ async fn four_node_four_drive_object_lock_worm_blocks_delete() -> TestResult {
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let retain_until = Timestamp::now() + SignedDuration::from_hours(24);
|
||||
let retain_until = Utc::now() + ChronoDuration::days(1);
|
||||
|
||||
let compliance_key = "compliance.bin";
|
||||
let compliance_version = put_object_with_retention(
|
||||
@@ -176,10 +176,7 @@ async fn four_node_default_retention_is_visible_and_non_lock_bucket_rejects_conf
|
||||
let retain_until = retention
|
||||
.retain_until_date()
|
||||
.ok_or("default retention omitted retain-until date")?;
|
||||
assert!(
|
||||
retain_until.secs() > Timestamp::now().as_second(),
|
||||
"default retention is not in the future"
|
||||
);
|
||||
assert!(retain_until.secs() > Utc::now().timestamp(), "default retention is not in the future");
|
||||
|
||||
let versioning = reader.get_bucket_versioning().bucket(&bucket).send().await?;
|
||||
assert_eq!(versioning.status().map(|status| status.as_str()), Some("Enabled"));
|
||||
|
||||
@@ -38,8 +38,8 @@ use aws_sdk_s3::types::{
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, ObjectLockLegalHoldStatus, ObjectLockRetentionMode,
|
||||
VersioningConfiguration,
|
||||
};
|
||||
use chrono::{DateTime, Utc};
|
||||
use http::{Method, StatusCode};
|
||||
use jiff::Timestamp;
|
||||
use serde_json::Value;
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::path::{Path, PathBuf};
|
||||
@@ -526,8 +526,8 @@ async fn put_multipart(client: &Client, bucket: &str, key: &str) -> TestResult<V
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
fn retention_deadline() -> TestResult<Timestamp> {
|
||||
Ok(Timestamp::now() + jiff::SignedDuration::from_hours(24))
|
||||
fn retention_deadline() -> TestResult<DateTime<Utc>> {
|
||||
DateTime::<Utc>::from_timestamp(Utc::now().timestamp() + 24 * 3600, 0).ok_or_else(|| "retention deadline out of range".into())
|
||||
}
|
||||
|
||||
/// Seed the deterministic inventory into the seed pool and record it.
|
||||
@@ -654,7 +654,7 @@ pub(crate) async fn seed_manifest(run: &TopologyRun) -> TestResult<TopologyManif
|
||||
let mut entry = ManifestEntry::body(&locked, &key, "compliance", Some(version_id.clone()), Some(version_id), &body);
|
||||
entry.retention = Some(RetentionRecord {
|
||||
mode: "COMPLIANCE".to_string(),
|
||||
retain_until_unix: retain_until.as_second(),
|
||||
retain_until_unix: retain_until.timestamp(),
|
||||
});
|
||||
entries.push(entry);
|
||||
}
|
||||
|
||||
@@ -31,7 +31,7 @@ use super::harness::{
|
||||
use crate::common::{
|
||||
AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, init_logging,
|
||||
};
|
||||
use crate::upgrade_compatibility_test::{UpgradeSourceCapabilities, wait_for_upgrade_write_readiness};
|
||||
use crate::upgrade_compatibility_test::wait_for_upgrade_write_readiness;
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use std::path::{Path, PathBuf};
|
||||
@@ -427,12 +427,7 @@ async fn assert_history_and_iam(dist: &DistCluster, seed: &UpgradeSeed, context:
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn compare_degraded_legacy_list_parts(
|
||||
dist: &mut DistCluster,
|
||||
seed: &UpgradeSeed,
|
||||
previous: &Path,
|
||||
source: &UpgradeSourceCapabilities,
|
||||
) -> TestResult {
|
||||
async fn compare_degraded_legacy_list_parts(dist: &mut DistCluster, seed: &UpgradeSeed, previous: &Path) -> TestResult {
|
||||
let decoded = String::from_utf8(base64_simd::URL_SAFE_NO_PAD.decode_to_vec(seed.multipart_upload.as_bytes())?)?;
|
||||
let (_, upload_dir) = decoded.split_once('.').ok_or("legacy upload ID lacks deployment delimiter")?;
|
||||
let hash = sha256_hex(format!("{}/legacy-multipart.bin", seed.history_bucket).as_bytes());
|
||||
@@ -494,22 +489,16 @@ async fn compare_degraded_legacy_list_parts(
|
||||
// Three online disks retain metadata quorum but have no part directory;
|
||||
// the fourth disk is offline. A legacy empty response is not a proof that
|
||||
// the unavailable participant has no part.
|
||||
let mut quorum_errors =
|
||||
vec![current.expect_err("current coordinator must not infer empty from missing directories and offline disks")];
|
||||
if source.requires_list_parts_read_quorum {
|
||||
quorum_errors.push(legacy.expect_err("previous coordinator must require evidence from unavailable disks"));
|
||||
} else {
|
||||
assert!(
|
||||
legacy
|
||||
.expect("previous coordinator should expose its legacy empty-list behavior")
|
||||
.parts()
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
for error in quorum_errors {
|
||||
assert_eq!(error.raw_response().expect("S3 response").status().as_u16(), 503);
|
||||
assert_eq!(error.as_service_error().and_then(|error| error.meta().code()), Some("ServiceUnavailable"));
|
||||
}
|
||||
let legacy = legacy.expect("previous coordinator should expose its legacy empty-list behavior");
|
||||
assert!(legacy.parts().is_empty());
|
||||
let current = current.expect_err("current coordinator must not infer empty from missing directories and offline disks");
|
||||
assert_eq!(current.raw_response().expect("current S3 response").status().as_u16(), 503);
|
||||
// Since #9149, SlowDownRead is reserved for GET/HEAD object reads; a
|
||||
// ListParts read-quorum failure is reported as ServiceUnavailable.
|
||||
assert_eq!(
|
||||
current.as_service_error().and_then(|error| error.meta().code()),
|
||||
Some("ServiceUnavailable")
|
||||
);
|
||||
dist.cluster.start_node_from_binary(3, previous).await?;
|
||||
wait_for_upgrade_write_readiness(&clients, &seed.history_bucket, "mixed-list-restored", CREDENTIAL_TIMEOUT).await?;
|
||||
for (path, before) in original_files {
|
||||
@@ -531,7 +520,7 @@ async fn compare_degraded_legacy_list_parts(
|
||||
assert_eq!(listed.parts()[0].e_tag(), Some(seed.multipart_etag.as_str()));
|
||||
}
|
||||
eprintln!(
|
||||
"MPU mixed degraded: one offline disk, three missing part directories; checked each coordinator against its release contract; original part restored on both"
|
||||
"MPU mixed degraded: one offline disk, three missing part directories; previous returned empty, current returned 503 SlowDownRead; original part restored on both"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -603,7 +592,6 @@ async fn replace_node_and_wait_for_write_readiness(dist: &mut DistCluster, seed:
|
||||
async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> TestResult {
|
||||
init_logging();
|
||||
let previous = source_binary()?;
|
||||
let source = UpgradeSourceCapabilities::from_binary(&previous).await?;
|
||||
let mut dist = DistCluster::new_stopped(DistLayout::FourNodeFourDisk).await?;
|
||||
dist.cluster.set_env("RUSTFS_STORAGE_CLASS_RRS", "EC:0");
|
||||
capture_upgrade_logs(&mut dist, "rolling-upgrade")?;
|
||||
@@ -613,7 +601,7 @@ async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> Te
|
||||
|
||||
replace_node_and_wait_for_write_readiness(&mut dist, &seed, 0).await?;
|
||||
assert_history_and_iam(&dist, &seed, "one-current-node").await?;
|
||||
compare_degraded_legacy_list_parts(&mut dist, &seed, &previous, &source).await?;
|
||||
compare_degraded_legacy_list_parts(&mut dist, &seed, &previous).await?;
|
||||
|
||||
for node_idx in [1, 2] {
|
||||
replace_node_and_wait_for_write_readiness(&mut dist, &seed, node_idx).await?;
|
||||
|
||||
@@ -12,22 +12,21 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Differential: two `rustfs` processes, one SDK script (rustfs/backlog#2740).
|
||||
//! Dual-stack differential: one `rustfs` binary, two processes, one SDK script (rustfs/backlog#2740).
|
||||
//!
|
||||
//! Side B is the binary under test. Side A is the binary under test too, unless
|
||||
//! `RUSTFS_E2E_REFERENCE_BINARY` names a released binary: then A is that binary, and a bad path
|
||||
//! fails the run instead of falling back (see `reference`). The report names which side is the
|
||||
//! reference and the `--version` each side ran. `RUSTFS_E2E_STACK_B_REGION`, when set, starts B
|
||||
//! in another region so that a difference can be injected by hand. Neither side is given the
|
||||
//! retired stack selector: the binary under test refuses to start with it, and the gateway stack
|
||||
//! is its only S3 stack. Both servers
|
||||
//! Stack A runs `RUSTFS_S3_STACK=legacy` of the binary under test, unless `RUSTFS_E2E_REFERENCE_BINARY`
|
||||
//! names a released binary: then A is that binary, started without `RUSTFS_S3_STACK` (older
|
||||
//! releases do not know it), and a bad path fails the run instead of falling back (see
|
||||
//! `reference`). The report names which side is the reference and the `--version` each side ran. Stack B runs the value of `RUSTFS_E2E_STACK_B`
|
||||
//! (default `legacy` until the gateway stack is assembled), and `RUSTFS_E2E_STACK_B_REGION`, when
|
||||
//! set, starts B in another region so that a difference can be injected by hand. Both servers
|
||||
//! answer the script in `scenarios`, which covers every operation of `impl S3 for FS`; `compare`
|
||||
//! reports each `(op, field, a, b)` on which the answers differ, and every run writes its report
|
||||
//! to `<target>/dual-stack/<run>.json`. Two processes, not two embedded servers: bucket metadata
|
||||
//! is process-global, so two stacks in one process would read each other's buckets.
|
||||
|
||||
mod compare;
|
||||
pub(crate) mod reference;
|
||||
mod reference;
|
||||
mod scenarios;
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, rustfs_binary_path, workspace_root};
|
||||
@@ -38,18 +37,20 @@ use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
type BoxError = Box<dyn std::error::Error + Send + Sync>;
|
||||
|
||||
/// Label of a side that runs the binary under test.
|
||||
const UNDER_TEST: &str = "under-test";
|
||||
const STACK_A: &str = "legacy";
|
||||
/// Label of side A in the report when it is the released reference binary.
|
||||
const STACK_A_REFERENCE: &str = "reference";
|
||||
const ENV_STACK_B: &str = "RUSTFS_E2E_STACK_B";
|
||||
const ENV_STACK_B_REGION: &str = "RUSTFS_E2E_STACK_B_REGION";
|
||||
/// Becomes `gateway` once rustfs/backlog#2734 T2.5 assembles the gateway stack.
|
||||
const DEFAULT_STACK_B: &str = "legacy";
|
||||
/// The region the negative control gives stack A explicitly, so an inherited `RUSTFS_REGION`
|
||||
/// cannot erase the injected difference. Not `us-east-1`: that region answers GetBucketLocation
|
||||
/// with an empty LocationConstraint and re-creating an owned bucket with 200, so it would not
|
||||
/// report its own name and would add CreateBucket differences to the control.
|
||||
const CONTROL_REGION_A: &str = "us-west-2";
|
||||
const CONTROL_REGION_B: &str = "eu-west-1";
|
||||
/// `normal` cases side A must answer with a 2xx. The comparison alone cannot tell a
|
||||
/// `normal` cases the legacy stack must answer with a 2xx. The comparison alone cannot tell a
|
||||
/// script that exercises real behavior from one in which both stacks fail the same way. Five of
|
||||
/// the 73 fail on legacy by design of this script: PutBucketReplication (no registered remote
|
||||
/// target), GetBucketReplication (so no configuration), PutBucketNotificationConfiguration (no
|
||||
@@ -59,6 +60,7 @@ const MIN_NORMAL_SUCCESSES: usize = 68;
|
||||
|
||||
struct PairConfig {
|
||||
side_a: SideA,
|
||||
stack_b: String,
|
||||
region_a: Option<String>,
|
||||
region_b: Option<String>,
|
||||
}
|
||||
@@ -67,6 +69,7 @@ impl PairConfig {
|
||||
fn from_env() -> Result<Self, BoxError> {
|
||||
Ok(Self {
|
||||
side_a: SideA::from_env()?,
|
||||
stack_b: std::env::var(ENV_STACK_B).unwrap_or_else(|_| DEFAULT_STACK_B.to_owned()),
|
||||
region_a: None,
|
||||
region_b: std::env::var(ENV_STACK_B_REGION).ok(),
|
||||
})
|
||||
@@ -74,16 +77,16 @@ impl PairConfig {
|
||||
|
||||
fn stack_a_label(&self) -> &'static str {
|
||||
match self.side_a {
|
||||
SideA::SameBinary => UNDER_TEST,
|
||||
SideA::SameBinary => STACK_A,
|
||||
SideA::Reference(_) => STACK_A_REFERENCE,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Starts one server of the binary under test, overriding its region only when asked.
|
||||
async fn start_under_test(region: Option<&str>) -> Result<RustFSTestEnvironment, BoxError> {
|
||||
/// Starts one server of the binary under test on `stack`, overriding its region only when asked.
|
||||
async fn start_stack(stack: &str, region: Option<&str>) -> Result<RustFSTestEnvironment, BoxError> {
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
let mut vars = Vec::new();
|
||||
let mut vars = vec![("RUSTFS_S3_STACK", stack)];
|
||||
if let Some(region) = region {
|
||||
vars.push(("RUSTFS_REGION", region));
|
||||
}
|
||||
@@ -91,7 +94,8 @@ async fn start_under_test(region: Option<&str>) -> Result<RustFSTestEnvironment,
|
||||
Ok(env)
|
||||
}
|
||||
|
||||
/// Starts one server of a released binary.
|
||||
/// Starts one server of a released binary. `RUSTFS_S3_STACK` is not passed: older releases do not
|
||||
/// know it and have exactly one stack.
|
||||
async fn start_reference(binary: &Path, region: Option<&str>) -> Result<RustFSTestEnvironment, BoxError> {
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
let mut vars = Vec::new();
|
||||
@@ -107,11 +111,11 @@ async fn start_reference(binary: &Path, region: Option<&str>) -> Result<RustFSTe
|
||||
async fn spawn_pair(config: &PairConfig) -> Result<(RustFSTestEnvironment, RustFSTestEnvironment), BoxError> {
|
||||
let side_a = async {
|
||||
match &config.side_a {
|
||||
SideA::SameBinary => start_under_test(config.region_a.as_deref()).await,
|
||||
SideA::SameBinary => start_stack(STACK_A, config.region_a.as_deref()).await,
|
||||
SideA::Reference(binary) => start_reference(binary, config.region_a.as_deref()).await,
|
||||
}
|
||||
};
|
||||
tokio::try_join!(side_a, start_under_test(config.region_b.as_deref()))
|
||||
tokio::try_join!(side_a, start_stack(&config.stack_b, config.region_b.as_deref()))
|
||||
}
|
||||
|
||||
fn report_directory() -> PathBuf {
|
||||
@@ -156,7 +160,7 @@ async fn run_differential(label: &str, config: &PairConfig) -> Run {
|
||||
schema: 2,
|
||||
run: format!("{label}-{millis}-{}", std::process::id()),
|
||||
stack_a: config.stack_a_label().to_owned(),
|
||||
stack_b: UNDER_TEST.to_owned(),
|
||||
stack_b: config.stack_b.clone(),
|
||||
reference_side: matches!(config.side_a, SideA::Reference(_)).then_some("a"),
|
||||
version_a,
|
||||
version_b,
|
||||
@@ -231,6 +235,7 @@ mod tests {
|
||||
init_logging();
|
||||
let config = PairConfig {
|
||||
side_a: SideA::SameBinary,
|
||||
stack_b: STACK_A.to_owned(),
|
||||
region_a: Some(CONTROL_REGION_A.to_owned()),
|
||||
region_b: Some(CONTROL_REGION_B.to_owned()),
|
||||
};
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
//! Which binary side A of the differential runs, and what it reports about itself.
|
||||
//!
|
||||
//! `RUSTFS_E2E_REFERENCE_BINARY` names a released `rustfs` binary to use as the reference. Unset,
|
||||
//! side A is the binary under test. Set, it must resolve to an
|
||||
//! side A is the binary under test on `RUSTFS_S3_STACK=legacy`. Set, it must resolve to an
|
||||
//! executable file or the run fails: a reference that was asked for and silently replaced by the
|
||||
//! binary under test would report an agreement with itself as agreement with the release.
|
||||
|
||||
@@ -28,9 +28,9 @@ pub(super) const ENV_REFERENCE_BINARY: &str = "RUSTFS_E2E_REFERENCE_BINARY";
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(super) enum SideA {
|
||||
/// Both sides are the binary under test.
|
||||
/// Both sides are the binary under test; A runs `RUSTFS_S3_STACK=legacy`.
|
||||
SameBinary,
|
||||
/// A is this released binary.
|
||||
/// A is this released binary, started without `RUSTFS_S3_STACK`.
|
||||
Reference(PathBuf),
|
||||
}
|
||||
|
||||
@@ -78,7 +78,7 @@ fn is_executable(_metadata: &std::fs::Metadata) -> bool {
|
||||
|
||||
/// What `<binary> --version` printed, trimmed. Fails when the binary does not run, exits
|
||||
/// non-zero, or prints nothing, because an empty version would report nothing as observed.
|
||||
pub(crate) async fn binary_version(binary: &Path) -> Result<String, BoxError> {
|
||||
pub(super) async fn binary_version(binary: &Path) -> Result<String, BoxError> {
|
||||
let output = tokio::process::Command::new(binary).arg("--version").output().await?;
|
||||
if !output.status.success() {
|
||||
return Err(format!("{} --version failed with {}", binary.display(), output.status).into());
|
||||
|
||||
@@ -14,42 +14,7 @@
|
||||
|
||||
//! Differential cases of the `bucket_access` batch (see `super`).
|
||||
|
||||
use super::super::{Recorder, none};
|
||||
use super::super::Recorder;
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
||||
|
||||
const BUCKET: &str = "a2-bucket-access";
|
||||
|
||||
pub(super) async fn cases(s: &mut Recorder, c: &Client) {
|
||||
s.step("CreateBucket", "a2-setup", c.create_bucket().bucket(BUCKET).send(), none)
|
||||
.await;
|
||||
versioning(s, c).await;
|
||||
}
|
||||
|
||||
async fn versioning(s: &mut Recorder, c: &Client) {
|
||||
let status = |o: &aws_sdk_s3::operation::get_bucket_versioning::GetBucketVersioningOutput| {
|
||||
vec![
|
||||
("status", format!("{:?}", o.status())),
|
||||
("mfa_delete", format!("{:?}", o.mfa_delete())),
|
||||
]
|
||||
};
|
||||
s.step(
|
||||
"GetBucketVersioning",
|
||||
"unconfigured",
|
||||
c.get_bucket_versioning().bucket(BUCKET).send(),
|
||||
status,
|
||||
)
|
||||
.await;
|
||||
let suspended = VersioningConfiguration::builder()
|
||||
.status(BucketVersioningStatus::Suspended)
|
||||
.build();
|
||||
let put = c.put_bucket_versioning().bucket(BUCKET).versioning_configuration(suspended);
|
||||
s.step("PutBucketVersioning", "a2-suspended", put.send(), none).await;
|
||||
s.step(
|
||||
"GetBucketVersioning",
|
||||
"suspended",
|
||||
c.get_bucket_versioning().bucket(BUCKET).send(),
|
||||
status,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
pub(super) async fn cases(_s: &mut Recorder, _c: &Client) {}
|
||||
|
||||
@@ -14,59 +14,7 @@
|
||||
|
||||
//! Differential cases of the `bucket_core` batch (see `super`).
|
||||
|
||||
use super::super::{Recorder, none};
|
||||
use super::super::Recorder;
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::types::{BucketLocationConstraint, CreateBucketConfiguration};
|
||||
|
||||
/// Names RustFS creates although the AWS grammar reserves them, and one it refuses.
|
||||
const RESERVED_NAMES: [(&str, &str); 5] = [
|
||||
("reserved-sthree", "sthree-dual-stack"),
|
||||
("reserved-s3alias", "dual-stack-s3alias"),
|
||||
("reserved-ol-s3", "dual-stack--ol-s3"),
|
||||
("reserved-x-s3", "dual-stack--x-s3"),
|
||||
("reserved-xn", "xn--dual-stack"),
|
||||
];
|
||||
|
||||
pub(super) async fn cases(s: &mut Recorder, c: &Client) {
|
||||
let located = |bucket: &str, location: &str| {
|
||||
let configuration = CreateBucketConfiguration::builder()
|
||||
.location_constraint(BucketLocationConstraint::from(location))
|
||||
.build();
|
||||
c.create_bucket().bucket(bucket).create_bucket_configuration(configuration)
|
||||
};
|
||||
s.step(
|
||||
"CreateBucket",
|
||||
"foreign-location",
|
||||
located("dual-stack-foreign", "ap-south-1").send(),
|
||||
none,
|
||||
)
|
||||
.await;
|
||||
s.step("CreateBucket", "eu-alias", located("dual-stack-eu", "EU").send(), none)
|
||||
.await;
|
||||
for (case, name) in RESERVED_NAMES {
|
||||
s.step("CreateBucket", case, c.create_bucket().bucket(name).send(), none)
|
||||
.await;
|
||||
}
|
||||
for (case, bucket, header) in FORCE_DELETES {
|
||||
// Each case deletes a bucket of its own; the create is setup, not a recorded step.
|
||||
let _ = c.create_bucket().bucket(bucket).send().await;
|
||||
let delete = c
|
||||
.delete_bucket()
|
||||
.bucket(bucket)
|
||||
.customize()
|
||||
.mutate_request(move |request| {
|
||||
request.headers_mut().insert(header.0, header.1);
|
||||
})
|
||||
.send();
|
||||
s.step("DeleteBucket", case, delete, none).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Force-delete headers under both spellings, with values each parser reads differently.
|
||||
const FORCE_DELETES: [(&str, &str, (&str, &str)); 5] = [
|
||||
("force-rustfs-true", "dual-stack-force-a", ("x-rustfs-force-delete", "true")),
|
||||
("force-rustfs-invalid", "dual-stack-force-b", ("x-rustfs-force-delete", "maybe")),
|
||||
("force-minio-true", "dual-stack-force-c", ("x-minio-force-delete", "true")),
|
||||
("force-minio-numeric", "dual-stack-force-d", ("x-minio-force-delete", "1")),
|
||||
("force-minio-invalid", "dual-stack-force-e", ("x-minio-force-delete", "maybe")),
|
||||
];
|
||||
pub(super) async fn cases(_s: &mut Recorder, _c: &Client) {}
|
||||
|
||||
+2
-118
@@ -13,124 +13,8 @@
|
||||
// limitations under the License.
|
||||
|
||||
//! Differential cases of the `bucket_replication_lifecycle` batch (see `super`).
|
||||
//!
|
||||
//! Replication has no case of its own: storing a replication configuration needs a remote target
|
||||
//! registered through the admin API, which this script does not drive.
|
||||
|
||||
use super::super::{Fields, RETAIN_UNTIL_SECS, Recorder, none};
|
||||
use super::super::Recorder;
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::primitives::DateTime;
|
||||
use aws_sdk_s3::types::{
|
||||
AbortIncompleteMultipartUpload, BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, LifecycleExpiration,
|
||||
LifecycleRule, LifecycleRuleAndOperator, LifecycleRuleFilter, NoncurrentVersionExpiration, ObjectLockConfiguration,
|
||||
ObjectLockEnabled, ObjectLockRetentionMode, ObjectLockRule, Tag,
|
||||
};
|
||||
|
||||
/// A name RustFS creates and serves that the AWS bucket grammar reserves.
|
||||
const RESERVED_BUCKET: &str = "sthree-dual-stack-a3";
|
||||
const LOCK_BUCKET: &str = "dual-stack-a3-lock";
|
||||
|
||||
fn lifecycle_rules(
|
||||
output: &aws_sdk_s3::operation::get_bucket_lifecycle_configuration::GetBucketLifecycleConfigurationOutput,
|
||||
) -> Fields {
|
||||
vec![("rules", format!("{:?}", output.rules()))]
|
||||
}
|
||||
|
||||
fn object_lock(output: &aws_sdk_s3::operation::get_object_lock_configuration::GetObjectLockConfigurationOutput) -> Fields {
|
||||
vec![("configuration", format!("{:?}", output.object_lock_configuration()))]
|
||||
}
|
||||
|
||||
pub(super) async fn cases(s: &mut Recorder, c: &Client) {
|
||||
s.step("CreateBucket", "a3-reserved-name", c.create_bucket().bucket(RESERVED_BUCKET).send(), none)
|
||||
.await;
|
||||
let get_lifecycle = || c.get_bucket_lifecycle_configuration().bucket(RESERVED_BUCKET);
|
||||
s.step("GetBucketLifecycleConfiguration", "a3-reserved-absent", get_lifecycle().send(), none)
|
||||
.await;
|
||||
let rule = LifecycleRule::builder()
|
||||
.id("members")
|
||||
.status(ExpirationStatus::Enabled)
|
||||
.filter(
|
||||
LifecycleRuleFilter::builder()
|
||||
.and(
|
||||
LifecycleRuleAndOperator::builder()
|
||||
.prefix("logs/")
|
||||
.tags(Tag::builder().key("tier").value("cold").build().expect("tag"))
|
||||
.object_size_greater_than(1)
|
||||
.build(),
|
||||
)
|
||||
.build(),
|
||||
)
|
||||
.expiration(LifecycleExpiration::builder().expired_object_delete_marker(true).build())
|
||||
.noncurrent_version_expiration(
|
||||
NoncurrentVersionExpiration::builder()
|
||||
.noncurrent_days(7)
|
||||
.newer_noncurrent_versions(2)
|
||||
.build(),
|
||||
)
|
||||
.abort_incomplete_multipart_upload(AbortIncompleteMultipartUpload::builder().days_after_initiation(3).build())
|
||||
.build()
|
||||
.expect("lifecycle rule");
|
||||
// A stored date is answered through a different conversion on each edge of the migration.
|
||||
let dated = LifecycleRule::builder()
|
||||
.id("dated")
|
||||
.status(ExpirationStatus::Disabled)
|
||||
.filter(LifecycleRuleFilter::builder().prefix("archive/").build())
|
||||
.expiration(
|
||||
LifecycleExpiration::builder()
|
||||
.date(DateTime::from_secs(RETAIN_UNTIL_SECS))
|
||||
.build(),
|
||||
)
|
||||
.build()
|
||||
.expect("dated lifecycle rule");
|
||||
let lifecycle = BucketLifecycleConfiguration::builder()
|
||||
.rules(rule)
|
||||
.rules(dated)
|
||||
.build()
|
||||
.expect("lifecycle configuration");
|
||||
let put = c
|
||||
.put_bucket_lifecycle_configuration()
|
||||
.bucket(RESERVED_BUCKET)
|
||||
.lifecycle_configuration(lifecycle);
|
||||
s.step("PutBucketLifecycleConfiguration", "a3-every-member", put.send(), none)
|
||||
.await;
|
||||
s.step(
|
||||
"GetBucketLifecycleConfiguration",
|
||||
"a3-every-member",
|
||||
get_lifecycle().send(),
|
||||
lifecycle_rules,
|
||||
)
|
||||
.await;
|
||||
let get_lock = |bucket: &str| c.get_object_lock_configuration().bucket(bucket);
|
||||
s.step("GetObjectLockConfiguration", "a3-reserved-absent", get_lock(RESERVED_BUCKET).send(), none)
|
||||
.await;
|
||||
|
||||
let create = c.create_bucket().bucket(LOCK_BUCKET).object_lock_enabled_for_bucket(true);
|
||||
s.step("CreateBucket", "a3-lock", create.send(), none).await;
|
||||
s.step(
|
||||
"GetObjectLockConfiguration",
|
||||
"a3-enabled-without-rule",
|
||||
get_lock(LOCK_BUCKET).send(),
|
||||
object_lock,
|
||||
)
|
||||
.await;
|
||||
let lock = ObjectLockConfiguration::builder()
|
||||
.object_lock_enabled(ObjectLockEnabled::Enabled)
|
||||
.rule(
|
||||
ObjectLockRule::builder()
|
||||
.default_retention(
|
||||
DefaultRetention::builder()
|
||||
.mode(ObjectLockRetentionMode::Compliance)
|
||||
.years(1)
|
||||
.build(),
|
||||
)
|
||||
.build(),
|
||||
)
|
||||
.build();
|
||||
let put = c
|
||||
.put_object_lock_configuration()
|
||||
.bucket(LOCK_BUCKET)
|
||||
.object_lock_configuration(lock);
|
||||
s.step("PutObjectLockConfiguration", "a3-years", put.send(), none).await;
|
||||
s.step("GetObjectLockConfiguration", "a3-years", get_lock(LOCK_BUCKET).send(), object_lock)
|
||||
.await;
|
||||
}
|
||||
pub(super) async fn cases(_s: &mut Recorder, _c: &Client) {}
|
||||
|
||||
@@ -12,99 +12,9 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Differential cases of the `listing_multipart` batch (see `super`): stored keys that need
|
||||
//! encoding, with and without `encoding-type=url`, ListObjects' marker, ListObjectsV2 pagination,
|
||||
//! and the listings' parameter refusals.
|
||||
//! Differential cases of the `listing_multipart` batch (see `super`).
|
||||
|
||||
use super::super::{Fields, Recorder, none};
|
||||
use super::super::Recorder;
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::EncodingType;
|
||||
|
||||
const BUCKET: &str = "differential-listing";
|
||||
const KEYS: [&str; 3] = ["dir a/file+b%.txt", "markup &<>\"'", "~tilde"];
|
||||
|
||||
fn v2_fields(o: &aws_sdk_s3::operation::list_objects_v2::ListObjectsV2Output) -> Fields {
|
||||
let keys: Vec<_> = o.contents().iter().map(|object| object.key()).collect();
|
||||
vec![
|
||||
("keys", format!("{keys:?}")),
|
||||
("encoding_type", format!("{:?}", o.encoding_type())),
|
||||
("key_count", format!("{:?}", o.key_count())),
|
||||
("is_truncated", format!("{:?}", o.is_truncated())),
|
||||
("has_next_token", format!("{}", o.next_continuation_token().is_some())),
|
||||
]
|
||||
}
|
||||
|
||||
fn v1_fields(o: &aws_sdk_s3::operation::list_objects::ListObjectsOutput) -> Fields {
|
||||
let keys: Vec<_> = o.contents().iter().map(|object| object.key()).collect();
|
||||
vec![
|
||||
("keys", format!("{keys:?}")),
|
||||
("marker", format!("{:?}", o.marker())),
|
||||
("next_marker", format!("{:?}", o.next_marker())),
|
||||
("encoding_type", format!("{:?}", o.encoding_type())),
|
||||
]
|
||||
}
|
||||
|
||||
pub(super) async fn cases(s: &mut Recorder, c: &Client) {
|
||||
s.step("ListObjectsV2", "setup-bucket", c.create_bucket().bucket(BUCKET).send(), none)
|
||||
.await;
|
||||
for (key, case) in KEYS.into_iter().zip(["setup-object-1", "setup-object-2", "setup-object-3"]) {
|
||||
let put = c
|
||||
.put_object()
|
||||
.bucket(BUCKET)
|
||||
.key(key)
|
||||
.body(ByteStream::from_static(b"x"))
|
||||
.send();
|
||||
s.step("ListObjectsV2", case, put, none).await;
|
||||
}
|
||||
|
||||
let v2 = || c.list_objects_v2().bucket(BUCKET);
|
||||
s.step("ListObjectsV2", "raw-keys", v2().send(), v2_fields).await;
|
||||
s.step("ListObjectsV2", "url-encoded", v2().encoding_type(EncodingType::Url).send(), v2_fields)
|
||||
.await;
|
||||
let first = s
|
||||
.step("ListObjectsV2", "first-page", v2().max_keys(2).send(), v2_fields)
|
||||
.await;
|
||||
let token = first.and_then(|page| page.next_continuation_token().map(str::to_owned));
|
||||
s.step("ListObjectsV2", "next-page", v2().set_continuation_token(token).send(), v2_fields)
|
||||
.await;
|
||||
s.step("ListObjectsV2", "negative-max-keys", v2().max_keys(-1).send(), none)
|
||||
.await;
|
||||
s.step("ListObjectsV2", "undecodable-token", v2().continuation_token("%%%").send(), none)
|
||||
.await;
|
||||
|
||||
let v1 = || c.list_objects().bucket(BUCKET).max_keys(1);
|
||||
s.step("ListObjects", "raw-marker", v1().send(), v1_fields).await;
|
||||
s.step(
|
||||
"ListObjects",
|
||||
"url-encoded-marker",
|
||||
v1().encoding_type(EncodingType::Url).send(),
|
||||
v1_fields,
|
||||
)
|
||||
.await;
|
||||
s.step("ListObjects", "request-marker", v1().marker(KEYS[0]).send(), v1_fields)
|
||||
.await;
|
||||
s.step("ListObjects", "negative-max-keys", v1().max_keys(-1).send(), none)
|
||||
.await;
|
||||
|
||||
let create = c.create_multipart_upload().bucket(BUCKET).key(KEYS[0]).send();
|
||||
s.step("ListMultipartUploads", "setup-upload", create, none).await;
|
||||
let uploads = || c.list_multipart_uploads().bucket(BUCKET);
|
||||
s.step("ListMultipartUploads", "raw-keys", uploads().send(), upload_fields)
|
||||
.await;
|
||||
let encoded = uploads().prefix("dir a/").encoding_type(EncodingType::Url).send();
|
||||
s.step("ListMultipartUploads", "url-encoded", encoded, upload_fields).await;
|
||||
s.step("ListMultipartUploads", "zero-max-uploads", uploads().max_uploads(0).send(), none)
|
||||
.await;
|
||||
let foreign = uploads().prefix("dir").key_marker("other").send();
|
||||
s.step("ListMultipartUploads", "foreign-key-marker", foreign, none).await;
|
||||
}
|
||||
|
||||
fn upload_fields(o: &aws_sdk_s3::operation::list_multipart_uploads::ListMultipartUploadsOutput) -> Fields {
|
||||
let keys: Vec<_> = o.uploads().iter().map(|upload| upload.key()).collect();
|
||||
vec![
|
||||
("keys", format!("{keys:?}")),
|
||||
("prefix", format!("{:?}", o.prefix())),
|
||||
("encoding_type", format!("{:?}", o.encoding_type())),
|
||||
]
|
||||
}
|
||||
pub(super) async fn cases(_s: &mut Recorder, _c: &Client) {}
|
||||
|
||||
@@ -14,106 +14,7 @@
|
||||
|
||||
//! Differential cases of the `object_meta` batch (see `super`).
|
||||
|
||||
use super::super::{Recorder, grants, none, tags};
|
||||
use super::super::Recorder;
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{AccessControlPolicy, Grant, Grantee, Owner, Permission, Type};
|
||||
|
||||
const BUCKET: &str = "diff-object-meta";
|
||||
const TAGGED: &str = "tagged";
|
||||
|
||||
pub(super) async fn cases(s: &mut Recorder, c: &Client) {
|
||||
s.step("CreateBucket", "object-meta", c.create_bucket().bucket(BUCKET).send(), none)
|
||||
.await;
|
||||
let put = c
|
||||
.put_object()
|
||||
.bucket(BUCKET)
|
||||
.key(TAGGED)
|
||||
.tagging("k=&odd%20key=v%2Fw")
|
||||
.body(ByteStream::from_static(b"object meta"));
|
||||
s.step("PutObject", "object-meta-tagged", put.send(), none).await;
|
||||
|
||||
let get_tags = |key: &str, version: Option<&str>| {
|
||||
c.get_object_tagging()
|
||||
.bucket(BUCKET)
|
||||
.key(key)
|
||||
.set_version_id(version.map(str::to_owned))
|
||||
};
|
||||
s.step("GetObjectTagging", "empty-and-encoded-tags", get_tags(TAGGED, None).send(), |o| {
|
||||
vec![("tags", tags(o.tag_set())), ("version", format!("{:?}", o.version_id()))]
|
||||
})
|
||||
.await;
|
||||
s.step(
|
||||
"GetObjectTagging",
|
||||
"malformed-version",
|
||||
get_tags(TAGGED, Some("not-a-version")).send(),
|
||||
none,
|
||||
)
|
||||
.await;
|
||||
let absent_version = "00000000-0000-0000-0000-000000000001";
|
||||
s.step("GetObjectTagging", "absent-version", get_tags(TAGGED, Some(absent_version)).send(), none)
|
||||
.await;
|
||||
s.step("GetObjectTagging", "nul-key", get_tags("a\0b", None).send(), none)
|
||||
.await;
|
||||
s.step(
|
||||
"DeleteObjectTagging",
|
||||
"nul-key",
|
||||
c.delete_object_tagging().bucket(BUCKET).key("a\0b").send(),
|
||||
none,
|
||||
)
|
||||
.await;
|
||||
s.step(
|
||||
"DeleteObjectTagging",
|
||||
"then-empty",
|
||||
c.delete_object_tagging().bucket(BUCKET).key(TAGGED).send(),
|
||||
none,
|
||||
)
|
||||
.await;
|
||||
s.step("GetObjectTagging", "after-delete", get_tags(TAGGED, None).send(), |o| {
|
||||
vec![("tags", tags(o.tag_set()))]
|
||||
})
|
||||
.await;
|
||||
|
||||
let grantee = Grantee::builder()
|
||||
.r#type(Type::CanonicalUser)
|
||||
.id("owner")
|
||||
.build()
|
||||
.expect("grantee");
|
||||
let policy = AccessControlPolicy::builder()
|
||||
.owner(Owner::builder().id("owner").build())
|
||||
.grants(Grant::builder().grantee(grantee).permission(Permission::Read).build())
|
||||
.build();
|
||||
let xml_grants = c.put_object_acl().bucket(BUCKET).key(TAGGED).access_control_policy(policy);
|
||||
s.step("PutObjectAcl", "xml-grants", xml_grants.send(), none).await;
|
||||
s.step(
|
||||
"GetObjectAcl",
|
||||
"after-xml-grants",
|
||||
c.get_object_acl().bucket(BUCKET).key(TAGGED).send(),
|
||||
|o| vec![("grants", grants(o.grants()))],
|
||||
)
|
||||
.await;
|
||||
s.step("GetObjectAcl", "nul-key", c.get_object_acl().bucket(BUCKET).key("a\0b").send(), none)
|
||||
.await;
|
||||
|
||||
s.step(
|
||||
"GetObjectLegalHold",
|
||||
"no-object-lock",
|
||||
c.get_object_legal_hold().bucket(BUCKET).key(TAGGED).send(),
|
||||
none,
|
||||
)
|
||||
.await;
|
||||
s.step(
|
||||
"GetObjectRetention",
|
||||
"no-object-lock",
|
||||
c.get_object_retention().bucket(BUCKET).key(TAGGED).send(),
|
||||
none,
|
||||
)
|
||||
.await;
|
||||
s.step(
|
||||
"GetObjectTorrent",
|
||||
"nul-key",
|
||||
c.get_object_torrent().bucket(BUCKET).key("a\0b").send(),
|
||||
none,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
pub(super) async fn cases(_s: &mut Recorder, _c: &Client) {}
|
||||
|
||||
@@ -14,50 +14,7 @@
|
||||
|
||||
//! Differential cases of the `object_misc` batch (see `super`).
|
||||
|
||||
use super::super::{Recorder, none};
|
||||
use super::super::Recorder;
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
||||
|
||||
const BUCKET: &str = "dual-stack-object-misc";
|
||||
const KEY: &str = "doomed.txt";
|
||||
|
||||
/// DeleteObject on a versioned bucket: the delete marker and its headers, the delete of that
|
||||
/// marker by version, and the refusals the delete raises before storage.
|
||||
pub(super) async fn cases(s: &mut Recorder, c: &Client) {
|
||||
s.step("CreateBucket", "object-misc-setup", c.create_bucket().bucket(BUCKET).send(), none)
|
||||
.await;
|
||||
let enabled = VersioningConfiguration::builder()
|
||||
.status(BucketVersioningStatus::Enabled)
|
||||
.build();
|
||||
let versioning = c.put_bucket_versioning().bucket(BUCKET).versioning_configuration(enabled);
|
||||
s.step("PutBucketVersioning", "object-misc-setup", versioning.send(), none)
|
||||
.await;
|
||||
let put = c.put_object().bucket(BUCKET).key(KEY).body(ByteStream::from_static(b"e2"));
|
||||
s.step("PutObject", "object-misc-setup", put.send(), none).await;
|
||||
|
||||
let marker = s
|
||||
.step(
|
||||
"DeleteObject",
|
||||
"creates-delete-marker",
|
||||
c.delete_object().bucket(BUCKET).key(KEY).send(),
|
||||
none,
|
||||
)
|
||||
.await
|
||||
.and_then(|output| output.version_id().map(str::to_owned));
|
||||
if let Some(marker) = marker {
|
||||
let delete = c.delete_object().bucket(BUCKET).key(KEY).version_id(marker);
|
||||
s.step("DeleteObject", "removes-delete-marker", delete.send(), none).await;
|
||||
}
|
||||
let delete = c.delete_object().bucket(BUCKET).key(KEY).version_id("not-a-version");
|
||||
s.step("DeleteObject", "malformed-version", delete.send(), none).await;
|
||||
let delete = c
|
||||
.delete_object()
|
||||
.bucket(BUCKET)
|
||||
.key(KEY)
|
||||
.customize()
|
||||
.mutate_request(|request| {
|
||||
request.headers_mut().insert("x-rustfs-force-delete", "maybe");
|
||||
});
|
||||
s.step("DeleteObject", "invalid-force-delete", delete.send(), none).await;
|
||||
}
|
||||
pub(super) async fn cases(_s: &mut Recorder, _c: &Client) {}
|
||||
|
||||
@@ -29,11 +29,10 @@ use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::ServerSideEncryption;
|
||||
use base64_simd::STANDARD as BASE64;
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use jiff::Timestamp;
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use serde_json;
|
||||
use std::process::{Child, Command};
|
||||
use std::time::Duration;
|
||||
@@ -387,7 +386,7 @@ pub async fn create_key_with_specific_id(key_dir: &str, key_id: &str) -> Result<
|
||||
"usage": "EncryptDecrypt",
|
||||
"status": "Active",
|
||||
"metadata": HashMap::<String, String>::new(),
|
||||
"created_at": format!("{}[UTC]", Timestamp::now().strftime("%Y-%m-%dT%H:%M:%S%.fZ")),
|
||||
"created_at": format!("{}[UTC]", chrono::Utc::now().to_rfc3339()),
|
||||
"rotated_at": serde_json::Value::Null,
|
||||
"created_by": "e2e-test",
|
||||
"encrypted_key_material": BASE64.encode_to_string(key_data),
|
||||
|
||||
@@ -31,15 +31,12 @@ pub mod chaos;
|
||||
#[cfg(test)]
|
||||
pub mod fake_s3_target;
|
||||
|
||||
// Differential (rustfs/backlog#2740): two processes, the binary under test against
|
||||
// itself or a released reference binary, the same SDK script for every S3 operation,
|
||||
// Dual-stack differential (rustfs/backlog#2740): one binary, two processes on
|
||||
// different RUSTFS_S3_STACK values, the same SDK script for every S3 operation,
|
||||
// compared on status, error code, allowlisted headers and typed body fields.
|
||||
#[cfg(test)]
|
||||
mod dual_stack_differential;
|
||||
|
||||
#[cfg(test)]
|
||||
mod persistence_parity;
|
||||
|
||||
// On-demand migration (backlog#2147): shared two-server environment, admin
|
||||
// wrappers, and the harness self-test (backlog#2151). Behavior scenarios are
|
||||
// added by later ODM tasks.
|
||||
|
||||
@@ -24,9 +24,9 @@
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use http::header::HOST;
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use std::error::Error;
|
||||
|
||||
/// Sends a SigV4-signed `GET` where the signature is computed over `sign_path`
|
||||
|
||||
@@ -21,9 +21,9 @@ mod tests {
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
||||
use http::header::HOST;
|
||||
use reqwest::StatusCode;
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use std::error::Error;
|
||||
use tracing::info;
|
||||
|
||||
|
||||
@@ -20,9 +20,9 @@ mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use http::header::HOST;
|
||||
use reqwest::StatusCode;
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use std::error::Error;
|
||||
use tracing::info;
|
||||
|
||||
|
||||
@@ -23,17 +23,17 @@ use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
ServerSideEncryption, ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule,
|
||||
};
|
||||
use chrono::{Duration as ChronoDuration, Utc};
|
||||
use flate2::{
|
||||
Compression,
|
||||
write::{GzEncoder, ZlibEncoder},
|
||||
};
|
||||
use http::HeaderValue;
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use jiff::{SignedDuration, Timestamp};
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use std::collections::HashMap;
|
||||
use std::error::Error;
|
||||
use std::io::Cursor;
|
||||
@@ -42,8 +42,8 @@ use tokio::io::AsyncWriteExt;
|
||||
use uuid::Uuid;
|
||||
|
||||
fn encode_post_policy(conditions: Vec<serde_json::Value>) -> String {
|
||||
let expiration = (Timestamp::now() + SignedDuration::from_hours(1))
|
||||
.strftime("%Y-%m-%dT%H:%M:%S.000Z")
|
||||
let expiration = (Utc::now() + ChronoDuration::hours(1))
|
||||
.format("%Y-%m-%dT%H:%M:%S.000Z")
|
||||
.to_string();
|
||||
let policy = serde_json::json!({
|
||||
"expiration": expiration,
|
||||
@@ -1543,30 +1543,6 @@ async fn test_anonymous_post_object_honors_success_action_status() -> Result<(),
|
||||
let uploaded = get_out.body.collect().await?.into_bytes();
|
||||
assert_eq!(uploaded.as_ref(), expected_body.as_slice(), "uploaded object body should match");
|
||||
|
||||
// An empty redirect is rejected when rendering the response, after storage succeeds.
|
||||
for (field, value, expected_status) in [
|
||||
("success_action_status", "200", reqwest::StatusCode::OK),
|
||||
("success_action_status", "204", reqwest::StatusCode::NO_CONTENT),
|
||||
("success_action_redirect", "", reqwest::StatusCode::BAD_REQUEST),
|
||||
] {
|
||||
let key = format!("{field}-{value}.txt");
|
||||
let form = reqwest::multipart::Form::new()
|
||||
.text("key", key.clone())
|
||||
.text(field, value)
|
||||
.part("file", reqwest::multipart::Part::bytes(expected_body.clone()).file_name("upload.txt"));
|
||||
let response = local_http_client()
|
||||
.post(format!("{}/{bucket}", env.url))
|
||||
.multipart(form)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(response.status(), expected_status);
|
||||
if value.is_empty() {
|
||||
assert!(response.text().await?.contains("<Code>InvalidArgument</Code>"));
|
||||
}
|
||||
let stored = admin_client.get_object().bucket(bucket).key(key).send().await?;
|
||||
assert_eq!(stored.body.collect().await?.into_bytes().as_ref(), expected_body.as_slice());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -283,7 +283,6 @@ async fn test_cold_degraded_object_mutations_require_only_storage_quorum() -> Te
|
||||
#[tokio::test]
|
||||
async fn test_degraded_cluster_read_quorum_follows_erasure_layout() -> TestResult {
|
||||
crate::common::init_logging();
|
||||
assert_single_node_cold_start_with_missing_volume().await?;
|
||||
|
||||
for (node_count, parity) in [(3, 1), (4, 2), (6, 3), (6, 2), (4, 0)] {
|
||||
let read_quorum = node_count - parity;
|
||||
@@ -431,138 +430,6 @@ async fn test_degraded_cluster_read_quorum_follows_erasure_layout() -> TestResul
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_single_node_cold_start_with_missing_volume() -> TestResult {
|
||||
for drive_count in [2, 3, 4] {
|
||||
let mut server = crate::chaos::DiskFaultHarness::new(drive_count).await?;
|
||||
server.set_env("RUSTFS_STORAGE_CLASS_STANDARD", format!("EC:{}", drive_count / 2));
|
||||
server.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false");
|
||||
server.set_env("RUSTFS_OBS_METRICS_EXPORT_ENABLED", "false");
|
||||
server.set_env("RUSTFS_HEALTH_ENDPOINT_ENABLE", "true");
|
||||
server.set_env("RUST_LOG", "warn");
|
||||
server.start_server_at_read_quorum().await?;
|
||||
let client = server.env.create_s3_client();
|
||||
let client = Client::from_conf(
|
||||
client
|
||||
.config()
|
||||
.to_builder()
|
||||
.retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(1))
|
||||
.build(),
|
||||
);
|
||||
let bucket = format!("missing-volume-{drive_count}");
|
||||
client.create_bucket().bucket(&bucket).send().await?;
|
||||
let small = vec![17; 4096];
|
||||
let large = vec![73; 8 * 1024 * 1024];
|
||||
for (key, body) in [("small", &small), ("large", &large)] {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.body(Bytes::copy_from_slice(body).into())
|
||||
.send()
|
||||
.await?;
|
||||
for disk in 0..drive_count {
|
||||
let census = crate::chaos::census_object_version_on_disk(server.disk_path(disk), &bucket, key, None)?;
|
||||
assert!(census.is_complete(), "seed shard must exist before restart: {census:?}");
|
||||
}
|
||||
}
|
||||
server.kill_server();
|
||||
let missing = drive_count - 1;
|
||||
server.take_disk_offline(missing)?;
|
||||
server.start_server_at_read_quorum().await?;
|
||||
|
||||
for (key, body) in [("small", &small), ("large", &large)] {
|
||||
assert_quorum_object_body(&client, &bucket, key, body).await?;
|
||||
let head = client.head_object().bucket(&bucket).key(key).send().await?;
|
||||
assert_eq!(head.content_length(), Some(i64::try_from(body.len())?));
|
||||
}
|
||||
let listing = client.list_objects_v2().bucket(&bucket).send().await?;
|
||||
let keys: Vec<_> = listing.contents().iter().filter_map(|object| object.key()).collect();
|
||||
assert_eq!(keys, ["large", "small"]);
|
||||
|
||||
let put = client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key("degraded-write")
|
||||
.body(Bytes::from_static(b"degraded write").into())
|
||||
.send()
|
||||
.await;
|
||||
if drive_count == 2 {
|
||||
let error = put.expect_err("one of two drives cannot commit a new object");
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(503));
|
||||
let error = client
|
||||
.delete_object()
|
||||
.bucket(&bucket)
|
||||
.key("large")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("one of two drives cannot commit an object deletion");
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(503));
|
||||
assert_quorum_object_body(&client, &bucket, "large", &large).await?;
|
||||
} else {
|
||||
put?;
|
||||
assert_quorum_object_body(&client, &bucket, "degraded-write", b"degraded write").await?;
|
||||
client.delete_object().bucket(&bucket).key("degraded-write").send().await?;
|
||||
}
|
||||
assert!(
|
||||
!server.disk_path(missing).exists(),
|
||||
"degraded startup and S3 operations must leave the root absent"
|
||||
);
|
||||
|
||||
if drive_count == 2 {
|
||||
server.kill_server();
|
||||
std::fs::create_dir(server.disk_path(missing))?;
|
||||
server.start_server_at_read_quorum().await?;
|
||||
for (key, body) in [("small", &small), ("large", &large)] {
|
||||
assert_quorum_object_body(&client, &bucket, key, body).await?;
|
||||
client.head_object().bucket(&bucket).key(key).send().await?;
|
||||
}
|
||||
assert_eq!(client.list_objects_v2().bucket(&bucket).send().await?.contents().len(), 2);
|
||||
let error = client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key("empty-root-write")
|
||||
.body(Bytes::from_static(b"must remain unavailable").into())
|
||||
.send()
|
||||
.await
|
||||
.expect_err("an empty replacement cannot restore write quorum");
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(503));
|
||||
assert!(
|
||||
!server
|
||||
.disk_path(missing)
|
||||
.join(crate::storage_api::RUSTFS_META_BUCKET)
|
||||
.join("format.json")
|
||||
.exists(),
|
||||
"read admission must not format the empty replacement"
|
||||
);
|
||||
}
|
||||
server.kill_server();
|
||||
server.bring_disk_online(missing)?;
|
||||
server.start_server_at_read_quorum().await?;
|
||||
for (key, body) in [("small", &small), ("large", &large)] {
|
||||
assert_quorum_object_body(&client, &bucket, key, body).await?;
|
||||
}
|
||||
for key in ["degraded-write", "empty-root-write"] {
|
||||
let error = client
|
||||
.head_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("rejected or deleted degraded write must stay absent after recovery");
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(404));
|
||||
}
|
||||
client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key("recovery-write")
|
||||
.body(Bytes::from_static(b"recovered write").into())
|
||||
.send()
|
||||
.await?;
|
||||
assert_quorum_object_body(&client, &bucket, "recovery-write", b"recovered write").await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cold_start_read_quorum_follows_erasure_layout() -> TestResult {
|
||||
crate::common::init_logging();
|
||||
|
||||
@@ -17,9 +17,9 @@ use aws_sdk_s3::primitives::ByteStream;
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use reqwest::StatusCode;
|
||||
use rustfs_config::{ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, ENV_NOTIFY_ENABLE};
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::pre_sign_v4;
|
||||
use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS;
|
||||
use s3s::Body;
|
||||
use std::collections::HashMap;
|
||||
use std::error::Error;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
@@ -29,7 +29,7 @@ use aws_sdk_s3::types::{
|
||||
DefaultRetention, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHold, ObjectLockLegalHoldStatus, ObjectLockMode,
|
||||
ObjectLockRetention, ObjectLockRetentionMode, ObjectLockRule,
|
||||
};
|
||||
use jiff::{SignedDuration, Timestamp};
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use tracing::info;
|
||||
|
||||
use crate::common::RustFSTestEnvironment;
|
||||
@@ -126,10 +126,10 @@ pub async fn put_object_with_retention(
|
||||
key: &str,
|
||||
data: &[u8],
|
||||
mode: ObjectLockRetentionMode,
|
||||
retain_until: Timestamp,
|
||||
retain_until: DateTime<Utc>,
|
||||
) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
|
||||
// AWS SDK requires UTC time without timezone offset (e.g., "2026-01-24T11:20:14Z")
|
||||
let retain_until_str = retain_until.strftime("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||||
let retain_until_str = retain_until.format("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||||
let retain_until_datetime =
|
||||
aws_sdk_s3::primitives::DateTime::from_str(&retain_until_str, aws_sdk_s3::primitives::DateTimeFormat::DateTime)?;
|
||||
|
||||
@@ -181,10 +181,10 @@ pub async fn put_object_retention(
|
||||
key: &str,
|
||||
version_id: Option<&str>,
|
||||
mode: ObjectLockRetentionMode,
|
||||
retain_until: Timestamp,
|
||||
retain_until: DateTime<Utc>,
|
||||
bypass_governance: bool,
|
||||
) -> Result<(), Box<SdkError<PutObjectRetentionError>>> {
|
||||
let retain_until_datetime = aws_sdk_s3::primitives::DateTime::from_secs(retain_until.as_second());
|
||||
let retain_until_datetime = aws_sdk_s3::primitives::DateTime::from_secs(retain_until.timestamp());
|
||||
|
||||
let retention = ObjectLockRetention::builder()
|
||||
.mode(mode.clone())
|
||||
@@ -253,6 +253,6 @@ pub async fn delete_object_with_bypass(
|
||||
}
|
||||
|
||||
/// Calculate a future retain_until date
|
||||
pub fn future_retain_until(days: i64) -> Timestamp {
|
||||
Timestamp::now() + SignedDuration::from_hours(days * 24)
|
||||
pub fn future_retain_until(days: i64) -> DateTime<Utc> {
|
||||
Utc::now() + Duration::days(days)
|
||||
}
|
||||
|
||||
@@ -34,7 +34,7 @@ use aws_sdk_s3::types::{
|
||||
CompletedMultipartUpload, CompletedPart, Delete, MetadataDirective, ObjectIdentifier, ObjectLockLegalHoldStatus,
|
||||
ObjectLockMode, ObjectLockRetentionMode,
|
||||
};
|
||||
use jiff::{SignedDuration, Timestamp};
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use tracing::info;
|
||||
|
||||
/// Initialize test logging
|
||||
@@ -67,7 +67,7 @@ async fn put_bucket_deny_policy(
|
||||
}
|
||||
|
||||
fn retention_timestamp(days: i64) -> aws_sdk_s3::primitives::DateTime {
|
||||
let retain_until = future_retain_until(days).strftime("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||||
let retain_until = future_retain_until(days).format("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||||
aws_sdk_s3::primitives::DateTime::from_str(&retain_until, aws_sdk_s3::primitives::DateTimeFormat::DateTime)
|
||||
.expect("retention timestamp should parse")
|
||||
}
|
||||
@@ -118,14 +118,13 @@ where
|
||||
);
|
||||
}
|
||||
|
||||
fn parse_s3_datetime(value: &aws_sdk_s3::primitives::DateTime) -> Timestamp {
|
||||
fn parse_s3_datetime(value: &aws_sdk_s3::primitives::DateTime) -> DateTime<Utc> {
|
||||
let formatted = value
|
||||
.fmt(DateTimeFormat::DateTime)
|
||||
.expect("S3 timestamp should format as RFC3339");
|
||||
let normalized = formatted
|
||||
.strip_suffix('Z')
|
||||
.map_or(formatted.clone(), |prefix| format!("{prefix}+00:00"));
|
||||
Timestamp::strptime("%Y-%m-%dT%H:%M:%S%.f%:z", normalized).expect("S3 timestamp should parse as RFC3339")
|
||||
DateTime::parse_from_rfc3339(&formatted)
|
||||
.expect("S3 timestamp should parse as RFC3339")
|
||||
.with_timezone(&Utc)
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
@@ -471,7 +470,7 @@ async fn test_get_object_retention_returns_configured_values() {
|
||||
let bucket = "test-get-retention";
|
||||
let key = "retained-object";
|
||||
let retain_until = future_retain_until(30);
|
||||
let retain_until_expected = retain_until.strftime("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||||
let retain_until_expected = retain_until.format("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||||
|
||||
env.create_object_lock_bucket(bucket).await.unwrap();
|
||||
|
||||
@@ -1925,7 +1924,7 @@ async fn test_copy_object_retention_uses_destination_policy() {
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let copy_started = Timestamp::now();
|
||||
let copy_started = Utc::now();
|
||||
client
|
||||
.copy_object()
|
||||
.copy_source(format!("{src_bucket}/{src_key}"))
|
||||
@@ -1948,7 +1947,7 @@ async fn test_copy_object_retention_uses_destination_policy() {
|
||||
assert_eq!(retention.mode().map(|value| value.as_str()), Some("GOVERNANCE"));
|
||||
let retain_until = parse_s3_datetime(retention.retain_until_date().expect("retain-until date should be present"));
|
||||
assert!(
|
||||
retain_until < copy_started + SignedDuration::from_hours(3 * 24),
|
||||
retain_until < copy_started + Duration::days(3),
|
||||
"destination default retention should not inherit the source's longer retention"
|
||||
);
|
||||
|
||||
@@ -2009,7 +2008,7 @@ async fn test_copy_object_retention_uses_destination_policy() {
|
||||
.expect("explicit retain-until date should be present"),
|
||||
);
|
||||
assert!(
|
||||
explicit_retain_until > Timestamp::now() + SignedDuration::from_hours(20 * 24),
|
||||
explicit_retain_until > Utc::now() + Duration::days(20),
|
||||
"explicit retention should override the shorter bucket default"
|
||||
);
|
||||
|
||||
@@ -2089,7 +2088,7 @@ async fn test_multipart_default_retention_fixed_at_create() {
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let create_started = Timestamp::now();
|
||||
let create_started = Utc::now();
|
||||
let create_output = client.create_multipart_upload().bucket(bucket).key(key).send().await.unwrap();
|
||||
let upload_id = create_output.upload_id().unwrap();
|
||||
|
||||
@@ -2132,7 +2131,7 @@ async fn test_multipart_default_retention_fixed_at_create() {
|
||||
assert_eq!(retention.mode().map(|value| value.as_str()), Some("GOVERNANCE"));
|
||||
let retain_until = parse_s3_datetime(retention.retain_until_date().expect("retain-until date should be present"));
|
||||
assert!(
|
||||
retain_until < create_started + SignedDuration::from_hours(3 * 24),
|
||||
retain_until < create_started + Duration::days(3),
|
||||
"CompleteMultipartUpload should keep the default retention calculated at create time"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -693,7 +693,7 @@ impl OdmTestEnv {
|
||||
.uri(uri)
|
||||
.header(http::header::HOST, authority)
|
||||
.header("x-amz-content-sha256", rustfs_signer::constants::UNSIGNED_PAYLOAD)
|
||||
.body(rustfs_s3_types::Body::empty())?;
|
||||
.body(s3s::Body::empty())?;
|
||||
let signed = rustfs_signer::sign_v4(request, 0, &self.rustfs.access_key, &self.rustfs.secret_key, "", "us-east-1");
|
||||
|
||||
let mut builder =
|
||||
|
||||
@@ -1,449 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Compare bytes produced by S3 writes to independent, single-drive volumes.
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, admin_request, rustfs_binary_path, signed_s3_request_with_headers};
|
||||
use crate::dual_stack_differential::reference::binary_version;
|
||||
use crate::fake_s3_target::{FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target};
|
||||
use crate::on_demand_migration::common::{ODM_ADMIN_ROUTE, ODM_MODULE_SWITCH_ENV, OdmSourceSpec};
|
||||
use crate::replication_extension_test::send_set_replication_target_request;
|
||||
use crate::storage_api::persistence_parity::{BucketMetadata, RUSTFS_META_BUCKET};
|
||||
use crate::upgrade_compatibility_test::{set_bucket_quota, write_multipart};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{BucketCannedAcl, Tag, Tagging};
|
||||
use http::{HeaderMap, Method};
|
||||
use md5::{Digest, Md5};
|
||||
use rustfs_filemeta::{FileMeta, MetaObject};
|
||||
use serde_value::Value;
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
type Error = Box<dyn std::error::Error + Send + Sync>;
|
||||
const CONFIGS: &str = "persisted-configs";
|
||||
const OBJECTS: &str = "persisted-objects";
|
||||
const FIRST_PART_SIZE: usize = 5 * 1024 * 1024;
|
||||
const LOGS: &str = "rustfs-access-logs";
|
||||
|
||||
fn bucket_metadata(volume: &Path, bucket: &str) -> Result<BucketMetadata, Error> {
|
||||
let key = BucketMetadata::new(bucket).save_file_path();
|
||||
let object = volume.join(RUSTFS_META_BUCKET).join(&key);
|
||||
let meta = FileMeta::load(&std::fs::read(object.join("xl.meta"))?)?;
|
||||
let info = meta.into_fileinfo(RUSTFS_META_BUCKET, &key, "", true, false, true)?;
|
||||
info.validate_for_metadata_read()?;
|
||||
let [part] = info.parts.as_slice() else { return Err("bucket metadata must have one part".into()) };
|
||||
let size = usize::try_from(info.size)?;
|
||||
if info.erasure.data_blocks != 1 || size != part.size || size > info.erasure.block_size {
|
||||
return Err("bucket metadata exceeds the single-drive, single-stripe fixture".into());
|
||||
}
|
||||
let framed = if info.inline_data() {
|
||||
info.data.ok_or("missing inline bucket metadata")?.to_vec()
|
||||
} else {
|
||||
let directory = info.data_dir.ok_or("missing bucket metadata data directory")?;
|
||||
std::fs::read(object.join(directory.to_string()).join(format!("part.{}", part.number)))?
|
||||
};
|
||||
let algorithm = info.erasure.get_checksum_info(part.number).algorithm;
|
||||
let (digest, payload) = framed.split_at_checked(algorithm.size()).ok_or("truncated bucket metadata")?;
|
||||
if digest.is_empty() || payload.len() != size || algorithm.hash_encode(payload).as_ref() != digest {
|
||||
return Err("invalid bucket metadata shard checksum or size".into());
|
||||
}
|
||||
BucketMetadata::check_header(payload)?;
|
||||
let fields = &payload[4..];
|
||||
if bucket == CONFIGS {
|
||||
// The production decoder defaults a missing field to empty. Require the ACL field on disk
|
||||
// so dropping it cannot look like the expected zero-byte document.
|
||||
let keys: BTreeMap<&str, serde::de::IgnoredAny> = rmp_serde::from_slice(fields)?;
|
||||
if !keys.contains_key("BucketAclConfigJSON") {
|
||||
return Err("missing BucketAclConfigJSON field".into());
|
||||
}
|
||||
}
|
||||
Ok(BucketMetadata::unmarshal(fields)?)
|
||||
}
|
||||
|
||||
type VersionFields = BTreeMap<String, Value>;
|
||||
|
||||
fn read_object(volume: &Path, key: &str) -> Result<(MetaObject, VersionFields), Error> {
|
||||
let meta = FileMeta::load(&std::fs::read(volume.join(OBJECTS).join(key).join("xl.meta"))?)?;
|
||||
let [version] = meta.versions.as_slice() else {
|
||||
return Err(format!("{key}: expected one version").into());
|
||||
};
|
||||
let object = version.parse_version_meta()?.object.ok_or("missing V2 object")?;
|
||||
if object.mod_time.is_none() || object.data_dir.is_none_or(|id| id.is_nil()) {
|
||||
return Err(format!("{key}: missing modification time or data directory").into());
|
||||
}
|
||||
Ok((object, nonvolatile_fields(&version.meta, key)?))
|
||||
}
|
||||
|
||||
fn nonvolatile_fields(raw: &[u8], key: &str) -> Result<VersionFields, Error> {
|
||||
// The production reader skips unknown keys and supplies defaults for some missing keys.
|
||||
// Keep the raw maps so both the version envelope and V2Obj preserve every field and value.
|
||||
let mut fields: VersionFields = rmp_serde::from_slice(raw)?;
|
||||
let Some(Value::Map(object_fields)) = fields.get_mut("V2Obj") else {
|
||||
return Err(format!("{key}: missing raw V2Obj map").into());
|
||||
};
|
||||
for volatile in ["MTime", "DDir"] {
|
||||
let value = object_fields
|
||||
.get_mut(&Value::String(volatile.to_owned()))
|
||||
.ok_or_else(|| format!("{key}: missing {volatile}"))?;
|
||||
*value = Value::Unit;
|
||||
}
|
||||
Ok(fields)
|
||||
}
|
||||
|
||||
async fn put_xml(env: &RustFSTestEnvironment, query: &str, xml: &str) -> Result<(), Error> {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("content-md5", base64_simd::STANDARD.encode_to_string(Md5::digest(xml)).parse()?);
|
||||
let response = signed_s3_request_with_headers(
|
||||
Method::PUT,
|
||||
&format!("{}/{CONFIGS}?{query}", env.url),
|
||||
Some(xml.to_owned()),
|
||||
Some("application/xml"),
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
&headers,
|
||||
)
|
||||
.await?;
|
||||
if !response.status().is_success() {
|
||||
return Err(format!("PUT ?{query} failed with {}", response.status()).into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
struct Snapshot {
|
||||
configs: BTreeMap<&'static str, Vec<u8>>,
|
||||
objects: BTreeMap<&'static str, VersionFields>,
|
||||
multipart_files: BTreeMap<String, Vec<u8>>,
|
||||
}
|
||||
|
||||
async fn capture(binary: &Path, target: &FakeS3Target, clock: &str, clock_library: &str) -> Result<Snapshot, Error> {
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
let queue = tempfile::tempdir()?;
|
||||
let clock_loader = if cfg!(target_os = "macos") {
|
||||
"DYLD_INSERT_LIBRARIES"
|
||||
} else {
|
||||
"LD_PRELOAD"
|
||||
};
|
||||
let options = [
|
||||
(clock_loader, clock_library),
|
||||
("DYLD_FORCE_FLAT_NAMESPACE", "1"),
|
||||
("FAKETIME", clock),
|
||||
("FAKETIME_DONT_FAKE_MONOTONIC", "1"),
|
||||
("TZ", "UTC"),
|
||||
("RUSTFS_NOTIFY_ENABLE", "true"),
|
||||
("RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY", "on"),
|
||||
("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY", target.endpoint()),
|
||||
(
|
||||
"RUSTFS_NOTIFY_WEBHOOK_QUEUE_DIR_PRIMARY",
|
||||
queue.path().to_str().ok_or("non-UTF8 queue path")?,
|
||||
),
|
||||
("RUSTFS_OUTBOUND_ALLOW_ORIGINS", target.endpoint()),
|
||||
("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", "true"),
|
||||
(ODM_MODULE_SWITCH_ENV, "true"),
|
||||
("RUSTFS_NEW_BUCKET_DURABILITY_MODE", "strict"),
|
||||
("RUSTFS_SCANNER_CYCLE", "1"),
|
||||
("RUSTFS_SCANNER_START_DELAY_SECS", "0"),
|
||||
];
|
||||
env.start_rustfs_server_from_binary_without_cleanup(binary, vec![], &options)
|
||||
.await?;
|
||||
let client = env.create_s3_client();
|
||||
write_configs(&env, &client, target).await?;
|
||||
write_objects(&client).await?;
|
||||
env.stop_server();
|
||||
read_snapshot(Path::new(&env.temp_dir))
|
||||
}
|
||||
|
||||
async fn write_configs(env: &RustFSTestEnvironment, client: &Client, target: &FakeS3Target) -> Result<(), Error> {
|
||||
env.create_test_bucket(LOGS).await?;
|
||||
env.create_test_bucket(OBJECTS).await?;
|
||||
env.create_test_bucket("persistence-warehouse").await?;
|
||||
env.create_test_bucket("persistence-migration").await?;
|
||||
client
|
||||
.create_bucket()
|
||||
.bucket(CONFIGS)
|
||||
.object_lock_enabled_for_bucket(true)
|
||||
.send()
|
||||
.await?;
|
||||
macro_rules! golden {
|
||||
($path:literal) => {
|
||||
include_str!(concat!("../../ecstore/tests/fixtures/bucket-config-goldens/", $path, ".in.xml"))
|
||||
};
|
||||
}
|
||||
let target_request = serde_json::json!({
|
||||
"endpoint": target.address(), "credentials": {"accessKey": FAKE_ACCESS_KEY, "secretKey": FAKE_SECRET_KEY},
|
||||
"targetbucket": "interop-dr", "type": "replication", "secure": false,
|
||||
"deploymentID": "0b6f3c2a-5e6a-4d2b-9c1e-7a8f9d0e1b2c"
|
||||
});
|
||||
let response = send_set_replication_target_request(env, CONFIGS, false, target_request).await?;
|
||||
if !response.status().is_success() {
|
||||
return Err(format!("set remote target failed with {}", response.status()).into());
|
||||
}
|
||||
let target_arn: String = response.json().await?;
|
||||
let replication = golden!("replication/rustfs").replace("arn:rustfs:replication:", "arn:minio:replication:");
|
||||
if !replication.contains(&format!("<Bucket>{target_arn}</Bucket>")) {
|
||||
return Err("registered target does not match the golden replication destination".into());
|
||||
}
|
||||
for (query, xml) in [
|
||||
("versioning", golden!("versioning/minio")),
|
||||
("notification", golden!("notification/rustfs")),
|
||||
("replication", replication.as_str()),
|
||||
("lifecycle", golden!("lifecycle/minio")),
|
||||
("object-lock", golden!("object_lock/rustfs")),
|
||||
("encryption", golden!("encryption/minio")),
|
||||
("tagging", golden!("tagging/rustfs")),
|
||||
("cors", golden!("cors/rustfs")),
|
||||
("logging", golden!("logging/rustfs")),
|
||||
("website", golden!("website/rustfs")),
|
||||
("accelerate", golden!("accelerate/rustfs")),
|
||||
("requestPayment", golden!("request_payment/rustfs")),
|
||||
("publicAccessBlock", golden!("public_access_block/rustfs")),
|
||||
] {
|
||||
put_xml(env, query, xml).await?;
|
||||
}
|
||||
client
|
||||
.put_bucket_acl()
|
||||
.bucket(CONFIGS)
|
||||
.acl(BucketCannedAcl::Private)
|
||||
.send()
|
||||
.await?;
|
||||
let acl = client.get_bucket_acl().bucket(CONFIGS).send().await?;
|
||||
let owner = acl.owner().and_then(|owner| owner.id()).ok_or("ACL owner missing")?;
|
||||
let [grant] = acl.grants() else { return Err("expected exactly one ACL grant".into()) };
|
||||
let grantee = grant.grantee().ok_or("missing ACL grantee")?;
|
||||
if owner.is_empty()
|
||||
|| grant.permission().map(|p| p.as_str()) != Some("FULL_CONTROL")
|
||||
|| grantee.r#type().as_str() != "CanonicalUser"
|
||||
|| grantee.id().is_some()
|
||||
{
|
||||
return Err("GET ACL changed its canonical owner grant".into());
|
||||
}
|
||||
let policy = serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Deny", "Principal": "*", "Action": "s3:DeleteObject",
|
||||
"Resource": format!("arn:aws:s3:::{CONFIGS}/protected/*"),
|
||||
}],
|
||||
});
|
||||
client
|
||||
.put_bucket_policy()
|
||||
.bucket(CONFIGS)
|
||||
.policy(policy.to_string())
|
||||
.send()
|
||||
.await?;
|
||||
set_bucket_quota(env, CONFIGS, 16 * 1024 * 1024).await?;
|
||||
let migration = OdmSourceSpec::for_fake_source(target, "migration-source");
|
||||
for (path, body) in [
|
||||
("/iceberg/v1/buckets/persistence-warehouse".to_owned(), None),
|
||||
(format!("{ODM_ADMIN_ROUTE}/persistence-migration"), Some(migration.to_json().to_string())),
|
||||
] {
|
||||
let (status, _) = admin_request(&env.url, Method::PUT, &path, body, &env.access_key, &env.secret_key).await?;
|
||||
if !status.is_success() {
|
||||
return Err(format!("configuring {path} failed with {status}").into());
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn write_objects(client: &Client) -> Result<(), Error> {
|
||||
for key in ["put", "tagged"] {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(OBJECTS)
|
||||
.key(key)
|
||||
.metadata("purpose", "byte-parity")
|
||||
.body(ByteStream::from_static(b"persisted object contents"))
|
||||
.send()
|
||||
.await?;
|
||||
}
|
||||
client
|
||||
.copy_object()
|
||||
.bucket(OBJECTS)
|
||||
.key("copy")
|
||||
.copy_source(format!("{OBJECTS}/put"))
|
||||
.send()
|
||||
.await?;
|
||||
let tags = Tagging::builder()
|
||||
.tag_set(Tag::builder().key("type").value("persisted").build()?)
|
||||
.build()?;
|
||||
client
|
||||
.put_object_tagging()
|
||||
.bucket(OBJECTS)
|
||||
.key("tagged")
|
||||
.tagging(tags)
|
||||
.send()
|
||||
.await?;
|
||||
write_multipart(client, OBJECTS, "multipart", &[vec![0x31; FIRST_PART_SIZE], vec![0x32; 100]]).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn read_snapshot(volume: &Path) -> Result<Snapshot, Error> {
|
||||
let b = bucket_metadata(volume, CONFIGS)?;
|
||||
let configs = BTreeMap::from([
|
||||
("acl", b.bucket_acl_config_json),
|
||||
("accelerate", b.accelerate_config_xml),
|
||||
("cors", b.cors_config_xml),
|
||||
("encryption", b.encryption_config_xml),
|
||||
("lifecycle", b.lifecycle_config_xml),
|
||||
("logging", b.logging_config_xml),
|
||||
("notification", b.notification_config_xml),
|
||||
("object_lock", b.object_lock_config_xml),
|
||||
("public_access_block", b.public_access_block_config_xml),
|
||||
("request_payment", b.request_payment_config_xml),
|
||||
("tagging", b.tagging_config_xml),
|
||||
("versioning", b.versioning_config_xml),
|
||||
("website", b.website_config_xml),
|
||||
("durability", b.durability_config_json),
|
||||
("policy", b.policy_config_json),
|
||||
("quota", b.quota_config_json),
|
||||
("replication", b.replication_config_xml),
|
||||
("targets", b.bucket_targets_config_json),
|
||||
("table", bucket_metadata(volume, "persistence-warehouse")?.table_bucket_config_json),
|
||||
("odm", bucket_metadata(volume, "persistence-migration")?.on_demand_migration_config_json),
|
||||
]);
|
||||
let mut objects = BTreeMap::new();
|
||||
for key in ["put", "copy", "tagged"] {
|
||||
let (_, fields) = read_object(volume, key)?;
|
||||
objects.insert(key, fields);
|
||||
}
|
||||
let (multipart, _) = read_object(volume, "multipart")?;
|
||||
if multipart.part_numbers != [1, 2] || multipart.part_sizes != [FIRST_PART_SIZE, 100] {
|
||||
return Err("xl.meta does not describe both uploaded parts".into());
|
||||
}
|
||||
let directory = volume
|
||||
.join(OBJECTS)
|
||||
.join("multipart")
|
||||
.join(multipart.data_dir.ok_or("missing multipart data directory")?.to_string());
|
||||
let mut multipart_files = BTreeMap::new();
|
||||
for entry in std::fs::read_dir(directory)? {
|
||||
let entry = entry?;
|
||||
let name = entry.file_name().into_string().map_err(|_| "invalid multipart file name")?;
|
||||
if name.starts_with("part.") {
|
||||
let bytes = std::fs::read(entry.path())?;
|
||||
if bytes.is_empty() {
|
||||
return Err(format!("empty multipart file {name}").into());
|
||||
}
|
||||
multipart_files.insert(name, bytes);
|
||||
}
|
||||
}
|
||||
for name in ["part.1", "part.2", "part.1.meta", "part.2.meta"] {
|
||||
if !multipart_files.contains_key(name) {
|
||||
return Err(format!("missing multipart file {name}").into());
|
||||
}
|
||||
}
|
||||
Ok(Snapshot {
|
||||
configs,
|
||||
objects,
|
||||
multipart_files,
|
||||
})
|
||||
}
|
||||
|
||||
fn save_xml_samples(root: &Path, side: &str, snapshot: &Snapshot) -> Result<(), Error> {
|
||||
let directory = root.join(side);
|
||||
std::fs::create_dir_all(&directory)?;
|
||||
let families: [&str; 13] = [
|
||||
"accelerate",
|
||||
"cors",
|
||||
"encryption",
|
||||
"lifecycle",
|
||||
"logging",
|
||||
"notification",
|
||||
"object_lock",
|
||||
"public_access_block",
|
||||
"replication",
|
||||
"request_payment",
|
||||
"tagging",
|
||||
"versioning",
|
||||
"website",
|
||||
];
|
||||
for family in families {
|
||||
let bytes = snapshot
|
||||
.configs
|
||||
.get(family)
|
||||
.ok_or_else(|| format!("missing XML family {family}"))?;
|
||||
std::fs::write(directory.join(format!("{family}.xml")), bytes)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn persistence_byte_parity() -> Result<(), Error> {
|
||||
let reference = PathBuf::from(
|
||||
std::env::var_os("RUSTFS_PERSISTENCE_REFERENCE_BINARY").ok_or("missing pinned RUSTFS_PERSISTENCE_REFERENCE_BINARY")?,
|
||||
);
|
||||
let candidate = rustfs_binary_path();
|
||||
if binary_version(&reference).await? == binary_version(&candidate).await? {
|
||||
return Err("reference and candidate binaries report identical build identities".into());
|
||||
}
|
||||
// The quota writer records the process wall clock inside its JSON document. Keep raw bytes
|
||||
// comparable while the SDK's signature clock and both servers' monotonic timers advance normally.
|
||||
let clock_library = std::env::var("RUSTFS_PERSISTENCE_CLOCK_LIBRARY")?;
|
||||
if !Path::new(&clock_library).is_absolute() || !Path::new(&clock_library).is_file() {
|
||||
return Err("RUSTFS_PERSISTENCE_CLOCK_LIBRARY must name an absolute libfaketime path".into());
|
||||
}
|
||||
let clock = time::OffsetDateTime::now_utc()
|
||||
.format(time::macros::format_description!("[year]-[month]-[day] [hour]:[minute]:[second]"))?;
|
||||
let target = FakeS3Target::start().await?;
|
||||
target.create_bucket("migration-source");
|
||||
target.create_bucket_with_object_lock("interop-dr");
|
||||
let reference = capture(&reference, &target, &clock, &clock_library).await?;
|
||||
let candidate = capture(&candidate, &target, &clock, &clock_library).await?;
|
||||
if let Some(directory) = std::env::var_os("RUSTFS_PERSISTENCE_XML_DIR") {
|
||||
save_xml_samples(Path::new(&directory), "reference", &reference)?;
|
||||
save_xml_samples(Path::new(&directory), "candidate", &candidate)?;
|
||||
}
|
||||
if reference.configs.len() != 20 || candidate.configs.len() != 20 {
|
||||
return Err("expected 13 XML and seven JSON fields on each side".into());
|
||||
}
|
||||
for (&field, expected) in &reference.configs {
|
||||
// Canned ACLs acknowledge the write and return one owner grant without persisting an ACL document.
|
||||
if expected.is_empty() != (field == "acl") || candidate.configs.get(field) != Some(expected) {
|
||||
return Err(format!("incorrect or different persisted bucket config: {field}").into());
|
||||
}
|
||||
}
|
||||
for (key, expected) in &reference.objects {
|
||||
if candidate.objects.get(key) != Some(expected) {
|
||||
return Err(format!("different persisted xl.meta fields: {key}").into());
|
||||
}
|
||||
}
|
||||
if candidate.multipart_files != reference.multipart_files {
|
||||
return Err("different multipart part files or part metadata".into());
|
||||
}
|
||||
eprintln!(
|
||||
"Compared {} bucket fields (canned ACL empty), {} full object records and multipart part files",
|
||||
reference.configs.len(),
|
||||
reference.objects.len()
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn version_comparison_preserves_unknown_fields_and_absent_values() -> Result<(), Error> {
|
||||
let mut sample = serde_json::json!({"Type": 1, "v": 1, "V2Obj": {"DDir": vec![1; 16], "MTime": 1}});
|
||||
let fields = |value: &serde_json::Value| nonvolatile_fields(&rmp_serde::to_vec(value)?, "fixture");
|
||||
let original = fields(&sample)?;
|
||||
sample["V2Obj"]["DDir"] = serde_json::json!(vec![2; 16]);
|
||||
sample["V2Obj"]["MTime"] = serde_json::json!(2);
|
||||
assert_eq!(fields(&sample)?, original);
|
||||
sample["V2Obj"]["Future"] = serde_json::Value::Null;
|
||||
let with_nil = fields(&sample)?;
|
||||
assert_ne!(with_nil, original);
|
||||
sample["V2Obj"]["Future"] = serde_json::json!(false);
|
||||
let with_value = fields(&sample)?;
|
||||
assert_ne!(with_value, with_nil);
|
||||
sample["FutureEnvelope"] = serde_json::json!(1);
|
||||
assert_ne!(fields(&sample)?, with_value);
|
||||
assert!(fields(&serde_json::json!({"V2Obj": {"MTime": 1}})).is_err());
|
||||
Ok(())
|
||||
}
|
||||
@@ -20,9 +20,9 @@
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use http::header::{CONTENT_LENGTH, HOST, TRANSFER_ENCODING};
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::sign_v4;
|
||||
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
|
||||
use s3s::Body;
|
||||
use std::error::Error;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpStream;
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
|
||||
use async_trait::async_trait;
|
||||
use rustfs_lock::{
|
||||
CacheFenceStamp, LockClient, LockError, LockId, LockInfo, LockRequest, LockResponse, LockStats, LockStatus, LockType, Result,
|
||||
LockClient, LockError, LockId, LockInfo, LockRequest, LockResponse, LockStats, LockStatus, LockType, Result,
|
||||
types::{LockMetadata, LockPriority},
|
||||
};
|
||||
use rustfs_protos::proto_gen::node_service::{BatchGenerallyLockRequest, GenerallyLockRequest, PingRequest};
|
||||
@@ -64,7 +64,6 @@ impl GrpcLockClient {
|
||||
deadlock_detection: false,
|
||||
suppress_contention_logs: false,
|
||||
refresh_interval: None,
|
||||
cache_fence_stamp_requested: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -125,15 +124,10 @@ impl LockClient for GrpcLockClient {
|
||||
|
||||
// Check if the lock acquisition was successful
|
||||
if resp.success {
|
||||
let info = Self::build_lock_info(request, resp.lock_info);
|
||||
match resp
|
||||
.cache_fence_stamp
|
||||
.as_deref()
|
||||
.and_then(|encoded| CacheFenceStamp::decode(encoded).ok())
|
||||
{
|
||||
Some(stamp) => Ok(LockResponse::success_with_cache_fence_stamp(info, std::time::Duration::ZERO, stamp)),
|
||||
None => Ok(LockResponse::success(info, std::time::Duration::ZERO)),
|
||||
}
|
||||
Ok(LockResponse::success(
|
||||
Self::build_lock_info(request, resp.lock_info),
|
||||
std::time::Duration::ZERO,
|
||||
))
|
||||
} else {
|
||||
// Lock acquisition failed
|
||||
Ok(LockResponse::failure(
|
||||
@@ -166,15 +160,7 @@ impl LockClient for GrpcLockClient {
|
||||
.enumerate()
|
||||
.map(|(idx, request)| match resp.results.get(idx) {
|
||||
Some(result) if result.success => {
|
||||
let info = Self::build_lock_info(request, result.lock_info.clone());
|
||||
match result
|
||||
.cache_fence_stamp
|
||||
.as_deref()
|
||||
.and_then(|encoded| CacheFenceStamp::decode(encoded).ok())
|
||||
{
|
||||
Some(stamp) => LockResponse::success_with_cache_fence_stamp(info, std::time::Duration::ZERO, stamp),
|
||||
None => LockResponse::success(info, std::time::Duration::ZERO),
|
||||
}
|
||||
LockResponse::success(Self::build_lock_info(request, result.lock_info.clone()), std::time::Duration::ZERO)
|
||||
}
|
||||
Some(result) => LockResponse::failure(
|
||||
result
|
||||
|
||||
@@ -22,11 +22,10 @@ use rustfs_protos::{
|
||||
models::PingBodyBuilder,
|
||||
proto_gen::node_service::{
|
||||
BatchGenerallyLockRequest, BatchGenerallyLockResponse, BatchReadVersionRequest, BatchReadVersionResponse,
|
||||
GenerallyLockRequest, GenerallyLockResponse, GenerallyLockResult, MrfLifecycleRequest, MrfLifecycleResponse, PingRequest,
|
||||
PingResponse, ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseReleaseResponse,
|
||||
ScannerPublicationLeaseRequest, ScannerPublicationLeaseResponse, SnapshotLeaseMutationResponse,
|
||||
SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest, SnapshotLeaseResponse,
|
||||
node_service_server::NodeService,
|
||||
GenerallyLockRequest, GenerallyLockResponse, GenerallyLockResult, PingRequest, PingResponse,
|
||||
ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseReleaseResponse, ScannerPublicationLeaseRequest,
|
||||
ScannerPublicationLeaseResponse, SnapshotLeaseMutationResponse, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest,
|
||||
SnapshotLeaseRequest, SnapshotLeaseResponse, node_service_server::NodeService,
|
||||
},
|
||||
};
|
||||
use std::pin::Pin;
|
||||
@@ -43,7 +42,6 @@ fn lock_result_from_response(response: rustfs_lock::LockResponse) -> GenerallyLo
|
||||
success: response.success,
|
||||
error_info: response.error,
|
||||
lock_info: response.lock_info.and_then(|info| serde_json::to_string(&info).ok()),
|
||||
cache_fence_stamp: response.cache_fence_stamp.map(|stamp| stamp.encode().to_vec().into()),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
@@ -181,7 +179,6 @@ impl NodeService for MinimalLockNodeService {
|
||||
success: result.success,
|
||||
error_info: result.error,
|
||||
lock_info: lock_info_json,
|
||||
cache_fence_stamp: result.cache_fence_stamp.map(|stamp| stamp.encode().to_vec().into()),
|
||||
..Default::default()
|
||||
}))
|
||||
}
|
||||
@@ -441,13 +438,6 @@ impl NodeService for MinimalLockNodeService {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn write_format_control(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::WriteFormatControlRequest>,
|
||||
) -> Result<Response<rustfs_protos::proto_gen::node_service::WriteAllResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn compare_and_update_file(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::CompareAndUpdateFileRequest>,
|
||||
@@ -573,15 +563,6 @@ impl NodeService for MinimalLockNodeService {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
type RenameDataBatchStream = ResponseStream<rustfs_protos::proto_gen::node_service::RenameDataBatchResult>;
|
||||
|
||||
async fn rename_data_batch(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::RenameDataBatchRequest>,
|
||||
) -> Result<Response<Self::RenameDataBatchStream>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn rename_multipart_data(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::RenameDataRequest>,
|
||||
@@ -1014,10 +995,6 @@ impl NodeService for MinimalLockNodeService {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn mrf_lifecycle(&self, _request: Request<MrfLifecycleRequest>) -> Result<Response<MrfLifecycleResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn background_heal_status(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::BackgroundHealStatusRequest>,
|
||||
|
||||
@@ -27,9 +27,9 @@ use http::Version;
|
||||
use http::header::HOST;
|
||||
use rcgen::generate_simple_self_signed;
|
||||
use reqwest::{Certificate, Client, Response, StatusCode};
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use std::error::Error;
|
||||
use std::path::Path;
|
||||
use std::process::Command;
|
||||
|
||||
@@ -412,7 +412,6 @@ struct ManualTransitionRunReport {
|
||||
skipped_delete_marker: u64,
|
||||
skipped_directory: u64,
|
||||
skipped_replication: u64,
|
||||
skipped_already_transitioned: u64,
|
||||
skipped_already_in_flight: u64,
|
||||
skipped_queue_full: u64,
|
||||
skipped_queue_closed: u64,
|
||||
@@ -1215,16 +1214,9 @@ async fn test_manual_transition_run_black_box_semantics() -> TestResult {
|
||||
assert!(!due.report.dry_run);
|
||||
assert!(due.report.lifecycle_config_found);
|
||||
assert_eq!(due.report.scanned, 1, "due report: {:#?}", due.report);
|
||||
// PUT can enqueue this object before the manual scan. Account for each
|
||||
// terminal outcome without depending on how fast the tier worker runs.
|
||||
assert_eq!(due.report.eligible, 1, "due report: {:#?}", due.report);
|
||||
assert_eq!(
|
||||
due.report.eligible + due.report.skipped_already_transitioned,
|
||||
1,
|
||||
"due report: {:#?}",
|
||||
due.report
|
||||
);
|
||||
assert_eq!(
|
||||
due.report.enqueued + due.report.skipped_already_in_flight + due.report.skipped_already_transitioned,
|
||||
due.report.enqueued + due.report.skipped_already_in_flight,
|
||||
1,
|
||||
"due report: {:#?}",
|
||||
due.report
|
||||
@@ -1233,34 +1225,12 @@ async fn test_manual_transition_run_black_box_semantics() -> TestResult {
|
||||
assert_eq!(due.report.skipped_delete_marker, 0);
|
||||
assert_eq!(due.report.skipped_directory, 0);
|
||||
assert_eq!(due.report.skipped_replication, 0);
|
||||
assert_eq!(due.report.skipped_queue_full, 0);
|
||||
assert_eq!(due.report.skipped_queue_closed, 0);
|
||||
assert_eq!(due.report.skipped_queue_timeout, 0);
|
||||
assert_eq!(due.report.tier_failure, 0);
|
||||
assert!(!due.report.cancelled);
|
||||
assert!(!due.report.truncated_by_limit);
|
||||
assert!(!due.report.truncated_by_duration);
|
||||
wait_for_transition(&hot_client, MANUAL_DUE_BUCKET, MANUAL_DUE_KEY, StdDuration::from_secs(90)).await?;
|
||||
let remote_count_after_due = cold_tier_object_count(&cold_client).await?;
|
||||
assert_eq!(remote_count_after_due, 1, "due object must have exactly one remote copy");
|
||||
|
||||
// Once HEAD observes the committed transition, a repeated manual scan must
|
||||
// report the completed object and leave both local reads and the tier intact.
|
||||
let repeated = manual_transition_run(&hot, MANUAL_DUE_BUCKET, "manual-due/", false).await?;
|
||||
assert_eq!(repeated.state, "completed", "repeated report: {:#?}", repeated.report);
|
||||
assert_eq!(repeated.report.scanned, 1, "repeated report: {:#?}", repeated.report);
|
||||
assert_eq!(repeated.report.eligible, 0, "repeated report: {:#?}", repeated.report);
|
||||
assert_eq!(repeated.report.enqueued, 0, "repeated report: {:#?}", repeated.report);
|
||||
assert_eq!(repeated.report.skipped_already_transitioned, 1);
|
||||
assert_eq!(repeated.report.skipped_already_in_flight, 0);
|
||||
assert_eq!(cold_tier_object_count(&cold_client).await?, remote_count_after_due);
|
||||
let due_object = hot_client
|
||||
.get_object()
|
||||
.bucket(MANUAL_DUE_BUCKET)
|
||||
.key(MANUAL_DUE_KEY)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(due_object.body.collect().await?.into_bytes().as_ref(), b"manual due object");
|
||||
|
||||
hot_client.create_bucket().bucket(MANUAL_DRY_RUN_BUCKET).send().await?;
|
||||
enable_bucket_versioning(&hot_client, MANUAL_DRY_RUN_BUCKET).await?;
|
||||
@@ -2316,14 +2286,6 @@ async fn test_manual_transition_run_continuation_token_resumes_without_raw_marke
|
||||
)
|
||||
.await?;
|
||||
|
||||
// Configuration updates enqueue existing due objects. Observe their commit
|
||||
// before paging so eligibility stays stable across the source restart.
|
||||
for idx in 0..2 {
|
||||
let key = format!("{MANUAL_CONTINUATION_PREFIX}obj-{idx:02}");
|
||||
wait_for_transition(&hot_client, MANUAL_CONTINUATION_BUCKET, &key, StdDuration::from_secs(90)).await?;
|
||||
}
|
||||
assert_eq!(cold_tier_object_count(&cold_client).await?, 2);
|
||||
|
||||
let first = manual_transition_run_with_max(&hot, MANUAL_CONTINUATION_BUCKET, MANUAL_CONTINUATION_PREFIX, true, 1).await?;
|
||||
assert_eq!(first.state, "partial", "first continuation page: {first:#?}");
|
||||
assert_eq!(first.mode, "enqueue_only");
|
||||
@@ -2331,10 +2293,8 @@ async fn test_manual_transition_run_continuation_token_resumes_without_raw_marke
|
||||
assert_eq!(first.report.prefix, MANUAL_CONTINUATION_PREFIX);
|
||||
assert!(first.report.dry_run);
|
||||
assert_eq!(first.report.scanned, 1, "first continuation page: {first:#?}");
|
||||
assert_eq!(first.report.eligible, 0, "first continuation page: {first:#?}");
|
||||
assert_eq!(first.report.dry_run_eligible, 0, "first continuation page: {first:#?}");
|
||||
assert_eq!(first.report.skipped_already_transitioned, 1);
|
||||
assert_eq!(first.report.enqueued, 0);
|
||||
assert_eq!(first.report.eligible, 1, "first continuation page: {first:#?}");
|
||||
assert_eq!(first.report.dry_run_eligible, 1, "first continuation page: {first:#?}");
|
||||
assert_eq!(first.report.tier_failure, 0);
|
||||
assert!(!first.report.cancelled);
|
||||
assert!(first.report.truncated_by_limit);
|
||||
@@ -2361,15 +2321,12 @@ async fn test_manual_transition_run_continuation_token_resumes_without_raw_marke
|
||||
.await?;
|
||||
assert_eq!(second.state, "completed", "second continuation page: {second:#?}");
|
||||
assert_eq!(second.report.scanned, 1, "second continuation page: {second:#?}");
|
||||
assert_eq!(second.report.eligible, 0, "second continuation page: {second:#?}");
|
||||
assert_eq!(second.report.dry_run_eligible, 0, "second continuation page: {second:#?}");
|
||||
assert_eq!(second.report.skipped_already_transitioned, 1);
|
||||
assert_eq!(second.report.enqueued, 0);
|
||||
assert_eq!(second.report.eligible, 1, "second continuation page: {second:#?}");
|
||||
assert_eq!(second.report.dry_run_eligible, 1, "second continuation page: {second:#?}");
|
||||
assert_eq!(second.report.tier_failure, 0);
|
||||
assert!(!second.report.cancelled);
|
||||
assert!(!second.report.truncated_by_limit);
|
||||
assert!(second.report.continuation_token.is_none());
|
||||
assert_eq!(cold_tier_object_count(&cold_client).await?, 2);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user