mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-17 21:44:21 +00:00
Compare commits
178 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 569b946454 | |||
| 75b3dce2e0 | |||
| 9df834a955 | |||
| 969191ebe6 | |||
| 17f1164581 | |||
| 6b60b64b81 | |||
| 3ab62090c2 | |||
| 1c4a7c2f8b | |||
| dc83d478f8 | |||
| a18231a2b3 | |||
| 0717dc1b0d | |||
| a68e299baa | |||
| f2003af73d | |||
| f47840b3c2 | |||
| 7fb9b3501f | |||
| cbe5b16758 | |||
| a33fa8a226 | |||
| bfff795635 | |||
| fb104ab2c1 | |||
| 251eee0cba | |||
| 676875576f | |||
| a54e2ea386 | |||
| 913edbc536 | |||
| 909b7c3c6f | |||
| 571d100a27 | |||
| e4ac38a011 | |||
| 849cdd6319 | |||
| dc7f4e943c | |||
| 0278e1cd0e | |||
| 94a138b994 | |||
| b3cdea0c15 | |||
| b849852863 | |||
| 489316e7a2 | |||
| f68ef7d5f6 | |||
| d3abf80237 | |||
| 70022e51c4 | |||
| 88515bb7d0 | |||
| a98bd9028d | |||
| 65057691b6 | |||
| 2fea17d57d | |||
| 2ca689f486 | |||
| b2c30d69c2 | |||
| fb9f19c6d2 | |||
| f4266a4576 | |||
| 00ecde1250 | |||
| e9da1d3096 | |||
| 2308dee4b2 | |||
| b2b4f0c491 | |||
| c5b4a85778 | |||
| 4ae180ea40 | |||
| c0763a43f5 | |||
| d461908bf0 | |||
| 5b8876ad0e | |||
| 75b293f094 | |||
| fb3e2818c7 | |||
| 4294c41ccf | |||
| 3cb0aa137e | |||
| a8cf62f8f3 | |||
| d0fd6425e4 | |||
| 92050cdca6 | |||
| 26db9b19af | |||
| 292974baff | |||
| 993464fbbb | |||
| cfdd4b2686 | |||
| 6f93ac5f47 | |||
| 620fdc286a | |||
| 58310c569b | |||
| e36c7452ac | |||
| 48e97253b5 | |||
| ce5bd8b117 | |||
| 93947f22ce | |||
| 8f361e3a6f | |||
| 3cee9c154b | |||
| 504cc06d0c | |||
| 44145ef327 | |||
| ba9e8342f8 | |||
| a41019802d | |||
| 4313812e24 | |||
| 5c58a56c4c | |||
| 8196603a76 | |||
| 1e62cbaeaf | |||
| da1817ae01 | |||
| c2b5fbb138 | |||
| 97a7639f59 | |||
| 77da63397c | |||
| 1467d32695 | |||
| 2c2af51a46 | |||
| 0567078f5b | |||
| b7e7a6156c | |||
| df4c4916c2 | |||
| fd8dc41f0c | |||
| 645bb63809 | |||
| 5f9e5d62df | |||
| da5414c345 | |||
| a279b4bb63 | |||
| 4182d55d11 | |||
| d4af2377fe | |||
| f5e8f17461 | |||
| 925e5d0c0c | |||
| 247aecf7b8 | |||
| fc22c11d1a | |||
| 0eec25e4c0 | |||
| c132329961 | |||
| 0f503fc613 | |||
| 0460ad63a0 | |||
| 28da2e12a8 | |||
| 41225da343 | |||
| 4053e694cb | |||
| a9dd5f0fd1 | |||
| 4bb272ed64 | |||
| 9ea8e8d37f | |||
| f2a43376de | |||
| ebd4bb2ba8 | |||
| ebe31dfdb5 | |||
| ed3a9e670d | |||
| e3858df4fa | |||
| 8e64a3aa2f | |||
| 092c8c272d | |||
| 3c59e4f306 | |||
| e97dc74d93 | |||
| e83b6815ed | |||
| 20ea63e27e | |||
| a984f42b0b | |||
| f9f52f9fad | |||
| af93bf4663 | |||
| 4db694f1c4 | |||
| 0e8fb73e57 | |||
| 05554bee00 | |||
| 8d5faa5750 | |||
| a9f308b075 | |||
| 2b0d74027a | |||
| 964cb463a2 | |||
| ef6af77026 | |||
| 707694c097 | |||
| f621f28d01 | |||
| d57e781375 | |||
| da07ad6a08 | |||
| 17081a6d23 | |||
| d36c3595ad | |||
| db5a910b69 | |||
| 77ce76fde9 | |||
| d645456a90 | |||
| fa13764bb7 | |||
| c221fb62a3 | |||
| ad94f6914a | |||
| 190f3686f4 | |||
| 941fb4fc11 | |||
| 0e44453639 | |||
| 0fde0f14a0 | |||
| aa5f145983 | |||
| ea56aa9238 | |||
| 8eb89dd542 | |||
| 4a1d542f2d | |||
| 75f53303e1 | |||
| f33271a743 | |||
| c83f47871d | |||
| f6d64bc1fb | |||
| d37640fbb9 | |||
| 9c394af489 | |||
| be2d3ca2cf | |||
| 1ce2e51358 | |||
| 95c7e4218e | |||
| c43e415b09 | |||
| ce9d0b153e | |||
| ef79dcbe19 | |||
| dea9ac7fe2 | |||
| ca5ac98c6c | |||
| 34be0a802f | |||
| 7e93d41c5d | |||
| eca95f2973 | |||
| 5a26862e2e | |||
| b8efcb8dfa | |||
| 704b984c1f | |||
| 598282bc63 | |||
| 45156f313e | |||
| a6cd90b6fb | |||
| 248c56ec1f | |||
| cb75405d8c |
@@ -31,3 +31,14 @@
|
||||
# the flag, so the two can no longer drift apart unnoticed.
|
||||
#
|
||||
# For CPU profiling, add `-C force-frame-pointers=yes` to that RUSTFLAGS value.
|
||||
|
||||
# Exact internode HTTP/2 PING ACK timeout accounting is also deliberately
|
||||
# default-off. Hyper exposes the exact timeout only through its unstable tracing
|
||||
# event, so diagnostic builds must opt in to both the feature and cfg:
|
||||
#
|
||||
# RUSTFLAGS="--cfg hyper_unstable_tracing" \
|
||||
# cargo build -p rustfs --features diagnostic-internode-http2-ping
|
||||
#
|
||||
# `crates/protos/build.rs` and `crates/obs/build.rs` fail fast if the feature and
|
||||
# cfg drift apart. Do not move this cfg into global rustflags: doing so would
|
||||
# recompile every dependency under an unstable, non-semver build contract.
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
# Rulings for the gateway conformance lane (.github/workflows/gateway-acceptance.yml).
|
||||
#
|
||||
# One row per case that fails on the GATEWAY stack for a reviewed reason: `case`; `ledger`, the
|
||||
# rustfs/backlog#2684 entry that accepts the difference (R-0xx or K-n); a one-line `reason`; and
|
||||
# `failure`, a prefix of the failure the ruling accepts, as the runner's report.json renders it
|
||||
# (`[expect/status] at /expect/status: exchange #1, attempt 1: expected status 400, observed 403`).
|
||||
# scripts/check_conformance_rulings.py fails the lane on a failed case without a row, on a ruled
|
||||
# case whose failures do not start with its `failure`, and on a row whose case no longer fails or
|
||||
# no longer exists. The legacy stack is a reference run and takes no rulings: the `rustfs` profile
|
||||
# describes the gateway edge, not the legacy one.
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=dde32c85d39a3c2eb1cad6912861883af2b3554fc6d04b86b3aeb2afbcc86c7e
|
||||
sha256-linux=155b526f75743796c18c4c5202bbef3a1770b0ea4ba18b32910317a957ad36f2
|
||||
sha256-darwin=24a8cfaa1b9125bd69d7d4a7ff752a8ab8ed350c761d90ae8472ef52393e2686
|
||||
sha256-linux=014257197fbbfbb48cab2864da9e10d96ee877d478fa29d119f8186532ea293d
|
||||
|
||||
@@ -1 +1 @@
|
||||
sha256=caccbc5bcbfbe1b7548fbd184f1a6ae3aedec7cb84ed862d62dddbe27e75271f
|
||||
sha256=fd5683273bba1eae3b4402a4e854a5d168ace845bd0e5fc801f8e336485a4394
|
||||
|
||||
@@ -70,6 +70,11 @@ s3s-footprint-check: ## Check the s3s dependency footprint ratchet stays frozen
|
||||
@echo "📦 Checking s3s footprint ratchet..."
|
||||
./scripts/check_s3s_footprint.sh
|
||||
|
||||
.PHONY: release-feature-boundary-check
|
||||
release-feature-boundary-check: ## Check no release-reachable feature enables a debug-only one
|
||||
@echo "🚧 Checking release feature boundaries..."
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_connect_release_boundary.py --features-only
|
||||
|
||||
.PHONY: test-count-ratchet-check
|
||||
test-count-ratchet-check: ## Check no workspace crate fell below its test floor or rose above its #[ignore] ceiling in the committed baseline
|
||||
@echo "🧪 Checking per-crate test-count ratchet..."
|
||||
|
||||
@@ -20,11 +20,11 @@ planning-docs-check: ## Check that no planning-type documents are committed
|
||||
./scripts/check_no_planning_docs.sh
|
||||
|
||||
.PHONY: pre-commit
|
||||
pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check test-count-ratchet-check test-seams-check test-retirements-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
|
||||
pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check release-feature-boundary-check test-count-ratchet-check test-seams-check test-retirements-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
|
||||
@echo "✅ All pre-commit checks passed!"
|
||||
|
||||
.PHONY: pre-pr
|
||||
pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check test-count-ratchet-check test-seams-check test-retirements-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check log-analyzer-rules-check offline-enrollment-e2e-check clippy-check test ## Run full pre-PR checks with clippy and tests
|
||||
pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check release-feature-boundary-check test-count-ratchet-check test-seams-check test-retirements-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check log-analyzer-rules-check offline-enrollment-e2e-check clippy-check test ## Run full pre-PR checks with clippy and tests
|
||||
@echo "✅ All pre-PR checks passed!"
|
||||
|
||||
.PHONY: dev-check
|
||||
|
||||
@@ -39,7 +39,11 @@ script-tests: ## Run shell script tests
|
||||
./scripts/test_entrypoint_directories.sh
|
||||
./scripts/test_internode_grpc_ab_bench.sh
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_internode_grpc_window.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_internode_rpc_timeout_boundary.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_internode_msgpack_peer_canary.py
|
||||
./scripts/test_internode_msgpack_peer_canary_runner.sh
|
||||
./scripts/test_object_batch_bench_enhanced.sh
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_inline_docker_lab.py
|
||||
./scripts/test_package_service_scripts.sh
|
||||
./scripts/test_hotpath_warp_ab_gate.sh
|
||||
./scripts/test_hotpath_warp_abba.sh
|
||||
@@ -99,6 +103,10 @@ script-tests: ## Run shell script tests
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_retain_validation_evidence.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_timing_report.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_junit_diff.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_dual_stack.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_mint_stack_diff.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_conformance_rulings.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/compat_matrix_report.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_runner_tools.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_api_coverage.py
|
||||
|
||||
+13
-2
@@ -99,6 +99,11 @@ threads-required = "num-test-threads"
|
||||
filter = '(package(rustfs-ecstore) & (test(/^core::pools_test::decommission_lock_order_tests::(active_decommission_migration_does_not_invert_capacity_and_delete_object(_tag)?_locks|decommission_moves_a_transitioned_version_expired_by_a_delete_all_rule_without_stalling)$/) | test(=services::integrity::tests::audit_migration_and_ack_recovery_preserve_source_and_reject_conflicts))) | (package(rustfs-heal) & test(=heal::mrf_queue::partial_write::tests::durable_retry_submission_error_keeps_checkpoint_and_uses_backoff))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# These eight-disk namespace-owner probes overlap durable cleanup with a lock contender. Reserve capacity so sibling disk fixtures cannot consume their lock and completion budgets.
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(rustfs-ecstore) & test(/^store::object::tests::(canceled_create_retains_usage_cleanup_namespace_owner|batch_delete_excludes_recovery_on_nonzero_pool|delete_excludes_recovery_with_(disjoint|identical|overlapping)_pool_lock_clients)$/)'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# These eight-disk decommission fixtures exercise concurrent retries,
|
||||
# free-version cleanup, and real capacity probes under production lock deadlines.
|
||||
# Reserve the run's capacity so unrelated IO cannot exhaust those budgets.
|
||||
@@ -136,8 +141,9 @@ setup = 'ecstore-large-stack'
|
||||
filter = '(package(rustfs) & test(/^(admin::handlers::replication::target_repair_tests::[a-z_]+|admin::handlers::plugin_attempt::v2_tests::delegation_v2_persists_and_vends_with_real_parent_intersection|admin::handlers::site_replication::tests::durable_iam_state_apply_iam_item_rejects_a_stale_recreate_after_a_replicated_delete|admin::service::federated_identity::tests::(binding_keeps_explicit_iam_mapping_strict|markerless_legacy_oidc_session_crosses_auth_and_iam_deletion_boundaries)|storage::scanner_dirty_journal::tests::(clear_callback_keeps_unknown_coverage_until_all_pending_work_is_confirmed|owned_journals_keep_peer_records_after_interleaved_writes_and_clear))$/)) | (package(rustfs-heal) & ((binary(heal_b5_versioned_regression_test) & test(/^serial_tests::(test_heal_unversioned_bucket_e2e|test_historical_null_recursive_heal_restores_physical_versions|test_recursive_deep_heal_reports_stale_delete_marker_repair)$/)) | (binary(usage_observation_read_repair_test) & test(=usage_observation_read_repair_survives_snapshot_cleanup)))) | (package(rustfs-test-utils) & test(=data_usage_snapshot_tests::data_usage_snapshot_recovery_wires_primary_backup_and_negative_cache))'
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
# The multipart refusal fixture composes real-store setup, abort, quota and legal-hold futures.
|
||||
[[profile.default.scripts]]
|
||||
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::complete::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
|
||||
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::complete::tests::completion_absent_binding_refusal_order_baseline|app::multipart_usecase::complete::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
# These handler/store fixtures and the relocated-pool GET compose deep storage
|
||||
@@ -405,6 +411,11 @@ threads-required = "num-test-threads"
|
||||
filter = '(package(rustfs-ecstore) & (test(/^core::pools_test::decommission_lock_order_tests::(active_decommission_migration_does_not_invert_capacity_and_delete_object(_tag)?_locks|decommission_moves_a_transitioned_version_expired_by_a_delete_all_rule_without_stalling)$/) | test(=services::integrity::tests::audit_migration_and_ack_recovery_preserve_source_and_reject_conflicts))) | (package(rustfs-heal) & test(=heal::mrf_queue::partial_write::tests::durable_retry_submission_error_keeps_checkpoint_and_uses_backoff))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# These eight-disk namespace-owner probes overlap durable cleanup with a lock contender. Reserve capacity so sibling disk fixtures cannot consume their lock and completion budgets.
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(rustfs-ecstore) & test(/^store::object::tests::(canceled_create_retains_usage_cleanup_namespace_owner|batch_delete_excludes_recovery_on_nonzero_pool|delete_excludes_recovery_with_(disjoint|identical|overlapping)_pool_lock_clients)$/)'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# These eight-disk decommission fixtures exercise concurrent retries,
|
||||
# free-version cleanup, and real capacity probes under production lock deadlines.
|
||||
# Reserve the run's capacity so unrelated IO cannot exhaust those budgets.
|
||||
@@ -455,7 +466,7 @@ filter = '(package(rustfs) & test(/^(admin::handlers::replication::target_repair
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
[[profile.ci.scripts]]
|
||||
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::complete::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
|
||||
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::complete::tests::completion_absent_binding_refusal_order_baseline|app::multipart_usecase::complete::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
[[profile.ci.scripts]]
|
||||
|
||||
@@ -14,5 +14,5 @@
|
||||
# header's rule clears (rustfs_s3_types's macro); fails closed
|
||||
# ecstore_files Rust files referencing s3s paths under crates/ecstore/src
|
||||
files=63
|
||||
s3_error_lines=217
|
||||
s3_error_lines=209
|
||||
ecstore_files=0
|
||||
|
||||
@@ -22,7 +22,7 @@ rustfs-connect-symbol-catalog 6 0
|
||||
rustfs-credentials 24 0
|
||||
rustfs-crypto 67 0
|
||||
rustfs-data-usage 58 0
|
||||
rustfs-ecstore 7858 2
|
||||
rustfs-ecstore 7858 3
|
||||
rustfs-extension-schema 23 0
|
||||
rustfs-filemeta 361 0
|
||||
rustfs-gateway-adapter 74 0
|
||||
|
||||
@@ -34,6 +34,10 @@ services:
|
||||
- RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT:-5}
|
||||
- RUSTFS_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_LOCK_ACQUIRE_TIMEOUT:-5}
|
||||
- RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
|
||||
- RUSTFS_DURABILITY_MODE=${RUSTFS_DURABILITY_MODE:-strict}
|
||||
- RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE=${RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS:-0}
|
||||
# Internode gRPC optimization knobs (grpc-optimization P0-P3). Defaults match the binary
|
||||
# defaults, so leaving these unset is a no-op; the A/B driver exports them to toggle a stage.
|
||||
- RUSTFS_INTERNODE_RPC_TCP_NODELAY=${RUSTFS_INTERNODE_RPC_TCP_NODELAY:-true}
|
||||
@@ -43,6 +47,8 @@ services:
|
||||
- RUSTFS_INTERNODE_CHANNEL_ISOLATION=${RUSTFS_INTERNODE_CHANNEL_ISOLATION:-false}
|
||||
- RUSTFS_INTERNODE_BULK_CHANNELS=${RUSTFS_INTERNODE_BULK_CHANNELS:-2}
|
||||
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY:-false}
|
||||
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED:-false}
|
||||
- RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED=${RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED:-false}
|
||||
- RUSTFS_INTERNODE_PREWARM=${RUSTFS_INTERNODE_PREWARM:-false}
|
||||
- RUSTFS_INTERNODE_OFFLINE_BYPASS=${RUSTFS_INTERNODE_OFFLINE_BYPASS:-false}
|
||||
- RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS=${RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS:-5}
|
||||
@@ -56,7 +62,7 @@ services:
|
||||
- node1_data_2:/data/rustfs2
|
||||
- node1_data_3:/data/rustfs3
|
||||
ports:
|
||||
- "9000:9000"
|
||||
- "${RUSTFS_NODE1_PORT:-9000}:9000"
|
||||
networks:
|
||||
- rustfs-cluster-net
|
||||
|
||||
@@ -81,6 +87,10 @@ services:
|
||||
- RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT:-5}
|
||||
- RUSTFS_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_LOCK_ACQUIRE_TIMEOUT:-5}
|
||||
- RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
|
||||
- RUSTFS_DURABILITY_MODE=${RUSTFS_DURABILITY_MODE:-strict}
|
||||
- RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE=${RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS:-0}
|
||||
# Internode gRPC optimization knobs (grpc-optimization P0-P3). Defaults match the binary
|
||||
# defaults, so leaving these unset is a no-op; the A/B driver exports them to toggle a stage.
|
||||
- RUSTFS_INTERNODE_RPC_TCP_NODELAY=${RUSTFS_INTERNODE_RPC_TCP_NODELAY:-true}
|
||||
@@ -90,6 +100,8 @@ services:
|
||||
- RUSTFS_INTERNODE_CHANNEL_ISOLATION=${RUSTFS_INTERNODE_CHANNEL_ISOLATION:-false}
|
||||
- RUSTFS_INTERNODE_BULK_CHANNELS=${RUSTFS_INTERNODE_BULK_CHANNELS:-2}
|
||||
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY:-false}
|
||||
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED:-false}
|
||||
- RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED=${RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED:-false}
|
||||
- RUSTFS_INTERNODE_PREWARM=${RUSTFS_INTERNODE_PREWARM:-false}
|
||||
- RUSTFS_INTERNODE_OFFLINE_BYPASS=${RUSTFS_INTERNODE_OFFLINE_BYPASS:-false}
|
||||
- RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS=${RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS:-5}
|
||||
@@ -103,7 +115,7 @@ services:
|
||||
- node2_data_2:/data/rustfs2
|
||||
- node2_data_3:/data/rustfs3
|
||||
ports:
|
||||
- "9001:9000"
|
||||
- "${RUSTFS_NODE2_PORT:-9001}:9000"
|
||||
networks:
|
||||
- rustfs-cluster-net
|
||||
|
||||
@@ -128,6 +140,10 @@ services:
|
||||
- RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT:-5}
|
||||
- RUSTFS_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_LOCK_ACQUIRE_TIMEOUT:-5}
|
||||
- RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
|
||||
- RUSTFS_DURABILITY_MODE=${RUSTFS_DURABILITY_MODE:-strict}
|
||||
- RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE=${RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS:-0}
|
||||
# Internode gRPC optimization knobs (grpc-optimization P0-P3). Defaults match the binary
|
||||
# defaults, so leaving these unset is a no-op; the A/B driver exports them to toggle a stage.
|
||||
- RUSTFS_INTERNODE_RPC_TCP_NODELAY=${RUSTFS_INTERNODE_RPC_TCP_NODELAY:-true}
|
||||
@@ -137,6 +153,8 @@ services:
|
||||
- RUSTFS_INTERNODE_CHANNEL_ISOLATION=${RUSTFS_INTERNODE_CHANNEL_ISOLATION:-false}
|
||||
- RUSTFS_INTERNODE_BULK_CHANNELS=${RUSTFS_INTERNODE_BULK_CHANNELS:-2}
|
||||
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY:-false}
|
||||
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED:-false}
|
||||
- RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED=${RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED:-false}
|
||||
- RUSTFS_INTERNODE_PREWARM=${RUSTFS_INTERNODE_PREWARM:-false}
|
||||
- RUSTFS_INTERNODE_OFFLINE_BYPASS=${RUSTFS_INTERNODE_OFFLINE_BYPASS:-false}
|
||||
- RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS=${RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS:-5}
|
||||
@@ -150,7 +168,7 @@ services:
|
||||
- node3_data_2:/data/rustfs2
|
||||
- node3_data_3:/data/rustfs3
|
||||
ports:
|
||||
- "9002:9000"
|
||||
- "${RUSTFS_NODE3_PORT:-9002}:9000"
|
||||
networks:
|
||||
- rustfs-cluster-net
|
||||
|
||||
@@ -175,6 +193,10 @@ services:
|
||||
- RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT:-5}
|
||||
- RUSTFS_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_LOCK_ACQUIRE_TIMEOUT:-5}
|
||||
- RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
|
||||
- RUSTFS_DURABILITY_MODE=${RUSTFS_DURABILITY_MODE:-strict}
|
||||
- RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE=${RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS:-0}
|
||||
# Internode gRPC optimization knobs (grpc-optimization P0-P3). Defaults match the binary
|
||||
# defaults, so leaving these unset is a no-op; the A/B driver exports them to toggle a stage.
|
||||
- RUSTFS_INTERNODE_RPC_TCP_NODELAY=${RUSTFS_INTERNODE_RPC_TCP_NODELAY:-true}
|
||||
@@ -184,6 +206,8 @@ services:
|
||||
- RUSTFS_INTERNODE_CHANNEL_ISOLATION=${RUSTFS_INTERNODE_CHANNEL_ISOLATION:-false}
|
||||
- RUSTFS_INTERNODE_BULK_CHANNELS=${RUSTFS_INTERNODE_BULK_CHANNELS:-2}
|
||||
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY:-false}
|
||||
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED:-false}
|
||||
- RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED=${RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED:-false}
|
||||
- RUSTFS_INTERNODE_PREWARM=${RUSTFS_INTERNODE_PREWARM:-false}
|
||||
- RUSTFS_INTERNODE_OFFLINE_BYPASS=${RUSTFS_INTERNODE_OFFLINE_BYPASS:-false}
|
||||
- RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS=${RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS:-5}
|
||||
@@ -197,7 +221,7 @@ services:
|
||||
- node4_data_2:/data/rustfs2
|
||||
- node4_data_3:/data/rustfs3
|
||||
ports:
|
||||
- "9003:9000"
|
||||
- "${RUSTFS_NODE4_PORT:-9003}:9000"
|
||||
networks:
|
||||
- rustfs-cluster-net
|
||||
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
# syntax=docker/dockerfile:1.6
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Server-only Linux validation image. The build context supplies a Git bundle.
|
||||
FROM rust:1.100.0-trixie AS build-tools
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates clang lld libssl-dev pkg-config protobuf-compiler python3 python3-boto3 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
ENV RUSTUP_TOOLCHAIN=1.100.0 \
|
||||
CARGO_BUILD_JOBS=4 \
|
||||
CARGO_NET_GIT_FETCH_WITH_CLI=true \
|
||||
CARGO_TARGET_DIR=/work/rustfs/target \
|
||||
RUSTFLAGS="-C link-arg=-fuse-ld=lld"
|
||||
|
||||
FROM build-tools AS build
|
||||
ARG SOURCE_REVISION
|
||||
COPY repository.bundle /tmp/repository.bundle
|
||||
COPY inspect_geometry.rs /tmp/inspect_geometry.rs
|
||||
RUN git clone /tmp/repository.bundle /work/rustfs \
|
||||
&& git -C /work/rustfs checkout --detach "$SOURCE_REVISION" \
|
||||
&& test "$(git -C /work/rustfs rev-parse HEAD)" = "$SOURCE_REVISION"
|
||||
WORKDIR /work/rustfs
|
||||
RUN --mount=type=cache,id=rustfs-inline-registry,target=/usr/local/cargo/registry \
|
||||
--mount=type=cache,id=rustfs-inline-git,target=/usr/local/cargo/git \
|
||||
--mount=type=cache,id=rustfs-inline-linux-target,target=/work/rustfs/target \
|
||||
cargo build --locked --release --bin rustfs \
|
||||
&& install -Dm755 target/release/rustfs /out/rustfs
|
||||
|
||||
# Build the diagnostic tool after preserving the clean-source server binary.
|
||||
RUN cp /tmp/inspect_geometry.rs crates/filemeta/examples/inline_lab_inspect.rs
|
||||
RUN --mount=type=cache,id=rustfs-inline-registry,target=/usr/local/cargo/registry \
|
||||
--mount=type=cache,id=rustfs-inline-git,target=/usr/local/cargo/git \
|
||||
--mount=type=cache,id=rustfs-inline-linux-target,target=/work/rustfs/target \
|
||||
cargo build --locked --release -p rustfs-filemeta --example inline_lab_inspect \
|
||||
&& install -Dm755 target/release/examples/inline_lab_inspect /out/inline_lab_inspect
|
||||
|
||||
FROM debian:trixie-slim AS runtime
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates curl libssl3t64 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
ARG SOURCE_REVISION
|
||||
LABEL org.opencontainers.image.revision="$SOURCE_REVISION" \
|
||||
rustfs.validation="inline"
|
||||
COPY --from=build /out/rustfs /usr/local/bin/rustfs
|
||||
COPY --from=build /out/inline_lab_inspect /usr/local/bin/inline_lab_inspect
|
||||
ENTRYPOINT ["/usr/local/bin/rustfs"]
|
||||
@@ -0,0 +1,5 @@
|
||||
FROM debian:trixie-slim
|
||||
COPY mc /usr/bin/mc
|
||||
RUN chmod 0755 /usr/bin/mc && /usr/bin/mc --version
|
||||
LABEL rustfs.validation="inline"
|
||||
ENTRYPOINT ["/usr/bin/mc"]
|
||||
@@ -0,0 +1,133 @@
|
||||
# Inline geometry Linux lab
|
||||
|
||||
**Use this when** validating inline placement, block geometry, Range reads and reader rollback in a task-owned local Docker cluster.
|
||||
|
||||
The adapter reuses the repository's four-node compose topology with 16 logical Docker volumes and EC12+4. It binds one S3 port to loopback, keeps strict durability and checks running binary hashes and persisted object geometry. Docker volumes share the host storage substrate. Performance observations describe this local virtualized lab.
|
||||
|
||||
## Build a frozen server image
|
||||
|
||||
Use the current stable toolchain pinned in the lab Dockerfile. Supply a Git bundle and the metadata inspector source as the build context. The server is compiled from a clean checkout before the diagnostic example is added.
|
||||
|
||||
```bash
|
||||
set -euo pipefail
|
||||
LAB_BUILD_DIR="$(mktemp -d)"
|
||||
LAB_REVISION="$(git rev-parse HEAD)"
|
||||
git bundle create "$LAB_BUILD_DIR/repository.bundle" HEAD
|
||||
cp crates/filemeta/examples/inline_lab_inspect.rs "$LAB_BUILD_DIR/inspect_geometry.rs"
|
||||
docker --context "$LAB_CONTEXT" build --target build-tools \
|
||||
-t rustfs-inline-build-tools:1.100 \
|
||||
-f .docker/inline-lab/Dockerfile .docker/inline-lab
|
||||
docker --context "$LAB_CONTEXT" build \
|
||||
--build-arg SOURCE_REVISION="$LAB_REVISION" \
|
||||
-t rustfs-inline-validation:local \
|
||||
-f .docker/inline-lab/Dockerfile "$LAB_BUILD_DIR"
|
||||
LAB_EXPORT_ID="$(docker --context "$LAB_CONTEXT" create \
|
||||
--label rustfs.validation=inline rustfs-inline-validation:local)"
|
||||
docker --context "$LAB_CONTEXT" cp \
|
||||
"$LAB_EXPORT_ID:/usr/local/bin/rustfs" "$LAB_BUILD_DIR/rustfs-linux"
|
||||
docker --context "$LAB_CONTEXT" rm "$LAB_EXPORT_ID"
|
||||
```
|
||||
|
||||
## Configure and probe
|
||||
|
||||
Select a new compose project and a private state directory. The adapter generates test credentials there. Existing resources require matching ownership labels. Configure checks the image's source revision and pins immutable image IDs. Each operation takes an exclusive state lock. Deployment waits for `/health/ready` on every node, covering storage, IAM, lock quorum and peer dependencies.
|
||||
|
||||
```bash
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" configure \
|
||||
--context "$LAB_CONTEXT" --project "$LAB_PROJECT" \
|
||||
--image rustfs-inline-validation:local --revision "$LAB_REVISION" \
|
||||
--binary "$LAB_BUILD_DIR/rustfs-linux" --port 19080
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" deploy \
|
||||
--bucket inline-geometry-probe --shard-bytes 0
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" probe \
|
||||
--bucket inline-geometry-probe
|
||||
```
|
||||
|
||||
The probe verifies complete bytes, HEAD sizes, Range contents, EC12+4 and the actual persisted block. It records inline markers for 4 KiB, 1 MiB, 1 MiB + 1 byte, 1.25 MiB, 1,572,856 bytes, 1,572,857 bytes and 1.5 MiB objects in `probe.json`. The server-only image includes the existing filemeta decoder through a small geometry inspector. Console assets and the administration CLI are outside this lab image.
|
||||
|
||||
## ABBA deployment hook
|
||||
|
||||
Use `inline_docker_lab.py ... deploy` in the existing `run_hotpath_warp_abba.sh` external deploy hook. Export `RUSTFS_ERASURE_SHARD_BYTES` according to `HOTPATH_ABBA_PHASE`. The adapter uses the supplied leg, phase, bucket, binary and evidence-file context, resets only owned lab volumes, checks running configuration and binary hashes, creates the fresh bucket and writes the runner's deployment evidence.
|
||||
|
||||
Use the same frozen binary for both sides of a geometry experiment. Fix the inline budget and encoding mode, then compare zero with one candidate shard value. Include the CPU, memory and image identities from every node in each result. The state `config.json` supplies the generated test credentials for the load driver. Set `DOCKER_CONTEXT` to the selected context when the driver captures Docker resource snapshots.
|
||||
|
||||
For content compatibility across an image or geometry change, configure the next selected image and artifact using the same state, then deploy with `--keep-data` and an existing bucket. This retains owned volumes. Compare existing objects and version IDs with the manifest prepared before the change.
|
||||
|
||||
## Cleanup
|
||||
|
||||
```bash
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" down --purge
|
||||
```
|
||||
|
||||
Cleanup checks ownership for containers, volumes and networks. A configuration cannot adopt another project's resources. Raw measurements, state, Git bundles and build outputs stay in task artifact directories.
|
||||
|
||||
Unit checks run with `scripts/python_bin.sh scripts/test_inline_docker_lab.py` and are included in `make script-tests`.
|
||||
|
||||
## Linux content-probe client
|
||||
|
||||
The PUT after-probe requires a working `mc`. For a macOS client affected by cgo initialization failures, compile the same frozen public mc module for Linux with CGO disabled, build it with `Dockerfile.mc`, and use `scripts/inline_docker_mc.sh` as the process-local `mc` command. Record the module version, checksum and resulting image ID with the run.
|
||||
|
||||
The adapter rewrites only the lab loopback alias endpoint to the private `node1` endpoint. Alias configuration stays in the probe's own `MC_CONFIG_DIR`, and `cat` streams raw bytes. The existing HEAD, GET and hash checks continue to execute.
|
||||
|
||||
```bash
|
||||
export RUSTFS_INLINE_LAB_STATE="$LAB_STATE"
|
||||
export RUSTFS_INLINE_MC_IMAGE="$LAB_MC_IMAGE"
|
||||
mkdir -p "$LAB_TOOLS"
|
||||
ln -s "$PWD/scripts/inline_docker_mc.sh" "$LAB_TOOLS/mc"
|
||||
export PATH="$LAB_TOOLS:$PATH"
|
||||
```
|
||||
|
||||
## Compare transport and placement independently
|
||||
|
||||
`deploy --rpc-mode json-compat` explicitly clears both MessagePack-only flags on every node. `deploy --rpc-mode msgpack-only` enables both flags for the selected homogeneous lab fleet. Deployment evidence records the mode and checks both running flags. The default is `json-compat`. The ABBA hook may select the same mode through `RUSTFS_INLINE_LAB_RPC_MODE`.
|
||||
|
||||
First compare the base and bounded-multi-block binaries at `--shard-bytes 0` in JSON compatibility mode. Then repeat with MessagePack-only on both sides. A same-binary transport ABBA isolates the encoding effect. Keep the 128 KiB budget, CPU sets, strict durability, load and object set fixed. Check actual markers and block sizes at the boundary sizes before timing. Retain baseline-drift and candidate gates, error logs and raw request distributions.
|
||||
|
||||
For a rolling downgrade, redeploy the current binary with `--keep-data --rpc-mode json-compat` before introducing an older node. Check old and new reads, version IDs and batch deletes against the persisted manifest. A running MessagePack-only flag pair establishes the lab configuration. Compatibility with a particular old reader requires the explicit reader test.
|
||||
|
||||
## Preserve a compatibility manifest
|
||||
|
||||
Run the seed once in a fresh lab bucket. It records full byte oracles and the observed version inventory, including any additional version matching a known body, and leaves a 5 MiB multipart part ready for the next reader. Each bucket has its own private `compat-<bucket>.json` manifest. Keep the same state and volumes across image changes. A fresh deployment or cleanup that resets volumes moves existing compatibility manifests into `compat-history`. Retained-data deployments preserve the active manifests.
|
||||
|
||||
```bash
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" compat-seed --bucket inline-compat
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" compat-check --bucket inline-compat
|
||||
# Configure the next frozen image and binary, then retain the data.
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" deploy \
|
||||
--bucket inline-compat --shard-bytes 0 --rpc-mode json-compat --keep-data
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" compat-check --bucket inline-compat
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" compat-finish-mpu --bucket inline-compat
|
||||
```
|
||||
|
||||
The check compares every persisted version ID, HEAD length, complete SHA256, cross-block Range and per-version geometry from `xl.meta`. The seed includes zero-length objects, null versions, the 192 KiB versioned admission boundary and the accumulated 128 KiB shard boundary. Seed a second bucket on the new writer before the downgrade, then check both manifests using the selected old reader. Complete an old upload on the new binary and a new upload on the old binary. After returning to the current fleet, `compat-delete --bucket <bucket>` deletes the recorded versions in one S3 batch and checks the exact remaining inventory and bytes.
|
||||
|
||||
Successful multipart-completion and batch-delete receipts are saved before subsequent content and geometry checks. If either check fails, rerun the same command with the same state directory to resume verification from the saved completion state. After a seed check fails with a saved manifest, use `compat-check` to resume verification.
|
||||
|
||||
For mixed-version testing, retain JSON compatibility mode throughout the rolling replacement. Verify each node's immutable image and binary, then perform manifest checks and new writes through both coordinator versions using `--coordinator node1` and `--coordinator node4`. The manifest records the actual writer coordinator revision from the running image. Stop one owned node to check read availability at EC12+4, restore it and check the manifests again. Record the exact oldest tested source revision, toolchain and build profile with the evidence. The version boundary is established by these executed cells.
|
||||
|
||||
An owned lab can replace just its fourth node through a compose image override. Select the frozen old image first and verify its revision label. Keep `compose.json` in JSON compatibility mode for this cell.
|
||||
|
||||
```bash
|
||||
set -euo pipefail
|
||||
OLD_IMAGE_ID="$(docker --context "$LAB_CONTEXT" image inspect "$OLD_IMAGE" --format '{{.Id}}')"
|
||||
test "$(docker --context "$LAB_CONTEXT" image inspect "$OLD_IMAGE_ID" --format '{{index .Config.Labels "org.opencontainers.image.revision"}}')" = "$OLD_REVISION"
|
||||
INLINE_OLD_IMAGE_ID="$OLD_IMAGE_ID" scripts/python_bin.sh - <<'PYTHON' > "$LAB_STATE/mixed-image.json"
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
override = {"services": {"node4": {"image": os.environ["INLINE_OLD_IMAGE_ID"]}}}
|
||||
json.dump(override, sys.stdout)
|
||||
PYTHON
|
||||
docker --context "$LAB_CONTEXT" compose --project-name "$LAB_PROJECT" \
|
||||
--file "$LAB_STATE/compose.json" --file "$LAB_STATE/mixed-image.json" \
|
||||
up --detach --no-deps --no-build --pull never node4
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" compat-check \
|
||||
--bucket inline-compat --coordinator node4
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" compat-seed \
|
||||
--bucket inline-mixed-writer --coordinator node4
|
||||
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" compat-check \
|
||||
--bucket inline-mixed-writer --coordinator node1
|
||||
```
|
||||
|
||||
Inspect all four running image IDs and binary hashes before recording the mixed-fleet result. Redeploy the configured current image with `--keep-data --rpc-mode json-compat` to restore a homogeneous fleet. The same manifest checks apply after stopping and restarting one owned node through the base compose file.
|
||||
@@ -0,0 +1,54 @@
|
||||
# Copyright 2026 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Both-stack leg
|
||||
description: Prove which S3 stack one e2e leg's servers selected and upload the leg for the summary (call with if always)
|
||||
|
||||
inputs:
|
||||
profile:
|
||||
description: The nextest profile the leg ran
|
||||
required: true
|
||||
stack:
|
||||
description: The stack the leg requested, legacy or gateway
|
||||
required: true
|
||||
logs:
|
||||
description: Directory or file holding the servers' output
|
||||
required: true
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Prove which stack the servers selected
|
||||
shell: bash
|
||||
env:
|
||||
PROFILE: ${{ inputs.profile }}
|
||||
STACK: ${{ inputs.stack }}
|
||||
LOGS: ${{ inputs.logs }}
|
||||
run: |
|
||||
# Outside the workspace: e2e_binary.py refuses a run whose source tree changed.
|
||||
mkdir -p "$RUNNER_TEMP/leg"
|
||||
junit="target/nextest/${PROFILE}/junit.xml"
|
||||
if [ -s "$junit" ]; then cp "$junit" "$RUNNER_TEMP/leg/junit.xml"; fi
|
||||
# A gateway leg stopped by its preflight has no output; the preflight's stack.json stays.
|
||||
if [ -e "$LOGS" ]; then python3 scripts/dual_stack.py stack --expect "$STACK" --out "$RUNNER_TEMP/leg/stack.json" "$LOGS"; fi
|
||||
|
||||
- name: Upload the leg
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: dual-stack-leg-${{ inputs.profile }}-${{ inputs.stack }}
|
||||
overwrite: true
|
||||
path: ${{ runner.temp }}/leg
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
@@ -0,0 +1,56 @@
|
||||
# Copyright 2026 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Both-stack summary
|
||||
description: Diff the legacy and gateway legs of this run (scripts/junit_diff.py) and upload one dated summary
|
||||
|
||||
inputs:
|
||||
profiles:
|
||||
description: Comma-separated nextest profiles whose two legs this run uploaded
|
||||
required: true
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Download the legs
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
pattern: dual-stack-leg-*
|
||||
path: legs
|
||||
|
||||
- name: Diff every profile and write the summary
|
||||
id: summary
|
||||
shell: bash
|
||||
env:
|
||||
PROFILES: ${{ inputs.profiles }}
|
||||
run: |
|
||||
# The shell runs with -e; the exit status is reported after the summary is written.
|
||||
set +e -u -o pipefail
|
||||
date="$(date -u +%F)"
|
||||
echo "date=${date}" >> "$GITHUB_OUTPUT"
|
||||
mkdir -p legs
|
||||
python3 scripts/dual_stack.py summary --profiles "$PROFILES" --legs legs --date "$date" --out-dir dual-stack
|
||||
status=$?
|
||||
if [ -f dual-stack/summary.md ]; then cat dual-stack/summary.md >> "$GITHUB_STEP_SUMMARY"; fi
|
||||
exit "$status"
|
||||
|
||||
- name: Upload the dated summary
|
||||
if: always() && steps.summary.outputs.date != ''
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: dual-stack-summary-${{ steps.summary.outputs.date }}
|
||||
overwrite: true
|
||||
path: dual-stack
|
||||
if-no-files-found: error
|
||||
retention-days: 30
|
||||
@@ -107,6 +107,10 @@ runs:
|
||||
git cat-file -e "$FOOTPRINT_BASE^{commit}" 2>/dev/null || git fetch --no-tags --depth=1 origin "$FOOTPRINT_BASE"
|
||||
./scripts/check_s3s_footprint.sh --base "$FOOTPRINT_BASE"
|
||||
|
||||
- name: Check release feature boundaries
|
||||
shell: bash
|
||||
run: python3 scripts/check_connect_release_boundary.py --features-only
|
||||
|
||||
- name: Check per-crate test-count ratchet
|
||||
shell: bash
|
||||
run: ./scripts/check_test_count_ratchet.sh
|
||||
|
||||
@@ -296,6 +296,34 @@
|
||||
"origin": "baseline",
|
||||
"alert_contract": "scheduled-validation"
|
||||
},
|
||||
{
|
||||
"workflow": ".github/workflows/rdma-feature-check.yml",
|
||||
"name": "S3-over-RDMA feature check",
|
||||
"roles": [
|
||||
"pr",
|
||||
"periodic",
|
||||
"manual"
|
||||
],
|
||||
"events": [
|
||||
"pull_request",
|
||||
"schedule",
|
||||
"workflow_dispatch"
|
||||
],
|
||||
"cadence": "weekly",
|
||||
"timezone": "UTC",
|
||||
"schedule": [
|
||||
"37 5 * * 1"
|
||||
],
|
||||
"owner": "@overtrue",
|
||||
"policy": "active",
|
||||
"monitor": "workflow",
|
||||
"max_age_hours": 192,
|
||||
"result_owner": "rdma-feature-check.yml",
|
||||
"disposition": "compiles the nixl feature and runs rustfs-nixl unit tests on Linux; the feature is off in every other build",
|
||||
"failure_owner": "workflow",
|
||||
"origin": "added",
|
||||
"alert_contract": "scheduled-validation"
|
||||
},
|
||||
{
|
||||
"workflow": ".github/workflows/rustfs-validation-sweep.yml",
|
||||
"name": "RustFS Validation Sweep",
|
||||
@@ -360,14 +388,15 @@
|
||||
"cadence": "nightly",
|
||||
"timezone": "UTC",
|
||||
"schedule": [
|
||||
"13 1 * * *"
|
||||
"13 1 * * *",
|
||||
"17 16 * * 0"
|
||||
],
|
||||
"owner": "@overtrue",
|
||||
"policy": "active",
|
||||
"monitor": "workflow",
|
||||
"max_age_hours": 36,
|
||||
"result_owner": "build.yml",
|
||||
"disposition": "Builds artifacts only; public release writes belong to release-publish.yml.",
|
||||
"disposition": "Daily cron builds Linux; Sunday 16:17 UTC builds macOS and Windows. Public release writes belong to release-publish.yml.",
|
||||
"failure_owner": "workflow",
|
||||
"origin": "baseline",
|
||||
"alert_contract": "scheduled-validation"
|
||||
@@ -764,6 +793,31 @@
|
||||
"monitor_reason": "Event-driven or operator-requested work has no periodic freshness obligation.",
|
||||
"origin": "baseline"
|
||||
},
|
||||
{
|
||||
"workflow": ".github/workflows/dual-stack-nightly.yml",
|
||||
"name": "dual-stack-nightly",
|
||||
"roles": [
|
||||
"periodic",
|
||||
"manual"
|
||||
],
|
||||
"events": [
|
||||
"workflow_dispatch",
|
||||
"schedule"
|
||||
],
|
||||
"cadence": "nightly",
|
||||
"timezone": "UTC",
|
||||
"schedule": [
|
||||
"13 3 * * *"
|
||||
],
|
||||
"owner": "@overtrue",
|
||||
"policy": "active",
|
||||
"monitor": "workflow",
|
||||
"max_age_hours": 36,
|
||||
"result_owner": "dual-stack-nightly.yml",
|
||||
"disposition": "both-stack JUnit diff of the two profiles only ci.yml runs; the gateway legs are report-only until T2.10",
|
||||
"failure_owner": "watchdog",
|
||||
"origin": "added"
|
||||
},
|
||||
{
|
||||
"workflow": ".github/workflows/e2e-distributed.yml",
|
||||
"name": "e2e-distributed",
|
||||
@@ -951,6 +1005,31 @@
|
||||
"origin": "baseline",
|
||||
"alert_contract": "scheduled-validation"
|
||||
},
|
||||
{
|
||||
"workflow": ".github/workflows/gateway-acceptance.yml",
|
||||
"name": "Gateway Acceptance",
|
||||
"roles": [
|
||||
"periodic",
|
||||
"manual"
|
||||
],
|
||||
"events": [
|
||||
"schedule",
|
||||
"workflow_dispatch"
|
||||
],
|
||||
"cadence": "nightly",
|
||||
"timezone": "UTC",
|
||||
"schedule": [
|
||||
"29 3 * * *"
|
||||
],
|
||||
"owner": "@overtrue",
|
||||
"policy": "active",
|
||||
"monitor": "workflow",
|
||||
"max_age_hours": 36,
|
||||
"result_owner": "gateway-acceptance.yml",
|
||||
"disposition": "gateway conformance corpus (rustfs profile) and client matrix against the candidate on both S3 stacks",
|
||||
"failure_owner": "watchdog",
|
||||
"origin": "added"
|
||||
},
|
||||
{
|
||||
"workflow": ".github/workflows/gateway-integration-sync.yml",
|
||||
"name": "Gateway Integration Sync",
|
||||
@@ -1023,7 +1102,7 @@
|
||||
},
|
||||
{
|
||||
"workflow": ".github/workflows/manual-main-packages.yml",
|
||||
"name": "Manual main packages",
|
||||
"name": "Manual Linux packages",
|
||||
"roles": [
|
||||
"manual"
|
||||
],
|
||||
@@ -1037,7 +1116,7 @@
|
||||
"policy": "active",
|
||||
"monitor": "none",
|
||||
"result_owner": "manual-main-packages.yml",
|
||||
"disposition": "Packages the origin/main tip as workflow artifacts only. No GitHub Release or download-channel writes.",
|
||||
"disposition": "Packages a git tag, or the origin/main tip when the ref input is empty, as workflow artifacts only. No GitHub Release or download-channel writes.",
|
||||
"monitor_reason": "Operator-requested packaging has no periodic freshness obligation.",
|
||||
"origin": "manual-main-packages"
|
||||
},
|
||||
|
||||
@@ -81,6 +81,6 @@ Rows using `macos-latest-xlarge` or a label starting with `sm-`, `dind-`, `ubicl
|
||||
|
||||
## Release stages
|
||||
|
||||
The [candidate release pipeline](../../docs/operations/release-pipeline.md) builds Linux first and defers paid desktop capacity until acceptance. Daily development builds also select Linux; use manual platform selection for desktop development artifacts. All six platforms remain required before publication. The pipeline records and promotes the exact tested bytes. Builder workflows have no public-release credentials. Keep periodic development builds, PR verification and field-test chains separate when their contracts differ; do not re-enable disabled workflows as part of a source-file migration.
|
||||
The [candidate release pipeline](../../docs/operations/release-pipeline.md) builds Linux first and defers paid desktop capacity until acceptance. The daily `build.yml` schedule selects Linux; the Sunday 16:17 UTC schedule selects macOS and Windows. Manual dispatch can still select any platform list. All six platforms remain required before publication. The pipeline records and promotes the exact tested bytes. Builder workflows have no public-release credentials. Keep periodic development builds, PR verification and field-test chains separate when their contracts differ; do not re-enable disabled workflows as part of a source-file migration.
|
||||
|
||||
Larger runners remain billed for public repositories; see [GitHub runner billing](https://docs.github.com/en/actions/concepts/runners/larger-runners#billing). Actual spending comes from organization billing, not runner labels or an estimated build duration.
|
||||
|
||||
@@ -12,7 +12,8 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Nightly development artifacts and reusable immutable-candidate builds.
|
||||
# Scheduled development artifacts and reusable immutable-candidate builds.
|
||||
# The daily cron builds Linux. Sunday 16:17 UTC builds macOS and Windows.
|
||||
# Public release writes live exclusively in release-publish.yml. Candidate callers
|
||||
# select Linux or desktop and pin release inputs before any matrix starts.
|
||||
|
||||
@@ -30,7 +31,8 @@ on:
|
||||
required: true
|
||||
type: string
|
||||
schedule:
|
||||
- cron: "13 1 * * *" # Nightly 01:13 UTC
|
||||
- cron: "13 1 * * *" # Daily Linux, 01:13 UTC
|
||||
- cron: "17 16 * * 0" # Weekly macOS and Windows, Sunday 16:17 UTC
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
publish:
|
||||
@@ -219,16 +221,33 @@ jobs:
|
||||
env:
|
||||
# via env, not interpolation: a dispatch input is free-form text and
|
||||
# would otherwise be pasted into the script for bash to evaluate.
|
||||
# Daily development builds cover Linux. Desktop remains available on
|
||||
# manual dispatch and is required after candidate field acceptance.
|
||||
# A direct schedule has no platforms input. Choose its subset from
|
||||
# github.event.schedule, which is fixed when that cron fires, so a
|
||||
# queued job cannot switch subsets because the runner clock moved.
|
||||
# Callers that pass platforms keep that list, including a scheduled
|
||||
# caller: inputs.platforms is set even though event_name stays schedule.
|
||||
RAW_PLATFORMS: ${{ inputs.platforms || 'linux' }}
|
||||
HAS_PLATFORM_INPUT: ${{ inputs.platforms && 'true' || 'false' }}
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
EVENT_SCHEDULE: ${{ github.event.schedule }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
selected="$RAW_PLATFORMS"
|
||||
selected="$(echo "${selected}" | tr -d '[:space:]')"
|
||||
if [[ -z "${selected}" ]]; then
|
||||
selected="all"
|
||||
if [[ "${EVENT_NAME}" == "schedule" && "${HAS_PLATFORM_INPUT}" != "true" ]]; then
|
||||
if [[ "${EVENT_SCHEDULE}" == "13 1 * * *" ]]; then
|
||||
selected="linux"
|
||||
elif [[ "${EVENT_SCHEDULE}" == "17 16 * * 0" ]]; then
|
||||
selected="desktop"
|
||||
else
|
||||
echo "Unrecognized schedule cron: ${EVENT_SCHEDULE}" >&2
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
selected="${RAW_PLATFORMS}"
|
||||
selected="$(echo "${selected}" | tr -d '[:space:]')"
|
||||
if [[ -z "${selected}" ]]; then
|
||||
selected="all"
|
||||
fi
|
||||
fi
|
||||
|
||||
# linux-aarch64-* need a real aarch64 host (placement rule: hardware)
|
||||
|
||||
@@ -118,7 +118,7 @@ jobs:
|
||||
printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
|
||||
echo "Base CI policy unavailable; running the full matrix."
|
||||
fi
|
||||
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=7$' "$GITHUB_OUTPUT"; then
|
||||
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=8$' "$GITHUB_OUTPUT"; then
|
||||
printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'gateway_smoke=true' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
|
||||
echo "Base policy does not cover this workflow contract; retaining the full matrix."
|
||||
fi
|
||||
@@ -469,7 +469,7 @@ jobs:
|
||||
# hold. `cargo check --release` is metadata-only and needs no
|
||||
# link-concurrency limit; it is cold because ci-dev holds no release
|
||||
# artifacts.
|
||||
- name: Reject short credentials in release builds
|
||||
- name: Reject debug-only features in release builds
|
||||
run: python3 scripts/check_connect_release_boundary.py
|
||||
|
||||
- name: Check offline enrollment E2E root boundary
|
||||
@@ -676,6 +676,54 @@ jobs:
|
||||
retention-days: 3
|
||||
if-no-files-found: warn
|
||||
|
||||
test-and-lint-gateway-stack:
|
||||
name: Test and Lint (gateway-stack)
|
||||
# The gateway S3 stack (rustfs/backlog#2734) sits behind the rustfs
|
||||
# `gateway-stack` feature, which no other pull-request lane lints or tests.
|
||||
# Unlike the dormant feature lanes above, this one runs for every code
|
||||
# change: each operation slice puts its handler tests and its refusal-matrix
|
||||
# row behind the feature, and a lane that ran only on demand would let them
|
||||
# rot behind a green pull request.
|
||||
if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
# Self-hosted for build scale and cache ownership: a workspace-scale Cargo
|
||||
# build that reads the ci-dev cache written on this image.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: ci-dev-ea7eec74fe14
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
# A feature that does not compile fails here; nothing below is skipped
|
||||
# into a pass.
|
||||
- name: Run clippy with gateway-stack
|
||||
run: cargo clippy --locked -p rustfs --all-targets --features gateway-stack -- -D warnings
|
||||
|
||||
# The gateway stack's own tests: the assembly, the operation table and
|
||||
# its completeness, the shared handler sequence, the stack selection, and
|
||||
# the embedded server driven through the whole middleware stack. The
|
||||
# filter names modules, never operations.
|
||||
- name: Run tests with gateway-stack
|
||||
env:
|
||||
# The same bounded link concurrency as the other feature lanes.
|
||||
CARGO_BUILD_JOBS: "2"
|
||||
run: |
|
||||
cargo nextest run --profile ci --locked -p rustfs --features gateway-stack --no-tests fail \
|
||||
--status-level all --final-status-level all \
|
||||
-E 'test(/gateway_stack|gateway_handlers|server::stack::/) | binary(embedded_gateway_stack_test)'
|
||||
|
||||
build-rustfs-debug-binary:
|
||||
name: Build RustFS Debug Binary
|
||||
if: contains(fromJSON('["full", "e2e"]'), needs.classify-changes.outputs.mode) && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
@@ -1564,6 +1612,7 @@ jobs:
|
||||
- fips-provider
|
||||
- test-and-lint-rio-v2
|
||||
- test-and-lint-protocols
|
||||
- test-and-lint-gateway-stack
|
||||
- build-rustfs-debug-binary
|
||||
- uring-integration
|
||||
- e2e-tests
|
||||
@@ -1599,6 +1648,7 @@ jobs:
|
||||
- core-contract
|
||||
- test-and-lint-rio-v2
|
||||
- test-and-lint-protocols
|
||||
- test-and-lint-gateway-stack
|
||||
- build-rustfs-debug-binary
|
||||
- build-rustfs-debug-binary-rio-v2
|
||||
- uring-integration
|
||||
|
||||
@@ -0,0 +1,242 @@
|
||||
# Copyright 2026 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Nightly both-stack diff of the two e2e profiles only ci.yml runs, e2e-smoke
|
||||
# and e2e-full (rustfs/backlog#2772, T3.1). The other profiles carry a `stack`
|
||||
# dimension in their own workflows, where the existing lane is the legacy leg.
|
||||
#
|
||||
# One build serves every leg. One preflight starts that binary with
|
||||
# RUSTFS_S3_STACK=gateway (scripts/dual_stack.py preflight); without a proven
|
||||
# gateway start the gateway legs do not run and the summary says `skipped:
|
||||
# gateway stack refuses startup: <why>`. Which stack a leg ran is read from the
|
||||
# servers' own start-up line. The gateway legs are report-only until T2.10.
|
||||
#
|
||||
# Left out of both stacks: `dual_stack_differential::` (servers with an explicit
|
||||
# stack), `fake_s3_target::`, the upgrade write-readiness cases and one deadline
|
||||
# case (no server), and `distributed_startup_regression_test::` (needs ci.yml's
|
||||
# hooks-binary manifest). Both e2e-full stacks use ci.yml's pinned mc and
|
||||
# Vault binaries and network fault-injection tools.
|
||||
|
||||
name: dual-stack-nightly
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
# 03:13 UTC nightly, clear of e2e-nightly (04:29), the ODM lane (05:23) and
|
||||
# e2e-distributed (05:53).
|
||||
- cron: "13 3 * * *"
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name != 'schedule' }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Build once and probe the gateway stack
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
# The runner and cache of ci.yml's build-rustfs-debug-binary.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 60
|
||||
outputs:
|
||||
stacks: ${{ steps.preflight.outputs.stacks }}
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: ci-dev
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
- name: Build the rustfs binary
|
||||
run: python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks
|
||||
|
||||
- name: Upload the binary
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: dual-stack-binary
|
||||
path: |
|
||||
target/debug/rustfs
|
||||
target/debug/rustfs.e2e.json
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
# Without a proven gateway start there is no gateway leg to run.
|
||||
- name: Gateway preflight
|
||||
id: preflight
|
||||
run: |
|
||||
if python3 scripts/dual_stack.py preflight --binary target/debug/rustfs --out "$RUNNER_TEMP/preflight/stack.json"; then
|
||||
echo 'stacks=["legacy","gateway"]' >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo 'stacks=["legacy"]' >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# The evidence stands in for a gateway leg that is skipped; a leg that runs overwrites it.
|
||||
- name: Upload the preflight evidence for e2e-smoke
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: dual-stack-leg-e2e-smoke-gateway
|
||||
path: ${{ runner.temp }}/preflight
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
|
||||
- name: Upload the preflight evidence for e2e-full
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: dual-stack-leg-e2e-full-gateway
|
||||
path: ${{ runner.temp }}/preflight
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
|
||||
leg:
|
||||
name: ${{ matrix.profile }} (${{ matrix.stack }}${{ matrix.stack == 'gateway' && ', report-only until T2.10' || '' }})
|
||||
needs: [ build ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 120
|
||||
# Report-only until T2.10: the gateway stack cannot serve the suites yet.
|
||||
continue-on-error: ${{ matrix.stack == 'gateway' }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
profile: [ e2e-smoke, e2e-full ]
|
||||
stack: ${{ fromJSON(needs.build.outputs.stacks) }}
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: ci-dev
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
- name: Download the binary
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: dual-stack-binary
|
||||
path: target/debug
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install awscurl
|
||||
run: |
|
||||
python3 -m pip install --user --upgrade pip "awscurl==0.44"
|
||||
echo "AWSCURL_PATH=$HOME/.local/bin/awscurl" >> "$GITHUB_ENV"
|
||||
test -x "$HOME/.local/bin/awscurl"
|
||||
|
||||
- name: Install network fault-injection tools
|
||||
if: matrix.profile == 'e2e-full'
|
||||
run: |
|
||||
sudo apt-get install -y iptables
|
||||
sudo -n iptables --version
|
||||
# The endpoint-blackhole heal scenario needs CAP_NET_ADMIN. Containerised
|
||||
# runners can run iptables but not touch the rule set; the test then logs
|
||||
# a skip instead of failing, so surface that here where it is visible.
|
||||
if ! sudo -n iptables -w 5 -S OUTPUT >/dev/null 2>&1; then
|
||||
echo "::warning::iptables cannot read the OUTPUT chain on this runner (no CAP_NET_ADMIN); the endpoint-blackhole heal scenario will be skipped"
|
||||
fi
|
||||
|
||||
- name: Install mc
|
||||
if: matrix.profile == 'e2e-full'
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
MC_VERSION: RELEASE.2025-08-13T08-35-41Z
|
||||
MC_SHA256: 01f866e9c5f9b87c2b09116fa5d7c06695b106242d829a8bb32990c00312e891
|
||||
run: |
|
||||
if ! command -v gh >/dev/null 2>&1; then
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y gh
|
||||
fi
|
||||
MC_BINARY="mc.linux-amd64.${MC_VERSION}"
|
||||
python3 .github/actions/setup/download_release_asset.py --repository minio/mc --tag "$MC_VERSION" --asset "$MC_BINARY" --output "$RUNNER_TEMP/mc" --sha256 "$MC_SHA256"
|
||||
echo "${MC_SHA256} $RUNNER_TEMP/mc" | sha256sum --check --status
|
||||
chmod +x "$RUNNER_TEMP/mc"
|
||||
echo "$RUNNER_TEMP" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Verify mc
|
||||
if: matrix.profile == 'e2e-full'
|
||||
run: mc --version
|
||||
|
||||
- name: Install Vault
|
||||
if: matrix.profile == 'e2e-full'
|
||||
run: |
|
||||
VAULT_VERSION="1.17.6"
|
||||
VAULT_ARCHIVE="vault_${VAULT_VERSION}_linux_amd64.zip"
|
||||
curl -fsSLo "$RUNNER_TEMP/$VAULT_ARCHIVE" "https://releases.hashicorp.com/vault/${VAULT_VERSION}/${VAULT_ARCHIVE}"
|
||||
echo "0cddc1fbbb88583b5ba5b845f9f8fae47c6fb39a6d48cd543c6ba6fd3ac1a669 $RUNNER_TEMP/$VAULT_ARCHIVE" | sha256sum --check --status
|
||||
unzip -q "$RUNNER_TEMP/$VAULT_ARCHIVE" -d "$RUNNER_TEMP/vault-bin"
|
||||
echo "RUSTFS_TEST_VAULT_BIN=$RUNNER_TEMP/vault-bin/vault" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Verify Vault
|
||||
if: matrix.profile == 'e2e-full'
|
||||
run: |
|
||||
"$RUSTFS_TEST_VAULT_BIN" version
|
||||
|
||||
- name: Run the profile on the ${{ matrix.stack }} stack
|
||||
env:
|
||||
PROFILE: ${{ matrix.profile }}
|
||||
RUSTFS_S3_STACK: ${{ matrix.stack }}
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/server-logs
|
||||
run: |
|
||||
chmod +x target/debug/rustfs
|
||||
filter='not test(/^(dual_stack_differential|fake_s3_target|distributed_startup_regression_test)::/)'
|
||||
filter="$filter & not test(/^upgrade_compatibility_test::upgrade_write_readiness_tests::/)"
|
||||
filter="$filter & not test(=list_objects_v2_pagination_test::tests::test_list_objects_v2_deadline_rejects_unresponsive_peer)"
|
||||
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile "$PROFILE" -p e2e_test -E "$filter"
|
||||
|
||||
- name: Prove the stack and upload the leg
|
||||
if: always()
|
||||
uses: ./.github/actions/dual-stack-leg
|
||||
with: { profile: "${{ matrix.profile }}", stack: "${{ matrix.stack }}", logs: "${{ runner.temp }}/server-logs" }
|
||||
|
||||
summary:
|
||||
name: Both-stack diff summary
|
||||
needs: [ build, leg ]
|
||||
if: github.repository == 'rustfs/rustfs' && always()
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: ./.github/actions/dual-stack-summary
|
||||
with:
|
||||
profiles: e2e-smoke,e2e-full
|
||||
@@ -53,11 +53,14 @@ concurrency:
|
||||
jobs:
|
||||
distributed:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Distributed 4-node 4-disk e2e
|
||||
name: Distributed 4-node 4-disk e2e (${{ matrix.stack }}${{ matrix.stack == 'gateway' && ', report-only until T2.10' || '' }})
|
||||
# GitHub-hosted VM: loop and tmpfs mounts work here. sm-standard-4 is an
|
||||
# ARC pod and rejects both (`mount -o loop` ENOENT, tmpfs "read-only").
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 180
|
||||
# The gateway leg is report-only until T2.10.
|
||||
continue-on-error: ${{ matrix.stack == 'gateway' }}
|
||||
strategy: { fail-fast: false, matrix: { stack: [legacy, gateway] } }
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
NO_PROXY: 127.0.0.1,localhost
|
||||
@@ -129,6 +132,11 @@ jobs:
|
||||
run: |
|
||||
python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks
|
||||
|
||||
# A gateway binary that refuses to start skips the suite; the summary says why.
|
||||
- name: Gateway preflight
|
||||
if: matrix.stack == 'gateway'
|
||||
run: python3 scripts/dual_stack.py preflight --binary target/debug/rustfs --out "$RUNNER_TEMP/leg/stack.json"
|
||||
|
||||
- name: Verify distributed e2e membership
|
||||
env:
|
||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-distributed-list.json
|
||||
@@ -140,6 +148,7 @@ jobs:
|
||||
env:
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-distributed-logs
|
||||
FILTER: ${{ inputs.filter }}
|
||||
RUSTFS_S3_STACK: ${{ matrix.stack }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
rm -f target/nextest/e2e-distributed/junit.xml
|
||||
@@ -150,11 +159,19 @@ jobs:
|
||||
fi
|
||||
test -s target/nextest/e2e-distributed/junit.xml
|
||||
|
||||
- name: Prove the stack and upload the leg
|
||||
if: always()
|
||||
uses: ./.github/actions/dual-stack-leg
|
||||
with:
|
||||
profile: e2e-distributed
|
||||
stack: ${{ matrix.stack }}
|
||||
logs: ${{ runner.temp }}/rustfs-e2e-distributed-logs
|
||||
|
||||
- name: Upload distributed e2e diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-distributed-${{ github.run_number }}
|
||||
name: e2e-distributed-${{ github.run_number }}-${{ matrix.stack }}
|
||||
path: |
|
||||
target/nextest/e2e-distributed/junit.xml
|
||||
${{ runner.temp }}/rustfs-e2e-distributed-list.json
|
||||
@@ -174,6 +191,20 @@ jobs:
|
||||
fi
|
||||
done
|
||||
|
||||
dual-stack-summary:
|
||||
name: Both-stack diff summary
|
||||
needs: [distributed]
|
||||
if: github.repository == 'rustfs/rustfs' && always()
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: ./.github/actions/dual-stack-summary
|
||||
with:
|
||||
profiles: e2e-distributed
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [distributed]
|
||||
|
||||
@@ -54,14 +54,18 @@ concurrency:
|
||||
jobs:
|
||||
repl-nightly:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Replication e2e (nightly)
|
||||
name: Replication e2e (nightly, ${{ matrix.stack }}${{ matrix.stack == 'gateway' && ', report-only until T2.10' || '' }})
|
||||
# dual-node tests spawn two full rustfs servers each; the extra headroom
|
||||
# over the per-PR sm-standard-2 e2e job keeps the parallel process fan-out
|
||||
# from starving the runner. Mirrors the ILM serial lane's runner class.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 45
|
||||
# The gateway leg is report-only until T2.10.
|
||||
continue-on-error: ${{ matrix.stack == 'gateway' }}
|
||||
strategy: { fail-fast: false, matrix: { stack: [legacy, gateway] } }
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
RUSTFS_BUILD_FEATURES: ${{ matrix.stack == 'gateway' && 'gateway-stack' || '' }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -93,7 +97,12 @@ jobs:
|
||||
# Build once and carry its source/binary identity into the test invocation.
|
||||
- name: Build rustfs binary
|
||||
run: |
|
||||
python3 scripts/e2e_binary.py build --bins
|
||||
python3 scripts/e2e_binary.py build --bins --features "$RUSTFS_BUILD_FEATURES"
|
||||
|
||||
# A gateway binary that refuses to start skips the suite; the summary says why.
|
||||
- name: Gateway preflight
|
||||
if: matrix.stack == 'gateway'
|
||||
run: python3 scripts/dual_stack.py preflight --binary target/debug/rustfs --out "$RUNNER_TEMP/leg/stack.json"
|
||||
|
||||
- name: Verify replication e2e membership
|
||||
env:
|
||||
@@ -105,13 +114,22 @@ jobs:
|
||||
- name: Run replication e2e nightly suite
|
||||
env:
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-repl-nightly-logs
|
||||
run: python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-repl-nightly -p e2e_test
|
||||
RUSTFS_S3_STACK: ${{ matrix.stack }}
|
||||
run: python3 scripts/e2e_binary.py run --features "$RUSTFS_BUILD_FEATURES" -- cargo nextest run --profile e2e-repl-nightly -p e2e_test
|
||||
|
||||
- name: Prove the stack and upload the leg
|
||||
if: always()
|
||||
uses: ./.github/actions/dual-stack-leg
|
||||
with:
|
||||
profile: e2e-repl-nightly
|
||||
stack: ${{ matrix.stack }}
|
||||
logs: ${{ runner.temp }}/rustfs-e2e-repl-nightly-logs
|
||||
|
||||
- name: Upload nextest junit report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-replication-nightly-junit-${{ github.run_number }}
|
||||
name: e2e-replication-nightly-junit-${{ github.run_number }}-${{ matrix.stack }}
|
||||
path: |
|
||||
target/nextest/e2e-repl-nightly/junit.xml
|
||||
${{ runner.temp }}/rustfs-e2e-repl-nightly-list.json
|
||||
@@ -121,9 +139,12 @@ jobs:
|
||||
|
||||
cluster-nightly:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Cluster fault e2e (nightly)
|
||||
name: Cluster fault e2e (nightly, ${{ matrix.stack }}${{ matrix.stack == 'gateway' && ', report-only until T2.10' || '' }})
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
# The gateway leg is report-only until T2.10.
|
||||
continue-on-error: ${{ matrix.stack == 'gateway' }}
|
||||
strategy: { fail-fast: false, matrix: { stack: [legacy, gateway] } }
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
@@ -144,6 +165,11 @@ jobs:
|
||||
run: |
|
||||
python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks
|
||||
|
||||
# A gateway binary that refuses to start skips the suite; the summary says why.
|
||||
- name: Gateway preflight
|
||||
if: matrix.stack == 'gateway'
|
||||
run: python3 scripts/dual_stack.py preflight --binary target/debug/rustfs --out "$RUNNER_TEMP/leg/stack.json"
|
||||
|
||||
- name: Verify cluster fault e2e membership
|
||||
env:
|
||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-nightly-list.json
|
||||
@@ -155,13 +181,22 @@ jobs:
|
||||
env:
|
||||
CARGO_BIN_EXE_rustfs: ${{ github.workspace }}/target/debug/rustfs
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-nightly-logs
|
||||
RUSTFS_S3_STACK: ${{ matrix.stack }}
|
||||
run: python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-nightly -p e2e_test
|
||||
|
||||
- name: Prove the stack and upload the leg
|
||||
if: always()
|
||||
uses: ./.github/actions/dual-stack-leg
|
||||
with:
|
||||
profile: e2e-nightly
|
||||
stack: ${{ matrix.stack }}
|
||||
logs: ${{ runner.temp }}/rustfs-e2e-nightly-logs
|
||||
|
||||
- name: Upload cluster fault diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-cluster-nightly-${{ github.run_number }}
|
||||
name: e2e-cluster-nightly-${{ github.run_number }}-${{ matrix.stack }}
|
||||
path: |
|
||||
target/nextest/e2e-nightly/junit.xml
|
||||
${{ runner.temp }}/rustfs-e2e-nightly-list.json
|
||||
@@ -171,12 +206,15 @@ jobs:
|
||||
|
||||
protocols-nightly:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Protocol e2e (nightly)
|
||||
name: Protocol e2e (nightly, ${{ matrix.stack }}${{ matrix.stack == 'gateway' && ', report-only until T2.10' || '' }})
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
# The gateway leg is report-only until T2.10.
|
||||
continue-on-error: ${{ matrix.stack == 'gateway' }}
|
||||
strategy: { fail-fast: false, matrix: { stack: [legacy, gateway] } }
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
RUSTFS_BUILD_FEATURES: ftps,webdav,sftp
|
||||
RUSTFS_BUILD_FEATURES: ${{ matrix.stack == 'gateway' && 'ftps,webdav,sftp,gateway-stack' || 'ftps,webdav,sftp' }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -200,6 +238,11 @@ jobs:
|
||||
- name: Build protocol server
|
||||
run: python3 scripts/e2e_binary.py build --features "$RUSTFS_BUILD_FEATURES"
|
||||
|
||||
# A gateway binary that refuses to start skips the suite; the summary says why.
|
||||
- name: Gateway preflight
|
||||
if: matrix.stack == 'gateway'
|
||||
run: python3 scripts/dual_stack.py preflight --binary target/debug/rustfs --out "$RUNNER_TEMP/leg/stack.json"
|
||||
|
||||
# The suite owns fixed protocol ports and serializes its internal cases.
|
||||
- name: Verify protocol e2e membership
|
||||
env:
|
||||
@@ -211,14 +254,28 @@ jobs:
|
||||
- name: Run protocol e2e nightly suite
|
||||
env:
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-protocol-e2e-logs
|
||||
run: >-
|
||||
python3 scripts/e2e_binary.py run --features "$RUSTFS_BUILD_FEATURES" -- cargo nextest run -j 1 --profile e2e-protocols -p e2e_test --no-capture
|
||||
RUSTFS_S3_STACK: ${{ matrix.stack }}
|
||||
# These servers set no log level of their own; the stack proof reads their start-up line.
|
||||
RUST_LOG: rustfs=info
|
||||
# The servers inherit nextest's stdout (--no-capture): keep it where the stack proof reads it.
|
||||
run: |
|
||||
set -o pipefail
|
||||
mkdir -p "$RUSTFS_E2E_LOG_DIR"
|
||||
python3 scripts/e2e_binary.py run --features "$RUSTFS_BUILD_FEATURES" -- cargo nextest run -j 1 --profile e2e-protocols -p e2e_test --no-capture 2>&1 | tee "$RUSTFS_E2E_LOG_DIR/nextest.log"
|
||||
|
||||
- name: Prove the stack and upload the leg
|
||||
if: always()
|
||||
uses: ./.github/actions/dual-stack-leg
|
||||
with:
|
||||
profile: e2e-protocols
|
||||
stack: ${{ matrix.stack }}
|
||||
logs: ${{ runner.temp }}/rustfs-protocol-e2e-logs
|
||||
|
||||
- name: Upload protocol diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-protocol-nightly-${{ github.run_number }}
|
||||
name: e2e-protocol-nightly-${{ github.run_number }}-${{ matrix.stack }}
|
||||
path: |
|
||||
target/nextest/e2e-protocols/junit.xml
|
||||
${{ runner.temp }}/rustfs-e2e-protocols-list.json
|
||||
@@ -226,6 +283,20 @@ jobs:
|
||||
retention-days: 7
|
||||
if-no-files-found: warn
|
||||
|
||||
dual-stack-summary:
|
||||
name: Both-stack diff summary
|
||||
needs: [repl-nightly, cluster-nightly, protocols-nightly]
|
||||
if: github.repository == 'rustfs/rustfs' && always()
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: ./.github/actions/dual-stack-summary
|
||||
with:
|
||||
profiles: e2e-repl-nightly,e2e-nightly,e2e-protocols
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [repl-nightly, cluster-nightly, protocols-nightly]
|
||||
|
||||
@@ -0,0 +1,395 @@
|
||||
# Copyright 2026 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Gateway black-box acceptance lanes (rustfs/backlog#2775 T3.4, #2776 T3.5).
|
||||
#
|
||||
# Runs two suites of the rustfs/gateway repository against the candidate binary,
|
||||
# once per S3 stack (RUSTFS_S3_STACK=legacy and =gateway), one server per stack:
|
||||
#
|
||||
# - the conformance corpus, `--profile rustfs`. That profile describes the
|
||||
# gateway edge, so scripts/check_conformance_rulings.py judges the gateway
|
||||
# stack against .config/conformance-rulings.toml (red: an unruled failure, a
|
||||
# ruled case failing differently, a stale ruling, a skip on the server's own
|
||||
# answer) and publishes the legacy run as a reference that is never judged:
|
||||
# it only has to serve, stay up and complete;
|
||||
# - the client matrix, ci/compat/run_matrix.sh --external, judged per stack by
|
||||
# scripts/compat_matrix_report.py (no `fail`, no `sut-unregistered`, every
|
||||
# client ran; a client that could not be installed is reported as skipped).
|
||||
# The legacy matrix is the baseline: every cell that passes on legacy must
|
||||
# pass on gateway (`compare`).
|
||||
#
|
||||
# The suites are checked out at the rustfs/gateway rev Cargo.toml pins, so the
|
||||
# suite and the framework under test never drift apart. The verdict of each
|
||||
# stack is computed from what the run left on disk (startup record, the stack
|
||||
# the server's own log names, runner exit code, liveness after the corpus,
|
||||
# report), never from the exit status of an earlier step. Needs
|
||||
# scripts/dual_stack.py and the `s3_stack` startup field of rustfs/rustfs#9367.
|
||||
#
|
||||
# The gateway stack is report-only until T2.10: its job may fail without failing
|
||||
# the workflow, and its summary says what it observed (did not start, or how many
|
||||
# cases ran, passed, failed and were answered 501). The stack comparison is
|
||||
# report-only for the same reason; its conformance half is information only.
|
||||
#
|
||||
# Not a pull-request gate. Scheduled failures are owned by the scheduled-validation
|
||||
# watchdog (.github/scheduled-validations.json).
|
||||
|
||||
name: Gateway Acceptance
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "29 3 * * *"
|
||||
workflow_dispatch:
|
||||
|
||||
# Matches ci.yml: these variables are hashed into the rust-cache key, so the
|
||||
# candidate build reads the same ci-dev cache as build-rustfs-debug-binary.
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
RUST_BACKTRACE: 1
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
plan:
|
||||
name: Read the gateway pin
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
outputs:
|
||||
gateway-rev: ${{ steps.pin.outputs.rev }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Read the rustfs/gateway rev from Cargo.toml
|
||||
id: pin
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 - <<'PY' >> "$GITHUB_OUTPUT"
|
||||
import re, sys, tomllib
|
||||
dependencies = tomllib.load(open("Cargo.toml", "rb"))["workspace"]["dependencies"]
|
||||
revs = {spec.get("rev") for spec in dependencies.values()
|
||||
if isinstance(spec, dict) and spec.get("git") == "https://github.com/rustfs/gateway.git"}
|
||||
if len(revs) != 1 or not re.fullmatch(r"[0-9a-f]{40}", str(next(iter(revs)))):
|
||||
sys.exit(f"Cargo.toml must pin rustfs/gateway at exactly one full rev, found {sorted(map(str, revs))}")
|
||||
print(f"rev={revs.pop()}")
|
||||
PY
|
||||
cat "$GITHUB_OUTPUT"
|
||||
|
||||
build-candidate:
|
||||
name: Build candidate binary
|
||||
# The same debug build as ci.yml's build-rustfs-debug-binary, reading the
|
||||
# ci-dev cache written on sm-standard-4.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: ci-dev
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
- name: Build debug binary
|
||||
run: python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks
|
||||
|
||||
- name: Upload candidate binary
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: gateway-acceptance-candidate-binary
|
||||
overwrite: true
|
||||
path: |
|
||||
target/debug/rustfs
|
||||
target/debug/rustfs.e2e.json
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
build-gateway-tools:
|
||||
name: Build gateway conformance runner and compat-sut
|
||||
needs: [ plan ]
|
||||
# The gateway workspace builds with its own rust-toolchain.toml, which rustup
|
||||
# installs on first use; nothing here names a toolchain.
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: Checkout rustfs/gateway at the pinned rev
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
repository: rustfs/gateway
|
||||
ref: ${{ needs.plan.outputs.gateway-rev }}
|
||||
path: gateway
|
||||
persist-credentials: false
|
||||
|
||||
- name: Build
|
||||
working-directory: gateway
|
||||
run: |
|
||||
cargo build --locked --release -p rustfs-gateway-conformance -p rustfs-gateway-compat-sut \
|
||||
--bin rustfs-gateway-conformance --bin compat-sut
|
||||
|
||||
- name: Upload tools
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: gateway-acceptance-tools
|
||||
overwrite: true
|
||||
path: |
|
||||
gateway/target/release/rustfs-gateway-conformance
|
||||
gateway/target/release/compat-sut
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
acceptance:
|
||||
name: ${{ matrix.stack == 'gateway' && 'Acceptance (gateway, report-only until T2.10)' || 'Acceptance (legacy)' }}
|
||||
needs: [ plan, build-candidate, build-gateway-tools ]
|
||||
# GitHub-hosted like the s3-tests targets of reference-parity.yml: the job
|
||||
# only runs prebuilt binaries and the pinned clients, none of which needs Docker.
|
||||
runs-on: ubuntu-latest
|
||||
# About 3x the 15 minutes the legacy job took in run 43325101234 (client install about
|
||||
# 8, matrix about 3, corpus about 3), so a hung server costs 45 runner minutes, not more.
|
||||
timeout-minutes: 45
|
||||
continue-on-error: ${{ matrix.stack == 'gateway' }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
stack: [ legacy, gateway ]
|
||||
env:
|
||||
STACK: ${{ matrix.stack }}
|
||||
ENDPOINT: http://127.0.0.1:9000
|
||||
# The conformance suite signs with these fixed fixture credentials (the AWS
|
||||
# documentation example pair), so the candidate is started with them and the
|
||||
# client matrix is handed the same pair.
|
||||
ACCESS_KEY: AKIAIOSFODNN7EXAMPLE
|
||||
SECRET_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Checkout rustfs/gateway at the pinned rev
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
repository: rustfs/gateway
|
||||
ref: ${{ needs.plan.outputs.gateway-rev }}
|
||||
path: gateway
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download candidate binary
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: gateway-acceptance-candidate-binary
|
||||
path: target/debug
|
||||
|
||||
- name: Download gateway tools
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: gateway-acceptance-tools
|
||||
path: tools
|
||||
|
||||
# Records `ready`, or what the process said when it exited, in lane/startup.txt. `ready`
|
||||
# also needs the server's own `server_starting` log event to name the requested stack
|
||||
# (its `s3_stack` field comes from rustfs/rustfs#9367; scripts/dual_stack.py reads it).
|
||||
- name: Start candidate (${{ matrix.stack }} stack)
|
||||
run: |
|
||||
set -uo pipefail
|
||||
chmod +x target/debug/rustfs tools/*
|
||||
mkdir -p lane "$RUNNER_TEMP"/data/rustfs{0..3}
|
||||
target/debug/rustfs --version | tee lane/candidate-version.txt
|
||||
# The four directories share the runner's one disk; the bypass is the CI case the
|
||||
# physical-disk guard sanctions (as in mint.yml).
|
||||
RUSTFS_S3_STACK="$STACK" RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true RUST_LOG=info target/debug/rustfs --address 127.0.0.1:9000 \
|
||||
--access-key "$ACCESS_KEY" --secret-key "$SECRET_KEY" "$RUNNER_TEMP"/data/rustfs{0..3} \
|
||||
> lane/rustfs.log 2>&1 &
|
||||
pid=$!
|
||||
echo "$pid" > lane/rustfs.pid
|
||||
# Liveness first: a ready answer counts only while this process is still running.
|
||||
for _ in $(seq 120); do
|
||||
if ! kill -0 "$pid" 2>/dev/null; then
|
||||
status=0
|
||||
wait "$pid" || status=$?
|
||||
echo "did not start: exit ${status}: $(grep -v '^[[:space:]]*$' lane/rustfs.log | tail -n 1)" > lane/startup.txt
|
||||
cat lane/startup.txt
|
||||
exit 1
|
||||
fi
|
||||
if curl -sf "$ENDPOINT/health/ready" >/dev/null; then
|
||||
if python3 scripts/dual_stack.py stack --expect "$STACK" --out lane/stack.json lane/rustfs.log > /dev/null; then
|
||||
echo ready > lane/startup.txt
|
||||
exit 0
|
||||
fi
|
||||
echo "did not start: stack not observed" > lane/startup.txt
|
||||
cat lane/startup.txt
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "did not start: /health/ready did not answer within 120 seconds" > lane/startup.txt
|
||||
exit 1
|
||||
|
||||
# The exit code is data for the judge step, not this step's result.
|
||||
- name: Run gateway conformance (rustfs profile)
|
||||
run: |
|
||||
set +e
|
||||
tools/rustfs-gateway-conformance run --root gateway/conformance --profile rustfs \
|
||||
--endpoint "$ENDPOINT" --allow-external-fixtures \
|
||||
--json lane/report.json > lane/conformance.log 2>&1
|
||||
echo "$?" > lane/runner-exit.txt
|
||||
if kill -0 "$(cat lane/rustfs.pid)" 2>/dev/null; then echo alive; else echo exited; fi > lane/alive-after-conformance.txt
|
||||
tail -n 20 lane/conformance.log
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
# Toolchains for the matrix's SDK driver programs, as the gateway's own
|
||||
# client-matrix.yml sets them up; the client versions come from
|
||||
# gateway/compat/versions.toml.
|
||||
- uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
|
||||
with:
|
||||
go-version: "1.26"
|
||||
cache: false
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
with:
|
||||
node-version: "22"
|
||||
- uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9 # v4.3.1
|
||||
with:
|
||||
dotnet-version: "8.0.x"
|
||||
- uses: actions/setup-java@cf277c60eb25467037889841efdb72551f06f6c3 # v4.9.1
|
||||
with:
|
||||
distribution: temurin
|
||||
java-version: "21"
|
||||
- name: Install Maven
|
||||
env:
|
||||
MAVEN_VERSION: "3.9.16"
|
||||
MAVEN_SHA512: "831a8591fe20c8243b1dbe7d71e3244f31d1665b0804b2e825e38cbbe5ce0cafb8338851f90780735568773e0a6cd07bbec107cda0b896b008b861075358b6f6"
|
||||
run: |
|
||||
set -euo pipefail
|
||||
archive="$RUNNER_TEMP/apache-maven.tar.gz"
|
||||
curl -sSfL -o "$archive" "https://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz"
|
||||
echo "${MAVEN_SHA512} ${archive}" | sha512sum -c -
|
||||
tar -xzf "$archive" -C "$RUNNER_TEMP"
|
||||
echo "$RUNNER_TEMP/apache-maven-${MAVEN_VERSION}/bin" >> "$GITHUB_PATH"
|
||||
|
||||
# One client at a time, so a client that cannot be installed is reported
|
||||
# as skipped with its reason instead of stopping the others.
|
||||
- name: Install pinned clients
|
||||
working-directory: gateway
|
||||
run: |
|
||||
set -uo pipefail
|
||||
python3 -m pip install --disable-pip-version-check "pyyaml==6.0.2"
|
||||
installed=()
|
||||
for client in $(python3 -c 'import tomllib; print(" ".join(sorted(tomllib.load(open("compat/versions.toml", "rb"))["clients"])))'); do
|
||||
if bash ci/compat/install_clients.sh --clients "$client" > "../lane/install-${client}.log" 2>&1; then
|
||||
installed+=("$client")
|
||||
else
|
||||
echo "${client}: skipped: $(tail -n 1 "../lane/install-${client}.log")" >> ../lane/skipped-clients.txt
|
||||
fi
|
||||
done
|
||||
echo "COMPAT_CLIENTS=$(IFS=,; echo "${installed[*]}")" >> "$GITHUB_ENV"
|
||||
cat ../lane/skipped-clients.txt 2>/dev/null || echo "every pinned client installed"
|
||||
|
||||
# The exit code is data for the judge step, not this step's result.
|
||||
- name: Run client matrix (external)
|
||||
working-directory: gateway
|
||||
env:
|
||||
GATEWAY_COMPAT_SUT_BIN: ${{ github.workspace }}/tools/compat-sut
|
||||
GATEWAY_COMPAT_RUN_DIR: ${{ github.workspace }}/lane/compat
|
||||
GATEWAY_COMPAT_MATRIX_OUT: ${{ github.workspace }}/lane/compat/matrix.json
|
||||
run: |
|
||||
set +e
|
||||
export GATEWAY_COMPAT_ACCESS_KEY="$ACCESS_KEY" GATEWAY_COMPAT_SECRET_KEY="$SECRET_KEY"
|
||||
PATH="$(go env GOPATH)/bin:$PATH"
|
||||
if [[ -f ../lane/skipped-clients.txt ]]; then
|
||||
bash ci/compat/run_matrix.sh --external "$ENDPOINT" --product rustfs --clients "$COMPAT_CLIENTS"
|
||||
else
|
||||
bash ci/compat/run_matrix.sh --external "$ENDPOINT" --product rustfs
|
||||
fi
|
||||
echo "$?" > ../lane/matrix-exit.txt
|
||||
|
||||
- name: Judge and summarize
|
||||
if: always()
|
||||
run: |
|
||||
set -uo pipefail
|
||||
status=0
|
||||
python3 scripts/check_conformance_rulings.py --stack "$STACK" --run-dir lane | tee -a "$GITHUB_STEP_SUMMARY" || status=1
|
||||
python3 scripts/compat_matrix_report.py check --stack "$STACK" --run-dir lane | tee -a "$GITHUB_STEP_SUMMARY" || status=1
|
||||
exit "$status"
|
||||
|
||||
- name: Upload lane results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: gateway-acceptance-lane-${{ matrix.stack }}
|
||||
overwrite: true
|
||||
path: |
|
||||
lane/*.txt
|
||||
lane/*.log
|
||||
lane/report.json
|
||||
lane/stack.json
|
||||
lane/compat/matrix.json
|
||||
lane/compat/probe.jsonl
|
||||
lane/compat/sut.log
|
||||
lane/compat/results/**
|
||||
if-no-files-found: error
|
||||
retention-days: 14
|
||||
|
||||
compare:
|
||||
name: Compare stacks (report-only until T2.10)
|
||||
needs: [ acceptance ]
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download lane results
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
pattern: gateway-acceptance-lane-*
|
||||
path: lanes
|
||||
|
||||
- name: Diff legacy and gateway
|
||||
run: |
|
||||
set -uo pipefail
|
||||
status=0
|
||||
{
|
||||
python3 scripts/check_conformance_rulings.py \
|
||||
--diff lanes/gateway-acceptance-lane-legacy lanes/gateway-acceptance-lane-gateway || status=1
|
||||
echo
|
||||
python3 scripts/compat_matrix_report.py diff \
|
||||
--legacy lanes/gateway-acceptance-lane-legacy --gateway lanes/gateway-acceptance-lane-gateway || status=1
|
||||
} > diff.md
|
||||
cat diff.md
|
||||
cat diff.md >> "$GITHUB_STEP_SUMMARY"
|
||||
exit "$status"
|
||||
@@ -1,11 +1,13 @@
|
||||
# Copyright 2026 RustFS Team
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
# Manual packaging of the current origin/main tip. Workflow artifacts only.
|
||||
# Dispatch uses this file from the selected branch. The build fetches
|
||||
# refs/heads/main and checks out that commit, not the dispatch ref or a tag.
|
||||
# Manual Linux zip/deb/rpm packaging. Workflow artifacts only.
|
||||
# Dispatch uses this file from the selected branch. An empty ref packages
|
||||
# the tip of refs/heads/main. A non-empty ref must already exist; the
|
||||
# resolver fetches that tag, branch, or commit so checkout is not limited
|
||||
# to the default branch. Nothing is published.
|
||||
|
||||
name: Manual main packages
|
||||
name: Manual Linux packages
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
@@ -19,10 +21,16 @@ on:
|
||||
- arm
|
||||
- all
|
||||
default: all
|
||||
version:
|
||||
description: "Binary and package version: MAJOR.MINOR.PATCH, or MAJOR.MINOR.PATCH-(alpha|beta|rc).N. No leading v."
|
||||
required: true
|
||||
ref:
|
||||
description: "Git tag to package. Leave empty to package the tip of origin/main. A branch name or full commit SHA is also accepted."
|
||||
required: false
|
||||
type: string
|
||||
default: ""
|
||||
version:
|
||||
description: "Package version: MAJOR.MINOR.PATCH, or MAJOR.MINOR.PATCH-(alpha|beta|rc).N. No leading v. Required for origin/main. For a tag, leave empty to derive it from the tag; an explicit value is used as entered."
|
||||
required: false
|
||||
type: string
|
||||
default: ""
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -34,48 +42,43 @@ env:
|
||||
|
||||
jobs:
|
||||
prepare:
|
||||
name: Resolve main tip and version
|
||||
name: Resolve source and version
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
outputs:
|
||||
source_sha: ${{ steps.pin.outputs.source_sha }}
|
||||
version: ${{ steps.version.outputs.version }}
|
||||
source_sha: ${{ steps.source.outputs.source_sha }}
|
||||
source_kind: ${{ steps.source.outputs.source_kind }}
|
||||
source_name: ${{ steps.source.outputs.source_name }}
|
||||
version: ${{ steps.source.outputs.version }}
|
||||
packaging: ${{ steps.source.outputs.packaging }}
|
||||
tool_sha: ${{ steps.source.outputs.tool_sha }}
|
||||
confirm_from: ${{ steps.source.outputs.confirm_from }}
|
||||
toolchain: ${{ steps.inputs.outputs.toolchain }}
|
||||
console: ${{ steps.inputs.outputs.console }}
|
||||
matrix: ${{ steps.matrix.outputs.matrix }}
|
||||
steps:
|
||||
- name: Checkout main
|
||||
# Tooling checkout. An empty ref then detaches this tree to the tip of
|
||||
# origin/main. A tag or other ref is fetched and pinned without
|
||||
# replacing this tree, because console resolution needs candidate.py.
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
ref: main
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Pin the tip of origin/main
|
||||
id: pin
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
GIT_TERMINAL_PROMPT=0 git fetch --no-tags origin refs/heads/main
|
||||
sha=$(git rev-parse FETCH_HEAD)
|
||||
git checkout --detach "$sha"
|
||||
test "$(git rev-parse HEAD)" = "$sha"
|
||||
printf 'source_sha=%s\n' "$sha" >> "$GITHUB_OUTPUT"
|
||||
printf 'Pinned origin/main %s\n' "$sha"
|
||||
|
||||
- name: Validate package version
|
||||
id: version
|
||||
- name: Resolve source and version
|
||||
id: source
|
||||
shell: bash
|
||||
env:
|
||||
SOURCE_REF: ${{ inputs.ref }}
|
||||
PACKAGE_VERSION: ${{ inputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
bash scripts/release/package_manual_artifacts.sh validate \
|
||||
--version "$PACKAGE_VERSION" \
|
||||
--arch x86_64 \
|
||||
--libc gnu | tee "$RUNNER_TEMP/package-version.txt"
|
||||
printf 'version=%s\n' "$PACKAGE_VERSION" >> "$GITHUB_OUTPUT"
|
||||
bash scripts/release/package_manual_artifacts.sh resolve-source \
|
||||
--source "$SOURCE_REF" \
|
||||
--version "$PACKAGE_VERSION" | tee -a "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Install stable Rust
|
||||
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
|
||||
@@ -134,22 +137,45 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix: ${{ fromJson(needs.prepare.outputs.matrix) }}
|
||||
steps:
|
||||
- name: Checkout the pinned origin/main commit
|
||||
- name: Checkout the pinned source commit
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
ref: ${{ needs.prepare.outputs.source_sha }}
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Confirm the checkout is that main commit
|
||||
# Tags cut before the manual packaging script still compile from their
|
||||
# own tree. The official zip/deb/rpm recipe, and confirm-source when the
|
||||
# tagged script does not implement it, come from the prepare checkout.
|
||||
- name: Checkout packaging recipe
|
||||
if: ${{ needs.prepare.outputs.packaging == 'tool' || needs.prepare.outputs.confirm_from == 'tool' }}
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
ref: ${{ needs.prepare.outputs.tool_sha }}
|
||||
path: packaging-tools
|
||||
fetch-depth: 1
|
||||
persist-credentials: false
|
||||
|
||||
- name: Confirm the pinned source
|
||||
shell: bash
|
||||
env:
|
||||
SOURCE_KIND: ${{ needs.prepare.outputs.source_kind }}
|
||||
SOURCE_NAME: ${{ needs.prepare.outputs.source_name }}
|
||||
SOURCE_SHA: ${{ needs.prepare.outputs.source_sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
test "$(git rev-parse HEAD)" = "$SOURCE_SHA"
|
||||
GIT_TERMINAL_PROMPT=0 git fetch --no-tags origin refs/heads/main
|
||||
git merge-base --is-ancestor "$SOURCE_SHA" FETCH_HEAD
|
||||
helper=scripts/release/package_manual_artifacts.sh
|
||||
if [[ ! -f "$helper" ]] || ! grep -q 'confirm-source)' "$helper"; then
|
||||
helper=packaging-tools/scripts/release/package_manual_artifacts.sh
|
||||
fi
|
||||
if [[ ! -f "$helper" ]] || ! grep -q 'confirm-source)' "$helper"; then
|
||||
printf 'no packaging script implements confirm-source\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
bash "$helper" confirm-source \
|
||||
--kind "$SOURCE_KIND" \
|
||||
--name "$SOURCE_NAME" \
|
||||
--sha "$SOURCE_SHA"
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
@@ -247,7 +273,15 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
if [[ -n "$BUILD_GLIBC" ]]; then
|
||||
python3 scripts/release/check_glibc.py "$BUILD_GLIBC" "$bin"
|
||||
glibc_check=scripts/release/check_glibc.py
|
||||
if [[ ! -f "$glibc_check" ]]; then
|
||||
glibc_check=packaging-tools/scripts/release/check_glibc.py
|
||||
fi
|
||||
if [[ ! -f "$glibc_check" ]]; then
|
||||
printf 'glibc check script not found\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
python3 "$glibc_check" "$BUILD_GLIBC" "$bin"
|
||||
fi
|
||||
|
||||
- name: Generate the CPU symbol catalog
|
||||
@@ -285,9 +319,23 @@ jobs:
|
||||
LIBC: ${{ matrix.libc }}
|
||||
BUILD_TARGET: ${{ matrix.target }}
|
||||
BUILD_FEATURES: ${{ matrix.features }}
|
||||
SOURCE_KIND: ${{ needs.prepare.outputs.source_kind }}
|
||||
SOURCE_SHA: ${{ needs.prepare.outputs.source_sha }}
|
||||
PACKAGING: ${{ needs.prepare.outputs.packaging }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
case "$PACKAGING" in
|
||||
source) script=scripts/release/package_manual_artifacts.sh ;;
|
||||
tool) script=packaging-tools/scripts/release/package_manual_artifacts.sh ;;
|
||||
*)
|
||||
printf 'unexpected packaging recipe: %q\n' "$PACKAGING" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
if [[ ! -f "$script" ]]; then
|
||||
printf 'packaging script not found: %s\n' "$script" >&2
|
||||
exit 1
|
||||
fi
|
||||
binary="target/${BUILD_TARGET}/release/rustfs"
|
||||
extra=()
|
||||
if [[ "$BUILD_TARGET" == *-linux-gnu && "$BUILD_FEATURES" == pyroscope ]]; then
|
||||
@@ -296,14 +344,14 @@ jobs:
|
||||
--zip-extra "target/${BUILD_TARGET}/release/rustfs.cpu-symbol-catalog.sha256"
|
||||
)
|
||||
fi
|
||||
bash scripts/release/package_manual_artifacts.sh package \
|
||||
bash "$script" package \
|
||||
--version "$PACKAGE_VERSION" \
|
||||
--arch "$ARCH" \
|
||||
--libc "$LIBC" \
|
||||
--binary "$binary" \
|
||||
--output dist \
|
||||
"${extra[@]}"
|
||||
printf 'Packaged %s from origin/main %s\n' "$PACKAGE_VERSION" "$SOURCE_SHA"
|
||||
printf 'Packaged %s from %s %s\n' "$PACKAGE_VERSION" "$SOURCE_KIND" "$SOURCE_SHA"
|
||||
|
||||
- name: Verify the packaged binary
|
||||
shell: bash
|
||||
|
||||
+89
-10
@@ -44,6 +44,13 @@
|
||||
# well inside the 120-minute budget and buys deterministic infra. The
|
||||
# docker-capable self-hosted `dind-sm-standard-2` label was the alternative but
|
||||
# has fewer cores and reintroduces fleet-state risk for no reliability gain.
|
||||
#
|
||||
# Stack dimension (rustfs/backlog#2774, T3.3): one image, run with RUSTFS_S3_STACK=legacy and
|
||||
# =gateway. `compare` (scripts/mint_stack_diff.py) holds the gate that every case legacy passes
|
||||
# passes on gateway with no new failure. The gateway leg and `compare` are report-only until
|
||||
# T2.10. A gateway server that refuses to start stops its leg at "Wait for RustFS ready", before
|
||||
# mint runs; the summary says `skipped: gateway stack refuses startup: <why>`. The stack is read
|
||||
# from the server's own start-up line (scripts/dual_stack.py stack).
|
||||
|
||||
name: mint
|
||||
|
||||
@@ -108,14 +115,8 @@ defaults:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
mint:
|
||||
# GitHub-hosted: reliably provides a running Docker daemon + buildx +
|
||||
# python3. See the header note (ci-2) for why the self-hosted sm-standard-4
|
||||
# label was abandoned (no docker.sock on the pod its only run landed on).
|
||||
# sm-standard-2 has no docker.sock either (weekly runs failed at "Enable
|
||||
# buildx" from 2026-09-27 to 2026-11-08), so keep this job on ubuntu-latest.
|
||||
# TODO(ci-8): scheduled-failure alerting (auto-open issue on a red weekly
|
||||
# run) is added by the ci-8 composite action; do not implement it here.
|
||||
image:
|
||||
name: Build the RustFS image once
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 120
|
||||
steps:
|
||||
@@ -134,6 +135,48 @@ jobs:
|
||||
--cache-to "type=gha,mode=max,scope=${BUILDX_CACHE_SCOPE}" \
|
||||
-t rustfs-ci \
|
||||
-f Dockerfile.source .
|
||||
docker save rustfs-ci --output "${RUNNER_TEMP}/rustfs-ci.tar"
|
||||
|
||||
- name: Upload the image
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: mint-image
|
||||
path: ${{ runner.temp }}/rustfs-ci.tar
|
||||
retention-days: 1
|
||||
|
||||
mint:
|
||||
name: mint (${{ matrix.stack }}${{ matrix.stack == 'gateway' && ', report-only until T2.10' || '' }})
|
||||
needs: [image]
|
||||
# Report-only until T2.10: the gateway stack cannot serve the suites yet.
|
||||
continue-on-error: ${{ matrix.stack == 'gateway' }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
stack: [legacy, gateway]
|
||||
env:
|
||||
STACK: ${{ matrix.stack }}
|
||||
# GitHub-hosted: reliably provides a running Docker daemon + buildx +
|
||||
# python3. See the header note (ci-2) for why the self-hosted sm-standard-4
|
||||
# label was abandoned (no docker.sock on the pod its only run landed on).
|
||||
# sm-standard-2 has no docker.sock either (weekly runs failed at "Enable
|
||||
# buildx" from 2026-09-27 to 2026-11-08), so keep this job on ubuntu-latest.
|
||||
# TODO(ci-8): scheduled-failure alerting (auto-open issue on a red weekly
|
||||
# run) is added by the ci-8 composite action; do not implement it here.
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 120
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download the image
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: mint-image
|
||||
path: ${{ runner.temp }}
|
||||
|
||||
- name: Load the image
|
||||
run: docker load --input "${RUNNER_TEMP}/rustfs-ci.tar"
|
||||
|
||||
- name: Start RustFS
|
||||
run: |
|
||||
@@ -151,6 +194,8 @@ jobs:
|
||||
-e RUSTFS_SECRET_KEY="${S3_SECRET_KEY}" \
|
||||
-e RUSTFS_VOLUMES="/data/rustfs{0...3}" \
|
||||
-e RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true \
|
||||
-e RUSTFS_S3_STACK="${STACK}" \
|
||||
-e RUST_LOG=rustfs=info \
|
||||
-v /tmp/rustfs-mint:/data \
|
||||
rustfs-ci
|
||||
|
||||
@@ -246,16 +291,50 @@ jobs:
|
||||
mkdir -p artifacts/rustfs-mint
|
||||
docker logs rustfs-mint > artifacts/rustfs-mint/rustfs.log 2>&1 || true
|
||||
|
||||
- name: Prove which stack the server selected
|
||||
if: always()
|
||||
run: python3 scripts/dual_stack.py stack --expect "${STACK}" --out artifacts/stack.json artifacts/rustfs-mint
|
||||
|
||||
- name: Upload artifacts
|
||||
if: always() && env.ACT != 'true'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: mint
|
||||
name: mint-${{ matrix.stack }}
|
||||
path: artifacts/**
|
||||
|
||||
compare:
|
||||
name: Compare stacks (report-only until T2.10)
|
||||
needs: [mint]
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download both legs
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
pattern: mint-{legacy,gateway}
|
||||
path: legs
|
||||
|
||||
# A leg that never uploaded leaves no directory: create both so that the
|
||||
# comparison reports "no result" for it instead of failing to find it.
|
||||
- name: Compare the stacks
|
||||
run: |
|
||||
# The shell runs with -e; the exit status is reported after the summary is written.
|
||||
set +e -u -o pipefail
|
||||
mkdir -p legs/mint-legacy legs/mint-gateway
|
||||
python3 scripts/mint_stack_diff.py legs/mint-legacy legs/mint-gateway | tee report.txt
|
||||
status="${PIPESTATUS[0]}"
|
||||
{ echo '## Mint: gateway versus legacy'; echo; echo '```'; cat report.txt; echo '```'; } >> "$GITHUB_STEP_SUMMARY"
|
||||
exit "$status"
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [mint]
|
||||
needs: [image, mint]
|
||||
# `always()` is required: without it this job is skipped when a needed
|
||||
# job fails. Alerts only for scheduled runs (backlog#1149 ci-8) — manual
|
||||
# dispatch failures are already watched by a human.
|
||||
|
||||
@@ -68,11 +68,14 @@ env:
|
||||
|
||||
jobs:
|
||||
minio-source:
|
||||
name: MinIO source (read-through, list-through, backfill)
|
||||
name: MinIO source (read-through, list-through, backfill, ${{ matrix.stack }}${{ matrix.stack == 'gateway' && ', report-only until T2.10' || '' }})
|
||||
# Keep this scheduled interoperability lane scoped to the upstream repo.
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 90
|
||||
# The gateway leg is report-only until T2.10.
|
||||
continue-on-error: ${{ matrix.stack == 'gateway' }}
|
||||
strategy: { fail-fast: false, matrix: { stack: [legacy, gateway] } }
|
||||
env:
|
||||
NO_PROXY: 127.0.0.1,localhost
|
||||
# Fixed credentials of the container this job starts and throws away;
|
||||
@@ -137,6 +140,11 @@ jobs:
|
||||
- name: Build the RustFS binary under test
|
||||
run: python3 scripts/e2e_binary.py build --bins
|
||||
|
||||
# A gateway binary that refuses to start skips the suite; the summary says why.
|
||||
- name: Gateway preflight
|
||||
if: matrix.stack == 'gateway'
|
||||
run: python3 scripts/dual_stack.py preflight --binary target/debug/rustfs --out "$RUNNER_TEMP/leg/stack.json"
|
||||
|
||||
# The lane selects tests by module, so a rename would quietly shrink it.
|
||||
# The committed digest in .config/e2e-odm-interop-selection.txt fails
|
||||
# closed on that.
|
||||
@@ -146,8 +154,19 @@ jobs:
|
||||
python3 ./scripts/check_test_wiring.py --check-profile e2e-odm-interop "${NEXTEST_LISTING}"
|
||||
|
||||
- name: Run the interop cases against MinIO
|
||||
env:
|
||||
RUSTFS_S3_STACK: ${{ matrix.stack }}
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-logs
|
||||
run: python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-odm-interop -p e2e_test --no-tests=fail
|
||||
|
||||
- name: Prove the stack and upload the leg
|
||||
if: always()
|
||||
uses: ./.github/actions/dual-stack-leg
|
||||
with:
|
||||
profile: e2e-odm-interop
|
||||
stack: ${{ matrix.stack }}
|
||||
logs: ${{ runner.temp }}/rustfs-e2e-logs
|
||||
|
||||
- name: Build the MinIO interop report
|
||||
if: always()
|
||||
uses: ./.github/actions/odm-interop-report
|
||||
@@ -171,7 +190,7 @@ jobs:
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: odm-interop-minio-${{ github.run_number }}-${{ github.run_attempt }}
|
||||
name: odm-interop-minio-${{ github.run_number }}-${{ github.run_attempt }}-${{ matrix.stack }}
|
||||
path: |
|
||||
artifacts/odm-interop/minio
|
||||
target/nextest/e2e-odm-interop/junit.xml
|
||||
@@ -297,6 +316,20 @@ jobs:
|
||||
target/nextest/e2e-odm-interop/junit.xml
|
||||
retention-days: 14
|
||||
|
||||
dual-stack-summary:
|
||||
name: Both-stack diff summary
|
||||
needs: [minio-source]
|
||||
if: github.repository == 'rustfs/rustfs' && always()
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: ./.github/actions/dual-stack-summary
|
||||
with:
|
||||
profiles: e2e-odm-interop
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [minio-source, cloud-source]
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
name: S3-over-RDMA feature check
|
||||
|
||||
# The `nixl` and `cuobject` features are off in every other build, so the code
|
||||
# behind them is compiled nowhere else. This job compiles the `nixl` half on
|
||||
# Linux and runs rustfs-nixl's unit tests; nixl-sys builds through its stub
|
||||
# fallback, so no NIXL SDK is needed.
|
||||
# The cuObject half needs the NVIDIA cuObject SDK and is not compiled here.
|
||||
# Not a required check and not part of the `Test and Lint` aggregate.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- "crates/nixl/**"
|
||||
- "crates/cuobject/**"
|
||||
- "rustfs/src/storage/rpc/nixl*"
|
||||
- "rustfs/src/app/object/cuobject.rs"
|
||||
- "rustfs/src/app/object/get.rs"
|
||||
- "rustfs/src/app/object/put.rs"
|
||||
- "rustfs/src/app/multipart_usecase/upload_part.rs"
|
||||
- "Cargo.toml"
|
||||
- "rustfs/Cargo.toml"
|
||||
- ".github/workflows/rdma-feature-check.yml"
|
||||
schedule:
|
||||
- cron: "37 5 * * 1"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: rdma-feature-check-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
nixl-check:
|
||||
name: cargo check and unit tests (nixl)
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: ci-feat-nixl
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
- name: Install only missing native build tools
|
||||
run: |
|
||||
packages=()
|
||||
command -v g++ >/dev/null || packages+=(g++)
|
||||
ldconfig -p | grep -q 'libclang' || packages+=(libclang-dev)
|
||||
if (( ${#packages[@]} )); then
|
||||
sudo apt-get update
|
||||
sudo apt-get install --no-install-recommends -y "${packages[@]}"
|
||||
fi
|
||||
|
||||
- name: Check the nixl feature build
|
||||
run: cargo check --locked -p rustfs --features nixl
|
||||
|
||||
# These unit tests use a fake sysfs and plain host memory; none needs an
|
||||
# RDMA device or the NIXL libraries.
|
||||
- name: Run the rustfs-nixl unit tests
|
||||
run: cargo test --locked -p rustfs-nixl --features sys
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [nixl-check]
|
||||
# Only scheduled runs open or update the tracking issue; pull-request and
|
||||
# manual runs stay quiet.
|
||||
if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Open or update failure-tracking issue
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}
|
||||
@@ -44,6 +44,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Vault KMS backends no longer fail when the server answers reads from a replica**: a Vault or OpenBao node that serves reads locally (an OpenBao standby, a Vault Enterprise performance standby) can return the state from before a write for a short while after acknowledging it. Against such a node the KV2 backend failed every key creation with `404 Key not found` while leaving the key stored without its description and tags, failed operations issued right after a create or update with `Key not found` or `Concurrent modification`, and the Transit backend failed a disable followed quickly by an enable. Key creation on the KV2 backend is now a single create-only write that carries the description and tags. Both backends now re-read a key record when the answer is older than a write the same RustFS node made in the previous five seconds, pause before re-reading after a lost check-and-set write and allow five attempts instead of three, and retry a Transit key delete that is refused because the deletion switch is not visible yet. A different RustFS node reading the same replica can still observe the earlier state until the replica catches up; see [OpenBao as the Vault KMS backend](docs/operations/kms-openbao-compatibility.md).
|
||||
|
||||
- **KMS keys could not be described by ARN through the path route**: `GET /rustfs/admin/v3/kms/keys/{key_id}` read the path segment verbatim, so a key id that has to be percent-encoded to fit in one segment — an AWS key ARN, with its `:` and `/` — was looked up, authorized and audited under its encoded spelling and answered `404`. The segment is now percent-decoded first; a segment that is not valid UTF-8 after decoding is refused with `400 InvalidArgument`. The `describe-key?keyId=` form was unaffected.
|
||||
|
||||
- **SSE-KMS objects written through the AWS KMS backend could not be read**: `GetObject`, `CopyObject` and every other read of an `aws:kms` object failed with `400 InvalidRequest` ("SSE-S3 requires RUSTFS_SSE_S3_MASTER_KEY to be set ...") on a deployment using `RUSTFS_KMS_BACKEND=aws`, although the write and `HeadObject` succeeded. The AWS backend stored the `GenerateDataKey` blob as it came from AWS, the read path took those unrecognized bytes for a locally wrapped key, and AWS was never asked to decrypt. The backend now stores the blob inside the same data-key envelope the other backends use, and a blob stored without one is routed to AWS when the AWS backend is active. Objects written by 1.0.4 and 1.0.5 are intact and become readable after the upgrade, with no rewrite. Setting `RUSTFS_SSE_S3_MASTER_KEY` was never a workaround. An object written by this release is not readable by an older one; the same was already true of every object the older release wrote itself.
|
||||
@@ -75,6 +77,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- **Per-pool erasure parity**: Erasure parity (STANDARD and reduced-redundancy) is now resolved independently for every pool instead of reusing the first pool's value. A heterogeneous topology — for example a 4-drive pool plus a 2-drive pool created during expansion — previously inherited the first pool's parity and could resolve to zero data shards in the smaller pool, panicking Reed-Solomon construction on write. Automatic parity now resolves per pool (for example `2+2` in the 4-drive pool and `1+1` in the 2-drive pool). Fixes #4801.
|
||||
|
||||
### Added
|
||||
- **Multiple bucket regions**: `RUSTFS_REGION` / `--region` accepts a comma-separated list. The first entry stays the server's own region (request signing and `GetBucketLocation` are unchanged); every entry is accepted as a `CreateBucket` `LocationConstraint`, so one deployment can stand in for buckets that live in several AWS regions (local development and test rigs) now that location constraints are validated (#8739). Regions outside the list are still rejected with `InvalidLocationConstraint`. A single region behaves exactly as before; a comma was never a valid region, so no existing value changes meaning.
|
||||
- **On-Demand Migration**: Lazy, pull-style migration of an existing S3-compatible bucket into RustFS. A local bucket is attached to an external source bucket; a GET for a key that does not exist locally fetches it from the source, streams it to the client, and stores it locally in the same pass, so every later read is served locally. The module is on by default; set `RUSTFS_ON_DEMAND_MIGRATION_ENABLED=false` on every node to turn it off. A bucket with no source configured behaves exactly as before — the runtime never intervenes on its reads and makes no outbound call. Operator guide at `docs/operations/on-demand-migration.md`.
|
||||
- Per-bucket configuration persisted as `on-demand-migration.json` in the bucket metadata: source provider (`s3`, `aws`, `minio`, `rustfs`, `r2`, `gcs`), endpoint, region, addressing style, credentials and TLS material, an optional key-prefix filter and source-prefix rewrite, and a policy block covering the inline size threshold, multipart part size, concurrency, queue capacity, timeouts, bandwidth limit and negative-cache TTL
|
||||
- Admin routes under `/rustfs/admin/v3/on-demand-migration/{bucket}`: `PUT` (with `?dry-run=true` to validate and probe the source without saving), `GET`, `DELETE`, `GET .../status`, plus `POST .../backfill?op=start|cancel` and `GET .../backfill` for the background full-backfill job with its resumable checkpoint. Authorized by the new `admin:GetBucketOnDemandMigration` and `admin:SetBucketOnDemandMigration` actions; every response redacts `secret_key` and `session_token`
|
||||
|
||||
Generated
+387
-97
File diff suppressed because it is too large
Load Diff
+17
-10
@@ -23,6 +23,8 @@ members = [
|
||||
"crates/connect-symbol-catalog", # Release-only CPU profile symbol catalog generator
|
||||
"crates/credentials", # Credential management system
|
||||
"crates/crypto", # Cryptography and security features
|
||||
"crates/cuobject", # NVIDIA cuObject S3-over-RDMA staging and transfers (optional)
|
||||
"crates/nixl", # NIXL/UCX S3-over-RDMA server side (optional)
|
||||
"crates/ecstore", # Erasure coding storage implementation
|
||||
"crates/e2e_test", # End-to-end test suite
|
||||
"crates/filemeta", # File metadata management
|
||||
@@ -78,7 +80,6 @@ repository = "https://github.com/rustfs/rustfs"
|
||||
rust-version = "1.99.0"
|
||||
version = "1.0.5"
|
||||
homepage = "https://rustfs.com"
|
||||
description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
|
||||
keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
|
||||
categories = ["web-programming", "development-tools", "filesystem", "network-programming"]
|
||||
authors = ["RustFS Team"]
|
||||
@@ -93,7 +94,6 @@ redundant_clone = "warn"
|
||||
|
||||
[workspace.dependencies]
|
||||
# RustFS Internal Crates
|
||||
rustfs = { path = "./rustfs", version = "1.0.5" }
|
||||
rustfs-gateway-adapter = { path = "crates/gateway-adapter", version = "1.0.5" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.5" }
|
||||
rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.5" }
|
||||
@@ -106,6 +106,8 @@ rustfs-config = { path = "./crates/config", version = "1.0.5" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.5" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.5" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.5" }
|
||||
rustfs-cuobject = { path = "crates/cuobject", version = "1.0.5" }
|
||||
rustfs-nixl = { path = "crates/nixl", version = "1.0.5" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.5" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.5" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.5" }
|
||||
@@ -149,14 +151,16 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.5" }
|
||||
# engine crates move onto from s3s (rustfs/backlog#2745). The one pin of the
|
||||
# gateway in this workspace; no crate may enable a `compat-s3s*` feature except
|
||||
# the s3s-facing edge of the rustfs binary.
|
||||
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "7915d06950f88611d883f6ab3d95efc90c65dec0", version = "0.40.0", default-features = false }
|
||||
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "7915d06950f88611d883f6ab3d95efc90c65dec0", version = "0.2.0", default-features = false }
|
||||
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "0935429d61b90c65883f0a277e0b50de4294b3d8", version = "0.42.0", default-features = false }
|
||||
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "0935429d61b90c65883f0a277e0b50de4294b3d8", version = "0.2.0", default-features = false }
|
||||
# The gateway framework itself, consumed for its extension traits by the ring-2
|
||||
# adapter. `default-features = false` keeps the `server` feature, and with it the
|
||||
# server crate, out of the build.
|
||||
rustfs-gateway = { git = "https://github.com/rustfs/gateway.git", rev = "7915d06950f88611d883f6ab3d95efc90c65dec0", version = "0.62.0", default-features = false }
|
||||
rustfs-gateway = { git = "https://github.com/rustfs/gateway.git", rev = "0935429d61b90c65883f0a277e0b50de4294b3d8", version = "0.66.0", default-features = false }
|
||||
# Test-only: builds the `EffectiveHost` a host-resolution test queries with.
|
||||
rustfs-gateway-http = { git = "https://github.com/rustfs/gateway.git", rev = "7915d06950f88611d883f6ab3d95efc90c65dec0", default-features = false }
|
||||
rustfs-gateway-http = { git = "https://github.com/rustfs/gateway.git", rev = "0935429d61b90c65883f0a277e0b50de4294b3d8", default-features = false }
|
||||
rustfs-gateway-core = { git = "https://github.com/rustfs/gateway.git", rev = "0935429d61b90c65883f0a277e0b50de4294b3d8", version = "0.63.0", default-features = false }
|
||||
rustfs-gateway-dialect-rustfs-admin = { git = "https://github.com/rustfs/gateway.git", rev = "0935429d61b90c65883f0a277e0b50de4294b3d8", version = "0.13.0", default-features = false }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-channel = "2.5.0"
|
||||
@@ -184,6 +188,7 @@ memchr = "2.8.3"
|
||||
minlz = "1.2.3"
|
||||
reqwest = "0.13.5"
|
||||
rustfs-kafka-async = { version = "1.4.1" }
|
||||
slab = { version = "0.4.12" }
|
||||
socket2 = { version = "0.6.5" }
|
||||
tokio = { version = "1.54.1" }
|
||||
tokio-rustls = { default-features = false, version = "0.26.6" }
|
||||
@@ -211,7 +216,7 @@ bytesize = "2.7.0"
|
||||
byteorder = "1.5.0"
|
||||
flatbuffers = "25.12.19"
|
||||
form_urlencoded = "1.2.2"
|
||||
prost = "0.14.5"
|
||||
prost = "0.14.6"
|
||||
quick-xml = "0.42.0"
|
||||
rmp = { version = "0.8.15" }
|
||||
rmp-serde = { version = "1.3.1" }
|
||||
@@ -233,7 +238,7 @@ blake2 = "0.11.0"
|
||||
chacha20poly1305 = { version = "0.11.0" }
|
||||
crc-fast = "1.10.0"
|
||||
hmac = { version = "0.13.0" }
|
||||
jsonwebtoken = { version = "11.1.0" }
|
||||
jsonwebtoken = { version = "12.0.0" }
|
||||
openidconnect = { default-features = false, version = "4.1.0" }
|
||||
pbkdf2 = "0.13.0"
|
||||
p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
|
||||
@@ -306,10 +311,12 @@ hostname = "0.4.2"
|
||||
ipnetwork = { version = "0.21.1" }
|
||||
lazy_static = "1.5.1"
|
||||
libc = "0.2.191"
|
||||
cuobject = { version = "0.1.0", default-features = false }
|
||||
libsystemd = "0.7.2"
|
||||
local-ip-address = "0.6.13"
|
||||
log = "0.4"
|
||||
memmap2 = "0.9.12"
|
||||
nixl-sys = { git = "https://github.com/houseme/nixl", rev = "cc636c83f13d692e85b67e55aed092413d16fb41" }
|
||||
lz4 = "1.28.1"
|
||||
lz4_flex = { version = "=0.14.0", default-features = false, features = ["std", "frame"] }
|
||||
matchit = "0.9.2"
|
||||
@@ -333,7 +340,7 @@ pretty_assertions = "1.4.1"
|
||||
rand = { version = "0.10.3" }
|
||||
ratelimit = "2.0.0"
|
||||
rayon = "1.12.0"
|
||||
rustfs-erasure-codec = { version = "9.0.0" }
|
||||
rustfs-erasure-codec = { version = "9.1.0" }
|
||||
reed-solomon-simd = "3.1.0"
|
||||
regex = { version = "1.13.1" }
|
||||
rumqttc = { package = "rumqttc-next", version = "0.34.0" }
|
||||
@@ -396,7 +403,7 @@ opentelemetry-stdout = { version = "0.33.1" }
|
||||
pyroscope = { version = "2.1.5" }
|
||||
|
||||
# FTP and SFTP
|
||||
libunftp = { version = "0.24.0" }
|
||||
libunftp = { version = "0.25.0" }
|
||||
unftp-core = "0.1.1"
|
||||
suppaftp = { version = "12.2.1" }
|
||||
rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
|
||||
@@ -170,10 +170,17 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
||||
;; \
|
||||
esac
|
||||
|
||||
# `RC_RELEASE=disabled` is reserved for local server-only validation when the
|
||||
# GitHub release asset is unavailable. The placeholder fails if invoked, so a
|
||||
# production image that needs the CLI must keep the default release download.
|
||||
# TARGETARCH is only provided by BuildKit. Fall back to TARGETPLATFORM (for
|
||||
# example set by docker-compose) and then to the build host, like the cargo
|
||||
# step above, so every path that builds this file passes a real architecture.
|
||||
RUN set -eu; \
|
||||
if [ "${RC_RELEASE}" = "disabled" ]; then \
|
||||
install -m 0755 /bin/false /usr/local/bin/rc; \
|
||||
exit 0; \
|
||||
fi; \
|
||||
rc_arch="${TARGETARCH:-}"; \
|
||||
if [ -z "${rc_arch}" ]; then \
|
||||
case "${TARGETPLATFORM:-}" in \
|
||||
|
||||
@@ -311,6 +311,10 @@ x env use rustfs
|
||||
rustfs --help
|
||||
```
|
||||
|
||||
### 7\. Docker Swarm with the SwarmCLI chart (Option 7)
|
||||
|
||||
Follow the [SwarmCLI chart guide](https://docs.rustfs.com/en/installation/cloud-native/swarmcli-chart) to run RustFS on Docker Swarm, as a single-node service or a distributed, erasure-coded cluster with one peer per node, with the open-source [SwarmCLI `rustfs` chart](https://github.com/Eldara-Tech/swarmcli-charts/tree/main/charts/rustfs). The chart keeps the root credentials in Docker secrets and refuses the default `rustfsadmin` keys, upgrades and rolls back as versioned releases, and can serve the S3 API over HTTPS through Traefik. Its Console stays off until you enable it. The chart and [SwarmCLI](https://github.com/Eldara-Tech/swarmcli) are maintained by Eldara Tech, not by the RustFS project.
|
||||
|
||||
---
|
||||
|
||||
### Accessing RustFS
|
||||
|
||||
@@ -239,6 +239,10 @@ x env use rustfs
|
||||
rustfs --help
|
||||
```
|
||||
|
||||
### 7\. 使用 SwarmCLI Chart 部署到 Docker Swarm (Option 7)
|
||||
|
||||
请按照 [SwarmCLI Chart 指南](https://docs.rustfs.com/zh/installation/cloud-native/swarmcli-chart),使用开源的 [SwarmCLI `rustfs` Chart](https://github.com/Eldara-Tech/swarmcli-charts/tree/main/charts/rustfs) 在 Docker Swarm 上运行 RustFS,可部署为单节点服务,或每个节点一个对等节点、采用纠删码的分布式集群。该 Chart 将 root 凭证保存在 Docker Secret 中并拒绝默认的 `rustfsadmin` 密钥,以带版本的发布进行升级和回滚,并可通过 Traefik 以 HTTPS 提供 S3 API。控制台默认关闭,需要时再启用。该 Chart 和 [SwarmCLI](https://github.com/Eldara-Tech/swarmcli) 由 Eldara Tech 维护,并非 RustFS 项目的一部分。
|
||||
|
||||
---
|
||||
|
||||
### 访问 RustFS
|
||||
|
||||
@@ -30,12 +30,11 @@ workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
hotpath = ["hotpath/hotpath"]
|
||||
hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"]
|
||||
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"]
|
||||
hotpath = []
|
||||
hotpath-alloc = ["hotpath"]
|
||||
hotpath-cpu = ["hotpath"]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
crc-fast = { workspace = true }
|
||||
http = { workspace = true }
|
||||
|
||||
@@ -30,13 +30,12 @@ workspace = true
|
||||
[features]
|
||||
default = []
|
||||
test-util = []
|
||||
hotpath = ["hotpath/hotpath", "hotpath/tokio"]
|
||||
hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"]
|
||||
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"]
|
||||
hotpath = []
|
||||
hotpath-alloc = ["hotpath"]
|
||||
hotpath-cpu = ["hotpath"]
|
||||
|
||||
[dependencies]
|
||||
arc-swap.workspace = true
|
||||
hotpath.workspace = true
|
||||
tokio = { workspace = true, features = ["fs", "rt-multi-thread"] }
|
||||
tonic = { workspace = true, features = ["gzip", "deflate"] }
|
||||
uuid = { workspace = true, features = ["v4", "fast-rng", "macro-diagnostics"] }
|
||||
|
||||
@@ -23,7 +23,80 @@ pub static GLOBAL_LOCAL_NODE_NAME: LazyLock<RwLock<String>> = LazyLock::new(|| R
|
||||
pub static GLOBAL_RUSTFS_HOST: LazyLock<RwLock<String>> = LazyLock::new(|| RwLock::new("".to_string()));
|
||||
pub static GLOBAL_RUSTFS_PORT: LazyLock<RwLock<String>> = LazyLock::new(|| RwLock::new("9000".to_string()));
|
||||
pub static GLOBAL_RUSTFS_ADDR: LazyLock<RwLock<String>> = LazyLock::new(|| RwLock::new("".to_string()));
|
||||
static NEXT_GRPC_CHANNEL_ID: AtomicU64 = AtomicU64::new(1);
|
||||
static NEXT_GRPC_CONNECTION_GENERATION: AtomicU64 = AtomicU64::new(1);
|
||||
|
||||
#[derive(Debug)]
|
||||
struct GrpcConnectionStateInner {
|
||||
channel_id: u64,
|
||||
generation: AtomicU64,
|
||||
}
|
||||
|
||||
/// Lock-free identity and physical-connection generation shared by every clone of one gRPC channel.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct GrpcConnectionState(Arc<GrpcConnectionStateInner>);
|
||||
|
||||
/// A point-in-time identity for the physical connection currently used by a gRPC channel.
|
||||
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
|
||||
pub struct GrpcConnectionSnapshot {
|
||||
pub channel_id: u64,
|
||||
pub generation: u64,
|
||||
}
|
||||
|
||||
impl GrpcConnectionState {
|
||||
pub fn new() -> Self {
|
||||
Self(Arc::new(GrpcConnectionStateInner {
|
||||
channel_id: next_nonzero_id(&NEXT_GRPC_CHANNEL_ID),
|
||||
generation: AtomicU64::new(0),
|
||||
}))
|
||||
}
|
||||
|
||||
/// Publish a successfully opened physical transport before tonic makes it request-ready.
|
||||
pub fn record_connected(&self) -> GrpcConnectionSnapshot {
|
||||
let generation = next_nonzero_id(&NEXT_GRPC_CONNECTION_GENERATION);
|
||||
self.0.generation.store(generation, Ordering::Release);
|
||||
GrpcConnectionSnapshot {
|
||||
channel_id: self.0.channel_id,
|
||||
generation,
|
||||
}
|
||||
}
|
||||
|
||||
/// Return `None` until the first physical transport has connected successfully.
|
||||
pub fn snapshot(&self) -> Option<GrpcConnectionSnapshot> {
|
||||
let generation = self.0.generation.load(Ordering::Acquire);
|
||||
(generation != 0).then_some(GrpcConnectionSnapshot {
|
||||
channel_id: self.0.channel_id,
|
||||
generation,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for GrpcConnectionState {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
fn next_nonzero_id(counter: &AtomicU64) -> u64 {
|
||||
loop {
|
||||
let id = counter.fetch_add(1, Ordering::Relaxed);
|
||||
if id != 0 {
|
||||
return id;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub static GLOBAL_CONN_MAP: LazyLock<RwLock<HashMap<String, Channel>>> = LazyLock::new(|| RwLock::new(HashMap::new()));
|
||||
|
||||
#[derive(Clone)]
|
||||
struct TrackedGrpcConnection {
|
||||
channel: Channel,
|
||||
connection_state: GrpcConnectionState,
|
||||
}
|
||||
|
||||
static GLOBAL_TRACKED_CONN_MAP: LazyLock<RwLock<HashMap<String, TrackedGrpcConnection>>> =
|
||||
LazyLock::new(|| RwLock::new(HashMap::new()));
|
||||
static GLOBAL_CONN_CACHE_WRITER: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
|
||||
pub static GLOBAL_ROOT_CERT: LazyLock<RwLock<Option<Vec<u8>>>> = LazyLock::new(|| RwLock::new(None));
|
||||
pub static GLOBAL_MTLS_IDENTITY: LazyLock<RwLock<Option<MtlsIdentityPem>>> = LazyLock::new(|| RwLock::new(None));
|
||||
pub static GLOBAL_OUTBOUND_TLS_GENERATION: LazyLock<AtomicU64> = LazyLock::new(|| AtomicU64::new(0));
|
||||
@@ -231,6 +304,16 @@ pub async fn evict_connection(addr: &str) {
|
||||
|
||||
/// Evict a stale/dead connection from the global connection cache with an explicit log level.
|
||||
pub async fn evict_connection_with_log_level(addr: &str, log_level: ConnectionEvictionLogLevel) {
|
||||
let _ = remove_connection_with_log_level(addr, log_level).await;
|
||||
}
|
||||
|
||||
/// Remove a cached connection and report whether it existed. Transport owners use this to
|
||||
/// attach a rebuild reason only to real cache transitions.
|
||||
pub async fn remove_connection_with_log_level(addr: &str, log_level: ConnectionEvictionLogLevel) -> bool {
|
||||
// Writes are rare and serialized so the optimized tracked cache and public compatibility map
|
||||
// cannot publish different channel clones for one key.
|
||||
let _writer = GLOBAL_CONN_CACHE_WRITER.lock().await;
|
||||
GLOBAL_TRACKED_CONN_MAP.write().await.remove(addr);
|
||||
let removed = GLOBAL_CONN_MAP.write().await.remove(addr);
|
||||
if removed.is_some() {
|
||||
match log_level {
|
||||
@@ -254,6 +337,7 @@ pub async fn evict_connection_with_log_level(addr: &str, log_level: ConnectionEv
|
||||
}
|
||||
}
|
||||
}
|
||||
removed.is_some()
|
||||
}
|
||||
|
||||
/// Get a cached gRPC connection for the given address.
|
||||
@@ -261,8 +345,37 @@ pub async fn cached_connection(addr: &str) -> Option<Channel> {
|
||||
GLOBAL_CONN_MAP.read().await.get(addr).cloned()
|
||||
}
|
||||
|
||||
/// Get a cached channel together with its shared physical-connection state, when tracked.
|
||||
pub async fn cached_connection_with_state(addr: &str) -> Option<(Channel, Option<GrpcConnectionState>)> {
|
||||
if let Some(cached) = GLOBAL_TRACKED_CONN_MAP.read().await.get(addr).cloned() {
|
||||
return Some((cached.channel, Some(cached.connection_state)));
|
||||
}
|
||||
// A miss may race a low-frequency publish/evict. Serialize only this fallback, recheck the
|
||||
// tracked map, then consult legacy entries injected through the compatibility API.
|
||||
let _writer = GLOBAL_CONN_CACHE_WRITER.lock().await;
|
||||
if let Some(cached) = GLOBAL_TRACKED_CONN_MAP.read().await.get(addr).cloned() {
|
||||
return Some((cached.channel, Some(cached.connection_state)));
|
||||
}
|
||||
cached_connection(addr).await.map(|channel| (channel, None))
|
||||
}
|
||||
|
||||
/// Cache a gRPC connection for the given address.
|
||||
pub async fn cache_connection(addr: String, channel: Channel) {
|
||||
let _writer = GLOBAL_CONN_CACHE_WRITER.lock().await;
|
||||
GLOBAL_TRACKED_CONN_MAP.write().await.remove(&addr);
|
||||
GLOBAL_CONN_MAP.write().await.insert(addr, channel);
|
||||
}
|
||||
|
||||
/// Cache a gRPC channel and its physical-connection state as one coherent entry.
|
||||
pub async fn cache_connection_with_state(addr: String, channel: Channel, connection_state: GrpcConnectionState) {
|
||||
let _writer = GLOBAL_CONN_CACHE_WRITER.lock().await;
|
||||
GLOBAL_TRACKED_CONN_MAP.write().await.insert(
|
||||
addr.clone(),
|
||||
TrackedGrpcConnection {
|
||||
channel: channel.clone(),
|
||||
connection_state,
|
||||
},
|
||||
);
|
||||
GLOBAL_CONN_MAP.write().await.insert(addr, channel);
|
||||
}
|
||||
|
||||
@@ -279,6 +392,8 @@ pub async fn has_cached_connection(addr: &str) -> bool {
|
||||
|
||||
/// Clear all cached connections. Useful for full cluster reset/recovery.
|
||||
pub async fn clear_all_connections() {
|
||||
let _writer = GLOBAL_CONN_CACHE_WRITER.lock().await;
|
||||
GLOBAL_TRACKED_CONN_MAP.write().await.clear();
|
||||
let mut map = GLOBAL_CONN_MAP.write().await;
|
||||
let count = map.len();
|
||||
map.clear();
|
||||
@@ -363,4 +478,46 @@ mod tls_snapshot_tests {
|
||||
assert!(!debug.contains("PRIVATE-KEY-MARKER"));
|
||||
assert!(debug.contains("[REDACTED]"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn conditional_connection_removal_preserves_legacy_evict_api() {
|
||||
let key = "http://conditional-eviction-test.invalid";
|
||||
cache_connection(
|
||||
key.to_owned(),
|
||||
tonic::transport::Endpoint::from_static("http://127.0.0.1:1").connect_lazy(),
|
||||
)
|
||||
.await;
|
||||
assert!(remove_connection_with_log_level(key, ConnectionEvictionLogLevel::Debug).await);
|
||||
assert!(!remove_connection_with_log_level(key, ConnectionEvictionLogLevel::Debug).await);
|
||||
let _: () = evict_connection_with_log_level(key, ConnectionEvictionLogLevel::Debug).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cached_connection_keeps_channel_and_transport_state_coherent() {
|
||||
let key = "http://tracked-connection-test.invalid";
|
||||
let channel = tonic::transport::Endpoint::from_static("http://127.0.0.1:1").connect_lazy();
|
||||
let state = GrpcConnectionState::new();
|
||||
assert!(state.snapshot().is_none());
|
||||
let first = state.record_connected();
|
||||
cache_connection_with_state(key.to_owned(), channel, state.clone()).await;
|
||||
|
||||
let (_, cached_state) = cached_connection_with_state(key).await.expect("tracked cache entry");
|
||||
let cached_state = cached_state.expect("tracked cache state");
|
||||
assert_eq!(cached_state.snapshot(), Some(first));
|
||||
let second = state.record_connected();
|
||||
assert_eq!(cached_state.snapshot(), Some(second));
|
||||
assert_eq!(first.channel_id, second.channel_id);
|
||||
assert_ne!(first.generation, second.generation);
|
||||
|
||||
cache_connection(
|
||||
key.to_owned(),
|
||||
tonic::transport::Endpoint::from_static("http://127.0.0.1:2").connect_lazy(),
|
||||
)
|
||||
.await;
|
||||
let (_, replacement_state) = cached_connection_with_state(key).await.expect("legacy replacement entry");
|
||||
assert!(replacement_state.is_none(), "legacy replacement must remove stale tracked state");
|
||||
|
||||
assert!(remove_connection_with_log_level(key, ConnectionEvictionLogLevel::Debug).await);
|
||||
assert!(cached_connection_with_state(key).await.is_none());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,7 +25,6 @@ keywords = ["configuration", "settings", "management", "rustfs", "Minio"]
|
||||
categories = ["web-programming", "development-tools", "config"]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
const-str = { workspace = true, optional = true, features = ["std", "proc"] }
|
||||
serde = { workspace = true, optional = true, features = ["derive"] }
|
||||
serde_json = { workspace = true, optional = true, features = ["raw_value"] }
|
||||
@@ -35,9 +34,9 @@ workspace = true
|
||||
|
||||
[features]
|
||||
default = ["constants"]
|
||||
hotpath = ["hotpath/hotpath"]
|
||||
hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"]
|
||||
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"]
|
||||
hotpath = []
|
||||
hotpath-alloc = ["hotpath"]
|
||||
hotpath-cpu = ["hotpath"]
|
||||
audit = ["dep:const-str", "constants"]
|
||||
constants = ["dep:const-str"]
|
||||
notify = ["dep:const-str", "constants"]
|
||||
|
||||
@@ -121,6 +121,11 @@ when restoring `/rustfs/console` for a custom OEM build.
|
||||
- `RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS` (canonical)
|
||||
- `RUSTFS_SCANNER_CYCLE_MAX_OBJECTS` (canonical)
|
||||
- `RUSTFS_SCANNER_CYCLE_MAX_DIRECTORIES` (canonical)
|
||||
- `RUSTFS_SCANNER_BUCKET_QUANTUM_DURATION_SECS` (canonical)
|
||||
- `RUSTFS_SCANNER_BUCKET_QUANTUM_OBJECTS` (canonical)
|
||||
- `RUSTFS_SCANNER_BUCKET_QUANTUM_DIRECTORIES` (canonical)
|
||||
- `RUSTFS_SCANNER_BUCKET_QUANTUM_ENTRIES` (canonical)
|
||||
- `RUSTFS_SCANNER_BUCKET_QUANTUM_NAME_BYTES` (canonical)
|
||||
- `RUSTFS_SCANNER_MAX_CONCURRENT_SET_SCANS` (canonical)
|
||||
- `RUSTFS_SCANNER_MAX_CONCURRENT_DISK_SCANS` (canonical)
|
||||
- `RUSTFS_SCANNER_WALKERS_PER_DISK` (canonical)
|
||||
@@ -130,6 +135,12 @@ Scanner cycle budget controls:
|
||||
- When `RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS` is unset, the finite default is 1800 seconds (30 minutes), matching the scanner benchmark guidance.
|
||||
- An explicit `0` preserves the compatibility behavior of an unbounded runtime budget. Object and directory budgets likewise remain unbounded when explicitly set to `0`.
|
||||
|
||||
Scanner bucket quantum controls:
|
||||
|
||||
- `RUSTFS_SCANNER_BUCKET_QUANTUM_DURATION_SECS`, `RUSTFS_SCANNER_BUCKET_QUANTUM_OBJECTS`, `RUSTFS_SCANNER_BUCKET_QUANTUM_DIRECTORIES`, `RUSTFS_SCANNER_BUCKET_QUANTUM_ENTRIES`, and `RUSTFS_SCANNER_BUCKET_QUANTUM_NAME_BYTES` cap one bucket admission. Their persistent keys are `scanner.bucket_quantum_duration`, `scanner.bucket_quantum_objects`, `scanner.bucket_quantum_directories`, `scanner.bucket_quantum_entries`, and `scanner.bucket_quantum_name_bytes`; duration is in whole seconds and the remaining keys are work counts or bytes.
|
||||
- All five defaults are `0` for controlled rollout. A positive value enables that dimension; `0` disables that dimension, and all five zero values disable the bucket quantum. Parent cycle limits remain in force. Invalid unsigned integers and durations outside the platform timer range are rejected; environment overrides take precedence but cannot hide invalid persisted values. Admin scanner status reports each effective value and source under `runtime_config.bucket_quantum_*`, using `bucket_quantum_duration_seconds` for duration.
|
||||
- Disabled defaults are an initial rollout state, not completion of usage-convergence acceptance. Production defaults require matched-topology A/B measurements and the remote progress/fairness checks in [Scanner runtime controls](../../docs/operations/scanner-runtime-controls.md#bucket-quantum-rollout-and-acceptance). These controls do not certify partial usage or acknowledge pending dirty work.
|
||||
|
||||
Scanner recovery retry controls:
|
||||
|
||||
- `RUSTFS_SCANNER_RECOVERY_BACKOFF_CAP_SECS` caps the wait between automatic startup recovery retries. The persistent admin key is `scanner.recovery_backoff_cap`; the environment takes precedence over the persistent value, and the unset default remains 600 seconds (10 minutes). Values must be positive integer seconds within the platform timer range; `0`, malformed values, and overflow are rejected. A valid environment override does not hide an invalid persistent value from `admin config set` validation.
|
||||
@@ -291,6 +302,8 @@ Drive timeout profile preset:
|
||||
|
||||
RustFS production guidance remains direct-attached local POSIX filesystems. Network-mounted filesystems (for example `nfs`, `cifs`, `smb2`, and `fuse.*`) are treated as unsupported by this startup guard. On Windows, endpoints on network drives (UNC shares such as `\\server\share` and mapped drive letters) are treated the same way, even though Windows reports the server's filesystem (usually `NTFS`) for them.
|
||||
|
||||
A local POSIX filesystem bind-mounted into a gVisor sandbox with directfs is a supported storage path. gVisor presents that bind mount as `9p`/`V9FS`, so the host filesystem type is not visible; the startup guard recognizes mountinfo options `trans=fd` and `directfs` and does not treat the mount as unsupported. Genuine 9p, virtio, and NFS shares remain unsupported.
|
||||
|
||||
## 📄 License
|
||||
|
||||
This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.
|
||||
|
||||
@@ -35,8 +35,8 @@ pub const DEFAULT_HEALTH_OBJECT_PROGRESS_TIMEOUT_MS: u64 = 30_000;
|
||||
/// Additional time beyond the configured object-lock acquisition deadline.
|
||||
pub const HEALTH_OBJECT_PROGRESS_LOCK_MARGIN_MS: u64 = 5_000;
|
||||
|
||||
/// Timeout for cluster health readiness collectors (milliseconds).
|
||||
/// This bounds expensive storage and lock quorum checks used by cluster probes.
|
||||
/// End-to-end budget for cluster health readiness probes (milliseconds).
|
||||
/// Includes cache/singleflight waiting, storage observations, and lock checks.
|
||||
pub const ENV_HEALTH_CLUSTER_TIMEOUT_MS: &str = "RUSTFS_HEALTH_CLUSTER_TIMEOUT_MS";
|
||||
pub const DEFAULT_HEALTH_CLUSTER_TIMEOUT_MS: u64 = 2000;
|
||||
|
||||
|
||||
@@ -121,9 +121,17 @@ pub const DEFAULT_INTERNODE_RPC_MSGPACK_ONLY: bool = false;
|
||||
pub const ENV_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED: &str = "RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED";
|
||||
pub const DEFAULT_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED: bool = false;
|
||||
|
||||
/// Permit current per-peer capability proof to replace fleet-wide confirmation.
|
||||
///
|
||||
/// This remains ineffective unless [`ENV_INTERNODE_RPC_MSGPACK_ONLY`] is also enabled. Unknown,
|
||||
/// unsupported, revoked, or stale connection generations keep dual-writing JSON.
|
||||
pub const ENV_INTERNODE_RPC_MSGPACK_PEER_CAPABILITY: &str = "RUSTFS_INTERNODE_RPC_MSGPACK_PEER_CAPABILITY";
|
||||
pub const DEFAULT_INTERNODE_RPC_MSGPACK_PEER_CAPABILITY: bool = false;
|
||||
|
||||
// Compile-time invariants: dual-write by default so the base build is byte-for-byte legacy behavior.
|
||||
const _: () = assert!(!DEFAULT_INTERNODE_RPC_MSGPACK_ONLY);
|
||||
const _: () = assert!(!DEFAULT_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED);
|
||||
const _: () = assert!(!DEFAULT_INTERNODE_RPC_MSGPACK_PEER_CAPABILITY);
|
||||
|
||||
/// Require target-bound v2 signatures on every internode gRPC request, rejecting the legacy
|
||||
/// constant-target fallback instead of accepting it (<https://github.com/rustfs/backlog/issues/1327>).
|
||||
@@ -294,6 +302,14 @@ pub const DEFAULT_INTERNODE_BULK_CONCURRENCY_LIMIT: usize = 64;
|
||||
pub const ENV_INTERNODE_BULK_BUFFER_SIZE: &str = "RUSTFS_INTERNODE_BULK_BUFFER_SIZE";
|
||||
pub const DEFAULT_INTERNODE_BULK_BUFFER_SIZE: usize = 1024;
|
||||
|
||||
/// Per-peer limit held through complete bulk-unary response decoding and validation. Zero keeps
|
||||
/// the compatibility behavior with no additional admission gate.
|
||||
pub const ENV_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT: &str = "RUSTFS_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT";
|
||||
pub const DEFAULT_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT: usize = 0;
|
||||
pub const ENV_INTERNODE_BULK_BODY_WAIT_TIMEOUT_MS: &str = "RUSTFS_INTERNODE_BULK_BODY_WAIT_TIMEOUT_MS";
|
||||
pub const DEFAULT_INTERNODE_BULK_BODY_WAIT_TIMEOUT_MS: u64 = 1_000;
|
||||
const _: () = assert!(DEFAULT_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT == 0);
|
||||
|
||||
/// Profile selector for conservative internode HTTP data-plane client tuning.
|
||||
pub const ENV_INTERNODE_HTTP_TUNING_PROFILE: &str = "RUSTFS_INTERNODE_HTTP_TUNING_PROFILE";
|
||||
pub const DEFAULT_INTERNODE_HTTP_TUNING_PROFILE: &str = "legacy";
|
||||
@@ -333,8 +349,130 @@ pub const DEFAULT_INTERNODE_DATA_TRANSPORT: &str = "tcp-http";
|
||||
/// Legacy alias for "tcp-http". Both values select the TCP/HTTP transport backend.
|
||||
pub const INTERNODE_DATA_TRANSPORT_TCP: &str = "tcp";
|
||||
|
||||
/// Internode reads whose data moves over NIXL/UCX (InfiniBand rails or RoCE by
|
||||
/// LID/GID, no IP needed); requests, writes and walks stay on TCP/HTTP. Requires
|
||||
/// a binary built with the `nixl` feature and `RUSTFS_NIXL_ENABLE`; until the
|
||||
/// NIXL agent is up, and for any read it cannot serve, it uses TCP/HTTP.
|
||||
pub const INTERNODE_DATA_TRANSPORT_NIXL: &str = "nixl";
|
||||
|
||||
/// Known internode transport backend names accepted by the config parser.
|
||||
pub const KNOWN_INTERNODE_DATA_TRANSPORT_BACKENDS: &[&str] = &[DEFAULT_INTERNODE_DATA_TRANSPORT, INTERNODE_DATA_TRANSPORT_TCP];
|
||||
pub const KNOWN_INTERNODE_DATA_TRANSPORT_BACKENDS: &[&str] = &[
|
||||
DEFAULT_INTERNODE_DATA_TRANSPORT,
|
||||
INTERNODE_DATA_TRANSPORT_TCP,
|
||||
INTERNODE_DATA_TRANSPORT_NIXL,
|
||||
];
|
||||
|
||||
/// Check the configured RDMA devices at startup and leave out any that is down,
|
||||
/// slower than its peers, or loses its link during a short watch.
|
||||
pub const ENV_RUSTFS_NIXL_DEVICE_CHECK: &str = "RUSTFS_NIXL_DEVICE_CHECK";
|
||||
pub const DEFAULT_RUSTFS_NIXL_DEVICE_CHECK: bool = true;
|
||||
/// Leave out a device whose link dropped this many times more since boot than
|
||||
/// the quietest device of its link layer; 0 skips this test.
|
||||
pub const ENV_RUSTFS_NIXL_DEVICE_MAX_LINK_DROPS: &str = "RUSTFS_NIXL_DEVICE_MAX_LINK_DROPS";
|
||||
pub const DEFAULT_RUSTFS_NIXL_DEVICE_MAX_LINK_DROPS: u64 = 16;
|
||||
/// How long the startup device check watches the link counters; 0 skips the watch.
|
||||
pub const ENV_RUSTFS_NIXL_DEVICE_FLAP_WINDOW_MS: &str = "RUSTFS_NIXL_DEVICE_FLAP_WINDOW_MS";
|
||||
pub const DEFAULT_RUSTFS_NIXL_DEVICE_FLAP_WINDOW_MS: u64 = 3_000;
|
||||
|
||||
/// NIXL internode reads in flight per remote shard stream.
|
||||
pub const ENV_RUSTFS_NIXL_INTERNODE_DEPTH: &str = "RUSTFS_NIXL_INTERNODE_DEPTH";
|
||||
pub const DEFAULT_RUSTFS_NIXL_INTERNODE_DEPTH: usize = 4;
|
||||
|
||||
/// Whether S3 GETs and PUTs carrying the NIXL headers are served over NIXL/UCX
|
||||
/// S3-over-RDMA (requires a build with the `nixl` feature and libnixl). The
|
||||
/// pipeline settings below (`RUSTFS_CUOBJECT_GET_FANOUT*`, `_INFLIGHT_WRITES`,
|
||||
/// `_SHARD_ARENA_BYTES`) apply to this transport as well.
|
||||
pub const ENV_RUSTFS_NIXL_ENABLE: &str = "RUSTFS_NIXL_ENABLE";
|
||||
pub const DEFAULT_RUSTFS_NIXL_ENABLE: bool = false;
|
||||
/// This node's NIXL agent name; defaults to `rustfs-<first local endpoint host>`.
|
||||
pub const ENV_RUSTFS_NIXL_AGENT_NAME: &str = "RUSTFS_NIXL_AGENT_NAME";
|
||||
/// Comma-separated UCX device names (`mlx5_9,mlx5_10`; NIXL adds the port); empty lets UCX use every device.
|
||||
pub const ENV_RUSTFS_NIXL_UCX_DEVICES: &str = "RUSTFS_NIXL_UCX_DEVICES";
|
||||
/// UCX progress workers of the agent.
|
||||
pub const ENV_RUSTFS_NIXL_NUM_WORKERS: &str = "RUSTFS_NIXL_NUM_WORKERS";
|
||||
pub const DEFAULT_RUSTFS_NIXL_NUM_WORKERS: u32 = 1;
|
||||
/// Staging chunk size per transfer descriptor to a client.
|
||||
pub const ENV_RUSTFS_NIXL_CHUNK_BYTES: &str = "RUSTFS_NIXL_CHUNK_BYTES";
|
||||
pub const DEFAULT_RUSTFS_NIXL_CHUNK_BYTES: usize = 16 * 1024 * 1024;
|
||||
/// Registered staging memory of the agent.
|
||||
pub const ENV_RUSTFS_NIXL_POOL_BYTES: &str = "RUSTFS_NIXL_POOL_BYTES";
|
||||
pub const DEFAULT_RUSTFS_NIXL_POOL_BYTES: usize = 1024 * 1024 * 1024;
|
||||
/// Transfers in flight per node at most.
|
||||
pub const ENV_RUSTFS_NIXL_MAX_INFLIGHT: &str = "RUSTFS_NIXL_MAX_INFLIGHT";
|
||||
pub const DEFAULT_RUSTFS_NIXL_MAX_INFLIGHT: usize = 256;
|
||||
/// A transfer that has not completed after this many seconds fails the request.
|
||||
pub const ENV_RUSTFS_NIXL_TRANSFER_TIMEOUT_SECS: &str = "RUSTFS_NIXL_TRANSFER_TIMEOUT_SECS";
|
||||
pub const DEFAULT_RUSTFS_NIXL_TRANSFER_TIMEOUT_SECS: u64 = 60;
|
||||
|
||||
/// Whether S3 GETs carrying an `x-amz-rdma-token` are served over cuObject S3-over-RDMA
|
||||
/// (requires a build with the `cuobject` feature and libcuobjserver).
|
||||
pub const ENV_RUSTFS_CUOBJECT_ENABLE: &str = "RUSTFS_CUOBJECT_ENABLE";
|
||||
pub const DEFAULT_RUSTFS_CUOBJECT_ENABLE: bool = false;
|
||||
|
||||
/// Comma-separated RDMA-capable IP addresses that get a cuObject server each.
|
||||
/// Defaults to the hosts of this node's own endpoints.
|
||||
pub const ENV_RUSTFS_CUOBJECT_ADDRS: &str = "RUSTFS_CUOBJECT_ADDRS";
|
||||
|
||||
/// UDP/RDMA control port of the cuObject servers.
|
||||
pub const ENV_RUSTFS_CUOBJECT_PORT: &str = "RUSTFS_CUOBJECT_PORT";
|
||||
pub const DEFAULT_RUSTFS_CUOBJECT_PORT: u16 = 18515;
|
||||
|
||||
/// Staging chunk size per RDMA write to a client.
|
||||
pub const ENV_RUSTFS_CUOBJECT_CHUNK_BYTES: &str = "RUSTFS_CUOBJECT_CHUNK_BYTES";
|
||||
pub const DEFAULT_RUSTFS_CUOBJECT_CHUNK_BYTES: usize = 16 * 1024 * 1024;
|
||||
|
||||
/// Registered staging memory per cuObject server.
|
||||
pub const ENV_RUSTFS_CUOBJECT_POOL_BYTES: &str = "RUSTFS_CUOBJECT_POOL_BYTES";
|
||||
pub const DEFAULT_RUSTFS_CUOBJECT_POOL_BYTES: usize = 1024 * 1024 * 1024;
|
||||
|
||||
/// Dynamically connected initiators per cuObject server (concurrent client connections).
|
||||
/// Staging chunks (RDMA writes) kept in flight per GET, so decoding never waits
|
||||
/// for the write of the previous chunk; each costs one chunk of the pool.
|
||||
pub const ENV_RUSTFS_CUOBJECT_INFLIGHT_WRITES: &str = "RUSTFS_CUOBJECT_INFLIGHT_WRITES";
|
||||
pub const DEFAULT_RUSTFS_CUOBJECT_INFLIGHT_WRITES: usize = 3;
|
||||
|
||||
/// Staging chunks (RDMA reads from the client buffer) kept in flight per
|
||||
/// RDMA PUT body, so ingest never waits on one chunk's round trip at a time;
|
||||
/// each costs one chunk of the pool.
|
||||
pub const ENV_RUSTFS_CUOBJECT_PUT_READ_AHEAD: &str = "RUSTFS_CUOBJECT_PUT_READ_AHEAD";
|
||||
pub const DEFAULT_RUSTFS_CUOBJECT_PUT_READ_AHEAD: usize = 4;
|
||||
|
||||
/// Concurrent erasure-block-aligned segments one cuObject GET is split into
|
||||
/// on the server (each segment decodes and RDMA-writes its own stripes), so a
|
||||
/// single stream keeps several stripes in flight. 1 disables the fan-out.
|
||||
pub const ENV_RUSTFS_CUOBJECT_GET_FANOUT: &str = "RUSTFS_CUOBJECT_GET_FANOUT";
|
||||
pub const DEFAULT_RUSTFS_CUOBJECT_GET_FANOUT: usize = 8;
|
||||
|
||||
/// Smallest segment worth its own producer (rounded up to whole erasure
|
||||
/// blocks): each segment pays a reader setup per part it touches, so short
|
||||
/// requests stay sequential.
|
||||
pub const ENV_RUSTFS_CUOBJECT_GET_FANOUT_MIN_BYTES: &str = "RUSTFS_CUOBJECT_GET_FANOUT_MIN_BYTES";
|
||||
pub const DEFAULT_RUSTFS_CUOBJECT_GET_FANOUT_MIN_BYTES: u64 = 192 * 1024 * 1024;
|
||||
|
||||
/// Segment producers a node runs across all its cuObject GETs: each new GET
|
||||
/// gets a fair share (budget / open GETs, at least 1, at most the fan-out)
|
||||
/// and never waits for one, so many-stream load degrades to sequential GETs
|
||||
/// instead of a reader-setup storm.
|
||||
pub const ENV_RUSTFS_CUOBJECT_GET_FANOUT_BUDGET: &str = "RUSTFS_CUOBJECT_GET_FANOUT_BUDGET";
|
||||
pub const DEFAULT_RUSTFS_CUOBJECT_GET_FANOUT_BUDGET: usize = 16;
|
||||
|
||||
/// Registered host memory the erasure decoder reads shards into so a
|
||||
/// cuObject GET can RDMA-write data shards straight from those buffers
|
||||
/// (zero-copy emit); 0 keeps the staging copy. Blocks are
|
||||
/// `RUSTFS_ERASURE_SHARD_BYTES` (4 MiB when unset); objects with larger
|
||||
/// shards fall back to heap buffers and the staging copy.
|
||||
pub const ENV_RUSTFS_CUOBJECT_SHARD_ARENA_BYTES: &str = "RUSTFS_CUOBJECT_SHARD_ARENA_BYTES";
|
||||
pub const DEFAULT_RUSTFS_CUOBJECT_SHARD_ARENA_BYTES: usize = 0;
|
||||
|
||||
pub const ENV_RUSTFS_CUOBJECT_NUM_DCIS: &str = "RUSTFS_CUOBJECT_NUM_DCIS";
|
||||
pub const DEFAULT_RUSTFS_CUOBJECT_NUM_DCIS: u32 = 128;
|
||||
|
||||
/// Complete cuObject transfers through the library's asynchronous mode: the
|
||||
/// write is submitted with a handle and the task waits on the channel's
|
||||
/// completion descriptor instead of holding a blocking thread for the whole
|
||||
/// transfer. `false` keeps one blocking library call per transfer.
|
||||
pub const ENV_RUSTFS_CUOBJECT_ASYNC_TRANSFERS: &str = "RUSTFS_CUOBJECT_ASYNC_TRANSFERS";
|
||||
pub const DEFAULT_RUSTFS_CUOBJECT_ASYNC_TRANSFERS: bool = true;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
@@ -379,8 +517,12 @@ mod tests {
|
||||
fn internode_channel_isolation_defaults_and_env_names() {
|
||||
// The isolation-off default is asserted at compile time next to the constant definition.
|
||||
assert_eq!(DEFAULT_INTERNODE_BULK_CHANNELS, 2);
|
||||
assert_eq!(DEFAULT_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT, 0);
|
||||
assert_eq!(DEFAULT_INTERNODE_BULK_BODY_WAIT_TIMEOUT_MS, 1_000);
|
||||
assert_eq!(ENV_INTERNODE_CHANNEL_ISOLATION, "RUSTFS_INTERNODE_CHANNEL_ISOLATION");
|
||||
assert_eq!(ENV_INTERNODE_BULK_CHANNELS, "RUSTFS_INTERNODE_BULK_CHANNELS");
|
||||
assert_eq!(ENV_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT, "RUSTFS_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT");
|
||||
assert_eq!(ENV_INTERNODE_BULK_BODY_WAIT_TIMEOUT_MS, "RUSTFS_INTERNODE_BULK_BODY_WAIT_TIMEOUT_MS");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -391,6 +533,7 @@ mod tests {
|
||||
ENV_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED,
|
||||
"RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED"
|
||||
);
|
||||
assert_eq!(ENV_INTERNODE_RPC_MSGPACK_PEER_CAPABILITY, "RUSTFS_INTERNODE_RPC_MSGPACK_PEER_CAPABILITY");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -465,6 +608,6 @@ mod tests {
|
||||
assert_eq!(ENV_RUSTFS_INTERNODE_DATA_TRANSPORT, "RUSTFS_INTERNODE_DATA_TRANSPORT");
|
||||
assert_eq!(DEFAULT_INTERNODE_DATA_TRANSPORT, "tcp-http");
|
||||
assert_eq!(INTERNODE_DATA_TRANSPORT_TCP, "tcp");
|
||||
assert_eq!(KNOWN_INTERNODE_DATA_TRANSPORT_BACKENDS, &["tcp-http", "tcp"]);
|
||||
assert_eq!(KNOWN_INTERNODE_DATA_TRANSPORT_BACKENDS, &["tcp-http", "tcp", "nixl"]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -117,6 +117,30 @@ pub const ENV_OBJECT_GET_SKIP_BITROT_VERIFY: &str = "RUSTFS_OBJECT_GET_SKIP_BITR
|
||||
/// Default: bitrot verification is enabled on GetObject reads (do not skip).
|
||||
pub const DEFAULT_OBJECT_GET_SKIP_BITROT_VERIFY: bool = false;
|
||||
|
||||
/// Comma-separated buckets that hold a content-addressed cache (an LLM KV
|
||||
/// cache) and serve PutObject, GetObject and HeadObject without distributed
|
||||
/// locks or authoritative bucket-metadata reads. See the ecstore
|
||||
/// `bucket::lockless_cache` module for what such a bucket gives up.
|
||||
///
|
||||
/// Default is empty: every bucket keeps the locked paths.
|
||||
pub const ENV_LOCKLESS_CACHE_BUCKETS: &str = "RUSTFS_LOCKLESS_CACHE_BUCKETS";
|
||||
|
||||
/// Smallest declared body, in bytes, whose ETag MD5 is computed on a
|
||||
/// blocking-pool thread while the body streams into the erasure encoder,
|
||||
/// instead of inline on the request's task. The ETag and Content-MD5
|
||||
/// checks are unchanged; `0` keeps every MD5 inline.
|
||||
///
|
||||
/// Each offloaded upload holds one blocking-pool thread until its body ends,
|
||||
/// and that pool also serves disk I/O, so size the threshold against the
|
||||
/// expected number of concurrent large uploads before enabling it.
|
||||
pub const ENV_OBJECT_ETAG_MD5_OFFLOAD_MIN_BYTES: &str = "RUSTFS_OBJECT_ETAG_MD5_OFFLOAD_MIN_BYTES";
|
||||
|
||||
/// Default: disabled; every ETag MD5 is computed inline. Set
|
||||
/// `RUSTFS_OBJECT_ETAG_MD5_OFFLOAD_MIN_BYTES` (for example `8388608`) to opt in.
|
||||
pub const DEFAULT_OBJECT_ETAG_MD5_OFFLOAD_MIN_BYTES: u64 = 0;
|
||||
|
||||
const _: () = assert!(DEFAULT_OBJECT_ETAG_MD5_OFFLOAD_MIN_BYTES == 0);
|
||||
|
||||
/// Create independent shard commitments for new writes after the fleet is upgraded.
|
||||
/// Existing protected objects and multipart uploads retain their protection.
|
||||
pub const ENV_SHARD_INTEGRITY_WRITE: &str = "RUSTFS_SHARD_INTEGRITY_WRITE";
|
||||
@@ -298,6 +322,37 @@ pub const ENV_OBJECT_DISK_READ_TIMEOUT: &str = "RUSTFS_OBJECT_DISK_READ_TIMEOUT"
|
||||
/// Default disk read timeout in seconds.
|
||||
pub const DEFAULT_OBJECT_DISK_READ_TIMEOUT: u64 = 10;
|
||||
|
||||
/// Environment variable for the smallest bitrot frame (hash + shard bytes)
|
||||
/// that is verified on a blocking thread instead of inline on the decode task
|
||||
/// (`0` disables off-thread verification).
|
||||
///
|
||||
/// The shard reads of one stripe complete on one task, so with large shards
|
||||
/// their hashes were computed one after another on one core: at 4 MiB shards
|
||||
/// a 12+4 stripe spent about 10 ms hashing serially. Frames at or above this
|
||||
/// size are hashed on the blocking pool so the shards of a stripe verify in
|
||||
/// parallel; small frames stay inline because a thread hop would cost more
|
||||
/// than the hash.
|
||||
pub const ENV_BITROT_VERIFY_OFFLOAD_MIN_BYTES: &str = "RUSTFS_BITROT_VERIFY_OFFLOAD_MIN_BYTES";
|
||||
/// Default smallest off-thread bitrot frame: 2 MiB (frames of the legacy 1 MiB
|
||||
/// block, and 1 MiB shards, stay inline).
|
||||
pub const DEFAULT_BITROT_VERIFY_OFFLOAD_MIN_BYTES: usize = 2 * 1024 * 1024;
|
||||
|
||||
/// Environment variable for the erasure shard size, in bytes, of newly written
|
||||
/// objects (`0`, the default, keeps the historical 1 MiB erasure block).
|
||||
///
|
||||
/// When set, the erasure block of a new object is `data shards x shard bytes`,
|
||||
/// so every shard read and write is exactly this large and aligned: 4 MiB on a
|
||||
/// 12+4 layout gives a 48 MiB block, 48x fewer stripes per object than today.
|
||||
/// Existing objects keep the block recorded in their `xl.meta`. System
|
||||
/// metadata under `.rustfs.sys` always uses the 1 MiB block. The value must be
|
||||
/// a multiple of 4 KiB and at most `MAX_ERASURE_SHARD_BYTES`; invalid values
|
||||
/// are ignored with a warning.
|
||||
pub const ENV_ERASURE_SHARD_BYTES: &str = "RUSTFS_ERASURE_SHARD_BYTES";
|
||||
/// Default erasure shard size override (`0` = derive from the 1 MiB block).
|
||||
pub const DEFAULT_ERASURE_SHARD_BYTES: usize = 0;
|
||||
/// Largest accepted erasure shard size override (64 MiB).
|
||||
pub const MAX_ERASURE_SHARD_BYTES: usize = 64 * 1024 * 1024;
|
||||
|
||||
/// Environment variable for the per-shard erasure write stall timeout (seconds).
|
||||
///
|
||||
/// A single shard write (or shard-writer shutdown) that makes no forward
|
||||
|
||||
@@ -48,7 +48,7 @@ pub const ENV_RUNTIME_DIAL9_S3_PREFIX: &str = "RUSTFS_RUNTIME_DIAL9_S3_PREFIX";
|
||||
pub const DEFAULT_WORKER_THREADS: usize = 16;
|
||||
pub const DEFAULT_MAX_BLOCKING_THREADS: usize = 1024;
|
||||
pub const DEFAULT_THREAD_PRINT_ENABLED: bool = false;
|
||||
pub const DEFAULT_THREAD_STACK_SIZE: usize = MI_B; // 1 MiB
|
||||
pub const DEFAULT_THREAD_STACK_SIZE: usize = 2 * MI_B; // 2 MiB
|
||||
pub const DEFAULT_THREAD_KEEP_ALIVE: u64 = 60; // seconds
|
||||
pub const DEFAULT_GLOBAL_QUEUE_INTERVAL: u32 = 31;
|
||||
pub const DEFAULT_THREAD_NAME: &str = "rustfs-worker";
|
||||
@@ -114,6 +114,14 @@ pub const ENV_OBJECT_FILE_CACHE_RECLAIM_READ_ENABLE: &str = "RUSTFS_OBJECT_FILE_
|
||||
pub const ENV_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD: &str = "RUSTFS_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD";
|
||||
pub const DEFAULT_OBJECT_FILE_CACHE_RECLAIM_WRITE_ENABLE: bool = false;
|
||||
pub const DEFAULT_OBJECT_FILE_CACHE_RECLAIM_READ_ENABLE: bool = true;
|
||||
|
||||
/// Bytes of page-cache read-ahead a ranged file stream keeps queued in front
|
||||
/// of its read position (`posix_fadvise(WILLNEED)`, Linux only; `0` disables).
|
||||
/// A shard stream served to a peer reads 4 MiB chunks one after another; the
|
||||
/// kernel's own read-ahead window ramps up slowly for large reads, so without
|
||||
/// this each chunk waits for its own device round trips.
|
||||
pub const ENV_OBJECT_STREAM_READ_AHEAD_BYTES: &str = "RUSTFS_OBJECT_STREAM_READ_AHEAD_BYTES";
|
||||
pub const DEFAULT_OBJECT_STREAM_READ_AHEAD_BYTES: usize = 16 * 1024 * 1024;
|
||||
pub const DEFAULT_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD: usize = 4 * 1024 * 1024;
|
||||
|
||||
/// Threshold for small object seek support in bytes.
|
||||
@@ -158,3 +166,10 @@ pub const ENV_OBJECT_DATA_CACHE_MIN_FREE_MEMORY_PERCENT: &str = "RUSTFS_OBJECT_D
|
||||
pub const ENV_OBJECT_DATA_CACHE_FILL_CONCURRENCY_PER_CPU: &str = "RUSTFS_OBJECT_DATA_CACHE_FILL_CONCURRENCY_PER_CPU";
|
||||
pub const ENV_OBJECT_DATA_CACHE_FILL_CONCURRENCY_MAX: &str = "RUSTFS_OBJECT_DATA_CACHE_FILL_CONCURRENCY_MAX";
|
||||
pub const ENV_OBJECT_DATA_CACHE_IDENTITY_KEYS_MAX: &str = "RUSTFS_OBJECT_DATA_CACHE_IDENTITY_KEYS_MAX";
|
||||
/// Fill the object data cache from the PUT body as it streams in, so the first
|
||||
/// GET after a write is a hit (write-once KV-cache chunks). Only objects that
|
||||
/// fit the cache entry limit and take the plain streaming PUT path (no
|
||||
/// compression, no SSE) are filled; the body is kept in memory once more
|
||||
/// during the PUT.
|
||||
pub const ENV_OBJECT_DATA_CACHE_FILL_ON_PUT: &str = "RUSTFS_OBJECT_DATA_CACHE_FILL_ON_PUT";
|
||||
pub const DEFAULT_OBJECT_DATA_CACHE_FILL_ON_PUT: bool = false;
|
||||
|
||||
@@ -44,6 +44,21 @@ pub const SCANNER_CYCLE_MAX_OBJECTS: &str = "cycle_max_objects";
|
||||
/// Scanner config key capping directories entered by one cycle.
|
||||
pub const SCANNER_CYCLE_MAX_DIRECTORIES: &str = "cycle_max_directories";
|
||||
|
||||
/// Per-bucket admission limit in seconds; zero disables this dimension.
|
||||
pub const SCANNER_BUCKET_QUANTUM_DURATION: &str = "bucket_quantum_duration";
|
||||
|
||||
/// Per-bucket admission limit in objects; zero disables this dimension.
|
||||
pub const SCANNER_BUCKET_QUANTUM_OBJECTS: &str = "bucket_quantum_objects";
|
||||
|
||||
/// Per-bucket admission limit in directories; zero disables this dimension.
|
||||
pub const SCANNER_BUCKET_QUANTUM_DIRECTORIES: &str = "bucket_quantum_directories";
|
||||
|
||||
/// Per-bucket admission limit in raw entries; zero disables this dimension.
|
||||
pub const SCANNER_BUCKET_QUANTUM_ENTRIES: &str = "bucket_quantum_entries";
|
||||
|
||||
/// Per-bucket admission limit in raw entry-name bytes; zero disables this dimension.
|
||||
pub const SCANNER_BUCKET_QUANTUM_NAME_BYTES: &str = "bucket_quantum_name_bytes";
|
||||
|
||||
/// Scanner config key setting the periodic bitrot scan cycle in seconds.
|
||||
pub const SCANNER_BITROT_CYCLE: &str = "bitrot_cycle";
|
||||
|
||||
@@ -93,6 +108,11 @@ pub const SCANNER_KEYS: &[&str] = &[
|
||||
SCANNER_CYCLE_MAX_DURATION,
|
||||
SCANNER_CYCLE_MAX_OBJECTS,
|
||||
SCANNER_CYCLE_MAX_DIRECTORIES,
|
||||
SCANNER_BUCKET_QUANTUM_DURATION,
|
||||
SCANNER_BUCKET_QUANTUM_OBJECTS,
|
||||
SCANNER_BUCKET_QUANTUM_DIRECTORIES,
|
||||
SCANNER_BUCKET_QUANTUM_ENTRIES,
|
||||
SCANNER_BUCKET_QUANTUM_NAME_BYTES,
|
||||
SCANNER_BITROT_CYCLE,
|
||||
SCANNER_IDLE_MODE,
|
||||
SCANNER_CACHE_SAVE_TIMEOUT,
|
||||
@@ -141,6 +161,36 @@ pub const ENV_SCANNER_CYCLE_MAX_OBJECTS: &str = "RUSTFS_SCANNER_CYCLE_MAX_OBJECT
|
||||
/// - Example: `export RUSTFS_SCANNER_CYCLE_MAX_DIRECTORIES=100000`
|
||||
pub const ENV_SCANNER_CYCLE_MAX_DIRECTORIES: &str = "RUSTFS_SCANNER_CYCLE_MAX_DIRECTORIES";
|
||||
|
||||
/// Per-bucket admission limit in seconds; zero disables this dimension.
|
||||
pub const ENV_SCANNER_BUCKET_QUANTUM_DURATION_SECS: &str = "RUSTFS_SCANNER_BUCKET_QUANTUM_DURATION_SECS";
|
||||
|
||||
/// Disabled until controlled-fleet acceptance selects production limits.
|
||||
pub const DEFAULT_SCANNER_BUCKET_QUANTUM_DURATION_SECS: u64 = 0;
|
||||
|
||||
/// Per-bucket admission limit in objects; zero disables this dimension.
|
||||
pub const ENV_SCANNER_BUCKET_QUANTUM_OBJECTS: &str = "RUSTFS_SCANNER_BUCKET_QUANTUM_OBJECTS";
|
||||
|
||||
/// Disabled until controlled-fleet acceptance selects production limits.
|
||||
pub const DEFAULT_SCANNER_BUCKET_QUANTUM_OBJECTS: u64 = 0;
|
||||
|
||||
/// Per-bucket admission limit in directories; zero disables this dimension.
|
||||
pub const ENV_SCANNER_BUCKET_QUANTUM_DIRECTORIES: &str = "RUSTFS_SCANNER_BUCKET_QUANTUM_DIRECTORIES";
|
||||
|
||||
/// Disabled until controlled-fleet acceptance selects production limits.
|
||||
pub const DEFAULT_SCANNER_BUCKET_QUANTUM_DIRECTORIES: u64 = 0;
|
||||
|
||||
/// Per-bucket admission limit in raw entries; zero disables this dimension.
|
||||
pub const ENV_SCANNER_BUCKET_QUANTUM_ENTRIES: &str = "RUSTFS_SCANNER_BUCKET_QUANTUM_ENTRIES";
|
||||
|
||||
/// Disabled until controlled-fleet acceptance selects production limits.
|
||||
pub const DEFAULT_SCANNER_BUCKET_QUANTUM_ENTRIES: u64 = 0;
|
||||
|
||||
/// Per-bucket admission limit in raw entry-name bytes; zero disables this dimension.
|
||||
pub const ENV_SCANNER_BUCKET_QUANTUM_NAME_BYTES: &str = "RUSTFS_SCANNER_BUCKET_QUANTUM_NAME_BYTES";
|
||||
|
||||
/// Disabled until controlled-fleet acceptance selects production limits.
|
||||
pub const DEFAULT_SCANNER_BUCKET_QUANTUM_NAME_BYTES: u64 = 0;
|
||||
|
||||
/// Environment variable that selects the scanner speed preset.
|
||||
/// Valid values: `fastest`, `fast`, `default`, `slow`, `slowest`.
|
||||
/// Controls the sleep factor, maximum sleep duration, and cycle interval.
|
||||
|
||||
@@ -55,6 +55,9 @@ pub const ENV_OBS_PUT_STAGE_METRICS_ENABLED: &str = "RUSTFS_OBS_PUT_STAGE_METRIC
|
||||
pub const ENV_OBS_GET_STAGE_LOCAL_SUMMARY_ENABLED: &str = "RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_ENABLED";
|
||||
pub const ENV_OBS_GET_STAGE_LOCAL_SUMMARY_SAMPLE_RATE: &str = "RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_SAMPLE_RATE";
|
||||
pub const ENV_OBS_GET_STAGE_LOCAL_SUMMARY_INTERVAL_SECS: &str = "RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_INTERVAL_SECS";
|
||||
/// Path of the sampled GetObject stage summary file (defaults to `/var/log/rustfs/get-stage-summary.log`).
|
||||
pub const ENV_OBS_GET_STAGE_LOCAL_SUMMARY_FILE: &str = "RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_FILE";
|
||||
pub const DEFAULT_OBS_GET_STAGE_LOCAL_SUMMARY_FILE: &str = "/var/log/rustfs/get-stage-summary.log";
|
||||
|
||||
pub const ENV_OBS_LOGGER_LEVEL: &str = "RUSTFS_OBS_LOGGER_LEVEL";
|
||||
pub const ENV_OBS_LOG_STDOUT_ENABLED: &str = "RUSTFS_OBS_LOG_STDOUT_ENABLED";
|
||||
@@ -178,6 +181,7 @@ mod tests {
|
||||
ENV_OBS_GET_STAGE_LOCAL_SUMMARY_INTERVAL_SECS,
|
||||
"RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_INTERVAL_SECS"
|
||||
);
|
||||
assert_eq!(ENV_OBS_GET_STAGE_LOCAL_SUMMARY_FILE, "RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_FILE");
|
||||
// Test log cleanup related env keys
|
||||
assert_eq!(ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES, "RUSTFS_OBS_LOG_MAX_TOTAL_SIZE_BYTES");
|
||||
assert_eq!(ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, "RUSTFS_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES");
|
||||
|
||||
@@ -26,12 +26,11 @@ categories = ["web-programming", "development-tools", "data-structures", "securi
|
||||
|
||||
[features]
|
||||
default = []
|
||||
hotpath = ["hotpath/hotpath"]
|
||||
hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"]
|
||||
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"]
|
||||
hotpath = []
|
||||
hotpath-alloc = ["hotpath"]
|
||||
hotpath-cpu = ["hotpath"]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
base64-simd = { workspace = true }
|
||||
hmac = { workspace = true }
|
||||
rand = { workspace = true, features = ["serde"] }
|
||||
|
||||
@@ -29,7 +29,6 @@ documentation = "https://docs.rs/rustfs-crypto/latest/rustfs_crypto/"
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
aes-gcm = { workspace = true, optional = true, features = ["rand_core"] }
|
||||
argon2 = { workspace = true, optional = true }
|
||||
chacha20poly1305 = { workspace = true, optional = true }
|
||||
@@ -50,9 +49,9 @@ time = { workspace = true, features = ["parsing", "formatting", "macros", "serde
|
||||
|
||||
[features]
|
||||
default = ["crypto", "fips"]
|
||||
hotpath = ["hotpath/hotpath"]
|
||||
hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"]
|
||||
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"]
|
||||
hotpath = []
|
||||
hotpath-alloc = ["hotpath"]
|
||||
hotpath-cpu = ["hotpath"]
|
||||
fips = []
|
||||
crypto = [
|
||||
"dep:aes-gcm",
|
||||
|
||||
@@ -18,9 +18,6 @@ use crate::Error;
|
||||
use crate::jwt::Claims;
|
||||
|
||||
pub fn decode(token: &str, token_secret: &[u8]) -> Result<TokenData<Claims>, Error> {
|
||||
Ok(jsonwebtoken::decode(
|
||||
token,
|
||||
&DecodingKey::from_secret(token_secret),
|
||||
&Validation::new(Algorithm::HS512),
|
||||
)?)
|
||||
let key = DecodingKey::from_secret(token_secret)?;
|
||||
Ok(jsonwebtoken::decode(token, &key, &Validation::new(Algorithm::HS512))?)
|
||||
}
|
||||
|
||||
@@ -18,9 +18,6 @@ use crate::Error;
|
||||
use crate::jwt::Claims;
|
||||
|
||||
pub fn encode(token_secret: &[u8], claims: &Claims) -> Result<String, Error> {
|
||||
Ok(jsonwebtoken::encode(
|
||||
&Header::new(Algorithm::HS512),
|
||||
claims,
|
||||
&EncodingKey::from_secret(token_secret),
|
||||
)?)
|
||||
let key = EncodingKey::from_secret(token_secret)?;
|
||||
Ok(jsonwebtoken::encode(&Header::new(Algorithm::HS512), claims, &key)?)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
[package]
|
||||
name = "rustfs-cuobject"
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
rust-version.workspace = true
|
||||
version.workspace = true
|
||||
homepage.workspace = true
|
||||
description = "RustFS staging pool and transfers for NVIDIA cuObject S3-over-RDMA (GPUDirect object loading)"
|
||||
keywords = ["rdma", "cuobject", "gpudirect", "s3"]
|
||||
categories = ["network-programming"]
|
||||
documentation = "https://docs.rs/rustfs-cuobject/latest/rustfs_cuobject/"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
bytes = { workspace = true, optional = true }
|
||||
cuobject = { workspace = true, default-features = false, optional = true }
|
||||
parking_lot = { workspace = true, optional = true }
|
||||
thiserror = { workspace = true, optional = true }
|
||||
tokio = { workspace = true, features = ["net", "rt-multi-thread", "sync", "time"], optional = true }
|
||||
|
||||
[features]
|
||||
# Compiles the cuObject server bindings (needs the NVIDIA cuObject SDK).
|
||||
server = ["dep:bytes", "dep:cuobject", "cuobject/server", "dep:parking_lot", "dep:thiserror", "dep:tokio"]
|
||||
@@ -0,0 +1,775 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Server side of NVIDIA cuObject S3-over-RDMA.
|
||||
//!
|
||||
//! A client that supports the protocol (NVIDIA cuObject, NIXL's object plugin,
|
||||
//! …) sends an ordinary S3 GET with an `x-amz-rdma-token` header describing a
|
||||
//! registered client buffer (host or GPU memory). Instead of streaming the
|
||||
//! body over HTTP, the server RDMA-writes the object bytes into that buffer
|
||||
//! through NVIDIA's `libcuobjserver` and answers with an empty body plus
|
||||
//! `x-amz-rdma-reply` and `x-amz-rdma-bytes-transferred`.
|
||||
//!
|
||||
//! The library bindings come from the [`cuobject`] crate. This crate owns
|
||||
//! RustFS's pool of registered, page-aligned host staging buffers: the regular
|
||||
//! read path fills a staging chunk and
|
||||
//! [`CuObjectServer::write_to_client_owned`] pushes it.
|
||||
//!
|
||||
//! Everything below needs the `server` feature, which compiles the
|
||||
//! NVIDIA cuObject server bindings; without it the crate is empty, so a
|
||||
//! workspace build does not need the cuObject SDK.
|
||||
#![cfg(feature = "server")]
|
||||
|
||||
use bytes::Bytes;
|
||||
use cuobject::{Channel, Completion, RegisteredBuffer, Server, ServerOptions, SgBuffer, SgEntry, Source};
|
||||
use std::collections::VecDeque;
|
||||
use std::net::IpAddr;
|
||||
use std::num::NonZeroUsize;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use thiserror::Error;
|
||||
use tokio::io::Interest;
|
||||
use tokio::io::unix::AsyncFd;
|
||||
use tokio::sync::Semaphore;
|
||||
|
||||
/// Header carrying the client's RDMA descriptor on a request.
|
||||
pub const RDMA_TOKEN_HEADER: &str = cuobject::TOKEN_HEADER;
|
||||
/// Header carrying the transfer status on the response (HTTP-style code).
|
||||
pub const RDMA_REPLY_HEADER: &str = cuobject::REPLY_HEADER;
|
||||
/// Header carrying the payload byte count moved over RDMA.
|
||||
pub const RDMA_BYTES_TRANSFERRED_HEADER: &str = cuobject::BYTES_TRANSFERRED_HEADER;
|
||||
/// Reply code for a declined transfer (the client then errors or falls back).
|
||||
pub const RDMA_REPLY_NOT_IMPLEMENTED: u16 = cuobject::REPLY_NOT_IMPLEMENTED;
|
||||
/// Largest single `handleGetObject` the library accepts.
|
||||
pub const MAX_RDMA_OP_BYTES: usize = cuobject::MAX_TRANSFER_BYTES;
|
||||
/// Default RDMA control port of the cuObject server.
|
||||
pub const DEFAULT_PORT: u16 = cuobject::DEFAULT_PORT;
|
||||
/// Most ranges one scatter-gather transfer may carry (the library's `max_sge`).
|
||||
pub const MAX_SG_RANGES: usize = cuobject::DEFAULT_MAX_SG_ENTRIES;
|
||||
|
||||
/// Version of the cuObject server headers the bindings were built against, as
|
||||
/// (major, minor).
|
||||
pub fn library_version() -> (u32, u32) {
|
||||
cuobject::header_version()
|
||||
}
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum CuObjectError {
|
||||
#[error("cuobject {op} failed: {source}")]
|
||||
Os {
|
||||
op: &'static str,
|
||||
#[source]
|
||||
source: std::io::Error,
|
||||
},
|
||||
#[error("cuobject rdma token is malformed: {0}")]
|
||||
Token(String),
|
||||
#[error("cuobject rdma token buffer ({buffer} bytes) is smaller than the response ({needed} bytes)")]
|
||||
TokenTooSmall { buffer: u64, needed: u64 },
|
||||
#[error("cuobject rejected the request: {0}")]
|
||||
Invalid(&'static str),
|
||||
#[error("cuobject transfer failed with verbs status {verbs_status}: {source}")]
|
||||
Transfer {
|
||||
verbs_status: i32,
|
||||
#[source]
|
||||
source: std::io::Error,
|
||||
},
|
||||
#[error("cuobject transfer moved {moved} of {expected} bytes")]
|
||||
ShortTransfer { moved: u64, expected: u64 },
|
||||
#[error("cuobject staging pool exhausted")]
|
||||
PoolExhausted,
|
||||
#[error("cuobject library offers no completion descriptor for channel {0}")]
|
||||
AsyncUnavailable(u16),
|
||||
#[error("cuobject read source failed: {0}")]
|
||||
Source(#[source] std::io::Error),
|
||||
}
|
||||
|
||||
impl From<cuobject::Error> for CuObjectError {
|
||||
fn from(error: cuobject::Error) -> Self {
|
||||
match error {
|
||||
cuobject::Error::Library { op, source } => Self::Os { op, source },
|
||||
cuobject::Error::Transfer { verbs_status, source } => Self::Transfer { verbs_status, source },
|
||||
cuobject::Error::InvalidArgument(reason) => Self::Invalid(reason),
|
||||
cuobject::Error::InvalidToken(reason) => Self::Token(reason.to_owned()),
|
||||
cuobject::Error::TokenTooSmall { buffer, needed } => Self::TokenTooSmall { buffer, needed },
|
||||
cuobject::Error::AsyncUnavailable(channel) => Self::AsyncUnavailable(channel),
|
||||
other => Self::Os {
|
||||
op: "cuobject",
|
||||
source: std::io::Error::other(other),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Parsed `x-amz-rdma-token`.
|
||||
///
|
||||
/// Wire format (cuObject RDMA descriptor protocol): colon-separated hex
|
||||
/// fields `addr:size:rkey:lid:dctn:gid_present:gid`; only the leading
|
||||
/// address and size are interpreted here, the library consumes the rest.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RdmaToken {
|
||||
inner: cuobject::RdmaToken,
|
||||
/// Start of the client buffer (a copy of the parsed field).
|
||||
pub remote_addr: u64,
|
||||
/// Length of the client buffer (a copy of the parsed field).
|
||||
pub buffer_len: u64,
|
||||
}
|
||||
|
||||
impl RdmaToken {
|
||||
pub fn parse(raw: &str) -> Result<Self, CuObjectError> {
|
||||
let inner = cuobject::RdmaToken::parse(raw)?;
|
||||
Ok(Self {
|
||||
remote_addr: inner.remote_addr(),
|
||||
buffer_len: inner.buffer_len(),
|
||||
inner,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn as_str(&self) -> &str {
|
||||
self.inner.as_str()
|
||||
}
|
||||
}
|
||||
|
||||
/// Tuning for one server instance.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct CuObjectConfig {
|
||||
pub port: u16,
|
||||
pub num_dcis: i32,
|
||||
pub cq_depth: u32,
|
||||
/// Size of one staging buffer; one RDMA write moves at most this much.
|
||||
pub chunk_bytes: usize,
|
||||
/// Total registered staging memory per server instance.
|
||||
pub pool_bytes: usize,
|
||||
/// Complete transfers through the library's asynchronous mode (submit,
|
||||
/// then wait on the channel's completion descriptor) instead of one
|
||||
/// blocking library call per transfer.
|
||||
pub async_transfers: bool,
|
||||
}
|
||||
|
||||
impl Default for CuObjectConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
port: DEFAULT_PORT,
|
||||
num_dcis: 128,
|
||||
cq_depth: 640,
|
||||
chunk_bytes: 64 * 1024 * 1024,
|
||||
pool_bytes: 1024 * 1024 * 1024,
|
||||
async_transfers: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct Staging {
|
||||
buffer: RegisteredBuffer<Arc<Server>>,
|
||||
free: parking_lot::Mutex<VecDeque<usize>>,
|
||||
permits: Semaphore,
|
||||
chunk_bytes: usize,
|
||||
}
|
||||
|
||||
/// One cuObject server bound to the RDMA device that owns an IP address.
|
||||
pub struct CuObjectServer {
|
||||
server: Arc<Server>,
|
||||
staging: Staging,
|
||||
addr: IpAddr,
|
||||
port: u16,
|
||||
/// One permit per library channel (DC initiator): a transfer waits here
|
||||
/// instead of failing with `EBUSY` when every channel is in use.
|
||||
channels: Arc<Semaphore>,
|
||||
async_transfers: bool,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for CuObjectServer {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("CuObjectServer")
|
||||
.field("addr", &self.addr)
|
||||
.field("port", &self.port)
|
||||
.field("chunk_bytes", &self.staging.chunk_bytes)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
/// Several byte ranges of one [`RegisteredArena`], sent in order as one
|
||||
/// RDMA operation (at most [`MAX_SG_RANGES`]).
|
||||
pub struct ArenaRanges {
|
||||
pub arena: Arc<RegisteredArena>,
|
||||
/// `(local_offset, len)` pairs, in stream order.
|
||||
pub ranges: Vec<(usize, usize)>,
|
||||
}
|
||||
|
||||
impl ArenaRanges {
|
||||
pub fn total_len(&self) -> usize {
|
||||
self.ranges.iter().map(|(_, len)| *len).sum()
|
||||
}
|
||||
}
|
||||
|
||||
/// A byte range of a [`RegisteredArena`] to send from.
|
||||
pub struct ArenaRange {
|
||||
pub arena: Arc<RegisteredArena>,
|
||||
pub local_offset: usize,
|
||||
pub len: usize,
|
||||
}
|
||||
|
||||
/// Host memory registered with one server, owned by the caller (see
|
||||
/// [`CuObjectServer::register_arena`]).
|
||||
pub struct RegisteredArena {
|
||||
buffer: RegisteredBuffer<Arc<Server>>,
|
||||
server: Arc<CuObjectServer>,
|
||||
id: u64,
|
||||
}
|
||||
|
||||
impl RegisteredArena {
|
||||
/// Length of the registered memory.
|
||||
pub fn len(&self) -> usize {
|
||||
self.buffer.len()
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.buffer.is_empty()
|
||||
}
|
||||
|
||||
/// Process-unique id of this registration.
|
||||
pub fn id(&self) -> u64 {
|
||||
self.id
|
||||
}
|
||||
|
||||
/// The server this memory is registered with.
|
||||
pub fn server(&self) -> &Arc<CuObjectServer> {
|
||||
&self.server
|
||||
}
|
||||
|
||||
/// Raw pointer to the start of the registered memory. The memory lives as
|
||||
/// long as this arena; callers hand out disjoint ranges of it and never
|
||||
/// touch a range while a transfer sends it.
|
||||
pub fn as_mut_ptr(&self) -> *mut u8 {
|
||||
self.buffer.as_mut_ptr()
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for RegisteredArena {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("RegisteredArena")
|
||||
.field("id", &self.id)
|
||||
.field("len", &self.len())
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
/// A reserved staging chunk; returned to the pool on drop.
|
||||
///
|
||||
/// The chunk's index is unique while it is reserved, so its range of the
|
||||
/// staging buffer has one owner. Transfers take the chunk by value, so no
|
||||
/// Rust borrow of the range overlaps an RDMA operation on it.
|
||||
pub struct StagingChunk {
|
||||
server: Arc<CuObjectServer>,
|
||||
index: usize,
|
||||
len: usize,
|
||||
}
|
||||
|
||||
impl StagingChunk {
|
||||
pub fn capacity(&self) -> usize {
|
||||
self.server.staging.chunk_bytes
|
||||
}
|
||||
|
||||
/// Bytes currently staged.
|
||||
pub fn len(&self) -> usize {
|
||||
self.len
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.len == 0
|
||||
}
|
||||
|
||||
/// Copies `data` at `self.len()` and advances the staged length.
|
||||
pub fn append(&mut self, data: &[u8]) -> Result<(), CuObjectError> {
|
||||
if self.len + data.len() > self.capacity() {
|
||||
return Err(CuObjectError::Invalid("staging append exceeds chunk"));
|
||||
}
|
||||
self.spare_mut()?[..data.len()].copy_from_slice(data);
|
||||
self.len += data.len();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn clear(&mut self) {
|
||||
self.len = 0;
|
||||
}
|
||||
|
||||
pub fn is_full(&self) -> bool {
|
||||
self.len >= self.capacity()
|
||||
}
|
||||
|
||||
/// The bytes staged so far.
|
||||
#[allow(unsafe_code)]
|
||||
pub fn staged(&self) -> Result<&[u8], CuObjectError> {
|
||||
// SAFETY: the range lies in this chunk (see the type docs); `&self`
|
||||
// excludes `spare_mut`, and transfers own the chunk they move.
|
||||
Ok(unsafe { self.server.staging.buffer.slice(self.start(), self.len)? })
|
||||
}
|
||||
|
||||
/// The unfilled remainder of the chunk, for a reader to fill directly.
|
||||
#[allow(unsafe_code)]
|
||||
pub fn spare_mut(&mut self) -> Result<&mut [u8], CuObjectError> {
|
||||
let len = self.capacity() - self.len;
|
||||
// SAFETY: the range lies in this chunk (see the type docs) and
|
||||
// `&mut self` excludes every other borrow of it.
|
||||
Ok(unsafe { self.server.staging.buffer.slice_mut(self.start() + self.len, len)? })
|
||||
}
|
||||
|
||||
/// Marks `n` more bytes of the spare area as staged.
|
||||
pub fn advance(&mut self, n: usize) -> Result<(), CuObjectError> {
|
||||
if self.len + n > self.capacity() {
|
||||
return Err(CuObjectError::Invalid("staging advance exceeds chunk"));
|
||||
}
|
||||
self.len += n;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn start(&self) -> usize {
|
||||
self.index * self.capacity()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for StagingChunk {
|
||||
fn drop(&mut self) {
|
||||
self.server.staging.free.lock().push_back(self.index);
|
||||
self.server.staging.permits.add_permits(1);
|
||||
}
|
||||
}
|
||||
|
||||
/// The local side of one GET, owned by the transfer until the DMA finished.
|
||||
enum Local {
|
||||
/// A range of the staging buffer, inside a chunk the transfer also owns.
|
||||
Staging { offset: usize, len: usize },
|
||||
/// A range of a caller-owned arena.
|
||||
Arena {
|
||||
arena: Arc<RegisteredArena>,
|
||||
offset: usize,
|
||||
len: usize,
|
||||
},
|
||||
/// Several ranges of a caller-owned arena, sent as one operation.
|
||||
ArenaList {
|
||||
arena: Arc<RegisteredArena>,
|
||||
entries: Vec<SgEntry>,
|
||||
},
|
||||
}
|
||||
|
||||
impl Local {
|
||||
fn len(&self) -> usize {
|
||||
match self {
|
||||
Self::Staging { len, .. } | Self::Arena { len, .. } => *len,
|
||||
Self::ArenaList { entries, .. } => entries.iter().map(|entry| entry.size).sum(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Why an asynchronous GET did not complete.
|
||||
enum AsyncFailure {
|
||||
/// The library has no completion descriptor; nothing was submitted.
|
||||
Unavailable,
|
||||
/// The GET failed, or was never submitted; nothing is in flight.
|
||||
Failed(CuObjectError),
|
||||
/// Waiting failed after submission: the NIC may still read the memory.
|
||||
InFlight(CuObjectError),
|
||||
}
|
||||
|
||||
impl From<cuobject::Error> for AsyncFailure {
|
||||
fn from(error: cuobject::Error) -> Self {
|
||||
Self::Failed(error.into())
|
||||
}
|
||||
}
|
||||
|
||||
/// The handle of the one GET in flight on an asynchronous channel.
|
||||
const ASYNC_HANDLE: NonZeroUsize = NonZeroUsize::MIN;
|
||||
|
||||
impl CuObjectServer {
|
||||
/// Creates the server and its registered staging pool.
|
||||
pub fn new(addr: IpAddr, config: &CuObjectConfig) -> Result<Arc<Self>, CuObjectError> {
|
||||
if config.chunk_bytes == 0 || config.chunk_bytes > MAX_RDMA_OP_BYTES || config.pool_bytes < config.chunk_bytes {
|
||||
return Err(CuObjectError::Invalid("invalid staging configuration"));
|
||||
}
|
||||
let mut options = ServerOptions::default();
|
||||
// A non-positive count keeps the library default.
|
||||
options.num_dcis = u32::try_from(config.num_dcis).unwrap_or(0);
|
||||
options.cq_depth = config.cq_depth;
|
||||
let server = Arc::new(Server::new(addr, config.port, &options)?);
|
||||
let chunks = config.pool_bytes / config.chunk_bytes;
|
||||
let buffer = RegisteredBuffer::new(Arc::clone(&server), chunks * config.chunk_bytes)?;
|
||||
Ok(Arc::new(Self {
|
||||
channels: Arc::new(Semaphore::new(usize::try_from(config.num_dcis).unwrap_or(1).max(1))),
|
||||
async_transfers: config.async_transfers,
|
||||
server,
|
||||
staging: Staging {
|
||||
buffer,
|
||||
free: parking_lot::Mutex::new((0..chunks).collect()),
|
||||
permits: Semaphore::new(chunks),
|
||||
chunk_bytes: config.chunk_bytes,
|
||||
},
|
||||
addr,
|
||||
port: config.port,
|
||||
}))
|
||||
}
|
||||
|
||||
pub fn addr(&self) -> IpAddr {
|
||||
self.addr
|
||||
}
|
||||
|
||||
pub fn port(&self) -> u16 {
|
||||
self.port
|
||||
}
|
||||
|
||||
pub fn chunk_bytes(&self) -> usize {
|
||||
self.staging.chunk_bytes
|
||||
}
|
||||
|
||||
/// Reserves one staging chunk, waiting up to `timeout` for a free one.
|
||||
pub async fn reserve(self: &Arc<Self>, timeout: Duration) -> Result<StagingChunk, CuObjectError> {
|
||||
let permit = tokio::time::timeout(timeout, self.staging.permits.acquire())
|
||||
.await
|
||||
.map_err(|_| CuObjectError::PoolExhausted)?
|
||||
.map_err(|_| CuObjectError::PoolExhausted)?;
|
||||
permit.forget();
|
||||
let index = self.staging.free.lock().pop_front().ok_or(CuObjectError::PoolExhausted)?;
|
||||
Ok(StagingChunk {
|
||||
server: Arc::clone(self),
|
||||
index,
|
||||
len: 0,
|
||||
})
|
||||
}
|
||||
|
||||
/// Reserves one staging chunk without waiting; `None` when the pool is empty.
|
||||
pub fn try_reserve(self: &Arc<Self>) -> Option<StagingChunk> {
|
||||
let permit = self.staging.permits.try_acquire().ok()?;
|
||||
permit.forget();
|
||||
let index = self.staging.free.lock().pop_front()?;
|
||||
Some(StagingChunk {
|
||||
server: Arc::clone(self),
|
||||
index,
|
||||
len: 0,
|
||||
})
|
||||
}
|
||||
|
||||
/// Staging chunks currently free in the pool (a hint; another reservation may win).
|
||||
pub fn available_chunks(&self) -> usize {
|
||||
self.staging.permits.available_permits()
|
||||
}
|
||||
|
||||
/// Registers `len` bytes of zero-filled, page-aligned host memory with this
|
||||
/// server for callers that manage the memory themselves (a shard arena the
|
||||
/// decoder reads into and the server RDMA-writes from without staging).
|
||||
pub fn register_arena(self: &Arc<Self>, len: usize) -> Result<Arc<RegisteredArena>, CuObjectError> {
|
||||
let buffer = RegisteredBuffer::new(Arc::clone(&self.server), len)?;
|
||||
static NEXT_ID: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);
|
||||
Ok(Arc::new(RegisteredArena {
|
||||
buffer,
|
||||
server: Arc::clone(self),
|
||||
id: NEXT_ID.fetch_add(1, std::sync::atomic::Ordering::Relaxed),
|
||||
}))
|
||||
}
|
||||
|
||||
/// RDMA-writes the staged bytes of `chunk` to `token.remote_addr +
|
||||
/// remote_offset`. The transfer owns the chunk until the DMA finished, so
|
||||
/// cancelling the awaiting future can never release the staging slot under
|
||||
/// an in-flight write. Returns the chunk with the byte count for reuse.
|
||||
pub async fn write_to_client_owned(
|
||||
self: &Arc<Self>,
|
||||
key: &str,
|
||||
token: &RdmaToken,
|
||||
remote_offset: u64,
|
||||
chunk: StagingChunk,
|
||||
) -> Result<(u64, StagingChunk), CuObjectError> {
|
||||
if chunk.is_empty() {
|
||||
return Ok((0, chunk));
|
||||
}
|
||||
let local = Local::Staging {
|
||||
offset: chunk.start(),
|
||||
len: chunk.len(),
|
||||
};
|
||||
self.get_owned(key, token, remote_offset, local, chunk).await
|
||||
}
|
||||
|
||||
/// RDMA-writes `source.len` bytes at `source.local_offset` of
|
||||
/// `source.arena` to `token.remote_addr + remote_offset`, keeping `keep`
|
||||
/// alive until the DMA finished (the caller passes whatever owns the arena
|
||||
/// bytes). The arena must have been registered with this server.
|
||||
pub async fn write_to_client_from_arena_owned<T: Send + 'static>(
|
||||
self: &Arc<Self>,
|
||||
key: &str,
|
||||
token: &RdmaToken,
|
||||
remote_offset: u64,
|
||||
source: ArenaRange,
|
||||
keep: T,
|
||||
) -> Result<(u64, T), CuObjectError> {
|
||||
if source.len == 0 {
|
||||
return Ok((0, keep));
|
||||
}
|
||||
let local = Local::Arena {
|
||||
arena: source.arena,
|
||||
offset: source.local_offset,
|
||||
len: source.len,
|
||||
};
|
||||
self.get_owned(key, token, remote_offset, local, keep).await
|
||||
}
|
||||
|
||||
/// RDMA-writes all of `source.ranges` (in order, as one operation) to
|
||||
/// `token.remote_addr + remote_offset`, keeping `keep` alive until the DMA
|
||||
/// finished. The arena must have been registered with this server.
|
||||
pub async fn write_to_client_sg_owned<T: Send + 'static>(
|
||||
self: &Arc<Self>,
|
||||
key: &str,
|
||||
token: &RdmaToken,
|
||||
remote_offset: u64,
|
||||
source: ArenaRanges,
|
||||
keep: T,
|
||||
) -> Result<(u64, T), CuObjectError> {
|
||||
if source.ranges.len() > MAX_SG_RANGES {
|
||||
return Err(CuObjectError::Invalid("too many scatter-gather ranges"));
|
||||
}
|
||||
if source.total_len() == 0 {
|
||||
return Ok((0, keep));
|
||||
}
|
||||
let entries = source
|
||||
.ranges
|
||||
.iter()
|
||||
.map(|&(offset, size)| SgEntry {
|
||||
offset: offset as u64,
|
||||
size,
|
||||
})
|
||||
.collect();
|
||||
let local = Local::ArenaList {
|
||||
arena: source.arena,
|
||||
entries,
|
||||
};
|
||||
self.get_owned(key, token, remote_offset, local, keep).await
|
||||
}
|
||||
|
||||
/// RDMA-reads `len` bytes from `token.remote_addr + remote_offset` into the
|
||||
/// (empty) `chunk`, owning it for the whole DMA like
|
||||
/// [`Self::write_to_client_owned`]. On success the chunk holds `len` staged
|
||||
/// bytes.
|
||||
pub async fn read_from_client_owned(
|
||||
self: &Arc<Self>,
|
||||
key: &str,
|
||||
token: &RdmaToken,
|
||||
remote_offset: u64,
|
||||
len: usize,
|
||||
mut chunk: StagingChunk,
|
||||
) -> Result<StagingChunk, CuObjectError> {
|
||||
chunk.clear();
|
||||
if len == 0 {
|
||||
return Ok(chunk);
|
||||
}
|
||||
if len > chunk.capacity() {
|
||||
return Err(CuObjectError::Invalid("read exceeds the staging chunk"));
|
||||
}
|
||||
let server = Arc::clone(self);
|
||||
let key = key.to_owned();
|
||||
let token = token.inner.clone();
|
||||
let channel_permit = self.channel_permit().await?;
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let _channel_permit = channel_permit;
|
||||
let mut channel = Channel::lease(Arc::clone(&server.server))?;
|
||||
let moved = channel.put(&key, &token, remote_offset, &server.staging.buffer, chunk.start(), len)?;
|
||||
check_moved(moved as u64, len as u64)?;
|
||||
chunk.advance(len)?;
|
||||
Ok(chunk)
|
||||
})
|
||||
.await
|
||||
.map_err(|error| join_error("read task", error))?
|
||||
}
|
||||
|
||||
async fn channel_permit(&self) -> Result<tokio::sync::OwnedSemaphorePermit, CuObjectError> {
|
||||
Arc::clone(&self.channels)
|
||||
.acquire_owned()
|
||||
.await
|
||||
.map_err(|_| CuObjectError::Invalid("server closed"))
|
||||
}
|
||||
|
||||
/// RDMA-writes `local` to the client and hands `keep` back once the DMA
|
||||
/// finished. The transfer runs in a task of its own that owns `local`,
|
||||
/// `keep` and the channel, so dropping this future never releases memory
|
||||
/// the NIC may still read.
|
||||
async fn get_owned<T: Send + 'static>(
|
||||
self: &Arc<Self>,
|
||||
key: &str,
|
||||
token: &RdmaToken,
|
||||
remote_offset: u64,
|
||||
local: Local,
|
||||
keep: T,
|
||||
) -> Result<(u64, T), CuObjectError> {
|
||||
let expected = local.len() as u64;
|
||||
let server = Arc::clone(self);
|
||||
let key = key.to_owned();
|
||||
let token = token.inner.clone();
|
||||
let channel_permit = self.channel_permit().await?;
|
||||
tokio::spawn(async move {
|
||||
if server.async_transfers {
|
||||
match server.get_async(&key, &token, remote_offset, &local).await {
|
||||
Ok(()) => {
|
||||
drop(channel_permit);
|
||||
return Ok((expected, keep));
|
||||
}
|
||||
Err(AsyncFailure::Unavailable) => {}
|
||||
Err(AsyncFailure::Failed(error)) => return Err(error),
|
||||
Err(AsyncFailure::InFlight(error)) => {
|
||||
// The NIC may still read this memory: never release it.
|
||||
std::mem::forget(local);
|
||||
std::mem::forget(keep);
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let _channel_permit = channel_permit;
|
||||
let mut channel = Channel::lease(Arc::clone(&server.server))?;
|
||||
let mut list = None;
|
||||
let source = server.source(&local, &mut list)?;
|
||||
let moved = channel.get(&key, &token, remote_offset, source)?;
|
||||
check_moved(moved as u64, expected)?;
|
||||
// `keep` (the chunk or the arena owner) is released only here.
|
||||
Ok((expected, keep))
|
||||
})
|
||||
.await
|
||||
.map_err(|error| join_error("write task", error))?
|
||||
})
|
||||
.await
|
||||
.map_err(|error| join_error("write task", error))?
|
||||
}
|
||||
|
||||
/// Moves `local` with the library's asynchronous mode: the GET is
|
||||
/// submitted on a channel of its own and the task waits on that channel's
|
||||
/// completion descriptor instead of holding a blocking thread.
|
||||
// SAFETY: the allowance covers only this function; its two unsafe calls
|
||||
// each state the invariant they rely on.
|
||||
#[allow(unsafe_code)]
|
||||
async fn get_async(
|
||||
&self,
|
||||
key: &str,
|
||||
token: &cuobject::RdmaToken,
|
||||
remote_offset: u64,
|
||||
local: &Local,
|
||||
) -> Result<(), AsyncFailure> {
|
||||
let mut channel = Channel::lease(Arc::clone(&self.server))?;
|
||||
let fd = match channel.completion_fd() {
|
||||
Ok(fd) => fd,
|
||||
Err(cuobject::Error::AsyncUnavailable(_)) => return Err(AsyncFailure::Unavailable),
|
||||
Err(error) => return Err(error.into()),
|
||||
};
|
||||
// SAFETY: the library owns the descriptor and keeps it open while the
|
||||
// server lives; `fd` holds the server, and nothing here closes or
|
||||
// replaces the descriptor while it is registered.
|
||||
let completions = unsafe { AsyncFd::register_with_interest(fd, Interest::READABLE) }.map_err(|error| {
|
||||
AsyncFailure::Failed(CuObjectError::Os {
|
||||
op: "completion fd",
|
||||
source: error.into_parts().1,
|
||||
})
|
||||
})?;
|
||||
let mut list = None;
|
||||
let source = self.source(local, &mut list).map_err(AsyncFailure::Failed)?;
|
||||
// SAFETY: the memory behind `source` belongs to `local`, owned by the
|
||||
// calling task together with the transfer's `keep`; staging chunks and
|
||||
// arena ranges given to a transfer are not borrowed by anyone else.
|
||||
// This function returns only after the completion carrying the handle
|
||||
// was drained, except through `InFlight`, where the channel and the
|
||||
// list are leaked here and the caller leaks `local` and `keep`.
|
||||
unsafe { channel.get_async(key, token, remote_offset, source, ASYNC_HANDLE) }?;
|
||||
let mut events = [Completion::default(); 8];
|
||||
loop {
|
||||
let mut ready = match completions.readable().await {
|
||||
Ok(ready) => ready,
|
||||
Err(source) => {
|
||||
std::mem::forget(list);
|
||||
std::mem::forget(channel);
|
||||
return Err(AsyncFailure::InFlight(CuObjectError::Os {
|
||||
op: "completion wait",
|
||||
source,
|
||||
}));
|
||||
}
|
||||
};
|
||||
loop {
|
||||
let landed = match channel.process_completions(&mut events) {
|
||||
Ok(landed) => landed,
|
||||
Err(error) => {
|
||||
std::mem::forget(list);
|
||||
std::mem::forget(channel);
|
||||
return Err(AsyncFailure::InFlight(error.into()));
|
||||
}
|
||||
};
|
||||
if let Some(done) = events[..landed].iter().find(|event| event.handle() == ASYNC_HANDLE.get()) {
|
||||
// A completion carries no byte count: success moved the whole range.
|
||||
return done
|
||||
.result()
|
||||
.map_err(|source| AsyncFailure::Failed(CuObjectError::Transfer { verbs_status: 0, source }));
|
||||
}
|
||||
if landed < events.len() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
ready.clear_ready();
|
||||
}
|
||||
}
|
||||
|
||||
/// The library view of `local`; a scatter-gather list is built into `list`.
|
||||
fn source<'a: 'l, 'l>(
|
||||
&'a self,
|
||||
local: &'a Local,
|
||||
list: &'l mut Option<SgBuffer<'a, Arc<Server>>>,
|
||||
) -> Result<Source<'l, Arc<Server>>, CuObjectError> {
|
||||
Ok(match local {
|
||||
Local::Staging { offset, len } => Source::Range {
|
||||
buffer: &self.staging.buffer,
|
||||
offset: *offset,
|
||||
len: *len,
|
||||
},
|
||||
Local::Arena { arena, offset, len } => Source::Range {
|
||||
buffer: &arena.buffer,
|
||||
offset: *offset,
|
||||
len: *len,
|
||||
},
|
||||
Local::ArenaList { arena, entries } => Source::List(list.insert(arena.buffer.sg_buffer(entries)?)),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn check_moved(moved: u64, expected: u64) -> Result<u64, CuObjectError> {
|
||||
if moved != expected {
|
||||
return Err(CuObjectError::ShortTransfer { moved, expected });
|
||||
}
|
||||
Ok(moved)
|
||||
}
|
||||
|
||||
fn join_error(op: &'static str, error: tokio::task::JoinError) -> CuObjectError {
|
||||
CuObjectError::Os {
|
||||
op,
|
||||
source: std::io::Error::other(error),
|
||||
}
|
||||
}
|
||||
|
||||
/// Copies owned bytes into staging; convenience for callers holding `Bytes`.
|
||||
pub fn fill_chunk(chunk: &mut StagingChunk, data: &Bytes) -> Result<(), CuObjectError> {
|
||||
chunk.append(data)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn token_parses_leading_fields_and_rejects_garbage() {
|
||||
let token =
|
||||
RdmaToken::parse("0000f52eddc00000:00100000:001824d2:0000:002873:1:00000000000000000000ffff0a0d5410").expect("token");
|
||||
assert_eq!(token.remote_addr, 0x0000_f52e_ddc0_0000);
|
||||
assert_eq!(token.buffer_len, 0x0010_0000);
|
||||
assert!(matches!(RdmaToken::parse("nothex:00100000:1:2:3:4:5"), Err(CuObjectError::Token(_))));
|
||||
assert!(RdmaToken::parse("0000f52eddc00000:00000000:1:2:3:4:5").is_err());
|
||||
assert!(RdmaToken::parse("0000f52eddc00000:00100000").is_err());
|
||||
}
|
||||
}
|
||||
@@ -39,5 +39,8 @@ serde = { workspace = true, features = ["derive"] }
|
||||
rmp-serde = { workspace = true }
|
||||
rustfs-filemeta = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
serde_json.workspace = true
|
||||
|
||||
[lib]
|
||||
doctest = false
|
||||
|
||||
@@ -474,6 +474,12 @@ pub struct BucketTargetUsageInfo {
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct BucketUsageInfo {
|
||||
pub size: u64,
|
||||
/// Provenance of this value. Missing and unknown states remain unproven.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub usage_value_state: Option<crate::BucketUsageValueState>,
|
||||
/// Certification of this exact value, independent of aggregation time.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub usage_certification: Option<crate::BucketUsageCertification>,
|
||||
/// Bucket identity certified by every set contributing to this snapshot.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub bucket_incarnation: Option<String>,
|
||||
@@ -580,6 +586,18 @@ pub struct DataUsageInfo {
|
||||
/// which last-known-good generation contributed to the view.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub usage_snapshot_set_states: Vec<DataUsageSnapshotSetState>,
|
||||
/// Complete mixed observations for admin display only. Legacy bucket maps
|
||||
/// and quota consumers continue to use the canonical values above.
|
||||
#[serde(
|
||||
default,
|
||||
skip_serializing_if = "HashMap::is_empty",
|
||||
deserialize_with = "crate::usage_observation::deserialize_bucket_observations"
|
||||
)]
|
||||
pub usage_snapshot_bucket_observations: HashMap<String, crate::DataUsageBucketObservation>,
|
||||
/// Response-only marker: a projected admin view must never be persisted or
|
||||
/// admitted as a quota/cache baseline, including after serialization.
|
||||
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
|
||||
pub usage_snapshot_admin_projection: bool,
|
||||
/// Buckets whose dirty-usage generation moved while this complete
|
||||
/// snapshot was being walked. Their `buckets_usage` values are the walk
|
||||
/// result, which the cycle could not certify as current; the next cycle
|
||||
@@ -709,7 +727,8 @@ pub fn observed_data_usage_is_newer(observed: &DataUsageInfo, authoritative: &Da
|
||||
/// This applies the structural, exact-baseline, and generation checks of
|
||||
/// [`observed_data_usage_is_newer`] without retaining the baseline's bucket maps.
|
||||
pub fn observed_data_usage_is_newer_than_identity(observed: &DataUsageInfo, authoritative: DataUsageSnapshotIdentity) -> bool {
|
||||
observed.usage_snapshot_converged == Some(false)
|
||||
!observed.usage_snapshot_admin_projection
|
||||
&& observed.usage_snapshot_converged == Some(false)
|
||||
&& (observed.is_complete_bucket_usage_snapshot() || observed.is_valid_partial_snapshot())
|
||||
&& observed.usage_snapshot_authoritative_baseline == Some(authoritative)
|
||||
&& data_usage_identity_is_newer(observed.snapshot_identity(), authoritative)
|
||||
@@ -2024,7 +2043,8 @@ impl DataUsageInfo {
|
||||
|
||||
/// Whether this snapshot authoritatively covers every reported bucket.
|
||||
pub fn is_complete_bucket_usage_snapshot(&self) -> bool {
|
||||
!self.usage_snapshot_bootstrap_pending
|
||||
!self.usage_snapshot_admin_projection
|
||||
&& !self.usage_snapshot_bootstrap_pending
|
||||
&& self.usage_snapshot_complete
|
||||
&& self.last_update.is_some()
|
||||
&& u64::try_from(self.buckets_usage.len()).ok() == Some(self.buckets_count)
|
||||
@@ -2034,7 +2054,8 @@ impl DataUsageInfo {
|
||||
/// admin display. Partial data is accepted only with unique set states,
|
||||
/// a plan digest for every state, and at least one usable generation.
|
||||
pub fn is_valid_partial_snapshot(&self) -> bool {
|
||||
if self.usage_snapshot_bootstrap_pending
|
||||
if self.usage_snapshot_admin_projection
|
||||
|| self.usage_snapshot_bootstrap_pending
|
||||
|| !self.usage_snapshot_partial
|
||||
|| self.usage_snapshot_converged != Some(false)
|
||||
|| self.last_update.is_none()
|
||||
@@ -2072,9 +2093,11 @@ impl DataUsageInfo {
|
||||
Ok(name) => name,
|
||||
Err(_) => return,
|
||||
};
|
||||
self.usage_snapshot_bucket_observations.remove(&bucket_name);
|
||||
|
||||
// Update bucket statistics
|
||||
if let Some(bucket_usage) = self.buckets_usage.get_mut(&bucket_name) {
|
||||
bucket_usage.mark_observed();
|
||||
bucket_usage.size += meta_object.size as u64;
|
||||
bucket_usage.objects_count += 1;
|
||||
bucket_usage.versions_count += 1; // Simplified: assume 1 version per object
|
||||
@@ -2127,6 +2150,7 @@ impl DataUsageInfo {
|
||||
Ok(name) => name,
|
||||
Err(_) => return,
|
||||
};
|
||||
self.usage_snapshot_bucket_observations.remove(&bucket_name);
|
||||
|
||||
// Calculate accurate statistics from all versions
|
||||
let mut total_size = 0u64;
|
||||
@@ -2155,6 +2179,7 @@ impl DataUsageInfo {
|
||||
|
||||
// Update bucket statistics
|
||||
if let Some(bucket_usage) = self.buckets_usage.get_mut(&bucket_name) {
|
||||
bucket_usage.mark_observed();
|
||||
bucket_usage.size += total_size;
|
||||
bucket_usage.objects_count += 1;
|
||||
bucket_usage.versions_count += versions_count;
|
||||
@@ -2335,6 +2360,9 @@ impl BucketUsageInfo {
|
||||
/// Add size summary to this bucket usage
|
||||
/// Merge another BucketUsageInfo into this one
|
||||
pub fn merge(&mut self, other: &BucketUsageInfo) {
|
||||
// Adding contributions produces a new value, not either input's
|
||||
// whole-bucket certificate. The scanner may certify the final sum.
|
||||
self.mark_observed();
|
||||
if self.bucket_incarnation != other.bucket_incarnation {
|
||||
self.bucket_incarnation = None;
|
||||
}
|
||||
|
||||
@@ -13,5 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
pub mod data_usage;
|
||||
pub mod usage_observation;
|
||||
|
||||
pub use data_usage::*;
|
||||
pub use usage_observation::*;
|
||||
|
||||
@@ -0,0 +1,265 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
//! Diagnostic provenance and response-only projections of complete bucket observations.
|
||||
|
||||
use crate::{BucketUsageInfo, DataUsageInfo, DataUsageSnapshotIdentity};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{collections::HashMap, time::SystemTime};
|
||||
|
||||
pub const DATA_USAGE_BUCKET_OBSERVATION_VERSION: u16 = 1;
|
||||
|
||||
/// Provenance of the returned bucket value, independent of quota authority.
|
||||
#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum BucketUsageValueState {
|
||||
Certified,
|
||||
Carried,
|
||||
Observed,
|
||||
#[serde(other)]
|
||||
Unknown,
|
||||
}
|
||||
|
||||
/// A certification of this exact bucket value, never its publication timestamp.
|
||||
#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)]
|
||||
pub struct BucketUsageCertification {
|
||||
pub epoch: u64,
|
||||
pub cycle: u64,
|
||||
pub certified_at: SystemTime,
|
||||
}
|
||||
|
||||
/// Immutable provenance of one complete traversal contribution.
|
||||
///
|
||||
/// The scanner validates its storage-owned sweep identity before constructing
|
||||
/// this public projection. Equal start/request plans are allowed when a fresh
|
||||
/// set contributes alongside another set's mixed traversal.
|
||||
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
|
||||
pub struct DataUsageBucketObservationSource {
|
||||
pub pool_index: u64,
|
||||
pub set_index: u64,
|
||||
pub bucket_incarnation: String,
|
||||
pub scanner_epoch: u64,
|
||||
pub scanner_cycle: u64,
|
||||
pub scan_plan_digest: [u8; 32],
|
||||
pub started_plan: [u8; 32],
|
||||
pub requested_plan: [u8; 32],
|
||||
pub completed_at: SystemTime,
|
||||
}
|
||||
|
||||
/// An additive companion; legacy bucket maps never contain this mixed value.
|
||||
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
|
||||
pub struct DataUsageBucketObservation {
|
||||
pub version: u16,
|
||||
pub usage: BucketUsageInfo,
|
||||
/// Exact enclosing candidate identity. Rewrapping preserves source times.
|
||||
pub snapshot_identity: DataUsageSnapshotIdentity,
|
||||
/// Oldest contribution time, so a newer set cannot hide an older value.
|
||||
pub completed_at: SystemTime,
|
||||
/// Strictly ordered, complete coverage of the enclosing live set inventory.
|
||||
pub sources: Vec<DataUsageBucketObservationSource>,
|
||||
}
|
||||
|
||||
/// Companion evolution must not make an otherwise usable canonical snapshot
|
||||
/// unreadable. Unsupported or malformed diagnostic entries never grant proof.
|
||||
pub(crate) fn deserialize_bucket_observations<'de, D>(
|
||||
deserializer: D,
|
||||
) -> Result<HashMap<String, DataUsageBucketObservation>, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
#[derive(Deserialize)]
|
||||
#[serde(untagged)]
|
||||
enum Entry {
|
||||
Observation(Box<DataUsageBucketObservation>),
|
||||
Ignored(serde::de::IgnoredAny),
|
||||
}
|
||||
Ok(HashMap::<String, Entry>::deserialize(deserializer)?
|
||||
.into_iter()
|
||||
.filter_map(|(bucket, entry)| match entry {
|
||||
Entry::Observation(observation) if observation.version == DATA_USAGE_BUCKET_OBSERVATION_VERSION => {
|
||||
Some((bucket, *observation))
|
||||
}
|
||||
Entry::Observation(_) | Entry::Ignored(_) => None,
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
impl BucketUsageInfo {
|
||||
/// Attach a certificate only after the caller proves complete bucket coverage.
|
||||
pub fn certify(&mut self, certification: BucketUsageCertification) {
|
||||
self.usage_value_state = Some(BucketUsageValueState::Certified);
|
||||
self.usage_certification = Some(certification);
|
||||
}
|
||||
|
||||
/// Preserve the certificate when carrying the exact prior bucket value.
|
||||
pub fn mark_carried(&mut self) {
|
||||
self.normalize_usage_provenance();
|
||||
self.usage_value_state = Some(BucketUsageValueState::Carried);
|
||||
}
|
||||
|
||||
/// The value changed without a complete bucket certificate.
|
||||
pub fn mark_observed(&mut self) {
|
||||
self.usage_value_state = Some(BucketUsageValueState::Observed);
|
||||
self.usage_certification = None;
|
||||
}
|
||||
|
||||
/// Future or legacy provenance cannot authorize a retained certificate.
|
||||
pub fn normalize_usage_provenance(&mut self) {
|
||||
if !matches!(
|
||||
self.usage_value_state,
|
||||
Some(BucketUsageValueState::Certified | BucketUsageValueState::Carried)
|
||||
) {
|
||||
self.usage_certification = None;
|
||||
}
|
||||
if self.usage_value_state == Some(BucketUsageValueState::Certified) && self.usage_certification.is_none() {
|
||||
self.usage_value_state = Some(BucketUsageValueState::Unknown);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl DataUsageBucketObservation {
|
||||
/// Verify the public provenance against the enclosing candidate and current
|
||||
/// storage-owned incarnation. This never grants publication or ACK authority.
|
||||
pub fn matches_snapshot(&self, snapshot: &DataUsageInfo, incarnation: &str) -> bool {
|
||||
let Some(root_generation) = snapshot.scanner_epoch.zip(snapshot.scanner_cycle) else {
|
||||
return false;
|
||||
};
|
||||
if self.version != DATA_USAGE_BUCKET_OBSERVATION_VERSION
|
||||
|| snapshot.usage_snapshot_admin_projection
|
||||
|| self.snapshot_identity != snapshot.snapshot_identity()
|
||||
|| incarnation.is_empty()
|
||||
|| self.usage.bucket_incarnation.as_deref() != Some(incarnation)
|
||||
|| self.sources.is_empty()
|
||||
|| self.usage.usage_certification.is_some()
|
||||
|| self.usage.usage_value_state != Some(BucketUsageValueState::Observed)
|
||||
|| self.sources.iter().map(|source| source.completed_at).min() != Some(self.completed_at)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let mut previous = None;
|
||||
let mut plan = None;
|
||||
for state in &snapshot.usage_snapshot_set_states {
|
||||
let key = (state.pool_index, state.set_index);
|
||||
if previous.is_some_and(|previous| previous >= key)
|
||||
|| state.scan_plan_digest.is_none()
|
||||
|| plan.is_some_and(|plan| Some(plan) != state.scan_plan_digest)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
previous = Some(key);
|
||||
plan = state.scan_plan_digest;
|
||||
}
|
||||
let mut sources = self.sources.iter();
|
||||
for state in snapshot.usage_snapshot_set_states.iter().filter(|state| !state.tombstone) {
|
||||
let Some(source) = sources.next() else {
|
||||
return false;
|
||||
};
|
||||
let Some(set_generation) = state.scanner_epoch.zip(state.scanner_cycle) else {
|
||||
return false;
|
||||
};
|
||||
if (source.pool_index, source.set_index) != (state.pool_index, state.set_index)
|
||||
|| source.bucket_incarnation != incarnation
|
||||
|| Some(source.scan_plan_digest) != state.scan_plan_digest
|
||||
|| (source.scanner_epoch, source.scanner_cycle) > set_generation
|
||||
|| set_generation > root_generation
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
sources.next().is_none()
|
||||
}
|
||||
}
|
||||
|
||||
impl DataUsageInfo {
|
||||
/// Overlay validated companions on an owned admin response only.
|
||||
///
|
||||
/// `incarnations` must come from current storage metadata, not the companion.
|
||||
/// The projection is marked explicitly so persistence and quota boundaries
|
||||
/// can reject accidental reuse. Invalid companions leave canonical values
|
||||
/// unchanged. Counter overflow rejects the whole projection.
|
||||
pub fn project_bucket_observations_for_admin(&mut self, incarnations: &HashMap<String, String>) -> usize {
|
||||
if self.usage_snapshot_admin_projection
|
||||
|| (!self.is_complete_bucket_usage_snapshot() && !self.is_valid_partial_snapshot())
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
let observations =
|
||||
self.usage_snapshot_bucket_observations
|
||||
.iter()
|
||||
.filter(|(bucket, observation)| {
|
||||
if self.usage_snapshot_complete
|
||||
&& (!self.usage_snapshot_pending_buckets.iter().any(|pending| pending == *bucket)
|
||||
|| u64::try_from(self.usage_snapshot_pending_buckets.len()).ok()
|
||||
!= Some(self.usage_snapshot_pending_bucket_count))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if self
|
||||
.buckets_usage
|
||||
.get(*bucket)
|
||||
.and_then(|usage| usage.usage_certification)
|
||||
.is_some_and(|certificate| {
|
||||
observation.sources.iter().all(|source| {
|
||||
(source.scanner_epoch, source.scanner_cycle) <= (certificate.epoch, certificate.cycle)
|
||||
})
|
||||
})
|
||||
{
|
||||
return false;
|
||||
}
|
||||
incarnations.get(*bucket).is_some_and(|incarnation| {
|
||||
self.buckets_usage
|
||||
.get(*bucket)
|
||||
.and_then(|usage| usage.bucket_incarnation.as_ref())
|
||||
.is_none_or(|canonical| canonical == incarnation)
|
||||
&& observation.matches_snapshot(self, incarnation)
|
||||
})
|
||||
})
|
||||
.collect::<HashMap<_, _>>();
|
||||
if observations.is_empty() {
|
||||
return 0;
|
||||
}
|
||||
let mut values = self
|
||||
.buckets_usage
|
||||
.iter()
|
||||
.filter(|(bucket, _)| !observations.contains_key(bucket))
|
||||
.map(|(_, usage)| usage)
|
||||
.chain(observations.values().map(|observation| &observation.usage));
|
||||
let totals = values.try_fold([0_u64; 4], |total, usage| {
|
||||
Some([
|
||||
total[0].checked_add(usage.size)?,
|
||||
total[1].checked_add(usage.objects_count)?,
|
||||
total[2].checked_add(usage.versions_count)?,
|
||||
total[3].checked_add(usage.delete_markers_count)?,
|
||||
])
|
||||
});
|
||||
let Some([size, objects, versions, markers]) = totals else {
|
||||
return 0;
|
||||
};
|
||||
let projected = observations.len();
|
||||
for (bucket, observation) in observations {
|
||||
self.buckets_usage.insert(bucket.clone(), observation.usage.clone());
|
||||
}
|
||||
self.bucket_sizes = self
|
||||
.buckets_usage
|
||||
.iter()
|
||||
.map(|(bucket, usage)| (bucket.clone(), usage.size))
|
||||
.collect();
|
||||
self.buckets_count = u64::try_from(self.buckets_usage.len()).unwrap_or(u64::MAX);
|
||||
self.objects_total_size = size;
|
||||
self.objects_total_count = objects;
|
||||
self.versions_total_count = versions;
|
||||
self.delete_markers_total_count = markers;
|
||||
self.replication_info.clear();
|
||||
self.tier_stats = None;
|
||||
self.unknown_tier_stats = None;
|
||||
self.usage_snapshot_complete = false;
|
||||
self.usage_snapshot_partial = true;
|
||||
self.usage_snapshot_converged = Some(false);
|
||||
self.usage_snapshot_observational = true;
|
||||
self.usage_snapshot_admin_projection = true;
|
||||
projected
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,323 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
use crate::DataUsageSnapshotSetState;
|
||||
use std::time::Duration;
|
||||
|
||||
const INCARNATION: &str = "11111111-1111-4111-8111-111111111111";
|
||||
|
||||
fn time(seconds: u64) -> SystemTime {
|
||||
SystemTime::UNIX_EPOCH + Duration::from_secs(seconds)
|
||||
}
|
||||
|
||||
fn usage(size: u64) -> BucketUsageInfo {
|
||||
BucketUsageInfo {
|
||||
size,
|
||||
objects_count: size,
|
||||
versions_count: size,
|
||||
bucket_incarnation: Some(INCARNATION.to_string()),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn snapshot() -> DataUsageInfo {
|
||||
let mut main = usage(10);
|
||||
main.certify(BucketUsageCertification {
|
||||
epoch: 2,
|
||||
cycle: 3,
|
||||
certified_at: time(30),
|
||||
});
|
||||
main.mark_carried();
|
||||
let mut snapshot = DataUsageInfo {
|
||||
last_update: Some(time(90)),
|
||||
scanner_epoch: Some(2),
|
||||
scanner_cycle: Some(9),
|
||||
buckets_count: 1,
|
||||
buckets_usage: HashMap::from([("bucket".to_string(), main)]),
|
||||
bucket_sizes: HashMap::from([("bucket".to_string(), 10)]),
|
||||
usage_snapshot_complete: true,
|
||||
usage_snapshot_converged: Some(false),
|
||||
usage_snapshot_pending_buckets: vec!["bucket".to_string()],
|
||||
usage_snapshot_pending_bucket_count: 1,
|
||||
usage_snapshot_set_states: (0..2)
|
||||
.map(|set_index| DataUsageSnapshotSetState {
|
||||
pool_index: 0,
|
||||
set_index,
|
||||
scanner_epoch: Some(2),
|
||||
scanner_cycle: Some(9),
|
||||
scan_plan_digest: Some([1; 32]),
|
||||
complete: true,
|
||||
..Default::default()
|
||||
})
|
||||
.collect(),
|
||||
..Default::default()
|
||||
};
|
||||
snapshot.calculate_totals();
|
||||
let mut observed = usage(25);
|
||||
observed.mark_observed();
|
||||
let observation = DataUsageBucketObservation {
|
||||
version: DATA_USAGE_BUCKET_OBSERVATION_VERSION,
|
||||
usage: observed,
|
||||
snapshot_identity: snapshot.snapshot_identity(),
|
||||
completed_at: time(40),
|
||||
sources: (0..2)
|
||||
.map(|set_index| DataUsageBucketObservationSource {
|
||||
pool_index: 0,
|
||||
set_index,
|
||||
bucket_incarnation: INCARNATION.to_string(),
|
||||
scanner_epoch: 2,
|
||||
scanner_cycle: 4 + set_index,
|
||||
scan_plan_digest: [1; 32],
|
||||
started_plan: [2; 32],
|
||||
requested_plan: [3; 32],
|
||||
completed_at: time(40 + set_index),
|
||||
})
|
||||
.collect(),
|
||||
};
|
||||
snapshot
|
||||
.usage_snapshot_bucket_observations
|
||||
.insert("bucket".to_string(), observation);
|
||||
snapshot
|
||||
}
|
||||
|
||||
fn incarnations() -> HashMap<String, String> {
|
||||
HashMap::from([("bucket".to_string(), INCARNATION.to_string())])
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provenance_is_additive_and_unknown_states_do_not_keep_certificates() {
|
||||
let legacy = usage(10);
|
||||
let bytes = serde_json::to_vec(&legacy).expect("encode a legacy bucket without provenance");
|
||||
let text = std::str::from_utf8(&bytes).expect("JSON is UTF-8");
|
||||
assert!(!text.contains("usage_value_state"));
|
||||
assert!(!text.contains("usage_certification"));
|
||||
let decoded: BucketUsageInfo = serde_json::from_slice(&bytes).expect("read legacy bucket");
|
||||
assert_eq!(decoded.usage_value_state, None);
|
||||
assert_eq!(decoded.usage_certification, None);
|
||||
|
||||
let mut wire = serde_json::to_value(legacy).expect("encode future bucket fixture");
|
||||
wire["usage_value_state"] = serde_json::json!("future_state");
|
||||
wire["usage_certification"] =
|
||||
serde_json::json!({"epoch":2,"cycle":3,"certified_at":{"secs_since_epoch":30,"nanos_since_epoch":0}});
|
||||
let mut future: BucketUsageInfo = serde_json::from_value(wire).expect("unknown diagnostic enum remains readable");
|
||||
future.normalize_usage_provenance();
|
||||
assert_eq!(future.usage_value_state, Some(BucketUsageValueState::Unknown));
|
||||
assert_eq!(future.usage_certification, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn carrying_preserves_time_but_adding_a_source_invalidates_the_certificate() {
|
||||
let mut bucket = snapshot().buckets_usage.remove("bucket").expect("canonical bucket");
|
||||
bucket.mark_carried();
|
||||
assert_eq!(bucket.usage_certification.map(|certification| certification.certified_at), Some(time(30)));
|
||||
bucket.merge(&usage(1));
|
||||
assert_eq!(bucket.size, 11);
|
||||
assert_eq!(bucket.usage_value_state, Some(BucketUsageValueState::Observed));
|
||||
assert_eq!(bucket.usage_certification, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn direct_object_accumulation_does_not_reuse_the_old_certificate() {
|
||||
let mut simple = snapshot();
|
||||
simple.add_object("bucket/object", &rustfs_filemeta::MetaObject::default());
|
||||
assert_eq!(simple.buckets_usage["bucket"].usage_value_state, Some(BucketUsageValueState::Observed));
|
||||
assert_eq!(simple.buckets_usage["bucket"].usage_certification, None);
|
||||
assert!(simple.usage_snapshot_bucket_observations.is_empty());
|
||||
|
||||
let mut versioned = snapshot();
|
||||
versioned.add_object_from_file_meta("bucket/object", &rustfs_filemeta::FileMeta::default());
|
||||
assert_eq!(versioned.buckets_usage["bucket"].usage_value_state, Some(BucketUsageValueState::Observed));
|
||||
assert_eq!(versioned.buckets_usage["bucket"].usage_certification, None);
|
||||
assert!(versioned.usage_snapshot_bucket_observations.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn future_or_malformed_companions_do_not_break_the_canonical_reader() {
|
||||
let canonical = snapshot();
|
||||
for entry in [
|
||||
serde_json::json!({"version": 2, "future_payload": {"new": true}}),
|
||||
serde_json::json!({"version": 1, "usage": "broken"}),
|
||||
] {
|
||||
let mut wire = serde_json::to_value(&canonical).expect("canonical JSON");
|
||||
wire["usage_snapshot_bucket_observations"]["bucket"] = entry;
|
||||
let decoded: DataUsageInfo = serde_json::from_value(wire).expect("diagnostics cannot break canonical data");
|
||||
assert_eq!(decoded.buckets_usage, canonical.buckets_usage);
|
||||
assert!(decoded.usage_snapshot_bucket_observations.is_empty());
|
||||
assert!(decoded.is_complete_bucket_usage_snapshot());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn complete_companion_projects_only_the_admin_copy_and_keeps_measurement_times() {
|
||||
let canonical = snapshot();
|
||||
let canonical_bytes = serde_json::to_vec(&canonical).expect("encode canonical candidate");
|
||||
let mut response = canonical.clone();
|
||||
assert_eq!(response.project_bucket_observations_for_admin(&incarnations()), 1);
|
||||
assert_eq!(response.buckets_usage["bucket"].size, 25);
|
||||
assert_eq!(response.buckets_usage["bucket"].usage_value_state, Some(BucketUsageValueState::Observed));
|
||||
assert_eq!(response.buckets_usage["bucket"].usage_certification, None);
|
||||
assert_eq!(response.objects_total_size, 25);
|
||||
assert_eq!(response.bucket_sizes["bucket"], 25);
|
||||
assert_eq!(response.last_update, canonical.last_update);
|
||||
assert_eq!(response.usage_snapshot_bucket_observations["bucket"].completed_at, time(40));
|
||||
assert!(response.usage_snapshot_admin_projection);
|
||||
assert!(response.usage_snapshot_partial);
|
||||
assert!(response.usage_snapshot_observational);
|
||||
assert_eq!(response.usage_snapshot_converged, Some(false));
|
||||
assert!(!response.is_complete_bucket_usage_snapshot());
|
||||
assert!(
|
||||
!response.is_valid_partial_snapshot(),
|
||||
"response-only values cannot be a storage candidate"
|
||||
);
|
||||
assert_eq!(serde_json::to_vec(&canonical).expect("reencode original"), canonical_bytes);
|
||||
assert_eq!(canonical.buckets_usage["bucket"].size, 10);
|
||||
assert_eq!(
|
||||
canonical.buckets_usage["bucket"]
|
||||
.usage_certification
|
||||
.map(|certificate| certificate.certified_at),
|
||||
Some(time(30))
|
||||
);
|
||||
assert_eq!(response.project_bucket_observations_for_admin(&incarnations()), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_companions_preserve_the_canonical_bucket() {
|
||||
for invalid in 0..11 {
|
||||
let mut candidate = snapshot();
|
||||
let observation = candidate
|
||||
.usage_snapshot_bucket_observations
|
||||
.get_mut("bucket")
|
||||
.expect("companion");
|
||||
match invalid {
|
||||
0 => observation.version += 1,
|
||||
1 => observation.snapshot_identity.scanner_cycle = Some(8),
|
||||
2 => {
|
||||
observation.sources.pop();
|
||||
}
|
||||
3 => observation.sources[1].set_index = 0,
|
||||
4 => observation.sources[1].scanner_cycle = 10,
|
||||
5 => observation.sources[1].scan_plan_digest = [9; 32],
|
||||
6 => observation.sources[1].bucket_incarnation = "other".to_string(),
|
||||
7 => observation.completed_at = time(41),
|
||||
8 => observation.usage.certify(BucketUsageCertification {
|
||||
epoch: 2,
|
||||
cycle: 9,
|
||||
certified_at: time(90),
|
||||
}),
|
||||
9 => candidate.usage_snapshot_set_states[1].scanner_cycle = None,
|
||||
10 => {
|
||||
candidate
|
||||
.buckets_usage
|
||||
.get_mut("bucket")
|
||||
.expect("canonical")
|
||||
.bucket_incarnation = Some("recreated".to_string())
|
||||
}
|
||||
_ => unreachable!("bounded fixture variants"),
|
||||
}
|
||||
let before = candidate.clone();
|
||||
assert_eq!(
|
||||
candidate.project_bucket_observations_for_admin(&incarnations()),
|
||||
0,
|
||||
"invalid case {invalid}"
|
||||
);
|
||||
assert_eq!(candidate, before, "invalid case {invalid} changed canonical data");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_fresh_set_can_contribute_without_refreshing_an_older_mixed_set() {
|
||||
let mut candidate = snapshot();
|
||||
let observation = candidate
|
||||
.usage_snapshot_bucket_observations
|
||||
.get_mut("bucket")
|
||||
.expect("companion");
|
||||
observation.sources[1].started_plan = observation.sources[1].requested_plan;
|
||||
observation.sources[1].completed_at = time(90);
|
||||
assert_eq!(candidate.project_bucket_observations_for_admin(&incarnations()), 1);
|
||||
assert_eq!(candidate.usage_snapshot_bucket_observations["bucket"].completed_at, time(40));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn companion_only_partial_keeps_legacy_maps_empty_until_admin_projection() {
|
||||
let mut candidate = snapshot();
|
||||
candidate.buckets_usage.clear();
|
||||
candidate.bucket_sizes.clear();
|
||||
candidate.buckets_count = 0;
|
||||
candidate.calculate_totals();
|
||||
candidate.usage_snapshot_complete = false;
|
||||
candidate.usage_snapshot_partial = true;
|
||||
candidate.usage_snapshot_observational = true;
|
||||
for state in &mut candidate.usage_snapshot_set_states {
|
||||
state.complete = false;
|
||||
}
|
||||
assert!(candidate.is_valid_partial_snapshot());
|
||||
let wire = serde_json::to_value(&candidate).expect("encode companion-only observation");
|
||||
assert_eq!(wire["buckets_usage"], serde_json::json!({}));
|
||||
assert_eq!(wire["bucket_sizes"], serde_json::json!({}));
|
||||
assert_eq!(wire["objects_total_size"], 0);
|
||||
assert_eq!(candidate.project_bucket_observations_for_admin(&incarnations()), 1);
|
||||
assert_eq!(candidate.objects_total_size, 25);
|
||||
assert!(candidate.usage_snapshot_partial);
|
||||
assert!(!candidate.usage_snapshot_complete);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn projection_rejects_counter_overflow_without_partial_mutation() {
|
||||
let mut candidate = snapshot();
|
||||
candidate.buckets_usage.insert("cold".to_string(), usage(u64::MAX));
|
||||
candidate.buckets_count = 2;
|
||||
let before = candidate.clone();
|
||||
assert_eq!(candidate.project_bucket_observations_for_admin(&incarnations()), 0);
|
||||
assert_eq!(candidate, before);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_newer_canonical_certificate_and_missing_pending_prevent_old_overlay() {
|
||||
for newer_certificate in [false, true] {
|
||||
let mut candidate = snapshot();
|
||||
if newer_certificate {
|
||||
candidate
|
||||
.buckets_usage
|
||||
.get_mut("bucket")
|
||||
.expect("canonical")
|
||||
.certify(BucketUsageCertification {
|
||||
epoch: 2,
|
||||
cycle: 9,
|
||||
certified_at: time(90),
|
||||
});
|
||||
} else {
|
||||
candidate.usage_snapshot_pending_buckets.clear();
|
||||
candidate.usage_snapshot_pending_bucket_count = 0;
|
||||
}
|
||||
assert_eq!(candidate.project_bucket_observations_for_admin(&incarnations()), 0);
|
||||
assert_eq!(candidate.buckets_usage["bucket"].size, 10);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provenance_and_companions_change_candidate_bytes_and_round_trip_as_maps() {
|
||||
let canonical = snapshot();
|
||||
let original = serde_json::to_vec(&canonical).expect("encode candidate");
|
||||
let mut changed = canonical.clone();
|
||||
changed
|
||||
.buckets_usage
|
||||
.get_mut("bucket")
|
||||
.expect("bucket")
|
||||
.usage_certification
|
||||
.as_mut()
|
||||
.expect("certificate")
|
||||
.certified_at = time(31);
|
||||
assert_ne!(serde_json::to_vec(&changed).expect("encode changed certificate"), original);
|
||||
let mut changed = canonical.clone();
|
||||
changed
|
||||
.usage_snapshot_bucket_observations
|
||||
.get_mut("bucket")
|
||||
.expect("observation")
|
||||
.sources[0]
|
||||
.started_plan = [8; 32];
|
||||
assert_ne!(serde_json::to_vec(&changed).expect("encode changed source"), original);
|
||||
let named = rmp_serde::to_vec_named(&canonical).expect("encode named map");
|
||||
let restored: DataUsageInfo = rmp_serde::from_slice(&named).expect("read named map with additive fields");
|
||||
assert_eq!(restored, canonical);
|
||||
}
|
||||
@@ -99,10 +99,8 @@ rustfs-filemeta.workspace = true
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
|
||||
aws-sdk-sts = { workspace = true, default-features = false, features = ["default-https-client", "rt-tokio"] }
|
||||
aws-config = { workspace = true }
|
||||
aws-smithy-http-client = { workspace = true, default-features = false, features = ["rustls-aws-lc"] }
|
||||
aws-smithy-types.workspace = true
|
||||
async-compression = { workspace = true, features = ["tokio", "bzip2", "lz4", "xz"] }
|
||||
async-trait = { workspace = true }
|
||||
flate2.workspace = true
|
||||
http.workspace = true
|
||||
@@ -129,24 +127,26 @@ rand = { workspace = true, features = ["serde"] }
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
hex-simd = { workspace = true }
|
||||
md-5 = { workspace = true }
|
||||
opentelemetry-proto = { workspace = true }
|
||||
prost.workspace = true
|
||||
sha2 = { workspace = true }
|
||||
rustfs-tokio-tar = { workspace = true }
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
zstd.workspace = true
|
||||
time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }
|
||||
suppaftp = { workspace = true, features = ["rustls-aws-lc-rs", "tokio-rustls-aws-lc-rs"] }
|
||||
rcgen.workspace = true
|
||||
local-ip-address.workspace = true
|
||||
anyhow.workspace = true
|
||||
rustls = { workspace = true, default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
|
||||
russh = { workspace = true, features = ["serde"] }
|
||||
russh-sftp = { workspace = true }
|
||||
zip.workspace = true
|
||||
clap = { workspace = true, features = ["derive", "env"] }
|
||||
|
||||
[dev-dependencies]
|
||||
async-compression = { workspace = true, features = ["tokio", "bzip2", "lz4", "xz"] }
|
||||
aws-config = { workspace = true }
|
||||
opentelemetry-proto = { workspace = true }
|
||||
russh = { workspace = true, features = ["serde"] }
|
||||
russh-sftp = { workspace = true }
|
||||
suppaftp = { workspace = true, features = ["rustls-aws-lc-rs", "tokio-rustls-aws-lc-rs"] }
|
||||
rustfs-tokio-tar = { workspace = true }
|
||||
zip.workspace = true
|
||||
same-file.workspace = true
|
||||
aws-smithy-http-client = { workspace = true, default-features = false, features = ["test-util"] }
|
||||
rustfs-heal.workspace = true
|
||||
|
||||
@@ -146,7 +146,11 @@ async fn create_service_account_for(
|
||||
Some(serde_json::json!({ "targetUser": parent }).to_string()),
|
||||
)
|
||||
.await?;
|
||||
let response: serde_json::Value = serde_json::from_str(&response)?;
|
||||
service_account_keys(&response)
|
||||
}
|
||||
|
||||
fn service_account_keys(response: &str) -> Result<(String, String), Box<dyn Error + Send + Sync>> {
|
||||
let response: serde_json::Value = serde_json::from_str(response)?;
|
||||
let access_key = response["credentials"]["accessKey"]
|
||||
.as_str()
|
||||
.ok_or("service account response should contain credentials.accessKey")?
|
||||
@@ -344,6 +348,134 @@ async fn test_update_service_account_enforces_owner_and_parent_scope() -> TestRe
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_delegated_service_account_creation_through_console_sessions() -> TestResult {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_without_cleanup(vec![]).await?;
|
||||
let bucket = "delegated-backups";
|
||||
let target = "backup-user";
|
||||
let operator = "backup-operator";
|
||||
let operator_secret = "operatorSecret123";
|
||||
let wildcard_admin = "backup-admin";
|
||||
let wildcard_secret = "adminSecret123";
|
||||
let s3 = env.create_s3_client();
|
||||
s3.create_bucket().bucket(bucket).send().await?;
|
||||
for (name, secret) in [
|
||||
(target, "backupSecret123"),
|
||||
(operator, operator_secret),
|
||||
(wildcard_admin, wildcard_secret),
|
||||
("other-user", "otherUserSecret123"),
|
||||
] {
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::PUT,
|
||||
&format!("/rustfs/admin/v3/add-user?accessKey={name}"),
|
||||
Some(serde_json::json!({"secretKey":secret,"status":"enabled"}).to_string()),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
let delegation = serde_json::json!({
|
||||
"Version":"2012-10-17",
|
||||
"Statement":[
|
||||
{"Effect":"Allow","Action":"admin:CreateServiceAccount"},
|
||||
{"Effect":"Allow","Action":"sts:AssumeRole","Resource":"arn:aws:s3:::*"},
|
||||
{"Effect":"Allow","Action":"admin:CreateServiceAccountForUser","Condition":{"StringEquals":{"svc:TargetUser":target}}}
|
||||
]
|
||||
});
|
||||
let wildcard = serde_json::json!({
|
||||
"Version":"2012-10-17",
|
||||
"Statement":[
|
||||
{"Effect":"Allow","Action":"admin:*"},
|
||||
{"Effect":"Allow","Action":"sts:AssumeRole","Resource":"arn:aws:s3:::*"}
|
||||
]
|
||||
});
|
||||
for (user, name, policy) in [
|
||||
(wildcard_admin, "backup-admin-wildcard", wildcard.to_string()),
|
||||
(target, "backup-writer", bucket_rw_policy(bucket)),
|
||||
(operator, "backup-key-manager", delegation.to_string()),
|
||||
] {
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::PUT,
|
||||
&format!("/rustfs/admin/v3/add-canned-policy?name={name}"),
|
||||
Some(policy),
|
||||
)
|
||||
.await?;
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::POST,
|
||||
"/rustfs/admin/v3/idp/builtin/policy/attach",
|
||||
Some(serde_json::json!({"policies":[name],"user":user}).to_string()),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
for (name, secret) in [
|
||||
(operator, operator_secret),
|
||||
(wildcard_admin, wildcard_secret),
|
||||
(env.access_key.as_str(), env.secret_key.as_str()),
|
||||
] {
|
||||
let session = build_test_sts_client(&env.url, name, secret, None, "delegated-create")
|
||||
.assume_role()
|
||||
.role_arn("arn:aws:iam::123456789012:role/console")
|
||||
.role_session_name("console")
|
||||
.send()
|
||||
.await?;
|
||||
let caller = session.credentials().ok_or("missing console session")?;
|
||||
let create = |user: &str| {
|
||||
admin_request_with_session_token(
|
||||
&env.url,
|
||||
http::Method::PUT,
|
||||
"/rustfs/admin/v3/add-service-account",
|
||||
Some(serde_json::json!({"targetUser":user}).to_string()),
|
||||
caller.access_key_id(),
|
||||
caller.secret_access_key(),
|
||||
Some(caller.session_token()),
|
||||
)
|
||||
};
|
||||
let (status, response) = create(target).await?;
|
||||
assert_eq!(status, StatusCode::OK, "create for target as {name}");
|
||||
let (key, secret) = service_account_keys(&response)?;
|
||||
let client = build_s3_client(&env.url, &key, &secret);
|
||||
wait_for_s3_put(&client, bucket, "backup", Duration::from_secs(10)).await?;
|
||||
let denied = client
|
||||
.create_bucket()
|
||||
.bucket("outside-backup-policy")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("target cannot create unrelated buckets");
|
||||
assert_eq!(denied.as_service_error().and_then(|e| e.code()), Some("AccessDenied"));
|
||||
if name != env.access_key {
|
||||
let (status, _) = create(&env.access_key).await?;
|
||||
assert_eq!(status, StatusCode::FORBIDDEN, "delegated targets exclude root");
|
||||
}
|
||||
if name == operator {
|
||||
let (status, _) = create("other-user").await?;
|
||||
assert_eq!(status, StatusCode::FORBIDDEN, "outside delegated target");
|
||||
}
|
||||
}
|
||||
let restricted = build_test_sts_client(&env.url, &env.access_key, &env.secret_key, None, "restricted-root")
|
||||
.assume_role()
|
||||
.role_arn("arn:aws:iam::123456789012:role/console")
|
||||
.role_session_name("restricted-console")
|
||||
.policy(r#"{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"admin:CreateServiceAccount"}]}"#)
|
||||
.send()
|
||||
.await?;
|
||||
let caller = restricted.credentials().ok_or("missing restricted root session")?;
|
||||
let (status, _) = admin_request_with_session_token(
|
||||
&env.url,
|
||||
http::Method::PUT,
|
||||
"/rustfs/admin/v3/add-service-account",
|
||||
Some(serde_json::json!({"targetUser":target}).to_string()),
|
||||
caller.access_key_id(),
|
||||
caller.secret_access_key(),
|
||||
Some(caller.session_token()),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(status, StatusCode::FORBIDDEN, "restricted root session requires delegated permission");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Full user -> policy -> service-account lifecycle, proving each management
|
||||
/// call takes effect on the data plane, not just that the endpoint answers 200.
|
||||
#[tokio::test]
|
||||
|
||||
@@ -135,6 +135,72 @@ mod tests {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A comma-separated region list widens the regions `CreateBucket`
|
||||
/// accepts without changing the server's own region (the first entry): a
|
||||
/// listed region creates the bucket, an unlisted one is still rejected,
|
||||
/// and `GetBucketLocation` keeps reporting the site region.
|
||||
#[tokio::test]
|
||||
async fn test_create_bucket_accepts_listed_regions() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec!["--region", "us-east-1,ap-southeast-2,us-west-2"])
|
||||
.await?;
|
||||
|
||||
for region in ["us-east-1", "ap-southeast-2", "us-west-2"] {
|
||||
let client = Client::from_conf(
|
||||
env.create_s3_client()
|
||||
.config()
|
||||
.to_builder()
|
||||
.region(Region::new(region))
|
||||
.build(),
|
||||
);
|
||||
let bucket = format!("additional-region-{region}");
|
||||
client
|
||||
.create_bucket()
|
||||
.bucket(&bucket)
|
||||
.create_bucket_configuration(
|
||||
CreateBucketConfiguration::builder()
|
||||
.location_constraint(BucketLocationConstraint::from(region))
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.expect("a listed region must create the bucket");
|
||||
let location = client.get_bucket_location().bucket(&bucket).send().await?;
|
||||
assert_eq!(
|
||||
location.location_constraint().map(|value| value.as_str()),
|
||||
Some(""),
|
||||
"GetBucketLocation keeps reporting the site region (us-east-1)"
|
||||
);
|
||||
}
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let error = client
|
||||
.create_bucket()
|
||||
.bucket("unlisted-region")
|
||||
.create_bucket_configuration(
|
||||
CreateBucketConfiguration::builder()
|
||||
.location_constraint(BucketLocationConstraint::from("eu-west-1"))
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a region missing from the region list must be rejected");
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(400));
|
||||
assert_eq!(
|
||||
error.as_service_error().and_then(ProvideErrorMetadata::code),
|
||||
Some("InvalidLocationConstraint")
|
||||
);
|
||||
let missing = client
|
||||
.head_bucket()
|
||||
.bucket("unlisted-region")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("rejected bucket must not exist");
|
||||
assert_eq!(missing.raw_response().map(|response| response.status().as_u16()), Some(404));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_create_bucket_uses_configured_region() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -14,7 +14,42 @@
|
||||
|
||||
//! Differential cases of the `bucket_access` batch (see `super`).
|
||||
|
||||
use super::super::Recorder;
|
||||
use super::super::{Recorder, none};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
||||
|
||||
pub(super) async fn cases(_s: &mut Recorder, _c: &Client) {}
|
||||
const BUCKET: &str = "a2-bucket-access";
|
||||
|
||||
pub(super) async fn cases(s: &mut Recorder, c: &Client) {
|
||||
s.step("CreateBucket", "a2-setup", c.create_bucket().bucket(BUCKET).send(), none)
|
||||
.await;
|
||||
versioning(s, c).await;
|
||||
}
|
||||
|
||||
async fn versioning(s: &mut Recorder, c: &Client) {
|
||||
let status = |o: &aws_sdk_s3::operation::get_bucket_versioning::GetBucketVersioningOutput| {
|
||||
vec![
|
||||
("status", format!("{:?}", o.status())),
|
||||
("mfa_delete", format!("{:?}", o.mfa_delete())),
|
||||
]
|
||||
};
|
||||
s.step(
|
||||
"GetBucketVersioning",
|
||||
"unconfigured",
|
||||
c.get_bucket_versioning().bucket(BUCKET).send(),
|
||||
status,
|
||||
)
|
||||
.await;
|
||||
let suspended = VersioningConfiguration::builder()
|
||||
.status(BucketVersioningStatus::Suspended)
|
||||
.build();
|
||||
let put = c.put_bucket_versioning().bucket(BUCKET).versioning_configuration(suspended);
|
||||
s.step("PutBucketVersioning", "a2-suspended", put.send(), none).await;
|
||||
s.step(
|
||||
"GetBucketVersioning",
|
||||
"suspended",
|
||||
c.get_bucket_versioning().bucket(BUCKET).send(),
|
||||
status,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
@@ -14,7 +14,59 @@
|
||||
|
||||
//! Differential cases of the `bucket_core` batch (see `super`).
|
||||
|
||||
use super::super::Recorder;
|
||||
use super::super::{Recorder, none};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::types::{BucketLocationConstraint, CreateBucketConfiguration};
|
||||
|
||||
pub(super) async fn cases(_s: &mut Recorder, _c: &Client) {}
|
||||
/// Names RustFS creates although the AWS grammar reserves them, and one it refuses.
|
||||
const RESERVED_NAMES: [(&str, &str); 5] = [
|
||||
("reserved-sthree", "sthree-dual-stack"),
|
||||
("reserved-s3alias", "dual-stack-s3alias"),
|
||||
("reserved-ol-s3", "dual-stack--ol-s3"),
|
||||
("reserved-x-s3", "dual-stack--x-s3"),
|
||||
("reserved-xn", "xn--dual-stack"),
|
||||
];
|
||||
|
||||
pub(super) async fn cases(s: &mut Recorder, c: &Client) {
|
||||
let located = |bucket: &str, location: &str| {
|
||||
let configuration = CreateBucketConfiguration::builder()
|
||||
.location_constraint(BucketLocationConstraint::from(location))
|
||||
.build();
|
||||
c.create_bucket().bucket(bucket).create_bucket_configuration(configuration)
|
||||
};
|
||||
s.step(
|
||||
"CreateBucket",
|
||||
"foreign-location",
|
||||
located("dual-stack-foreign", "ap-south-1").send(),
|
||||
none,
|
||||
)
|
||||
.await;
|
||||
s.step("CreateBucket", "eu-alias", located("dual-stack-eu", "EU").send(), none)
|
||||
.await;
|
||||
for (case, name) in RESERVED_NAMES {
|
||||
s.step("CreateBucket", case, c.create_bucket().bucket(name).send(), none)
|
||||
.await;
|
||||
}
|
||||
for (case, bucket, header) in FORCE_DELETES {
|
||||
// Each case deletes a bucket of its own; the create is setup, not a recorded step.
|
||||
let _ = c.create_bucket().bucket(bucket).send().await;
|
||||
let delete = c
|
||||
.delete_bucket()
|
||||
.bucket(bucket)
|
||||
.customize()
|
||||
.mutate_request(move |request| {
|
||||
request.headers_mut().insert(header.0, header.1);
|
||||
})
|
||||
.send();
|
||||
s.step("DeleteBucket", case, delete, none).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Force-delete headers under both spellings, with values each parser reads differently.
|
||||
const FORCE_DELETES: [(&str, &str, (&str, &str)); 5] = [
|
||||
("force-rustfs-true", "dual-stack-force-a", ("x-rustfs-force-delete", "true")),
|
||||
("force-rustfs-invalid", "dual-stack-force-b", ("x-rustfs-force-delete", "maybe")),
|
||||
("force-minio-true", "dual-stack-force-c", ("x-minio-force-delete", "true")),
|
||||
("force-minio-numeric", "dual-stack-force-d", ("x-minio-force-delete", "1")),
|
||||
("force-minio-invalid", "dual-stack-force-e", ("x-minio-force-delete", "maybe")),
|
||||
];
|
||||
|
||||
+118
-2
@@ -13,8 +13,124 @@
|
||||
// limitations under the License.
|
||||
|
||||
//! Differential cases of the `bucket_replication_lifecycle` batch (see `super`).
|
||||
//!
|
||||
//! Replication has no case of its own: storing a replication configuration needs a remote target
|
||||
//! registered through the admin API, which this script does not drive.
|
||||
|
||||
use super::super::Recorder;
|
||||
use super::super::{Fields, RETAIN_UNTIL_SECS, Recorder, none};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::primitives::DateTime;
|
||||
use aws_sdk_s3::types::{
|
||||
AbortIncompleteMultipartUpload, BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, LifecycleExpiration,
|
||||
LifecycleRule, LifecycleRuleAndOperator, LifecycleRuleFilter, NoncurrentVersionExpiration, ObjectLockConfiguration,
|
||||
ObjectLockEnabled, ObjectLockRetentionMode, ObjectLockRule, Tag,
|
||||
};
|
||||
|
||||
pub(super) async fn cases(_s: &mut Recorder, _c: &Client) {}
|
||||
/// A name RustFS creates and serves that the AWS bucket grammar reserves.
|
||||
const RESERVED_BUCKET: &str = "sthree-dual-stack-a3";
|
||||
const LOCK_BUCKET: &str = "dual-stack-a3-lock";
|
||||
|
||||
fn lifecycle_rules(
|
||||
output: &aws_sdk_s3::operation::get_bucket_lifecycle_configuration::GetBucketLifecycleConfigurationOutput,
|
||||
) -> Fields {
|
||||
vec![("rules", format!("{:?}", output.rules()))]
|
||||
}
|
||||
|
||||
fn object_lock(output: &aws_sdk_s3::operation::get_object_lock_configuration::GetObjectLockConfigurationOutput) -> Fields {
|
||||
vec![("configuration", format!("{:?}", output.object_lock_configuration()))]
|
||||
}
|
||||
|
||||
pub(super) async fn cases(s: &mut Recorder, c: &Client) {
|
||||
s.step("CreateBucket", "a3-reserved-name", c.create_bucket().bucket(RESERVED_BUCKET).send(), none)
|
||||
.await;
|
||||
let get_lifecycle = || c.get_bucket_lifecycle_configuration().bucket(RESERVED_BUCKET);
|
||||
s.step("GetBucketLifecycleConfiguration", "a3-reserved-absent", get_lifecycle().send(), none)
|
||||
.await;
|
||||
let rule = LifecycleRule::builder()
|
||||
.id("members")
|
||||
.status(ExpirationStatus::Enabled)
|
||||
.filter(
|
||||
LifecycleRuleFilter::builder()
|
||||
.and(
|
||||
LifecycleRuleAndOperator::builder()
|
||||
.prefix("logs/")
|
||||
.tags(Tag::builder().key("tier").value("cold").build().expect("tag"))
|
||||
.object_size_greater_than(1)
|
||||
.build(),
|
||||
)
|
||||
.build(),
|
||||
)
|
||||
.expiration(LifecycleExpiration::builder().expired_object_delete_marker(true).build())
|
||||
.noncurrent_version_expiration(
|
||||
NoncurrentVersionExpiration::builder()
|
||||
.noncurrent_days(7)
|
||||
.newer_noncurrent_versions(2)
|
||||
.build(),
|
||||
)
|
||||
.abort_incomplete_multipart_upload(AbortIncompleteMultipartUpload::builder().days_after_initiation(3).build())
|
||||
.build()
|
||||
.expect("lifecycle rule");
|
||||
// A stored date is answered through a different conversion on each edge of the migration.
|
||||
let dated = LifecycleRule::builder()
|
||||
.id("dated")
|
||||
.status(ExpirationStatus::Disabled)
|
||||
.filter(LifecycleRuleFilter::builder().prefix("archive/").build())
|
||||
.expiration(
|
||||
LifecycleExpiration::builder()
|
||||
.date(DateTime::from_secs(RETAIN_UNTIL_SECS))
|
||||
.build(),
|
||||
)
|
||||
.build()
|
||||
.expect("dated lifecycle rule");
|
||||
let lifecycle = BucketLifecycleConfiguration::builder()
|
||||
.rules(rule)
|
||||
.rules(dated)
|
||||
.build()
|
||||
.expect("lifecycle configuration");
|
||||
let put = c
|
||||
.put_bucket_lifecycle_configuration()
|
||||
.bucket(RESERVED_BUCKET)
|
||||
.lifecycle_configuration(lifecycle);
|
||||
s.step("PutBucketLifecycleConfiguration", "a3-every-member", put.send(), none)
|
||||
.await;
|
||||
s.step(
|
||||
"GetBucketLifecycleConfiguration",
|
||||
"a3-every-member",
|
||||
get_lifecycle().send(),
|
||||
lifecycle_rules,
|
||||
)
|
||||
.await;
|
||||
let get_lock = |bucket: &str| c.get_object_lock_configuration().bucket(bucket);
|
||||
s.step("GetObjectLockConfiguration", "a3-reserved-absent", get_lock(RESERVED_BUCKET).send(), none)
|
||||
.await;
|
||||
|
||||
let create = c.create_bucket().bucket(LOCK_BUCKET).object_lock_enabled_for_bucket(true);
|
||||
s.step("CreateBucket", "a3-lock", create.send(), none).await;
|
||||
s.step(
|
||||
"GetObjectLockConfiguration",
|
||||
"a3-enabled-without-rule",
|
||||
get_lock(LOCK_BUCKET).send(),
|
||||
object_lock,
|
||||
)
|
||||
.await;
|
||||
let lock = ObjectLockConfiguration::builder()
|
||||
.object_lock_enabled(ObjectLockEnabled::Enabled)
|
||||
.rule(
|
||||
ObjectLockRule::builder()
|
||||
.default_retention(
|
||||
DefaultRetention::builder()
|
||||
.mode(ObjectLockRetentionMode::Compliance)
|
||||
.years(1)
|
||||
.build(),
|
||||
)
|
||||
.build(),
|
||||
)
|
||||
.build();
|
||||
let put = c
|
||||
.put_object_lock_configuration()
|
||||
.bucket(LOCK_BUCKET)
|
||||
.object_lock_configuration(lock);
|
||||
s.step("PutObjectLockConfiguration", "a3-years", put.send(), none).await;
|
||||
s.step("GetObjectLockConfiguration", "a3-years", get_lock(LOCK_BUCKET).send(), object_lock)
|
||||
.await;
|
||||
}
|
||||
|
||||
@@ -12,9 +12,99 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Differential cases of the `listing_multipart` batch (see `super`).
|
||||
//! Differential cases of the `listing_multipart` batch (see `super`): stored keys that need
|
||||
//! encoding, with and without `encoding-type=url`, ListObjects' marker, ListObjectsV2 pagination,
|
||||
//! and the listings' parameter refusals.
|
||||
|
||||
use super::super::Recorder;
|
||||
use super::super::{Fields, Recorder, none};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::EncodingType;
|
||||
|
||||
pub(super) async fn cases(_s: &mut Recorder, _c: &Client) {}
|
||||
const BUCKET: &str = "differential-listing";
|
||||
const KEYS: [&str; 3] = ["dir a/file+b%.txt", "markup &<>\"'", "~tilde"];
|
||||
|
||||
fn v2_fields(o: &aws_sdk_s3::operation::list_objects_v2::ListObjectsV2Output) -> Fields {
|
||||
let keys: Vec<_> = o.contents().iter().map(|object| object.key()).collect();
|
||||
vec![
|
||||
("keys", format!("{keys:?}")),
|
||||
("encoding_type", format!("{:?}", o.encoding_type())),
|
||||
("key_count", format!("{:?}", o.key_count())),
|
||||
("is_truncated", format!("{:?}", o.is_truncated())),
|
||||
("has_next_token", format!("{}", o.next_continuation_token().is_some())),
|
||||
]
|
||||
}
|
||||
|
||||
fn v1_fields(o: &aws_sdk_s3::operation::list_objects::ListObjectsOutput) -> Fields {
|
||||
let keys: Vec<_> = o.contents().iter().map(|object| object.key()).collect();
|
||||
vec![
|
||||
("keys", format!("{keys:?}")),
|
||||
("marker", format!("{:?}", o.marker())),
|
||||
("next_marker", format!("{:?}", o.next_marker())),
|
||||
("encoding_type", format!("{:?}", o.encoding_type())),
|
||||
]
|
||||
}
|
||||
|
||||
pub(super) async fn cases(s: &mut Recorder, c: &Client) {
|
||||
s.step("ListObjectsV2", "setup-bucket", c.create_bucket().bucket(BUCKET).send(), none)
|
||||
.await;
|
||||
for (key, case) in KEYS.into_iter().zip(["setup-object-1", "setup-object-2", "setup-object-3"]) {
|
||||
let put = c
|
||||
.put_object()
|
||||
.bucket(BUCKET)
|
||||
.key(key)
|
||||
.body(ByteStream::from_static(b"x"))
|
||||
.send();
|
||||
s.step("ListObjectsV2", case, put, none).await;
|
||||
}
|
||||
|
||||
let v2 = || c.list_objects_v2().bucket(BUCKET);
|
||||
s.step("ListObjectsV2", "raw-keys", v2().send(), v2_fields).await;
|
||||
s.step("ListObjectsV2", "url-encoded", v2().encoding_type(EncodingType::Url).send(), v2_fields)
|
||||
.await;
|
||||
let first = s
|
||||
.step("ListObjectsV2", "first-page", v2().max_keys(2).send(), v2_fields)
|
||||
.await;
|
||||
let token = first.and_then(|page| page.next_continuation_token().map(str::to_owned));
|
||||
s.step("ListObjectsV2", "next-page", v2().set_continuation_token(token).send(), v2_fields)
|
||||
.await;
|
||||
s.step("ListObjectsV2", "negative-max-keys", v2().max_keys(-1).send(), none)
|
||||
.await;
|
||||
s.step("ListObjectsV2", "undecodable-token", v2().continuation_token("%%%").send(), none)
|
||||
.await;
|
||||
|
||||
let v1 = || c.list_objects().bucket(BUCKET).max_keys(1);
|
||||
s.step("ListObjects", "raw-marker", v1().send(), v1_fields).await;
|
||||
s.step(
|
||||
"ListObjects",
|
||||
"url-encoded-marker",
|
||||
v1().encoding_type(EncodingType::Url).send(),
|
||||
v1_fields,
|
||||
)
|
||||
.await;
|
||||
s.step("ListObjects", "request-marker", v1().marker(KEYS[0]).send(), v1_fields)
|
||||
.await;
|
||||
s.step("ListObjects", "negative-max-keys", v1().max_keys(-1).send(), none)
|
||||
.await;
|
||||
|
||||
let create = c.create_multipart_upload().bucket(BUCKET).key(KEYS[0]).send();
|
||||
s.step("ListMultipartUploads", "setup-upload", create, none).await;
|
||||
let uploads = || c.list_multipart_uploads().bucket(BUCKET);
|
||||
s.step("ListMultipartUploads", "raw-keys", uploads().send(), upload_fields)
|
||||
.await;
|
||||
let encoded = uploads().prefix("dir a/").encoding_type(EncodingType::Url).send();
|
||||
s.step("ListMultipartUploads", "url-encoded", encoded, upload_fields).await;
|
||||
s.step("ListMultipartUploads", "zero-max-uploads", uploads().max_uploads(0).send(), none)
|
||||
.await;
|
||||
let foreign = uploads().prefix("dir").key_marker("other").send();
|
||||
s.step("ListMultipartUploads", "foreign-key-marker", foreign, none).await;
|
||||
}
|
||||
|
||||
fn upload_fields(o: &aws_sdk_s3::operation::list_multipart_uploads::ListMultipartUploadsOutput) -> Fields {
|
||||
let keys: Vec<_> = o.uploads().iter().map(|upload| upload.key()).collect();
|
||||
vec![
|
||||
("keys", format!("{keys:?}")),
|
||||
("prefix", format!("{:?}", o.prefix())),
|
||||
("encoding_type", format!("{:?}", o.encoding_type())),
|
||||
]
|
||||
}
|
||||
|
||||
@@ -14,7 +14,106 @@
|
||||
|
||||
//! Differential cases of the `object_meta` batch (see `super`).
|
||||
|
||||
use super::super::Recorder;
|
||||
use super::super::{Recorder, grants, none, tags};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{AccessControlPolicy, Grant, Grantee, Owner, Permission, Type};
|
||||
|
||||
pub(super) async fn cases(_s: &mut Recorder, _c: &Client) {}
|
||||
const BUCKET: &str = "diff-object-meta";
|
||||
const TAGGED: &str = "tagged";
|
||||
|
||||
pub(super) async fn cases(s: &mut Recorder, c: &Client) {
|
||||
s.step("CreateBucket", "object-meta", c.create_bucket().bucket(BUCKET).send(), none)
|
||||
.await;
|
||||
let put = c
|
||||
.put_object()
|
||||
.bucket(BUCKET)
|
||||
.key(TAGGED)
|
||||
.tagging("k=&odd%20key=v%2Fw")
|
||||
.body(ByteStream::from_static(b"object meta"));
|
||||
s.step("PutObject", "object-meta-tagged", put.send(), none).await;
|
||||
|
||||
let get_tags = |key: &str, version: Option<&str>| {
|
||||
c.get_object_tagging()
|
||||
.bucket(BUCKET)
|
||||
.key(key)
|
||||
.set_version_id(version.map(str::to_owned))
|
||||
};
|
||||
s.step("GetObjectTagging", "empty-and-encoded-tags", get_tags(TAGGED, None).send(), |o| {
|
||||
vec![("tags", tags(o.tag_set())), ("version", format!("{:?}", o.version_id()))]
|
||||
})
|
||||
.await;
|
||||
s.step(
|
||||
"GetObjectTagging",
|
||||
"malformed-version",
|
||||
get_tags(TAGGED, Some("not-a-version")).send(),
|
||||
none,
|
||||
)
|
||||
.await;
|
||||
let absent_version = "00000000-0000-0000-0000-000000000001";
|
||||
s.step("GetObjectTagging", "absent-version", get_tags(TAGGED, Some(absent_version)).send(), none)
|
||||
.await;
|
||||
s.step("GetObjectTagging", "nul-key", get_tags("a\0b", None).send(), none)
|
||||
.await;
|
||||
s.step(
|
||||
"DeleteObjectTagging",
|
||||
"nul-key",
|
||||
c.delete_object_tagging().bucket(BUCKET).key("a\0b").send(),
|
||||
none,
|
||||
)
|
||||
.await;
|
||||
s.step(
|
||||
"DeleteObjectTagging",
|
||||
"then-empty",
|
||||
c.delete_object_tagging().bucket(BUCKET).key(TAGGED).send(),
|
||||
none,
|
||||
)
|
||||
.await;
|
||||
s.step("GetObjectTagging", "after-delete", get_tags(TAGGED, None).send(), |o| {
|
||||
vec![("tags", tags(o.tag_set()))]
|
||||
})
|
||||
.await;
|
||||
|
||||
let grantee = Grantee::builder()
|
||||
.r#type(Type::CanonicalUser)
|
||||
.id("owner")
|
||||
.build()
|
||||
.expect("grantee");
|
||||
let policy = AccessControlPolicy::builder()
|
||||
.owner(Owner::builder().id("owner").build())
|
||||
.grants(Grant::builder().grantee(grantee).permission(Permission::Read).build())
|
||||
.build();
|
||||
let xml_grants = c.put_object_acl().bucket(BUCKET).key(TAGGED).access_control_policy(policy);
|
||||
s.step("PutObjectAcl", "xml-grants", xml_grants.send(), none).await;
|
||||
s.step(
|
||||
"GetObjectAcl",
|
||||
"after-xml-grants",
|
||||
c.get_object_acl().bucket(BUCKET).key(TAGGED).send(),
|
||||
|o| vec![("grants", grants(o.grants()))],
|
||||
)
|
||||
.await;
|
||||
s.step("GetObjectAcl", "nul-key", c.get_object_acl().bucket(BUCKET).key("a\0b").send(), none)
|
||||
.await;
|
||||
|
||||
s.step(
|
||||
"GetObjectLegalHold",
|
||||
"no-object-lock",
|
||||
c.get_object_legal_hold().bucket(BUCKET).key(TAGGED).send(),
|
||||
none,
|
||||
)
|
||||
.await;
|
||||
s.step(
|
||||
"GetObjectRetention",
|
||||
"no-object-lock",
|
||||
c.get_object_retention().bucket(BUCKET).key(TAGGED).send(),
|
||||
none,
|
||||
)
|
||||
.await;
|
||||
s.step(
|
||||
"GetObjectTorrent",
|
||||
"nul-key",
|
||||
c.get_object_torrent().bucket(BUCKET).key("a\0b").send(),
|
||||
none,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
@@ -14,7 +14,50 @@
|
||||
|
||||
//! Differential cases of the `object_misc` batch (see `super`).
|
||||
|
||||
use super::super::Recorder;
|
||||
use super::super::{Recorder, none};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
||||
|
||||
pub(super) async fn cases(_s: &mut Recorder, _c: &Client) {}
|
||||
const BUCKET: &str = "dual-stack-object-misc";
|
||||
const KEY: &str = "doomed.txt";
|
||||
|
||||
/// DeleteObject on a versioned bucket: the delete marker and its headers, the delete of that
|
||||
/// marker by version, and the refusals the delete raises before storage.
|
||||
pub(super) async fn cases(s: &mut Recorder, c: &Client) {
|
||||
s.step("CreateBucket", "object-misc-setup", c.create_bucket().bucket(BUCKET).send(), none)
|
||||
.await;
|
||||
let enabled = VersioningConfiguration::builder()
|
||||
.status(BucketVersioningStatus::Enabled)
|
||||
.build();
|
||||
let versioning = c.put_bucket_versioning().bucket(BUCKET).versioning_configuration(enabled);
|
||||
s.step("PutBucketVersioning", "object-misc-setup", versioning.send(), none)
|
||||
.await;
|
||||
let put = c.put_object().bucket(BUCKET).key(KEY).body(ByteStream::from_static(b"e2"));
|
||||
s.step("PutObject", "object-misc-setup", put.send(), none).await;
|
||||
|
||||
let marker = s
|
||||
.step(
|
||||
"DeleteObject",
|
||||
"creates-delete-marker",
|
||||
c.delete_object().bucket(BUCKET).key(KEY).send(),
|
||||
none,
|
||||
)
|
||||
.await
|
||||
.and_then(|output| output.version_id().map(str::to_owned));
|
||||
if let Some(marker) = marker {
|
||||
let delete = c.delete_object().bucket(BUCKET).key(KEY).version_id(marker);
|
||||
s.step("DeleteObject", "removes-delete-marker", delete.send(), none).await;
|
||||
}
|
||||
let delete = c.delete_object().bucket(BUCKET).key(KEY).version_id("not-a-version");
|
||||
s.step("DeleteObject", "malformed-version", delete.send(), none).await;
|
||||
let delete = c
|
||||
.delete_object()
|
||||
.bucket(BUCKET)
|
||||
.key(KEY)
|
||||
.customize()
|
||||
.mutate_request(|request| {
|
||||
request.headers_mut().insert("x-rustfs-force-delete", "maybe");
|
||||
});
|
||||
s.step("DeleteObject", "invalid-force-delete", delete.send(), none).await;
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
|
||||
use async_trait::async_trait;
|
||||
use rustfs_lock::{
|
||||
LockClient, LockError, LockId, LockInfo, LockRequest, LockResponse, LockStats, LockStatus, LockType, Result,
|
||||
CacheFenceStamp, LockClient, LockError, LockId, LockInfo, LockRequest, LockResponse, LockStats, LockStatus, LockType, Result,
|
||||
types::{LockMetadata, LockPriority},
|
||||
};
|
||||
use rustfs_protos::proto_gen::node_service::{BatchGenerallyLockRequest, GenerallyLockRequest, PingRequest};
|
||||
@@ -64,6 +64,7 @@ impl GrpcLockClient {
|
||||
deadlock_detection: false,
|
||||
suppress_contention_logs: false,
|
||||
refresh_interval: None,
|
||||
cache_fence_stamp_requested: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -124,10 +125,15 @@ impl LockClient for GrpcLockClient {
|
||||
|
||||
// Check if the lock acquisition was successful
|
||||
if resp.success {
|
||||
Ok(LockResponse::success(
|
||||
Self::build_lock_info(request, resp.lock_info),
|
||||
std::time::Duration::ZERO,
|
||||
))
|
||||
let info = Self::build_lock_info(request, resp.lock_info);
|
||||
match resp
|
||||
.cache_fence_stamp
|
||||
.as_deref()
|
||||
.and_then(|encoded| CacheFenceStamp::decode(encoded).ok())
|
||||
{
|
||||
Some(stamp) => Ok(LockResponse::success_with_cache_fence_stamp(info, std::time::Duration::ZERO, stamp)),
|
||||
None => Ok(LockResponse::success(info, std::time::Duration::ZERO)),
|
||||
}
|
||||
} else {
|
||||
// Lock acquisition failed
|
||||
Ok(LockResponse::failure(
|
||||
@@ -160,7 +166,15 @@ impl LockClient for GrpcLockClient {
|
||||
.enumerate()
|
||||
.map(|(idx, request)| match resp.results.get(idx) {
|
||||
Some(result) if result.success => {
|
||||
LockResponse::success(Self::build_lock_info(request, result.lock_info.clone()), std::time::Duration::ZERO)
|
||||
let info = Self::build_lock_info(request, result.lock_info.clone());
|
||||
match result
|
||||
.cache_fence_stamp
|
||||
.as_deref()
|
||||
.and_then(|encoded| CacheFenceStamp::decode(encoded).ok())
|
||||
{
|
||||
Some(stamp) => LockResponse::success_with_cache_fence_stamp(info, std::time::Duration::ZERO, stamp),
|
||||
None => LockResponse::success(info, std::time::Duration::ZERO),
|
||||
}
|
||||
}
|
||||
Some(result) => LockResponse::failure(
|
||||
result
|
||||
|
||||
@@ -22,10 +22,11 @@ use rustfs_protos::{
|
||||
models::PingBodyBuilder,
|
||||
proto_gen::node_service::{
|
||||
BatchGenerallyLockRequest, BatchGenerallyLockResponse, BatchReadVersionRequest, BatchReadVersionResponse,
|
||||
GenerallyLockRequest, GenerallyLockResponse, GenerallyLockResult, PingRequest, PingResponse,
|
||||
ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseReleaseResponse, ScannerPublicationLeaseRequest,
|
||||
ScannerPublicationLeaseResponse, SnapshotLeaseMutationResponse, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest,
|
||||
SnapshotLeaseRequest, SnapshotLeaseResponse, node_service_server::NodeService,
|
||||
GenerallyLockRequest, GenerallyLockResponse, GenerallyLockResult, MrfLifecycleRequest, MrfLifecycleResponse, PingRequest,
|
||||
PingResponse, ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseReleaseResponse,
|
||||
ScannerPublicationLeaseRequest, ScannerPublicationLeaseResponse, SnapshotLeaseMutationResponse,
|
||||
SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest, SnapshotLeaseResponse,
|
||||
node_service_server::NodeService,
|
||||
},
|
||||
};
|
||||
use std::pin::Pin;
|
||||
@@ -42,6 +43,7 @@ fn lock_result_from_response(response: rustfs_lock::LockResponse) -> GenerallyLo
|
||||
success: response.success,
|
||||
error_info: response.error,
|
||||
lock_info: response.lock_info.and_then(|info| serde_json::to_string(&info).ok()),
|
||||
cache_fence_stamp: response.cache_fence_stamp.map(|stamp| stamp.encode().to_vec().into()),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
@@ -179,6 +181,7 @@ impl NodeService for MinimalLockNodeService {
|
||||
success: result.success,
|
||||
error_info: result.error,
|
||||
lock_info: lock_info_json,
|
||||
cache_fence_stamp: result.cache_fence_stamp.map(|stamp| stamp.encode().to_vec().into()),
|
||||
..Default::default()
|
||||
}))
|
||||
}
|
||||
@@ -438,6 +441,13 @@ impl NodeService for MinimalLockNodeService {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn write_format_control(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::WriteFormatControlRequest>,
|
||||
) -> Result<Response<rustfs_protos::proto_gen::node_service::WriteAllResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn compare_and_update_file(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::CompareAndUpdateFileRequest>,
|
||||
@@ -995,6 +1005,10 @@ impl NodeService for MinimalLockNodeService {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn mrf_lifecycle(&self, _request: Request<MrfLifecycleRequest>) -> Result<Response<MrfLifecycleResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn background_heal_status(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::BackgroundHealStatusRequest>,
|
||||
|
||||
@@ -412,6 +412,7 @@ struct ManualTransitionRunReport {
|
||||
skipped_delete_marker: u64,
|
||||
skipped_directory: u64,
|
||||
skipped_replication: u64,
|
||||
skipped_already_transitioned: u64,
|
||||
skipped_already_in_flight: u64,
|
||||
skipped_queue_full: u64,
|
||||
skipped_queue_closed: u64,
|
||||
@@ -1214,9 +1215,16 @@ async fn test_manual_transition_run_black_box_semantics() -> TestResult {
|
||||
assert!(!due.report.dry_run);
|
||||
assert!(due.report.lifecycle_config_found);
|
||||
assert_eq!(due.report.scanned, 1, "due report: {:#?}", due.report);
|
||||
assert_eq!(due.report.eligible, 1, "due report: {:#?}", due.report);
|
||||
// PUT can enqueue this object before the manual scan. Account for each
|
||||
// terminal outcome without depending on how fast the tier worker runs.
|
||||
assert_eq!(
|
||||
due.report.enqueued + due.report.skipped_already_in_flight,
|
||||
due.report.eligible + due.report.skipped_already_transitioned,
|
||||
1,
|
||||
"due report: {:#?}",
|
||||
due.report
|
||||
);
|
||||
assert_eq!(
|
||||
due.report.enqueued + due.report.skipped_already_in_flight + due.report.skipped_already_transitioned,
|
||||
1,
|
||||
"due report: {:#?}",
|
||||
due.report
|
||||
@@ -1225,12 +1233,34 @@ async fn test_manual_transition_run_black_box_semantics() -> TestResult {
|
||||
assert_eq!(due.report.skipped_delete_marker, 0);
|
||||
assert_eq!(due.report.skipped_directory, 0);
|
||||
assert_eq!(due.report.skipped_replication, 0);
|
||||
assert_eq!(due.report.skipped_queue_full, 0);
|
||||
assert_eq!(due.report.skipped_queue_closed, 0);
|
||||
assert_eq!(due.report.skipped_queue_timeout, 0);
|
||||
assert_eq!(due.report.tier_failure, 0);
|
||||
assert!(!due.report.cancelled);
|
||||
assert!(!due.report.truncated_by_limit);
|
||||
assert!(!due.report.truncated_by_duration);
|
||||
wait_for_transition(&hot_client, MANUAL_DUE_BUCKET, MANUAL_DUE_KEY, StdDuration::from_secs(90)).await?;
|
||||
let remote_count_after_due = cold_tier_object_count(&cold_client).await?;
|
||||
assert_eq!(remote_count_after_due, 1, "due object must have exactly one remote copy");
|
||||
|
||||
// Once HEAD observes the committed transition, a repeated manual scan must
|
||||
// report the completed object and leave both local reads and the tier intact.
|
||||
let repeated = manual_transition_run(&hot, MANUAL_DUE_BUCKET, "manual-due/", false).await?;
|
||||
assert_eq!(repeated.state, "completed", "repeated report: {:#?}", repeated.report);
|
||||
assert_eq!(repeated.report.scanned, 1, "repeated report: {:#?}", repeated.report);
|
||||
assert_eq!(repeated.report.eligible, 0, "repeated report: {:#?}", repeated.report);
|
||||
assert_eq!(repeated.report.enqueued, 0, "repeated report: {:#?}", repeated.report);
|
||||
assert_eq!(repeated.report.skipped_already_transitioned, 1);
|
||||
assert_eq!(repeated.report.skipped_already_in_flight, 0);
|
||||
assert_eq!(cold_tier_object_count(&cold_client).await?, remote_count_after_due);
|
||||
let due_object = hot_client
|
||||
.get_object()
|
||||
.bucket(MANUAL_DUE_BUCKET)
|
||||
.key(MANUAL_DUE_KEY)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(due_object.body.collect().await?.into_bytes().as_ref(), b"manual due object");
|
||||
|
||||
hot_client.create_bucket().bucket(MANUAL_DRY_RUN_BUCKET).send().await?;
|
||||
enable_bucket_versioning(&hot_client, MANUAL_DRY_RUN_BUCKET).await?;
|
||||
@@ -2286,6 +2316,14 @@ async fn test_manual_transition_run_continuation_token_resumes_without_raw_marke
|
||||
)
|
||||
.await?;
|
||||
|
||||
// Configuration updates enqueue existing due objects. Observe their commit
|
||||
// before paging so eligibility stays stable across the source restart.
|
||||
for idx in 0..2 {
|
||||
let key = format!("{MANUAL_CONTINUATION_PREFIX}obj-{idx:02}");
|
||||
wait_for_transition(&hot_client, MANUAL_CONTINUATION_BUCKET, &key, StdDuration::from_secs(90)).await?;
|
||||
}
|
||||
assert_eq!(cold_tier_object_count(&cold_client).await?, 2);
|
||||
|
||||
let first = manual_transition_run_with_max(&hot, MANUAL_CONTINUATION_BUCKET, MANUAL_CONTINUATION_PREFIX, true, 1).await?;
|
||||
assert_eq!(first.state, "partial", "first continuation page: {first:#?}");
|
||||
assert_eq!(first.mode, "enqueue_only");
|
||||
@@ -2293,8 +2331,10 @@ async fn test_manual_transition_run_continuation_token_resumes_without_raw_marke
|
||||
assert_eq!(first.report.prefix, MANUAL_CONTINUATION_PREFIX);
|
||||
assert!(first.report.dry_run);
|
||||
assert_eq!(first.report.scanned, 1, "first continuation page: {first:#?}");
|
||||
assert_eq!(first.report.eligible, 1, "first continuation page: {first:#?}");
|
||||
assert_eq!(first.report.dry_run_eligible, 1, "first continuation page: {first:#?}");
|
||||
assert_eq!(first.report.eligible, 0, "first continuation page: {first:#?}");
|
||||
assert_eq!(first.report.dry_run_eligible, 0, "first continuation page: {first:#?}");
|
||||
assert_eq!(first.report.skipped_already_transitioned, 1);
|
||||
assert_eq!(first.report.enqueued, 0);
|
||||
assert_eq!(first.report.tier_failure, 0);
|
||||
assert!(!first.report.cancelled);
|
||||
assert!(first.report.truncated_by_limit);
|
||||
@@ -2321,12 +2361,15 @@ async fn test_manual_transition_run_continuation_token_resumes_without_raw_marke
|
||||
.await?;
|
||||
assert_eq!(second.state, "completed", "second continuation page: {second:#?}");
|
||||
assert_eq!(second.report.scanned, 1, "second continuation page: {second:#?}");
|
||||
assert_eq!(second.report.eligible, 1, "second continuation page: {second:#?}");
|
||||
assert_eq!(second.report.dry_run_eligible, 1, "second continuation page: {second:#?}");
|
||||
assert_eq!(second.report.eligible, 0, "second continuation page: {second:#?}");
|
||||
assert_eq!(second.report.dry_run_eligible, 0, "second continuation page: {second:#?}");
|
||||
assert_eq!(second.report.skipped_already_transitioned, 1);
|
||||
assert_eq!(second.report.enqueued, 0);
|
||||
assert_eq!(second.report.tier_failure, 0);
|
||||
assert!(!second.report.cancelled);
|
||||
assert!(!second.report.truncated_by_limit);
|
||||
assert!(second.report.continuation_token.is_none());
|
||||
assert_eq!(cold_tier_object_count(&cold_client).await?, 2);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -126,6 +126,7 @@ test-util = []
|
||||
e2e-test-hooks = []
|
||||
|
||||
[dependencies]
|
||||
slab.workspace = true
|
||||
hashbrown.workspace = true
|
||||
hotpath.workspace = true
|
||||
rustfs-filemeta.workspace = true
|
||||
|
||||
@@ -194,8 +194,8 @@ pub mod bucket {
|
||||
table_catalog_path_hash, unreadable_config_refusal,
|
||||
};
|
||||
pub use crate::bucket::metadata::{
|
||||
BUCKET_DURABILITY_CONFIG, BUCKET_ON_DEMAND_MIGRATION_CONFIG, MAX_SWIFT_METADATA_COUNT, MAX_SWIFT_METADATA_VALUE_SIZE,
|
||||
SWIFT_ACCOUNT_META_TAG_PREFIX, SWIFT_CONTAINER_META_TAG_PREFIX, SWIFT_TAGGING_MARKER,
|
||||
BUCKET_DURABILITY_CONFIG, BUCKET_ETAG_CONFIG, BUCKET_ON_DEMAND_MIGRATION_CONFIG, MAX_SWIFT_METADATA_COUNT,
|
||||
MAX_SWIFT_METADATA_VALUE_SIZE, SWIFT_ACCOUNT_META_TAG_PREFIX, SWIFT_CONTAINER_META_TAG_PREFIX, SWIFT_TAGGING_MARKER,
|
||||
};
|
||||
pub use crate::bucket::metadata::{MAX_BUCKET_TAGS, is_swift_bucket_metadata_tag_key, validate_tag_set_entries};
|
||||
}
|
||||
@@ -208,10 +208,21 @@ pub mod bucket {
|
||||
|
||||
pub mod durability {
|
||||
pub use crate::bucket::durability::{
|
||||
BUCKET_DURABILITY_MODE_NONE, BUCKET_DURABILITY_MODE_RELAXED, BUCKET_DURABILITY_MODE_STRICT, BucketDurabilityConfig,
|
||||
BUCKET_DURABILITY_MODE_FILE_SYNC, BUCKET_DURABILITY_MODE_NONE, BUCKET_DURABILITY_MODE_RELAXED,
|
||||
BUCKET_DURABILITY_MODE_STRICT, BucketDurabilityConfig,
|
||||
};
|
||||
}
|
||||
|
||||
pub mod etag {
|
||||
pub use crate::bucket::etag::{
|
||||
BUCKET_ETAG_ALGORITHM_MD5, BUCKET_ETAG_ALGORITHM_XXH3, BucketEtagConfig, bucket_uses_xxh3_etag,
|
||||
};
|
||||
}
|
||||
|
||||
pub mod lockless_cache {
|
||||
pub use crate::bucket::lockless_cache::is_lockless_cache_bucket;
|
||||
}
|
||||
|
||||
pub mod metadata_sys {
|
||||
pub use crate::bucket::metadata_sys::{
|
||||
BUCKET_CONFIG_PUBLISH_HOOK, BucketConfigPublishHook, BucketMetadataMutationGuard, BucketMetadataSys,
|
||||
@@ -220,7 +231,7 @@ pub mod bucket {
|
||||
acquire_scanner_bucket_incarnation_read_fence, capture_bucket_metadata_incarnation, delete, delete_if_incarnation,
|
||||
delete_if_incarnation_at, delete_under_transaction_lock, get, get_accelerate_config, get_bucket_policy,
|
||||
get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
|
||||
get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
|
||||
get_etag_config, get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
|
||||
get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config,
|
||||
get_on_demand_migration_config_in, get_public_access_block_config, get_quota_config, get_replication_config,
|
||||
get_request_payment_config, get_s3_tagging_config, get_sse_config, get_tagging_config, get_versioning_config,
|
||||
@@ -382,7 +393,8 @@ pub mod cluster {
|
||||
|
||||
pub mod compression {
|
||||
pub use crate::io_support::compress::{
|
||||
MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible, is_disk_compression_enabled, is_multipart_disk_compression_enabled,
|
||||
DISK_COMPRESSION_EXCLUDED_CONTENT_TYPES, MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible, is_disk_compression_enabled,
|
||||
is_multipart_disk_compression_enabled,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -492,7 +504,7 @@ pub mod disk {
|
||||
QUOTA_MUTATION_FENCE_METADATA_SUFFIX, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_TMP_BUCKET,
|
||||
ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, SnapshotLeaseToken,
|
||||
UpdateMetadataOpts, VolumeInfo, WalkDirOptions, is_quota_mutation_fence_path, new_disk,
|
||||
validate_batch_read_version_item_count,
|
||||
validate_batch_read_version_item_count, validate_format_control_path,
|
||||
};
|
||||
pub use crate::disk::{
|
||||
BackgroundMemoryBudget, PhysicalDiskBinding, PhysicalDiskPlan, PhysicalDiskSample, PhysicalDiskSampler,
|
||||
@@ -555,6 +567,10 @@ pub mod runtime {
|
||||
global_lock_client, global_lock_clients, global_tier_config_mgr, local_disk_map_read, object_store_handle, region,
|
||||
rustfs_port, setup_is_dist_erasure, setup_is_erasure, setup_is_erasure_sd, transition_state_handle,
|
||||
};
|
||||
pub use crate::runtime::usage_watch::{
|
||||
SCANNER_USAGE_WATCH_COVERAGE_VERSION, SCANNER_USAGE_WATCH_LEASE_MS, ScannerUsageWatchKnown, ScannerUsageWatchObservation,
|
||||
ScannerUsageWatchRequest,
|
||||
};
|
||||
}
|
||||
|
||||
pub mod layout {
|
||||
@@ -594,9 +610,14 @@ pub mod integrity {
|
||||
}
|
||||
|
||||
pub mod object {
|
||||
pub use crate::erasure::codec::shard_buf::{
|
||||
ArenaBlock, RegisteredSlabArena, ShardArena, ShardBuf, install_shard_arena, shard_arena,
|
||||
};
|
||||
pub use crate::erasure::codec::shard_emit::ShardEmitter;
|
||||
pub use crate::object_api::{
|
||||
BLOCK_SIZE_V2, ERASURE_ALGORITHM, EncryptionResolutionError, EncryptionResolutionErrorKind, GetObjectBodyCacheHook,
|
||||
GetObjectBodyCacheHookLookup, GetObjectBodySource, GetObjectReadCondition, GetObjectReadMetadata, GetObjectReader,
|
||||
GetObjectSink, GetObjectSinkSegmentFactory, GetObjectSinkSegments, GetObjectSinkStream, GetObjectSinkWriter,
|
||||
NamespaceLockFence, ObjectEncryptionResolver, ObjectInfo, ObjectLockConfigSnapshot, ObjectMutationHook, ObjectOptions,
|
||||
PutObjReader, QuotaAdmission, RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest,
|
||||
SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, ScannerPublicationCommitScope, ScannerPublicationCommitStartError,
|
||||
@@ -632,12 +653,31 @@ pub mod rebalance {
|
||||
|
||||
pub mod rio {
|
||||
pub use crate::io_support::rio::{
|
||||
DecryptReader, DynReader, EncryptReader, HardLimitReader, HashReader, ReadStream, Reader, WriteEncryption, WritePlan,
|
||||
boxed_reader, compression_metadata_value, validate_compression_scheme, wrap_reader,
|
||||
DecryptReader, DynReader, EncryptReader, EtagAlgorithm, HardLimitReader, HashReader, ReadStream, Reader, WriteEncryption,
|
||||
WritePlan, boxed_reader, compression_metadata_value, validate_compression_scheme, wrap_reader,
|
||||
};
|
||||
}
|
||||
|
||||
pub mod rpc {
|
||||
#[cfg(all(feature = "test-util", feature = "internode-transport"))]
|
||||
pub mod test_util {
|
||||
/// Build a remote disk using a controlled transport and no health monitor.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns the normal remote-disk construction error for the supplied endpoint.
|
||||
pub async fn remote_disk_with_transport(
|
||||
endpoint: &crate::layout::endpoint::Endpoint,
|
||||
transport: std::sync::Arc<dyn super::InternodeDataTransport>,
|
||||
) -> crate::disk::error::Result<crate::disk::DiskStore> {
|
||||
let options = crate::disk::DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
};
|
||||
let disk = crate::cluster::rpc::remote_disk::RemoteDisk::new(endpoint, &options, transport).await?;
|
||||
Ok(std::sync::Arc::new(crate::disk::Disk::Remote(Box::new(disk))))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "internode-transport")]
|
||||
pub use crate::cluster::rpc::http_auth::ExternalStreamAuth;
|
||||
pub use crate::cluster::rpc::http_auth::with_rpc_background_request;
|
||||
@@ -646,6 +686,15 @@ pub mod rpc {
|
||||
InternodeDataTransport, InternodeDataTransportCapabilities, NsScannerCapabilityRequest, NsScannerStreamRequest,
|
||||
ReadStreamRequest, TcpHttpInternodeDataTransport, WalkDirStreamRequest, WriteStreamRequest,
|
||||
};
|
||||
/// Internode shard reads over NIXL: the receive-pool seam the binary fills
|
||||
/// and the wire names its serving handler shares with the transport.
|
||||
pub mod nixl {
|
||||
pub use crate::cluster::rpc::nixl_internode::{
|
||||
NIXL_AGENT_QUERY, NIXL_BUFFER_QUERY, NIXL_BUFFERS_QUERY, NIXL_BYTES_HEADER, NIXL_METADATA_HEADER,
|
||||
NIXL_METADATA_SHA256_QUERY, NIXL_NONCE_QUERY, NixlRecvBuffer, NixlRecvPool, READ_FILE_NIXL_PATH,
|
||||
WRITE_FILE_NIXL_PATH, install_nixl_recv_pool,
|
||||
};
|
||||
}
|
||||
pub use crate::cluster::rpc::{
|
||||
AuthenticatedChannel, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, MAX_NETWORK_PROBE_BYTES, MAX_NETWORK_PROBE_DURATION,
|
||||
NetworkPeerProbeClient, NetworkPeerProbeError, NetworkPeerProbeMeasurement, NetworkPeerTarget, PEER_RESTDRY_RUN,
|
||||
@@ -657,13 +706,17 @@ pub mod rpc {
|
||||
gen_tonic_replay_scope_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor,
|
||||
node_service_time_out_client, node_service_time_out_client_no_auth, normalize_tonic_rpc_audience,
|
||||
rpc_auth_is_resource_exhausted, rpc_background_admission, set_tonic_canonical_body_digest, sign_ns_scanner_capability,
|
||||
sign_ns_scanner_capability_with_checkpoint_coverage, sign_ns_scanner_capability_with_tier_registry_generation,
|
||||
sign_put_file_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
|
||||
tonic_rpc_auth_failure_reason, verify_ns_scanner_capability, verify_ns_scanner_capability_with_checkpoint_coverage,
|
||||
verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer, verify_put_file_capability,
|
||||
verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
|
||||
verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned, verify_tonic_rpc_response_proof,
|
||||
verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
|
||||
sign_ns_scanner_capability_with_checkpoint_coverage, sign_ns_scanner_capability_with_durable_progress,
|
||||
sign_ns_scanner_capability_with_tier_registry_generation, sign_ns_scanner_capability_with_usage_capabilities,
|
||||
sign_ns_scanner_usage_watch_capability, sign_put_file_capability, sign_tonic_rpc_response_proof,
|
||||
tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_boot_epoch_response_headers_with_msgpack_capability,
|
||||
tonic_msgpack_manifest_capability_requested, tonic_rpc_auth_failure_reason, verify_ns_scanner_capability,
|
||||
verify_ns_scanner_capability_with_checkpoint_coverage, verify_ns_scanner_capability_with_durable_progress,
|
||||
verify_ns_scanner_capability_with_tier_registry_generation, verify_ns_scanner_capability_with_usage_capabilities,
|
||||
verify_ns_scanner_usage_watch_capability, verify_put_file_auth_trailer, verify_put_file_capability, verify_rpc_signature,
|
||||
verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest,
|
||||
verify_tonic_mutation_body_digest_reject_unsigned, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
|
||||
verify_tonic_rpc_signature_with_bootstrap,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -26,6 +26,8 @@ use crate::bucket::target::BucketTargetType;
|
||||
use crate::bucket::target::{self, BucketTarget, BucketTargets, Credentials};
|
||||
use crate::bucket::versioning_sys::BucketVersioningSys;
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use aws_sdk_s3::config::interceptors::BeforeDeserializationInterceptorContextRef;
|
||||
use aws_sdk_s3::config::{ConfigBag, Intercept, RuntimeComponents};
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::error::SdkError;
|
||||
use aws_sdk_s3::operation::complete_multipart_upload::CompleteMultipartUploadOutput;
|
||||
@@ -53,8 +55,9 @@ use http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
|
||||
use reqwest::Client as HttpClient;
|
||||
use rustfs_utils::http::{
|
||||
AMZ_BUCKET_REPLICATION_STATUS, AMZ_OBJECT_LOCK_BYPASS_GOVERNANCE, AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_MODE,
|
||||
AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_OBJECT_TAGGING_LOWER, AMZ_STORAGE_CLASS, AMZ_WEBSITE_REDIRECT_LOCATION, is_amz_header,
|
||||
is_minio_header, is_rustfs_header, is_standard_header, is_storageclass_header,
|
||||
AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_OBJECT_TAGGING, AMZ_OBJECT_TAGGING_LOWER, AMZ_STORAGE_CLASS,
|
||||
AMZ_TAGGING_DIRECTIVE_LOWER, AMZ_WEBSITE_REDIRECT_LOCATION, is_amz_header, is_minio_header, is_rustfs_header,
|
||||
is_standard_header, is_storageclass_header,
|
||||
};
|
||||
use rustfs_utils::http::{
|
||||
SUFFIX_FORCE_DELETE, SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_ETAG, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_PROXY_REQUEST,
|
||||
@@ -132,6 +135,36 @@ impl From<&BucketTarget> for RemoteS3EndpointSpec {
|
||||
|
||||
pub type HeadObjectSdkError = Box<SdkError<HeadObjectError>>;
|
||||
|
||||
/// Captures the `x-amz-tagging` response header of a replication HEAD, which
|
||||
/// the SDK drops while deserializing `HeadObjectOutput`.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
struct HeadObjectTaggingCapture(Arc<std::sync::Mutex<Option<String>>>);
|
||||
|
||||
impl HeadObjectTaggingCapture {
|
||||
fn take(&self) -> Option<String> {
|
||||
self.0.lock().unwrap_or_else(std::sync::PoisonError::into_inner).take()
|
||||
}
|
||||
}
|
||||
|
||||
impl Intercept for HeadObjectTaggingCapture {
|
||||
fn name(&self) -> &'static str {
|
||||
"HeadObjectTaggingCapture"
|
||||
}
|
||||
|
||||
fn read_before_deserialization(
|
||||
&self,
|
||||
context: &BeforeDeserializationInterceptorContextRef<'_>,
|
||||
_runtime_components: &RuntimeComponents,
|
||||
_cfg: &mut ConfigBag,
|
||||
) -> Result<(), aws_sdk_s3::error::BoxError> {
|
||||
// Overwrite on every attempt so a retried HEAD reports the tags of the
|
||||
// response that was actually deserialized.
|
||||
*self.0.lock().unwrap_or_else(std::sync::PoisonError::into_inner) =
|
||||
context.response().headers().get(AMZ_OBJECT_TAGGING_LOWER).map(str::to_string);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether an edited bucket target still addresses the same remote service
|
||||
/// (endpoint, bucket, path style, TLS and identity), so a verdict learned
|
||||
/// about that service stays valid across the edit.
|
||||
@@ -2100,12 +2133,19 @@ impl TargetClient {
|
||||
// object with an identical ETag, and the worker concludes the object
|
||||
// already converged — so it never actually replicates it.
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_PROXY_REQUEST, "false");
|
||||
self.client
|
||||
// Ask for the tag set itself (MinIO `x-amz-tagging-directive: ACCESS`):
|
||||
// a plain HEAD reports only the tag count, which cannot tell the
|
||||
// replica's tags from the source's.
|
||||
headers.insert(HeaderName::from_static(AMZ_TAGGING_DIRECTIVE_LOWER), HeaderValue::from_static("ACCESS"));
|
||||
let target_tagging = HeadObjectTaggingCapture::default();
|
||||
let mut output = self
|
||||
.client
|
||||
.head_object()
|
||||
.bucket(bucket)
|
||||
.key(object)
|
||||
.set_version_id(resolve_read_api_version_id(version_id))
|
||||
.customize()
|
||||
.interceptor(target_tagging.clone())
|
||||
.map_request(move |mut req| {
|
||||
for (k, v) in headers.clone().into_iter() {
|
||||
if let Some(key_str) = k.map(|k| k.as_str().to_string()) {
|
||||
@@ -2117,7 +2157,17 @@ impl TargetClient {
|
||||
})
|
||||
.send()
|
||||
.await
|
||||
.map_err(Box::new)
|
||||
.map_err(Box::new)?;
|
||||
// The SDK's typed output has no field for the tag set; carry it where
|
||||
// the replication comparison reads it. A target that ignores the
|
||||
// directive leaves the key absent, which compares as "tags unknown".
|
||||
if let Some(tagging) = target_tagging.take() {
|
||||
output
|
||||
.metadata
|
||||
.get_or_insert_default()
|
||||
.insert(AMZ_OBJECT_TAGGING.to_string(), tagging);
|
||||
}
|
||||
Ok(output)
|
||||
}
|
||||
|
||||
/// Candidate replicas by content identity on a target that mints its own
|
||||
@@ -4556,6 +4606,62 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct TaggingHeadConnector {
|
||||
target_tagging: Option<&'static str>,
|
||||
tagging_directives: Arc<std::sync::Mutex<Vec<Option<String>>>>,
|
||||
}
|
||||
|
||||
impl SmithyHttpConnector for TaggingHeadConnector {
|
||||
fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
|
||||
self.tagging_directives
|
||||
.lock()
|
||||
.expect("recorded directive lock should not be poisoned")
|
||||
.push(request.headers().get(AMZ_TAGGING_DIRECTIVE_LOWER).map(str::to_string));
|
||||
let mut response = HttpResponse::new(
|
||||
aws_smithy_runtime_api::http::StatusCode::try_from(200_u16).expect("200 should be a valid response status"),
|
||||
SdkBody::empty(),
|
||||
);
|
||||
response.headers_mut().insert("x-amz-meta-owner", "team-a");
|
||||
if let Some(tagging) = self.target_tagging {
|
||||
response.headers_mut().insert(AMZ_OBJECT_TAGGING_LOWER, tagging);
|
||||
}
|
||||
HttpConnectorFuture::ready(Ok(response))
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn head_object_requests_and_reports_the_target_tag_set() {
|
||||
for target_tagging in [Some("env=prod&team=a"), None] {
|
||||
let tagging_directives = Arc::new(std::sync::Mutex::new(Vec::new()));
|
||||
let connector = SharedHttpConnector::new(TaggingHeadConnector {
|
||||
target_tagging,
|
||||
tagging_directives: Arc::clone(&tagging_directives),
|
||||
});
|
||||
let http_client = http_client_fn(move |_settings, _components| connector.clone());
|
||||
let (mut client, _) = recording_target_client();
|
||||
client.client = Arc::new(s3_client_for_test(443, Some(http_client)));
|
||||
|
||||
let head = client
|
||||
.head_object("target-bucket", "object", None)
|
||||
.await
|
||||
.expect("canned HEAD should succeed");
|
||||
|
||||
assert_eq!(
|
||||
*tagging_directives
|
||||
.lock()
|
||||
.expect("recorded directive lock should not be poisoned"),
|
||||
vec![Some("ACCESS".to_string())],
|
||||
"the replication HEAD must ask the target for its tag set"
|
||||
);
|
||||
let metadata = head.metadata.expect("user metadata should be parsed");
|
||||
assert_eq!(metadata.get("owner").map(String::as_str), Some("team-a"));
|
||||
// Absent means "tags unknown": the comparison then keeps treating a
|
||||
// tagged replica as different instead of trusting the count alone.
|
||||
assert_eq!(metadata.get(AMZ_OBJECT_TAGGING).map(String::as_str), target_tagging);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn put_object_sends_source_version_id_query_to_target() {
|
||||
// MinIO reads the replicated version only from the `versionId` query
|
||||
|
||||
@@ -22,12 +22,13 @@
|
||||
//! `.minio.sys`) can never carry an override; they stay pinned to `strict`.
|
||||
//! See docs/operations/durability-modes.md for the semantics.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde::{Deserialize, Deserializer, Serialize, Serializer};
|
||||
|
||||
/// Valid per-bucket durability tier names. `legacy-off` is deliberately not
|
||||
/// representable per bucket: it exists only for the process-wide legacy
|
||||
/// switch (`RUSTFS_DRIVE_SYNC_ENABLE=false`).
|
||||
pub const BUCKET_DURABILITY_MODE_STRICT: &str = "strict";
|
||||
pub const BUCKET_DURABILITY_MODE_FILE_SYNC: &str = "file-sync";
|
||||
pub const BUCKET_DURABILITY_MODE_RELAXED: &str = "relaxed";
|
||||
pub const BUCKET_DURABILITY_MODE_NONE: &str = "none";
|
||||
|
||||
@@ -35,14 +36,69 @@ pub const BUCKET_DURABILITY_MODE_NONE: &str = "none";
|
||||
///
|
||||
/// An absent entry, an empty payload, or an empty `mode` string all mean
|
||||
/// "no override": the bucket follows the process-wide durability mode.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct BucketDurabilityConfig {
|
||||
/// Requested tier: `strict` | `relaxed` | `none` (case-insensitive,
|
||||
/// Requested tier: `strict` | `file-sync` | `relaxed` | `none` (case-insensitive,
|
||||
/// whitespace-tolerant). Empty means "inherit the global mode".
|
||||
#[serde(default)]
|
||||
pub mode: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct BucketDurabilityConfigWireRef<'a> {
|
||||
mode: &'a str,
|
||||
#[serde(skip_serializing_if = "is_false")]
|
||||
file_sync: bool,
|
||||
}
|
||||
|
||||
fn is_false(value: &bool) -> bool {
|
||||
!*value
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct BucketDurabilityConfigWire {
|
||||
#[serde(default)]
|
||||
mode: String,
|
||||
#[serde(default)]
|
||||
file_sync: bool,
|
||||
}
|
||||
|
||||
impl Serialize for BucketDurabilityConfig {
|
||||
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: Serializer,
|
||||
{
|
||||
let file_sync = self.mode.trim().eq_ignore_ascii_case(BUCKET_DURABILITY_MODE_FILE_SYNC);
|
||||
BucketDurabilityConfigWireRef {
|
||||
mode: if file_sync {
|
||||
BUCKET_DURABILITY_MODE_STRICT
|
||||
} else {
|
||||
&self.mode
|
||||
},
|
||||
file_sync,
|
||||
}
|
||||
.serialize(serializer)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for BucketDurabilityConfig {
|
||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
let wire = BucketDurabilityConfigWire::deserialize(deserializer)?;
|
||||
let mode = if wire.file_sync {
|
||||
if wire.mode.trim().eq_ignore_ascii_case(BUCKET_DURABILITY_MODE_STRICT) {
|
||||
BUCKET_DURABILITY_MODE_FILE_SYNC.to_string()
|
||||
} else {
|
||||
BUCKET_DURABILITY_MODE_STRICT.to_string()
|
||||
}
|
||||
} else {
|
||||
wire.mode
|
||||
};
|
||||
Ok(Self { mode })
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketDurabilityConfig {
|
||||
pub fn new(mode: &str) -> Self {
|
||||
Self { mode: mode.to_string() }
|
||||
@@ -52,7 +108,10 @@ impl BucketDurabilityConfig {
|
||||
pub fn is_valid_mode(mode: &str) -> bool {
|
||||
matches!(
|
||||
mode.trim().to_ascii_lowercase().as_str(),
|
||||
BUCKET_DURABILITY_MODE_STRICT | BUCKET_DURABILITY_MODE_RELAXED | BUCKET_DURABILITY_MODE_NONE
|
||||
BUCKET_DURABILITY_MODE_STRICT
|
||||
| BUCKET_DURABILITY_MODE_FILE_SYNC
|
||||
| BUCKET_DURABILITY_MODE_RELAXED
|
||||
| BUCKET_DURABILITY_MODE_NONE
|
||||
)
|
||||
}
|
||||
|
||||
@@ -117,6 +176,7 @@ mod tests {
|
||||
#[test]
|
||||
fn valid_modes_are_recognized() {
|
||||
assert!(BucketDurabilityConfig::is_valid_mode("strict"));
|
||||
assert!(BucketDurabilityConfig::is_valid_mode("file-sync"));
|
||||
assert!(BucketDurabilityConfig::is_valid_mode("relaxed"));
|
||||
assert!(BucketDurabilityConfig::is_valid_mode("none"));
|
||||
assert!(BucketDurabilityConfig::is_valid_mode(" RELAXED "));
|
||||
@@ -130,6 +190,7 @@ mod tests {
|
||||
fn normalized_mode_canonicalizes_or_rejects() {
|
||||
assert_eq!(BucketDurabilityConfig::new(" Relaxed ").normalized_mode().as_deref(), Some("relaxed"));
|
||||
assert_eq!(BucketDurabilityConfig::new("strict").normalized_mode().as_deref(), Some("strict"));
|
||||
assert_eq!(BucketDurabilityConfig::new(" FILE-SYNC ").normalized_mode().as_deref(), Some("file-sync"));
|
||||
assert_eq!(BucketDurabilityConfig::default().normalized_mode(), None);
|
||||
assert_eq!(BucketDurabilityConfig::new("bogus").normalized_mode(), None);
|
||||
}
|
||||
@@ -141,11 +202,34 @@ mod tests {
|
||||
let back: BucketDurabilityConfig = serde_json::from_slice(&json).expect("deserialize");
|
||||
assert_eq!(back, cfg);
|
||||
|
||||
let file_sync = BucketDurabilityConfig::new("file-sync");
|
||||
let json = serde_json::to_vec(&file_sync).expect("serialize file-sync");
|
||||
assert_eq!(json, br#"{"mode":"strict","file_sync":true}"#);
|
||||
let back: BucketDurabilityConfig = serde_json::from_slice(&json).expect("deserialize file-sync");
|
||||
assert_eq!(back.normalized_mode().as_deref(), Some("file-sync"));
|
||||
|
||||
let malformed = serde_json::from_slice::<BucketDurabilityConfig>(br#"{"mode":"none","file_sync":true}"#)
|
||||
.expect("deserialize malformed file-sync");
|
||||
assert_eq!(malformed.normalized_mode().as_deref(), Some("strict"));
|
||||
|
||||
// Empty object deserializes to the "inherit" default.
|
||||
let empty: BucketDurabilityConfig = serde_json::from_slice(b"{}").expect("deserialize empty");
|
||||
assert_eq!(empty.normalized_mode(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn file_sync_encoding_fails_closed_on_legacy_readers() {
|
||||
#[derive(Deserialize)]
|
||||
struct LegacyConfig {
|
||||
#[serde(default)]
|
||||
mode: String,
|
||||
}
|
||||
|
||||
let json = serde_json::to_vec(&BucketDurabilityConfig::new("file-sync")).expect("serialize file-sync");
|
||||
let legacy: LegacyConfig = serde_json::from_slice(&json).expect("legacy reader should ignore the marker");
|
||||
assert_eq!(legacy.mode, "strict", "legacy readers must apply the stronger strict fallback");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_bucket_default_inherits_when_unset() {
|
||||
temp_env::with_var_unset(ENV_NEW_BUCKET_DURABILITY_MODE, || {
|
||||
@@ -157,6 +241,7 @@ mod tests {
|
||||
fn new_bucket_default_honors_explicit_tiers() {
|
||||
for mode in [
|
||||
BUCKET_DURABILITY_MODE_STRICT,
|
||||
BUCKET_DURABILITY_MODE_FILE_SYNC,
|
||||
BUCKET_DURABILITY_MODE_RELAXED,
|
||||
BUCKET_DURABILITY_MODE_NONE,
|
||||
] {
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Per-bucket ETag algorithm.
|
||||
//!
|
||||
//! A bucket can derive the ETags of objects written without Content-MD5 from
|
||||
//! XXH3-128 instead of MD5, for clients that never treat the ETag as an MD5
|
||||
//! (an LLM KV cache). The choice is stored as a RustFS extension entry
|
||||
//! (`etag.json`) in the bucket metadata file and published to an in-memory
|
||||
//! registry whenever the metadata is installed, so the write path looks it up
|
||||
//! without I/O. Buckets without the entry keep MD5 ETags.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashSet;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{OnceLock, PoisonError, RwLock};
|
||||
|
||||
pub const BUCKET_ETAG_ALGORITHM_MD5: &str = "md5";
|
||||
pub const BUCKET_ETAG_ALGORITHM_XXH3: &str = "xxh3";
|
||||
|
||||
/// JSON payload stored under the `etag.json` bucket metadata entry. An absent
|
||||
/// entry, an empty payload or an invalid algorithm all mean MD5 ETags.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct BucketEtagConfig {
|
||||
/// `md5` | `xxh3` (case-insensitive, whitespace-tolerant).
|
||||
#[serde(default)]
|
||||
pub algorithm: String,
|
||||
}
|
||||
|
||||
impl BucketEtagConfig {
|
||||
pub fn new(algorithm: &str) -> Self {
|
||||
Self {
|
||||
algorithm: algorithm.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether `algorithm` names a supported ETag algorithm.
|
||||
pub fn is_valid_algorithm(algorithm: &str) -> bool {
|
||||
matches!(
|
||||
algorithm.trim().to_ascii_lowercase().as_str(),
|
||||
BUCKET_ETAG_ALGORITHM_MD5 | BUCKET_ETAG_ALGORITHM_XXH3
|
||||
)
|
||||
}
|
||||
|
||||
/// The canonical (trimmed, lowercase) algorithm name, or `None` when the
|
||||
/// config does not carry a valid one.
|
||||
pub fn normalized_algorithm(&self) -> Option<String> {
|
||||
let algorithm = self.algorithm.trim().to_ascii_lowercase();
|
||||
if Self::is_valid_algorithm(&algorithm) {
|
||||
Some(algorithm)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether this config selects XXH3 ETags.
|
||||
pub fn uses_xxh3(&self) -> bool {
|
||||
self.normalized_algorithm().as_deref() == Some(BUCKET_ETAG_ALGORITHM_XXH3)
|
||||
}
|
||||
}
|
||||
|
||||
/// Buckets whose ETags use XXH3, kept in step with the installed bucket
|
||||
/// metadata. `ANY` lets the write path skip the lock while no bucket opts in.
|
||||
static XXH3_BUCKETS: OnceLock<RwLock<HashSet<String>>> = OnceLock::new();
|
||||
static ANY: AtomicBool = AtomicBool::new(false);
|
||||
|
||||
fn xxh3_buckets() -> &'static RwLock<HashSet<String>> {
|
||||
XXH3_BUCKETS.get_or_init(|| RwLock::new(HashSet::new()))
|
||||
}
|
||||
|
||||
/// Records whether `bucket` uses XXH3 ETags.
|
||||
pub fn set_bucket_xxh3_etag(bucket: &str, xxh3: bool) {
|
||||
if bucket.is_empty() {
|
||||
return;
|
||||
}
|
||||
let mut buckets = xxh3_buckets().write().unwrap_or_else(PoisonError::into_inner);
|
||||
if xxh3 {
|
||||
buckets.insert(bucket.to_string());
|
||||
} else {
|
||||
buckets.remove(bucket);
|
||||
}
|
||||
ANY.store(!buckets.is_empty(), Ordering::Release);
|
||||
}
|
||||
|
||||
/// Whether objects written to `bucket` without Content-MD5 get XXH3 ETags.
|
||||
pub fn bucket_uses_xxh3_etag(bucket: &str) -> bool {
|
||||
if !ANY.load(Ordering::Acquire) {
|
||||
return false;
|
||||
}
|
||||
xxh3_buckets().read().unwrap_or_else(PoisonError::into_inner).contains(bucket)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn valid_algorithms_are_recognized() {
|
||||
assert!(BucketEtagConfig::is_valid_algorithm("md5"));
|
||||
assert!(BucketEtagConfig::is_valid_algorithm(" XXH3 "));
|
||||
assert!(!BucketEtagConfig::is_valid_algorithm(""));
|
||||
assert!(!BucketEtagConfig::is_valid_algorithm("sha256"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalized_algorithm_canonicalizes_or_rejects() {
|
||||
assert_eq!(BucketEtagConfig::new(" Xxh3 ").normalized_algorithm().as_deref(), Some("xxh3"));
|
||||
assert_eq!(BucketEtagConfig::new("MD5").normalized_algorithm().as_deref(), Some("md5"));
|
||||
assert_eq!(BucketEtagConfig::new("crc32").normalized_algorithm(), None);
|
||||
assert!(BucketEtagConfig::new("xxh3").uses_xxh3());
|
||||
assert!(!BucketEtagConfig::new("md5").uses_xxh3());
|
||||
assert!(!BucketEtagConfig::default().uses_xxh3());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn json_round_trip() {
|
||||
let config = BucketEtagConfig::new("xxh3");
|
||||
let json = serde_json::to_vec(&config).expect("serialize");
|
||||
assert_eq!(json, br#"{"algorithm":"xxh3"}"#);
|
||||
assert_eq!(serde_json::from_slice::<BucketEtagConfig>(&json).expect("parse"), config);
|
||||
assert_eq!(
|
||||
serde_json::from_slice::<BucketEtagConfig>(b"{}").expect("parse"),
|
||||
BucketEtagConfig::default()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registry_tracks_xxh3_buckets() {
|
||||
let bucket = "etag-registry-test-bucket";
|
||||
assert!(!bucket_uses_xxh3_etag(bucket));
|
||||
set_bucket_xxh3_etag(bucket, true);
|
||||
assert!(bucket_uses_xxh3_etag(bucket));
|
||||
assert!(!bucket_uses_xxh3_etag("etag-registry-test-other"));
|
||||
set_bucket_xxh3_etag(bucket, false);
|
||||
assert!(!bucket_uses_xxh3_etag(bucket));
|
||||
set_bucket_xxh3_etag("", true);
|
||||
assert!(!bucket_uses_xxh3_etag(""));
|
||||
}
|
||||
}
|
||||
@@ -4066,6 +4066,11 @@ pub async fn enqueue_transition_immediate(oi: &ObjectInfo, src: LcEventSrc) {
|
||||
}
|
||||
|
||||
pub(crate) async fn enqueue_immediate_expiry(api: Arc<ECStore>, oi: &ObjectInfo, src: LcEventSrc, opts: &ObjectOptions) {
|
||||
// Without a lifecycle document there is nothing to expire; skip the
|
||||
// metadata-transaction lock and incarnation read that only validate it.
|
||||
if !metadata_boundary::cached_metadata_may_have_lifecycle(&api, &oi.bucket).await {
|
||||
return;
|
||||
}
|
||||
let configs = match metadata_boundary::get_expiry_configs_for_options(&api, &oi.bucket, opts).await {
|
||||
Ok(configs) => configs,
|
||||
Err(err) => {
|
||||
@@ -13569,6 +13574,105 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn immediate_expiry_without_lifecycle_skips_metadata_transaction() {
|
||||
let (_paths, ecstore) = setup_test_env().await;
|
||||
let bucket = format!("lifecycle-absent-expiry-{}", Uuid::new_v4().simple());
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
let oi = ecstore
|
||||
.put_object(
|
||||
&bucket,
|
||||
"object",
|
||||
&mut PutObjReader::from_vec(b"payload".to_vec()),
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("create object without lifecycle");
|
||||
let state = install_unconsumed_runtime_expiry_worker(&ecstore, 4).await;
|
||||
let transaction = ecstore
|
||||
.new_ns_lock(
|
||||
crate::disk::RUSTFS_META_BUCKET,
|
||||
&metadata_sys::bucket_metadata_transaction_lock_key(&bucket),
|
||||
)
|
||||
.await
|
||||
.expect("metadata transaction namespace");
|
||||
let opts = ObjectOptions::default();
|
||||
|
||||
// A configuration writer outlasts the hook's lock wait; without
|
||||
// lifecycle there is no incarnation to validate.
|
||||
let writer = transaction
|
||||
.get_write_lock(StdDuration::from_secs(5))
|
||||
.await
|
||||
.expect("hold metadata transaction");
|
||||
tokio::time::timeout(
|
||||
StdDuration::from_secs(1),
|
||||
super::enqueue_immediate_expiry(Arc::clone(&ecstore), &oi, LcEventSrc::S3PutObject, &opts),
|
||||
)
|
||||
.await
|
||||
.expect("no lifecycle must not wait for the metadata transaction");
|
||||
assert_eq!(state.read().await.stats.pending_tasks(), 0, "nothing to expire");
|
||||
drop(writer);
|
||||
|
||||
// An unreadable lifecycle document keeps the authoritative path, which
|
||||
// reports the metadata failure instead of treating it as absent.
|
||||
let sys = metadata_sys::bucket_metadata_sys_of(&ecstore.ctx).expect("metadata system");
|
||||
let sys = sys.read().await.clone();
|
||||
let mut metadata = (*sys.get(&bucket).await.expect("bucket metadata")).clone();
|
||||
metadata.lifecycle_config_xml = b"<LifecycleConfiguration>".to_vec();
|
||||
metadata.lifecycle_config = None;
|
||||
sys.persist_and_set(metadata).await.expect("persist unreadable lifecycle");
|
||||
let observed = Arc::new(StdMutex::new(Vec::new()));
|
||||
let observed_events = Arc::clone(&observed);
|
||||
let _observer = set_lifecycle_observability_observer(move |event, state, reason| {
|
||||
observed_events
|
||||
.lock()
|
||||
.expect("lifecycle observer should not poison")
|
||||
.push((event, state, reason));
|
||||
});
|
||||
super::enqueue_immediate_expiry(Arc::clone(&ecstore), &oi, LcEventSrc::S3PutObject, &opts).await;
|
||||
assert!(
|
||||
observed.lock().expect("observed events should not poison").contains(&(
|
||||
EVENT_LIFECYCLE_EVALUATION_FAILED,
|
||||
"failed",
|
||||
Some("metadata_unavailable")
|
||||
)),
|
||||
"an unreadable lifecycle document must keep the authoritative path"
|
||||
);
|
||||
assert_eq!(state.read().await.stats.pending_tasks(), 0, "nothing enqueued");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn cached_lifecycle_check_keeps_creation_pending_and_uncached_buckets() {
|
||||
let (_paths, ecstore) = setup_test_env().await;
|
||||
let bucket = format!("lifecycle-cached-check-{}", Uuid::new_v4().simple());
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
let sys = metadata_sys::bucket_metadata_sys_of(&ecstore.ctx).expect("metadata system");
|
||||
let sys = sys.read().await.clone();
|
||||
assert!(
|
||||
!super::metadata_boundary::cached_metadata_may_have_lifecycle(&ecstore, &bucket).await,
|
||||
"a cached bucket without lifecycle skips the lookup"
|
||||
);
|
||||
|
||||
let mut pending = (*sys.get(&bucket).await.expect("bucket metadata")).clone();
|
||||
pending.lock_enabled = true;
|
||||
pending.bucket_incarnation_sidecar = true;
|
||||
pending.bucket_creation_committed = false;
|
||||
assert!(pending.needs_bucket_creation_commit(), "fixture is creation-pending");
|
||||
sys.set(bucket.clone(), Arc::new(pending)).await;
|
||||
assert!(
|
||||
super::metadata_boundary::cached_metadata_may_have_lifecycle(&ecstore, &bucket).await,
|
||||
"a creation-pending bucket keeps the migration path"
|
||||
);
|
||||
|
||||
assert!(sys.remove(&bucket).await, "cached entry removed");
|
||||
assert!(
|
||||
super::metadata_boundary::cached_metadata_may_have_lifecycle(&ecstore, &bucket).await,
|
||||
"an uncached bucket keeps the authoritative path"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn lifecycle_expiry_fails_closed_on_corrupt_object_lock_metadata() {
|
||||
|
||||
@@ -55,6 +55,25 @@ pub(crate) async fn lifecycle_expiry_allowed(
|
||||
.table_bucket_enabled())
|
||||
}
|
||||
|
||||
/// `get_expiry_configs_for_options` takes the lifecycle rules from this same
|
||||
/// cached entry and reads the bucket incarnation only to validate it, so a
|
||||
/// cached entry without any lifecycle document cannot produce an expiry.
|
||||
/// Uncached, fabricated and creation-pending buckets keep the full path.
|
||||
pub(crate) async fn cached_metadata_may_have_lifecycle(api: &crate::store::ECStore, bucket: &str) -> bool {
|
||||
let Ok(sys) = metadata_sys::bucket_metadata_sys_of(&api.ctx) else {
|
||||
return true;
|
||||
};
|
||||
let sys = sys.read().await.clone();
|
||||
match sys.get(bucket).await {
|
||||
Ok(metadata) => {
|
||||
metadata.needs_bucket_creation_commit()
|
||||
|| metadata.lifecycle_config.is_some()
|
||||
|| !metadata.lifecycle_config_xml.is_empty()
|
||||
}
|
||||
Err(_) => true,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn get_expiry_configs(api: &crate::store::ECStore, bucket: &str) -> Result<LifecycleExpiryConfigs> {
|
||||
get_expiry_configs_for_options(api, bucket, &crate::object_api::ObjectOptions::default()).await
|
||||
}
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Lockless cache buckets.
|
||||
//!
|
||||
//! A bucket named in `RUSTFS_LOCKLESS_CACHE_BUCKETS` holds a content-addressed
|
||||
//! cache (an LLM KV cache): a key always names the same bytes, and a lost or
|
||||
//! unreadable object only costs a recompute. PutObject, GetObject and HeadObject
|
||||
//! on such a bucket take no distributed locks and read no bucket metadata from
|
||||
//! disk, so concurrent writers into one bucket no longer meet on bucket-wide
|
||||
//! lock keys.
|
||||
//!
|
||||
//! What a listed bucket gives up, by design:
|
||||
//! - Writes of one key are not serialized. Racing commits of the same bytes can
|
||||
//! leave the object without read quorum until it is rewritten or healed,
|
||||
//! which a cache treats as a miss.
|
||||
//! - Reads are not fenced against a concurrent overwrite or delete.
|
||||
//! - Writes are not fenced against bucket deletion, Object Lock, quota,
|
||||
//! versioning, replication or table-catalog changes made while they run. A
|
||||
//! PUT falls back to the locked path whenever the cached bucket state shows
|
||||
//! any of those features, but listed buckets must not enable them.
|
||||
//!
|
||||
//! The list is read once per process; RustFS system buckets are never lockless.
|
||||
|
||||
use super::utils::is_meta_bucketname;
|
||||
use std::collections::HashSet;
|
||||
use std::sync::OnceLock;
|
||||
|
||||
/// Buckets named in a comma-separated list, without blanks or system buckets.
|
||||
fn parse_lockless_cache_buckets(value: &str) -> HashSet<String> {
|
||||
value
|
||||
.split(',')
|
||||
.map(str::trim)
|
||||
.filter(|bucket| !bucket.is_empty() && !is_meta_bucketname(bucket))
|
||||
.map(str::to_owned)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Whether `bucket` is configured as a lockless cache bucket.
|
||||
pub fn is_lockless_cache_bucket(bucket: &str) -> bool {
|
||||
static BUCKETS: OnceLock<HashSet<String>> = OnceLock::new();
|
||||
BUCKETS
|
||||
.get_or_init(|| {
|
||||
rustfs_utils::get_env_opt_str(rustfs_config::ENV_LOCKLESS_CACHE_BUCKETS)
|
||||
.map(|value| parse_lockless_cache_buckets(&value))
|
||||
.unwrap_or_default()
|
||||
})
|
||||
.contains(bucket)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parse_keeps_named_user_buckets_only() {
|
||||
let buckets = parse_lockless_cache_buckets(" kvcache-a,,kvcache-b , .rustfs.sys ,");
|
||||
assert_eq!(
|
||||
buckets,
|
||||
HashSet::from(["kvcache-a".to_owned(), "kvcache-b".to_owned()]),
|
||||
"blanks and system buckets must be dropped"
|
||||
);
|
||||
assert!(parse_lockless_cache_buckets("").is_empty());
|
||||
}
|
||||
}
|
||||
@@ -390,6 +390,7 @@ pub const BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG: &str = "public-access-block.xml";
|
||||
pub const BUCKET_ACL_CONFIG: &str = "bucket-acl.json";
|
||||
pub const BUCKET_TABLE_CONFIG: &str = "table-bucket.json";
|
||||
pub const BUCKET_DURABILITY_CONFIG: &str = "durability.json";
|
||||
pub const BUCKET_ETAG_CONFIG: &str = "etag.json";
|
||||
pub const BUCKET_ON_DEMAND_MIGRATION_CONFIG: &str = "on-demand-migration.json";
|
||||
pub const BUCKET_TABLE_RESERVED_PREFIX: &str = ".rustfs-table";
|
||||
pub const BUCKET_TABLE_CATALOG_META_PREFIX: &str = "s3tables/catalog";
|
||||
@@ -581,6 +582,7 @@ pub struct BucketMetadata {
|
||||
pub table_bucket_config_json: Vec<u8>,
|
||||
pub durability_config_json: Vec<u8>,
|
||||
pub on_demand_migration_config_json: Vec<u8>,
|
||||
pub etag_config_json: Vec<u8>,
|
||||
|
||||
pub policy_config_updated_at: OffsetDateTime,
|
||||
pub object_lock_config_updated_at: OffsetDateTime,
|
||||
@@ -603,6 +605,7 @@ pub struct BucketMetadata {
|
||||
pub table_bucket_config_updated_at: OffsetDateTime,
|
||||
pub durability_config_updated_at: OffsetDateTime,
|
||||
pub on_demand_migration_config_updated_at: OffsetDateTime,
|
||||
pub etag_config_updated_at: OffsetDateTime,
|
||||
|
||||
pub new_field_updated_at: OffsetDateTime,
|
||||
|
||||
@@ -657,6 +660,7 @@ impl Default for BucketMetadata {
|
||||
table_bucket_config_json: Default::default(),
|
||||
durability_config_json: Default::default(),
|
||||
on_demand_migration_config_json: Default::default(),
|
||||
etag_config_json: Default::default(),
|
||||
policy_config_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
object_lock_config_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
encryption_config_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
@@ -678,6 +682,7 @@ impl Default for BucketMetadata {
|
||||
table_bucket_config_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
durability_config_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
on_demand_migration_config_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
etag_config_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
new_field_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
policy_config: Default::default(),
|
||||
notification_config: Default::default(),
|
||||
@@ -804,6 +809,29 @@ impl BucketMetadata {
|
||||
}
|
||||
}
|
||||
|
||||
/// Parsed per-bucket ETag algorithm, if one is stored. Invalid payloads
|
||||
/// keep MD5 ETags after logging a parse failure.
|
||||
pub fn etag_config(&self) -> Option<super::etag::BucketEtagConfig> {
|
||||
if self.etag_config_json.is_empty() {
|
||||
return None;
|
||||
}
|
||||
match serde_json::from_slice(&self.etag_config_json) {
|
||||
Ok(cfg) => Some(cfg),
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
event = "bucket_metadata_parse_failed",
|
||||
component = "ecstore",
|
||||
subsystem = "bucket_metadata",
|
||||
bucket = %self.name,
|
||||
config = "etag",
|
||||
error = %e,
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode from msgp bytes. Field order follows MinIO BucketMetadata for compatibility.
|
||||
pub fn decode_from<R: Read>(&mut self, rd: &mut R) -> Result<()> {
|
||||
let mut fields = rmp::decode::read_map_len(rd)?;
|
||||
@@ -864,6 +892,7 @@ impl BucketMetadata {
|
||||
"OnDemandMigrationConfigJSON" | "OnDemandMigrationConfigJson" => {
|
||||
self.on_demand_migration_config_json = read_msgp_bin(rd)?
|
||||
}
|
||||
"EtagConfigJSON" | "EtagConfigJson" => self.etag_config_json = read_msgp_bin(rd)?,
|
||||
"CorsConfigUpdatedAt" => self.cors_config_updated_at = read_msgp_time_value(rd)?,
|
||||
"LoggingConfigUpdatedAt" => self.logging_config_updated_at = read_msgp_time_value(rd)?,
|
||||
"WebsiteConfigUpdatedAt" => self.website_config_updated_at = read_msgp_time_value(rd)?,
|
||||
@@ -874,6 +903,7 @@ impl BucketMetadata {
|
||||
"TableBucketConfigUpdatedAt" => self.table_bucket_config_updated_at = read_msgp_time_value(rd)?,
|
||||
"DurabilityConfigUpdatedAt" => self.durability_config_updated_at = read_msgp_time_value(rd)?,
|
||||
"OnDemandMigrationConfigUpdatedAt" => self.on_demand_migration_config_updated_at = read_msgp_time_value(rd)?,
|
||||
"EtagConfigUpdatedAt" => self.etag_config_updated_at = read_msgp_time_value(rd)?,
|
||||
"BucketCreationCommitted" => {
|
||||
self.bucket_creation_committed = read_msgp_bool(rd)?;
|
||||
self.bucket_creation_commit_record_present = true;
|
||||
@@ -892,8 +922,8 @@ impl BucketMetadata {
|
||||
pub fn encode_to<W: Write>(&self, wr: &mut W) -> Result<()> {
|
||||
// Preserve the absence of a legacy commit record across config rewrites.
|
||||
let commit_record_present = self.bucket_creation_commit_record_present || self.bucket_creation_committed;
|
||||
// Map size: MinIO fields (25) + RustFS extensions (21) + optional commit record.
|
||||
let map_len: u32 = 46 + u32::from(commit_record_present);
|
||||
// Map size: MinIO fields (25) + RustFS extensions (23) + optional commit record.
|
||||
let map_len: u32 = 48 + u32::from(commit_record_present);
|
||||
rmp::encode::write_map_len(wr, map_len)?;
|
||||
|
||||
// MinIO field order (same as Go struct)
|
||||
@@ -954,6 +984,7 @@ impl BucketMetadata {
|
||||
write_bin_field(wr, "TableBucketConfigJSON", &self.table_bucket_config_json)?;
|
||||
write_bin_field(wr, "DurabilityConfigJSON", &self.durability_config_json)?;
|
||||
write_bin_field(wr, "OnDemandMigrationConfigJSON", &self.on_demand_migration_config_json)?;
|
||||
write_bin_field(wr, "EtagConfigJSON", &self.etag_config_json)?;
|
||||
rmp::encode::write_str(wr, "CorsConfigUpdatedAt")?;
|
||||
write_msgp_time(wr, self.cors_config_updated_at)?;
|
||||
rmp::encode::write_str(wr, "LoggingConfigUpdatedAt")?;
|
||||
@@ -974,6 +1005,8 @@ impl BucketMetadata {
|
||||
write_msgp_time(wr, self.durability_config_updated_at)?;
|
||||
rmp::encode::write_str(wr, "OnDemandMigrationConfigUpdatedAt")?;
|
||||
write_msgp_time(wr, self.on_demand_migration_config_updated_at)?;
|
||||
rmp::encode::write_str(wr, "EtagConfigUpdatedAt")?;
|
||||
write_msgp_time(wr, self.etag_config_updated_at)?;
|
||||
if commit_record_present {
|
||||
rmp::encode::write_str(wr, "BucketCreationCommitted")?;
|
||||
rmp::encode::write_bool(wr, self.bucket_creation_committed)?;
|
||||
@@ -1082,6 +1115,9 @@ impl BucketMetadata {
|
||||
if self.on_demand_migration_config_updated_at == OffsetDateTime::UNIX_EPOCH {
|
||||
self.on_demand_migration_config_updated_at = self.created
|
||||
}
|
||||
if self.etag_config_updated_at == OffsetDateTime::UNIX_EPOCH {
|
||||
self.etag_config_updated_at = self.created
|
||||
}
|
||||
}
|
||||
|
||||
/// Replace one config payload and stamp its `*_config_updated_at` with the
|
||||
@@ -1228,6 +1264,10 @@ impl BucketMetadata {
|
||||
self.on_demand_migration_config_json = data;
|
||||
self.on_demand_migration_config_updated_at = updated;
|
||||
}
|
||||
BUCKET_ETAG_CONFIG => {
|
||||
self.etag_config_json = data;
|
||||
self.etag_config_updated_at = updated;
|
||||
}
|
||||
_ => return Err(Error::other(format!("config file not found : {config_file}"))),
|
||||
}
|
||||
|
||||
@@ -2720,6 +2760,28 @@ mod test {
|
||||
|
||||
/// HP-5b (rustfs/backlog#938): the durability override is a RustFS
|
||||
/// extension entry and must survive an encode/decode round trip.
|
||||
#[test]
|
||||
fn etag_config_round_trips_and_tracks_updates() {
|
||||
let mut bm = BucketMetadata::new("etag-bucket");
|
||||
assert!(bm.etag_config().is_none());
|
||||
bm.update_config(BUCKET_ETAG_CONFIG, br#"{"algorithm":"xxh3"}"#.to_vec())
|
||||
.expect("set etag config");
|
||||
assert_ne!(bm.etag_config_updated_at, OffsetDateTime::UNIX_EPOCH);
|
||||
assert!(bm.etag_config().expect("parsed").uses_xxh3());
|
||||
|
||||
let back = BucketMetadata::unmarshal(&bm.marshal_msg().expect("marshal")).expect("unmarshal");
|
||||
assert_eq!(back.etag_config_json, bm.etag_config_json);
|
||||
assert_eq!(back.etag_config_updated_at.unix_timestamp(), bm.etag_config_updated_at.unix_timestamp());
|
||||
// The durability entry is independent of the ETag entry.
|
||||
assert!(back.durability_config_json.is_empty());
|
||||
|
||||
bm.update_config(BUCKET_ETAG_CONFIG, Vec::new()).expect("clear etag config");
|
||||
assert!(bm.etag_config().is_none());
|
||||
|
||||
bm.etag_config_json = b"not-json".to_vec();
|
||||
assert!(bm.etag_config().is_none(), "garbage must keep MD5 ETags");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn durability_config_round_trips_and_tracks_updates() {
|
||||
let mut bm = BucketMetadata::new("durability-bucket");
|
||||
|
||||
@@ -440,6 +440,25 @@ fn clear_bucket_durability(bucket: &str) {
|
||||
crate::disk::local::bucket_durability::set(bucket, None);
|
||||
}
|
||||
|
||||
/// Publish the bucket's ETag algorithm to the registry the write path consults.
|
||||
fn sync_bucket_etag(bucket: &str, bm: &BucketMetadata) {
|
||||
let xxh3 = bm.etag_config().is_some_and(|cfg| cfg.uses_xxh3());
|
||||
crate::bucket::etag::set_bucket_xxh3_etag(bucket, xxh3);
|
||||
}
|
||||
|
||||
/// Publish every per-bucket override kept outside the metadata cache
|
||||
/// (durability tier, ETag algorithm) on each install path.
|
||||
fn sync_bucket_overrides(bucket: &str, bm: &BucketMetadata) {
|
||||
sync_bucket_durability(bucket, bm);
|
||||
sync_bucket_etag(bucket, bm);
|
||||
}
|
||||
|
||||
/// Drop every per-bucket override when the bucket's metadata leaves the cache.
|
||||
fn clear_bucket_overrides(bucket: &str) {
|
||||
clear_bucket_durability(bucket);
|
||||
crate::bucket::etag::set_bucket_xxh3_etag(bucket, false);
|
||||
}
|
||||
|
||||
/// Publish application-owned bytes on every cache install path.
|
||||
fn sync_on_demand_migration(bucket: &str, bm: &BucketMetadata) {
|
||||
if let Some(hook) = BUCKET_CONFIG_PUBLISH_HOOK.get() {
|
||||
@@ -1587,6 +1606,16 @@ pub async fn get_durability_config(
|
||||
Ok((bm.durability_config(), bm.durability_config_updated_at))
|
||||
}
|
||||
|
||||
/// The bucket's ETag algorithm config with its update time. `Ok((None, ..))`
|
||||
/// means the bucket keeps MD5 ETags.
|
||||
pub async fn get_etag_config(bucket: &str) -> Result<(Option<crate::bucket::etag::BucketEtagConfig>, OffsetDateTime)> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||
|
||||
let (bm, _) = bucket_meta_sys.get_config(bucket).await?;
|
||||
Ok((bm.etag_config(), bm.etag_config_updated_at))
|
||||
}
|
||||
|
||||
/// The bucket's on-demand migration config with its update time, or
|
||||
/// `Ok(None)` when the bucket has none. Bytes are opaque to the metadata owner.
|
||||
pub async fn get_on_demand_migration_config(bucket: &str) -> Result<Option<(Vec<u8>, OffsetDateTime)>> {
|
||||
@@ -2451,7 +2480,7 @@ impl BucketMetadataSys {
|
||||
self.missing_buckets.insert(bucket.to_string(), ()).await;
|
||||
if removed {
|
||||
BucketTargetSys::get().delete(bucket).await;
|
||||
clear_bucket_durability(bucket);
|
||||
clear_bucket_overrides(bucket);
|
||||
clear_on_demand_migration(bucket);
|
||||
}
|
||||
}
|
||||
@@ -2489,7 +2518,7 @@ impl BucketMetadataSys {
|
||||
self.fabricated_metadata.write().await.remove(bucket);
|
||||
self.missing_buckets.invalidate(bucket).await;
|
||||
sync_bucket_target_sys(bucket, &bm).await;
|
||||
sync_bucket_durability(bucket, &bm);
|
||||
sync_bucket_overrides(bucket, &bm);
|
||||
sync_on_demand_migration(bucket, &bm);
|
||||
}
|
||||
MetadataLoadMode::Initial => {
|
||||
@@ -2537,7 +2566,7 @@ impl BucketMetadataSys {
|
||||
self.missing_buckets.invalidate(bucket).await;
|
||||
if removed {
|
||||
BucketTargetSys::get().delete(bucket).await;
|
||||
clear_bucket_durability(bucket);
|
||||
clear_bucket_overrides(bucket);
|
||||
clear_on_demand_migration(bucket);
|
||||
}
|
||||
return Ok(());
|
||||
@@ -2560,7 +2589,7 @@ impl BucketMetadataSys {
|
||||
self.fabricated_metadata.write().await.remove(bucket);
|
||||
self.missing_buckets.invalidate(bucket).await;
|
||||
sync_bucket_target_sys(bucket, &metadata).await;
|
||||
sync_bucket_durability(bucket, &metadata);
|
||||
sync_bucket_overrides(bucket, &metadata);
|
||||
sync_on_demand_migration(bucket, &metadata);
|
||||
Ok(())
|
||||
}
|
||||
@@ -2590,7 +2619,7 @@ impl BucketMetadataSys {
|
||||
self.fabricated_metadata.write().await.remove(&bucket);
|
||||
self.missing_buckets.invalidate(&bucket).await;
|
||||
sync_bucket_target_sys(&bucket, &bm).await;
|
||||
sync_bucket_durability(&bucket, &bm);
|
||||
sync_bucket_overrides(&bucket, &bm);
|
||||
sync_on_demand_migration(&bucket, &bm);
|
||||
}
|
||||
}
|
||||
@@ -2614,7 +2643,7 @@ impl BucketMetadataSys {
|
||||
super::config_parse_mode::forget_bucket_config_parse_state(bucket);
|
||||
if removed {
|
||||
BucketTargetSys::get().delete(bucket).await;
|
||||
clear_bucket_durability(bucket);
|
||||
clear_bucket_overrides(bucket);
|
||||
clear_on_demand_migration(bucket);
|
||||
}
|
||||
removed || removed_fabricated
|
||||
@@ -2957,7 +2986,7 @@ impl BucketMetadataSys {
|
||||
self.fabricated_metadata.write().await.remove(bucket);
|
||||
self.missing_buckets.invalidate(bucket).await;
|
||||
sync_bucket_target_sys(bucket, &bm).await;
|
||||
sync_bucket_durability(bucket, &bm);
|
||||
sync_bucket_overrides(bucket, &bm);
|
||||
sync_on_demand_migration(bucket, &bm);
|
||||
return Ok((bm, true));
|
||||
} else {
|
||||
@@ -3408,7 +3437,7 @@ impl BucketMetadataSys {
|
||||
self.fabricated_metadata.write().await.remove(bucket);
|
||||
self.missing_buckets.invalidate(bucket).await;
|
||||
sync_bucket_target_sys(bucket, &metadata).await;
|
||||
sync_bucket_durability(bucket, &metadata);
|
||||
sync_bucket_overrides(bucket, &metadata);
|
||||
sync_on_demand_migration(bucket, &metadata);
|
||||
Ok(BucketMetadataAuthority::Authoritative(metadata))
|
||||
}
|
||||
@@ -6686,6 +6715,41 @@ mod tests {
|
||||
target_sys.delete(bucket).await;
|
||||
}
|
||||
|
||||
/// Installing bucket metadata publishes the bucket's ETag algorithm to the
|
||||
/// write-path registry; metadata without it, an invalid payload or
|
||||
/// clearing the bucket's overrides withdraws XXH3.
|
||||
#[test]
|
||||
fn metadata_sync_publishes_and_clears_etag_algorithm() {
|
||||
use crate::bucket::etag::bucket_uses_xxh3_etag;
|
||||
|
||||
let bucket = "metadata-sync-etag";
|
||||
|
||||
let mut bm = BucketMetadata::new(bucket);
|
||||
bm.etag_config_json = br#"{"algorithm":"xxh3"}"#.to_vec();
|
||||
sync_bucket_overrides(bucket, &bm);
|
||||
assert!(bucket_uses_xxh3_etag(bucket));
|
||||
|
||||
sync_bucket_overrides(bucket, &BucketMetadata::new(bucket));
|
||||
assert!(!bucket_uses_xxh3_etag(bucket));
|
||||
|
||||
let mut bm = BucketMetadata::new(bucket);
|
||||
bm.etag_config_json = br#"{"algorithm":"md5"}"#.to_vec();
|
||||
sync_bucket_overrides(bucket, &bm);
|
||||
assert!(!bucket_uses_xxh3_etag(bucket));
|
||||
|
||||
let mut bm = BucketMetadata::new(bucket);
|
||||
bm.etag_config_json = b"not-json".to_vec();
|
||||
sync_bucket_overrides(bucket, &bm);
|
||||
assert!(!bucket_uses_xxh3_etag(bucket));
|
||||
|
||||
let mut bm = BucketMetadata::new(bucket);
|
||||
bm.etag_config_json = br#"{"algorithm":"XXH3"}"#.to_vec();
|
||||
sync_bucket_overrides(bucket, &bm);
|
||||
assert!(bucket_uses_xxh3_etag(bucket));
|
||||
clear_bucket_overrides(bucket);
|
||||
assert!(!bucket_uses_xxh3_etag(bucket));
|
||||
}
|
||||
|
||||
/// HP-5b (rustfs/backlog#938): installing bucket metadata publishes the
|
||||
/// durability override to the disk-layer registry, and clearing the
|
||||
/// config (or an invalid payload) withdraws it.
|
||||
|
||||
@@ -19,7 +19,9 @@ pub mod bucket_target_sys;
|
||||
pub mod config_parse_mode;
|
||||
pub mod durability;
|
||||
pub mod error;
|
||||
pub mod etag;
|
||||
pub mod lifecycle;
|
||||
pub mod lockless_cache;
|
||||
pub mod metadata;
|
||||
pub mod metadata_sys;
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -20,6 +20,8 @@ use crate::test_metrics::CapturingRecorder;
|
||||
use rustfs_data_usage::{BucketUsageInfo, DataUsageInfo};
|
||||
use std::time::SystemTime;
|
||||
|
||||
mod native_compatibility;
|
||||
|
||||
async fn stop_reporting_fixture(store: &crate::store::ECStore) {
|
||||
if let Some(cancel) = store.ctx.background_cancel_token() {
|
||||
cancel.cancel();
|
||||
@@ -90,6 +92,10 @@ async fn install_reporting_snapshot(
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
#[serial]
|
||||
async fn durable_quota_reporting_retires_degraded_floors_without_changing_ledger_admission() {
|
||||
assert_durable_quota_reporting_preserves_admission().await;
|
||||
}
|
||||
|
||||
async fn assert_durable_quota_reporting_preserves_admission() {
|
||||
let recorder = CapturingRecorder::default();
|
||||
let _recorder = metrics::set_default_local_recorder(&recorder);
|
||||
for protocol in [1, 2] {
|
||||
@@ -196,6 +202,11 @@ async fn durable_quota_reporting_retires_degraded_floors_without_changing_ledger
|
||||
.expect("fixture cache cleanup");
|
||||
crate::data_usage::invalidate_data_usage_snapshot_cache().await;
|
||||
stop_reporting_fixture(&store).await;
|
||||
for dir in _dirs {
|
||||
let path = dir.path().to_path_buf();
|
||||
dir.close().expect("remove durable quota reporting fixture");
|
||||
assert!(!path.exists());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -203,6 +214,10 @@ async fn durable_quota_reporting_retires_degraded_floors_without_changing_ledger
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn snapshot_quota_admission_retains_degraded_floors_until_migration() {
|
||||
assert_snapshot_quota_preserves_degraded_floors().await;
|
||||
}
|
||||
|
||||
async fn assert_snapshot_quota_preserves_degraded_floors() {
|
||||
for source in ["missing", "legacy", "incomplete", "observed"] {
|
||||
let (_dirs, store, _) = isolated_durable_quota_store().await;
|
||||
let bucket = format!("quota-compat-{}", Uuid::new_v4().simple());
|
||||
@@ -233,12 +248,21 @@ async fn snapshot_quota_admission_retains_degraded_floors_until_migration() {
|
||||
.expect("fixture cache cleanup");
|
||||
crate::data_usage::invalidate_data_usage_snapshot_cache().await;
|
||||
stop_reporting_fixture(&store).await;
|
||||
for dir in _dirs {
|
||||
let path = dir.path().to_path_buf();
|
||||
dir.close().expect("remove legacy quota reporting fixture");
|
||||
assert!(!path.exists());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
#[serial]
|
||||
async fn quota_without_limit_keeps_the_zero_usage_io_fast_path() {
|
||||
assert_unlimited_quota_preserves_fast_path().await;
|
||||
}
|
||||
|
||||
async fn assert_unlimited_quota_preserves_fast_path() {
|
||||
let (_dirs, store, sys) = isolated_durable_quota_store().await;
|
||||
let bucket = format!("quota-unlimited-{}", Uuid::new_v4().simple());
|
||||
create_bucket_with_quota_json(
|
||||
@@ -259,6 +283,11 @@ async fn quota_without_limit_keeps_the_zero_usage_io_fast_path() {
|
||||
assert_eq!(recorder.counter_value(METRIC_QUOTA_USAGE_REPORT_UNAVAILABLE_TOTAL, &[]), 0);
|
||||
assert_eq!(crate::data_usage::reload_unknown_bucket_usage_calls_for_test(&bucket), 0);
|
||||
stop_reporting_fixture(&store).await;
|
||||
for dir in _dirs {
|
||||
let path = dir.path().to_path_buf();
|
||||
dir.close().expect("remove unlimited quota reporting fixture");
|
||||
assert!(!path.exists());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
+265
@@ -0,0 +1,265 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use super::*;
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use rustfs_data_usage::{
|
||||
DATA_USAGE_BUCKET_OBSERVATION_VERSION, DataUsageBucketObservation, DataUsageBucketObservationSource,
|
||||
DataUsageSnapshotSetState,
|
||||
};
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
#[serial]
|
||||
async fn usage_quota_modes_preserve_actual_admission_decisions() {
|
||||
let (dirs, store, sys) = isolated_durable_quota_store().await;
|
||||
let bucket = format!("usage-no-quota-{}", Uuid::new_v4().simple());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("unconfigured quota bucket");
|
||||
let check = QuotaChecker::new(sys)
|
||||
.check_quota(&bucket, QuotaOperation::PutObject, u64::MAX)
|
||||
.await
|
||||
.expect("no quota");
|
||||
assert!(check.allowed && check.quota_limit.is_none());
|
||||
store
|
||||
.put_object(
|
||||
&bucket,
|
||||
"allowed",
|
||||
&mut PutObjReader::from_vec(vec![0x5a; 512]),
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("real no-quota write");
|
||||
stop_reporting_fixture(&store).await;
|
||||
let paths = dirs.iter().map(|dir| dir.path().to_path_buf()).collect::<Vec<_>>();
|
||||
drop(store);
|
||||
drop(dirs);
|
||||
assert!(paths.iter().all(|path| !path.exists()));
|
||||
|
||||
assert_unlimited_quota_preserves_fast_path().await;
|
||||
assert_snapshot_quota_preserves_degraded_floors().await;
|
||||
assert_durable_quota_reporting_preserves_admission().await;
|
||||
println!(
|
||||
"usage_assertion: {}",
|
||||
serde_json::json!({
|
||||
"case_id": "USAGE-19", "assertion_id": "USAGE-19-A01", "status": "PASS",
|
||||
"evidence": {
|
||||
"quota_modes": ["unconfigured", "unlimited", "legacy-snapshot", "durable-v1", "durable-v2"],
|
||||
"durable_snapshot_sources": ["missing", "legacy", "incomplete", "observed", "complete"],
|
||||
"durable_storage_decisions_per_fixture": ["reject-8192", "accept-512", "accept-3584", "reject-1"],
|
||||
"legacy_missing_baseline_fails_closed": true,
|
||||
"legacy_grounded_baselines": ["legacy", "incomplete", "observed"]
|
||||
}
|
||||
})
|
||||
);
|
||||
println!(
|
||||
"usage_cleanup: {}",
|
||||
serde_json::json!({"case_id": "USAGE-19", "status": "PASS", "fixtures_removed": true})
|
||||
);
|
||||
}
|
||||
|
||||
async fn persist_companion_fixture(
|
||||
store: Arc<crate::store::ECStore>,
|
||||
bucket: &str,
|
||||
primary: bool,
|
||||
companion_size: u64,
|
||||
) -> (String, Vec<u8>) {
|
||||
let incarnation = store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("real companion bucket identity");
|
||||
let completed_at = SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(20);
|
||||
let mut snapshot = DataUsageInfo {
|
||||
last_update: Some(completed_at),
|
||||
scanner_epoch: Some(2),
|
||||
scanner_cycle: Some(3),
|
||||
usage_snapshot_complete: primary,
|
||||
usage_snapshot_partial: !primary,
|
||||
usage_snapshot_observational: true,
|
||||
usage_snapshot_converged: Some(false),
|
||||
usage_snapshot_authoritative_baseline: Some(DataUsageInfo::default().snapshot_identity()),
|
||||
usage_snapshot_pending_buckets: vec![bucket.to_owned()],
|
||||
usage_snapshot_pending_bucket_count: 1,
|
||||
usage_snapshot_set_states: vec![DataUsageSnapshotSetState {
|
||||
pool_index: 0,
|
||||
set_index: 0,
|
||||
scanner_epoch: Some(2),
|
||||
scanner_cycle: Some(3),
|
||||
scan_plan_digest: Some([1; 32]),
|
||||
complete: primary,
|
||||
..Default::default()
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
if primary {
|
||||
snapshot.buckets_count = 1;
|
||||
snapshot.buckets_usage.insert(
|
||||
bucket.into(),
|
||||
BucketUsageInfo {
|
||||
size: 1000,
|
||||
objects_count: 1,
|
||||
versions_count: 1,
|
||||
bucket_incarnation: Some(incarnation.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
snapshot.bucket_sizes.insert(bucket.into(), 1000);
|
||||
snapshot.calculate_totals();
|
||||
}
|
||||
let mut usage = BucketUsageInfo {
|
||||
size: companion_size,
|
||||
objects_count: 1,
|
||||
versions_count: 1,
|
||||
bucket_incarnation: Some(incarnation.to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
usage.mark_observed();
|
||||
snapshot.usage_snapshot_bucket_observations.insert(
|
||||
bucket.into(),
|
||||
DataUsageBucketObservation {
|
||||
version: DATA_USAGE_BUCKET_OBSERVATION_VERSION,
|
||||
usage,
|
||||
snapshot_identity: snapshot.snapshot_identity(),
|
||||
completed_at,
|
||||
sources: vec![DataUsageBucketObservationSource {
|
||||
pool_index: 0,
|
||||
set_index: 0,
|
||||
bucket_incarnation: incarnation.to_string(),
|
||||
scanner_epoch: 2,
|
||||
scanner_cycle: 3,
|
||||
scan_plan_digest: [1; 32],
|
||||
started_plan: [2; 32],
|
||||
requested_plan: [3; 32],
|
||||
completed_at,
|
||||
}],
|
||||
},
|
||||
);
|
||||
let mut admin = snapshot.clone();
|
||||
assert_eq!(
|
||||
admin.project_bucket_observations_for_admin(&HashMap::from([(bucket.to_owned(), incarnation.to_string())])),
|
||||
1,
|
||||
"the companion must be valid, not discarded as malformed"
|
||||
);
|
||||
assert_eq!(admin.buckets_usage[bucket].size, companion_size);
|
||||
let path = format!(
|
||||
"{}/{}",
|
||||
crate::disk::BUCKET_META_PREFIX,
|
||||
rustfs_data_usage::DATA_USAGE_OBSERVED_OBJECT_NAME
|
||||
);
|
||||
let bytes = serde_json::to_vec(&snapshot).expect("encode valid companion snapshot");
|
||||
crate::config::com::save_config(store, &path, bytes.clone())
|
||||
.await
|
||||
.expect("persist companion beside canonical fields");
|
||||
crate::data_usage::invalidate_data_usage_snapshot_cache().await;
|
||||
(path, bytes)
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
#[serial]
|
||||
async fn usage_companions_do_not_expand_legacy_fallback_or_durable_admission() {
|
||||
let mut decisions = Vec::new();
|
||||
for protocol in [0, 1, 2] {
|
||||
for (primary, companion_size) in [(true, 0), (true, 1_000_000), (false, 1_000_000)] {
|
||||
let (dirs, store, sys) = isolated_durable_quota_store().await;
|
||||
let bucket = format!("usage-companion-quota-{}", Uuid::new_v4().simple());
|
||||
let quota = match protocol {
|
||||
0 => legacy_quota_json(4096),
|
||||
1 => serde_json::to_vec(&BucketQuota::new(Some(4096))).unwrap(),
|
||||
_ => serde_json::to_vec(&BucketQuota::new_sharded(4096)).unwrap(),
|
||||
};
|
||||
create_bucket_with_quota_json(&store, &bucket, quota).await;
|
||||
let (path, bytes) = persist_companion_fixture(store.clone(), &bucket, primary, companion_size).await;
|
||||
let checker = if protocol == 0 {
|
||||
QuotaChecker::new(Arc::new(RwLock::new(BucketMetadataSys::new(store.clone()))))
|
||||
} else {
|
||||
QuotaChecker::new(sys)
|
||||
};
|
||||
if protocol == 0 {
|
||||
let small = checker.check_quota(&bucket, QuotaOperation::PutObject, 512).await;
|
||||
if primary {
|
||||
let small = small.expect("grounded primary fallback");
|
||||
assert!(small.allowed && !small.uses_durable_reservations);
|
||||
assert_eq!(small.current_usage, Some(1000));
|
||||
assert!(
|
||||
!checker
|
||||
.check_quota(&bucket, QuotaOperation::PutObject, 4096)
|
||||
.await
|
||||
.expect("legacy limit")
|
||||
.allowed
|
||||
);
|
||||
} else {
|
||||
assert!(
|
||||
matches!(small, Err(QuotaError::UsageUnavailable { .. })),
|
||||
"a companion cannot invent a legacy baseline"
|
||||
);
|
||||
}
|
||||
} else {
|
||||
assert!(
|
||||
checker
|
||||
.check_quota(&bucket, QuotaOperation::PutObject, 512)
|
||||
.await
|
||||
.expect("durable checker")
|
||||
.uses_durable_reservations
|
||||
);
|
||||
if protocol == 1 {
|
||||
assert!(matches!(
|
||||
store
|
||||
.put_object(
|
||||
&bucket,
|
||||
"bootstrap",
|
||||
&mut PutObjReader::from_vec(vec![0x5a; 512]),
|
||||
&ObjectOptions::default()
|
||||
)
|
||||
.await,
|
||||
Err(StorageError::SlowDown)
|
||||
));
|
||||
crate::bucket::quota::reservation::retry_quota_reconciliation_for_test(store.clone(), &bucket)
|
||||
.await
|
||||
.expect("real v1 ledger bootstrap");
|
||||
}
|
||||
store
|
||||
.put_object(&bucket, "fits", &mut PutObjReader::from_vec(vec![0x5a; 4096]), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("ledger admits exactly the physical quota regardless of companion");
|
||||
assert!(
|
||||
matches!(
|
||||
store
|
||||
.put_object(&bucket, "exceeds", &mut PutObjReader::from_vec(vec![0x5a; 1]), &ObjectOptions::default())
|
||||
.await,
|
||||
Err(StorageError::QuotaExceeded { .. })
|
||||
),
|
||||
"a low companion cannot expand the ledger"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
crate::config::com::read_config(store.clone(), &path)
|
||||
.await
|
||||
.expect("retained scanner evidence"),
|
||||
bytes
|
||||
);
|
||||
decisions
|
||||
.push(serde_json::json!({"protocol": protocol, "primary_baseline": primary, "companion_size": companion_size}));
|
||||
crate::data_usage::prepare_bucket_usage_for_namespace_change(&bucket, None)
|
||||
.await
|
||||
.expect("clear fixture caches");
|
||||
crate::data_usage::invalidate_data_usage_snapshot_cache().await;
|
||||
stop_reporting_fixture(&store).await;
|
||||
let paths = dirs.iter().map(|dir| dir.path().to_path_buf()).collect::<Vec<_>>();
|
||||
drop(store);
|
||||
drop(dirs);
|
||||
assert!(paths.iter().all(|path| !path.exists()));
|
||||
}
|
||||
}
|
||||
println!(
|
||||
"usage_assertion: {}",
|
||||
serde_json::json!({
|
||||
"case_id": "USAGE-19", "assertion_id": "USAGE-19-A02", "status": "PASS",
|
||||
"evidence": {"validated_companion_fixtures": decisions, "legacy_primary_bytes": 1000, "durable_limit_bytes": 4096, "companion_only_legacy_admission": "UsageUnavailable"}
|
||||
})
|
||||
);
|
||||
println!(
|
||||
"usage_cleanup: {}",
|
||||
serde_json::json!({"case_id": "USAGE-19", "status": "PASS", "fixtures_removed": true})
|
||||
);
|
||||
}
|
||||
@@ -826,12 +826,17 @@ pub(crate) async fn begin(
|
||||
) -> Result<QuotaContext> {
|
||||
let snapshot_admission = opts.quota_admission;
|
||||
let data_movement = opts.data_movement;
|
||||
if crate::bucket::utils::is_meta_bucketname(bucket) {
|
||||
let meta_bucket_fast_path_started = rustfs_io_metrics::put_stage_timer();
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_META_BUCKET_FAST_PATH,
|
||||
meta_bucket_fast_path_started,
|
||||
);
|
||||
// A lockless cache PUT is only taken when the bucket has no quota, so it
|
||||
// reserves nothing and needs no metadata-transaction lock.
|
||||
let lockless_cache = opts.is_lockless_cache_write() && snapshot_admission.is_none();
|
||||
if lockless_cache || crate::bucket::utils::is_meta_bucketname(bucket) {
|
||||
if !lockless_cache {
|
||||
let meta_bucket_fast_path_started = rustfs_io_metrics::put_stage_timer();
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_META_BUCKET_FAST_PATH,
|
||||
meta_bucket_fast_path_started,
|
||||
);
|
||||
}
|
||||
return Ok(QuotaContext {
|
||||
store: None,
|
||||
bucket: bucket.to_string(),
|
||||
|
||||
@@ -52,9 +52,9 @@ use super::replication_resync_boundary::{
|
||||
#[cfg(test)]
|
||||
use super::replication_resync_boundary::{RESYNC_META_FORMAT, RESYNC_META_VERSION, WIRE_ZERO_TIME_UNIX};
|
||||
use super::replication_storage_boundary::{
|
||||
AdvancedGetOptions, EcstoreObjectOperations, GetObjectReader, HTTPPreconditions, HTTPRangeSpec, ObjectInfo, ObjectOptions,
|
||||
ObjectToDelete, ReplicationObjectIO, ReplicationStatusWritebackCondition, ReplicationStatusWritebackMode, ReplicationStorage,
|
||||
StatObjectOptions, StorageObjectInfoOrErr, WalkOptions,
|
||||
EcstoreObjectOperations, GetObjectReader, HTTPPreconditions, HTTPRangeSpec, ObjectInfo, ObjectOptions, ObjectToDelete,
|
||||
ReplicationObjectIO, ReplicationStatusWritebackCondition, ReplicationStatusWritebackMode, ReplicationStorage,
|
||||
StorageObjectInfoOrErr, WalkOptions,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use super::replication_storage_boundary::{NamespaceLockFence, NamespaceLockSignalTestFence, ReplicationDeletedObject};
|
||||
@@ -89,9 +89,9 @@ use rmp_serde;
|
||||
use rustfs_gateway_types::persistence::PersistedReplicationConfiguration;
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_utils::http::{
|
||||
AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_MODE, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_TAGGING_DIRECTIVE,
|
||||
SUFFIX_REPLICATION_RESET, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TARGET_VERSION_ARN_PREFIX, has_internal_suffix,
|
||||
insert_str, replication_target_versions,
|
||||
AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_MODE, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, SUFFIX_REPLICATION_RESET,
|
||||
SUFFIX_REPLICATION_RESET_ARN_PREFIX, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TARGET_VERSION_ARN_PREFIX,
|
||||
has_internal_suffix, insert_str, replication_target_versions,
|
||||
};
|
||||
use rustfs_utils::{DEFAULT_SIP_HASH_KEY, get_env_usize, sip_hash};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
@@ -4116,6 +4116,17 @@ fn replication_status_writeback_mode(state_update_needed: bool) -> ReplicationSt
|
||||
}
|
||||
}
|
||||
|
||||
/// Source-object metadata a replication worker publishes with its terminal
|
||||
/// status.
|
||||
#[derive(Clone, Copy, Default)]
|
||||
struct ReplicationStatusWriteback<'a> {
|
||||
replication_internal: Option<&'a String>,
|
||||
/// `(arn, version id the target assigned)` for targets minting their own ids.
|
||||
target_version_ledger: &'a [(String, String)],
|
||||
/// `(arn, resync stamp)` for targets this attempt resynced.
|
||||
resync_stamps: &'a [(String, String)],
|
||||
}
|
||||
|
||||
/// Publish a worker's status with a storage-enforced compare-and-set token.
|
||||
/// `put_object_metadata` checks the token while it owns the object write lock,
|
||||
/// avoiding both a read/write race and any need to hold a hot object lock
|
||||
@@ -4123,12 +4134,16 @@ fn replication_status_writeback_mode(state_update_needed: bool) -> ReplicationSt
|
||||
fn replication_status_writeback_options(
|
||||
roi: &ReplicateObjectInfo,
|
||||
replication_lock_guard: &rustfs_lock::NamespaceLockGuard,
|
||||
new_replication_internal: Option<&String>,
|
||||
target_version_ledger: &[(String, String)],
|
||||
writeback: ReplicationStatusWriteback<'_>,
|
||||
mode: ReplicationStatusWritebackMode,
|
||||
) -> ObjectOptions {
|
||||
let ReplicationStatusWriteback {
|
||||
replication_internal,
|
||||
target_version_ledger,
|
||||
resync_stamps,
|
||||
} = writeback;
|
||||
let mut eval_metadata = HashMap::new();
|
||||
if let Some(status) = new_replication_internal {
|
||||
if let Some(status) = replication_internal {
|
||||
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_STATUS, status.clone());
|
||||
}
|
||||
// One durable key per drifting target: the version id it assigned to this
|
||||
@@ -4141,6 +4156,12 @@ fn replication_status_writeback_options(
|
||||
assigned_version_id.clone(),
|
||||
);
|
||||
}
|
||||
// The per-target existing-object resync stamp. Without it the object keeps
|
||||
// answering "not yet resynced" for the target's reset id, and the scanner
|
||||
// re-queues it on every cycle.
|
||||
for (arn, stamp) in resync_stamps {
|
||||
insert_str(&mut eval_metadata, &format!("{SUFFIX_REPLICATION_RESET_ARN_PREFIX}{arn}"), stamp.clone());
|
||||
}
|
||||
let mut write_opts = ObjectOptions {
|
||||
version_id: roi.version_id.map(|version_id| version_id.to_string()),
|
||||
eval_metadata: Some(eval_metadata),
|
||||
@@ -4162,13 +4183,11 @@ async fn persist_replication_state_if_current<S: ReplicationStorage>(
|
||||
roi: &ReplicateObjectInfo,
|
||||
storage: &Arc<S>,
|
||||
replication_lock_guard: &rustfs_lock::NamespaceLockGuard,
|
||||
new_replication_internal: Option<&String>,
|
||||
target_version_ledger: &[(String, String)],
|
||||
writeback: ReplicationStatusWriteback<'_>,
|
||||
mode: ReplicationStatusWritebackMode,
|
||||
object_info: &mut ObjectInfo,
|
||||
) -> Result<ReplicationStatePersistOutcome> {
|
||||
let write_opts =
|
||||
replication_status_writeback_options(roi, replication_lock_guard, new_replication_internal, target_version_ledger, mode);
|
||||
let write_opts = replication_status_writeback_options(roi, replication_lock_guard, writeback, mode);
|
||||
match storage.put_object_metadata(&roi.bucket, &roi.name, &write_opts).await {
|
||||
Ok(updated) => {
|
||||
*object_info = updated;
|
||||
@@ -4334,6 +4353,12 @@ pub(crate) async fn replicate_object_with_outcome<S: ReplicationStorage>(
|
||||
.filter(|target| !target.is_empty())
|
||||
.filter_map(|target| Some((target.arn.clone(), target.target_version_id.clone()?)))
|
||||
.collect();
|
||||
let resync_stamps: Vec<(String, String)> = rinfos
|
||||
.targets
|
||||
.iter()
|
||||
.filter(|target| !target.is_empty() && !target.resync_timestamp.is_empty())
|
||||
.map(|target| (target.arn.clone(), target.resync_timestamp.clone()))
|
||||
.collect();
|
||||
let state_update_needed = roi.replication_status_internal != new_replication_internal
|
||||
|| rinfos.replication_resynced()
|
||||
|| !target_version_ledger.is_empty();
|
||||
@@ -4343,8 +4368,11 @@ pub(crate) async fn replicate_object_with_outcome<S: ReplicationStorage>(
|
||||
&roi,
|
||||
&storage,
|
||||
&obj_lock_guard,
|
||||
new_replication_internal.as_ref(),
|
||||
&target_version_ledger,
|
||||
ReplicationStatusWriteback {
|
||||
replication_internal: new_replication_internal.as_ref(),
|
||||
target_version_ledger: &target_version_ledger,
|
||||
resync_stamps: &resync_stamps,
|
||||
},
|
||||
writeback_mode,
|
||||
&mut object_info,
|
||||
)
|
||||
@@ -5005,8 +5033,6 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
return rinfo;
|
||||
}
|
||||
|
||||
let _sopts = replicate_all_stat_options(&object_info, &bucket, &tgt_client);
|
||||
|
||||
let Some((replication_action, object_info)) = resolve_replicate_all_action(
|
||||
ReplicateAllActionContext {
|
||||
siblings: &storage,
|
||||
@@ -5090,6 +5116,13 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
return rinfo;
|
||||
}
|
||||
|
||||
// The transfer is this target's resync of the object. Stamp it like the
|
||||
// already-converged outcome above, or the next scanner cycle admits the
|
||||
// object for the same reset id again.
|
||||
if self.op_type == ReplicationType::ExistingObject && !tgt_client.reset_id.is_empty() {
|
||||
apply_replication_resync_timestamp(&mut rinfo, &tgt_client.reset_id);
|
||||
}
|
||||
|
||||
rinfo
|
||||
}
|
||||
|
||||
@@ -5238,34 +5271,6 @@ fn note_replicate_all_target_bucket_empty(bucket: &str, tgt_client: &TargetClien
|
||||
});
|
||||
}
|
||||
|
||||
/// Build the stat options for the target metadata comparison, logging (without
|
||||
/// failing) when the tagging directive header cannot be set.
|
||||
fn replicate_all_stat_options(object_info: &ObjectInfo, bucket: &str, tgt_client: &TargetClient) -> StatObjectOptions {
|
||||
let mut sopts = StatObjectOptions {
|
||||
version_id: object_info.version_id.map(|v| v.to_string()).unwrap_or_default(),
|
||||
internal: AdvancedGetOptions {
|
||||
replication_proxy_request: "false".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
if let Err(err) = sopts.set(AMZ_TAGGING_DIRECTIVE, "ACCESS") {
|
||||
debug!(
|
||||
event = EVENT_RESYNC_RUNTIME_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
arn = %tgt_client.arn,
|
||||
error = %err,
|
||||
reason = "tagging_directive_header_invalid",
|
||||
"Skipped replication tagging directive header detail"
|
||||
);
|
||||
}
|
||||
|
||||
sopts
|
||||
}
|
||||
|
||||
/// Record a failed payload transfer: mark the outcome FAILED, log the target
|
||||
/// operation failure, and take the target offline when the error is a network
|
||||
/// failure.
|
||||
@@ -6395,8 +6400,7 @@ mod tests {
|
||||
let opts = replication_status_writeback_options(
|
||||
&ReplicateObjectInfo::default(),
|
||||
&guard,
|
||||
None,
|
||||
&[],
|
||||
ReplicationStatusWriteback::default(),
|
||||
ReplicationStatusWritebackMode::ValidateOnly,
|
||||
);
|
||||
forced_lost.store(true, std::sync::atomic::Ordering::Release);
|
||||
@@ -6415,6 +6419,76 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn terminal_writeback_persists_resync_stamp_that_ends_existing_object_resync() {
|
||||
let lock = rustfs_lock::NamespaceLock::new(
|
||||
"replication-status-writeback-resync-stamp".to_string(),
|
||||
Arc::new(rustfs_lock::LocalClient::new()),
|
||||
);
|
||||
let guard = lock
|
||||
.get_write_lock(
|
||||
rustfs_lock::ObjectKey::new("bucket", "/[replicate]/object"),
|
||||
"worker-a",
|
||||
std::time::Duration::from_secs(2),
|
||||
)
|
||||
.await
|
||||
.expect("replication lock should be acquired");
|
||||
let arn = "arn:rustfs:replication::target:bucket";
|
||||
let reset_before = OffsetDateTime::now_utc();
|
||||
let mut rinfo = ReplicatedTargetInfo {
|
||||
arn: arn.to_string(),
|
||||
replication_status: ReplicationStatusType::Completed,
|
||||
..Default::default()
|
||||
};
|
||||
apply_replication_resync_timestamp(&mut rinfo, "reset-1");
|
||||
let resync_decision = |user_defined: &HashMap<String, String>, reset_id: &str| {
|
||||
super::super::replication_object_decision_boundary::resync_target_for_object(
|
||||
&super::super::replication_object_decision_boundary::ReplicationResyncTargetObject {
|
||||
mod_time: Some(reset_before - time::Duration::hours(1)),
|
||||
user_defined,
|
||||
},
|
||||
arn,
|
||||
reset_id,
|
||||
Some(reset_before),
|
||||
ReplicationStatusType::Completed,
|
||||
)
|
||||
.replicate
|
||||
};
|
||||
let persisted = |resync_stamps: &[(String, String)]| {
|
||||
replication_status_writeback_options(
|
||||
&ReplicateObjectInfo::default(),
|
||||
&guard,
|
||||
ReplicationStatusWriteback {
|
||||
resync_stamps,
|
||||
..Default::default()
|
||||
},
|
||||
ReplicationStatusWritebackMode::Update,
|
||||
)
|
||||
.eval_metadata
|
||||
.expect("writeback always carries evaluated metadata")
|
||||
};
|
||||
|
||||
let unstamped = persisted(&[]);
|
||||
assert!(
|
||||
resync_decision(&unstamped, "reset-1"),
|
||||
"an object older than the reset date stays admitted until it carries the stamp"
|
||||
);
|
||||
|
||||
let stamped = persisted(&[(rinfo.arn.clone(), rinfo.resync_timestamp.clone())]);
|
||||
let suffix = format!("{SUFFIX_REPLICATION_RESET_ARN_PREFIX}{arn}");
|
||||
for prefix in [
|
||||
rustfs_utils::http::RUSTFS_INTERNAL_PREFIX,
|
||||
rustfs_utils::http::MINIO_INTERNAL_PREFIX,
|
||||
] {
|
||||
assert_eq!(stamped.get(&format!("{prefix}{suffix}")), Some(&rinfo.resync_timestamp));
|
||||
}
|
||||
assert!(
|
||||
!resync_decision(&stamped, "reset-1"),
|
||||
"the persisted stamp must end the resync for the same reset id"
|
||||
);
|
||||
assert!(resync_decision(&stamped, "reset-2"), "a new reset id must admit the object again");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unavailable_object_target_is_persisted_as_failed() {
|
||||
let arn = "arn:object-target";
|
||||
|
||||
@@ -33,7 +33,6 @@ pub(crate) use crate::storage_api_contracts::object::{
|
||||
};
|
||||
pub(crate) use crate::storage_api_contracts::range::HTTPRangeSpec;
|
||||
pub(crate) use rustfs_replication::{DeletedObject as ReplicationDeletedObject, ObjectToDelete as ReplicationObjectToDelete};
|
||||
pub(crate) use rustfs_s3_client::api_get_options::{AdvancedGetOptions, StatObjectOptions};
|
||||
|
||||
type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
|
||||
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -60,7 +60,7 @@ use time::OffsetDateTime;
|
||||
use tracing::{error, info, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
const SIGNATURE_HEADER: &str = "x-rustfs-signature";
|
||||
pub(crate) const SIGNATURE_HEADER: &str = "x-rustfs-signature";
|
||||
pub const RPC_BACKGROUND_ADMISSION_HEADER: &str = "x-rustfs-rpc-background-admission";
|
||||
const RPC_BACKGROUND_ADMISSION_V1: &str = "v1";
|
||||
const RPC_BACKGROUND_ADMISSION_DOMAIN: &[u8] = b"rustfs-rpc-background-admission-v1\0";
|
||||
@@ -78,6 +78,9 @@ pub const RPC_BOOT_EPOCH_CHALLENGE_HEADER: &str = "x-rustfs-rpc-boot-epoch-chall
|
||||
pub const RPC_BOOT_EPOCH_PROOF_HEADER: &str = "x-rustfs-rpc-boot-epoch-proof";
|
||||
pub(crate) const RPC_REPLAY_CACHE_CAPABILITY_HEADER: &str = "x-rustfs-rpc-replay-cache-capability";
|
||||
pub(crate) const RPC_REPLAY_CACHE_CAPABILITY_PROOF_HEADER: &str = "x-rustfs-rpc-replay-cache-capability-proof";
|
||||
pub(crate) const RPC_MSGPACK_MANIFEST_CAPABILITY_REQUEST_HEADER: &str = "x-rustfs-rpc-msgpack-manifest-request";
|
||||
pub(crate) const RPC_MSGPACK_MANIFEST_CAPABILITY_HEADER: &str = "x-rustfs-rpc-msgpack-manifest-capability";
|
||||
pub(crate) const RPC_MSGPACK_MANIFEST_CAPABILITY_PROOF_HEADER: &str = "x-rustfs-rpc-msgpack-manifest-capability-proof";
|
||||
const RPC_REPLAY_SCOPE_VERSION_V3: &str = "3";
|
||||
const RPC_RESPONSE_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-response-proof-v1\0";
|
||||
|
||||
@@ -124,6 +127,8 @@ const RPC_REPLAY_SCOPE_DOMAIN: &[u8] = b"rustfs-rpc-replay-scope-v3\0";
|
||||
const RPC_BOOT_EPOCH_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-boot-epoch-proof-v1\0";
|
||||
const RPC_REPLAY_CACHE_CAPABILITY_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-replay-cache-capability-proof-v1\0";
|
||||
const RPC_REPLAY_CACHE_CAPABILITY_V1: &str = "dynamic-replay-cache-v1";
|
||||
const RPC_MSGPACK_MANIFEST_CAPABILITY_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-msgpack-manifest-capability-proof-v1\0";
|
||||
pub(crate) const RPC_MSGPACK_MANIFEST_CAPABILITY_V1: &str = "msgpack-compat-manifest-v1";
|
||||
const HTTP_PUT_FILE_AUTH_DOMAIN: &[u8] = b"rustfs-http-put-file-auth-v1\0";
|
||||
const HTTP_PUT_FILE_CAPABILITY_AUTH_DOMAIN: &[u8] = b"rustfs-http-put-file-capability-v1\0";
|
||||
const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
|
||||
@@ -139,6 +144,8 @@ const REPLAY_CACHE_AUTO_MEMORY_PERCENT: u64 = 13;
|
||||
const REPLAY_CACHE_METADATA_BYTES: u64 = 1024 * 1024;
|
||||
const REPLAY_CACHE_TABLE_OVERHEAD_BYTES: u64 = 512;
|
||||
const NS_SCANNER_CAPABILITY_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-capability-v3";
|
||||
const NS_SCANNER_COMPLETE_OBSERVATION_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-complete-observation-v1";
|
||||
const NS_SCANNER_DURABLE_PROGRESS_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-durable-progress-v1";
|
||||
const NS_SCANNER_CHECKPOINT_COVERAGE_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-checkpoint-coverage-v1";
|
||||
const NS_SCANNER_TIER_REGISTRY_GENERATION_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-tier-registry-generation-v1";
|
||||
pub const TONIC_RPC_PREFIX: &str = "/node_service.NodeService";
|
||||
@@ -795,6 +802,8 @@ fn update_ns_scanner_capability_mac(
|
||||
server_epoch: Uuid,
|
||||
supports_tier_registry_generation: bool,
|
||||
supports_checkpoint_coverage: bool,
|
||||
supports_durable_progress: bool,
|
||||
supports_complete_observation: bool,
|
||||
) {
|
||||
mac.update(NS_SCANNER_CAPABILITY_AUTH_DOMAIN);
|
||||
mac.update(&NS_SCANNER_PROTOCOL_VERSION.to_be_bytes());
|
||||
@@ -809,14 +818,22 @@ fn update_ns_scanner_capability_mac(
|
||||
if supports_checkpoint_coverage {
|
||||
mac.update(NS_SCANNER_CHECKPOINT_COVERAGE_AUTH_DOMAIN);
|
||||
}
|
||||
if supports_durable_progress {
|
||||
mac.update(NS_SCANNER_DURABLE_PROGRESS_AUTH_DOMAIN);
|
||||
}
|
||||
if supports_complete_observation {
|
||||
mac.update(NS_SCANNER_COMPLETE_OBSERVATION_AUTH_DOMAIN);
|
||||
}
|
||||
}
|
||||
|
||||
fn generate_ns_scanner_capability_proof(
|
||||
fn generate_ns_scanner_capability_proof_with_observation(
|
||||
secret: &str,
|
||||
challenge: Uuid,
|
||||
server_epoch: Uuid,
|
||||
supports_tier_registry_generation: bool,
|
||||
supports_checkpoint_coverage: bool,
|
||||
supports_durable_progress: bool,
|
||||
supports_complete_observation: bool,
|
||||
) -> std::io::Result<Vec<u8>> {
|
||||
if challenge.is_nil() || server_epoch.is_nil() {
|
||||
return Err(std::io::Error::other("Invalid namespace scanner capability scope"));
|
||||
@@ -829,17 +846,22 @@ fn generate_ns_scanner_capability_proof(
|
||||
server_epoch,
|
||||
supports_tier_registry_generation,
|
||||
supports_checkpoint_coverage,
|
||||
supports_durable_progress,
|
||||
supports_complete_observation,
|
||||
);
|
||||
Ok(mac.finalize().into_bytes().to_vec())
|
||||
}
|
||||
|
||||
fn verify_ns_scanner_capability_proof(
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn verify_ns_scanner_capability_proof_with_observation(
|
||||
secret: &str,
|
||||
challenge: Uuid,
|
||||
server_epoch: Uuid,
|
||||
proof: &[u8],
|
||||
supports_tier_registry_generation: bool,
|
||||
supports_checkpoint_coverage: bool,
|
||||
supports_durable_progress: bool,
|
||||
supports_complete_observation: bool,
|
||||
) -> std::io::Result<()> {
|
||||
if challenge.is_nil() || server_epoch.is_nil() {
|
||||
return Err(std::io::Error::other("Invalid namespace scanner capability scope"));
|
||||
@@ -852,11 +874,35 @@ fn verify_ns_scanner_capability_proof(
|
||||
server_epoch,
|
||||
supports_tier_registry_generation,
|
||||
supports_checkpoint_coverage,
|
||||
supports_durable_progress,
|
||||
supports_complete_observation,
|
||||
);
|
||||
mac.verify_slice(proof)
|
||||
.map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid namespace scanner capability proof"))
|
||||
}
|
||||
|
||||
fn generate_ns_scanner_capability_proof(
|
||||
secret: &str,
|
||||
challenge: Uuid,
|
||||
server_epoch: Uuid,
|
||||
tier: bool,
|
||||
checkpoint: bool,
|
||||
progress: bool,
|
||||
) -> std::io::Result<Vec<u8>> {
|
||||
generate_ns_scanner_capability_proof_with_observation(secret, challenge, server_epoch, tier, checkpoint, progress, false)
|
||||
}
|
||||
fn verify_ns_scanner_capability_proof(
|
||||
secret: &str,
|
||||
challenge: Uuid,
|
||||
server_epoch: Uuid,
|
||||
proof: &[u8],
|
||||
tier: bool,
|
||||
checkpoint: bool,
|
||||
progress: bool,
|
||||
) -> std::io::Result<()> {
|
||||
verify_ns_scanner_capability_proof_with_observation(secret, challenge, server_epoch, proof, tier, checkpoint, progress, false)
|
||||
}
|
||||
|
||||
pub fn sign_ns_scanner_capability(challenge: Uuid, server_epoch: Uuid) -> std::io::Result<Vec<u8>> {
|
||||
sign_ns_scanner_capability_with_tier_registry_generation(challenge, server_epoch, false)
|
||||
}
|
||||
@@ -878,6 +924,22 @@ pub fn sign_ns_scanner_capability_with_checkpoint_coverage(
|
||||
server_epoch: Uuid,
|
||||
supports_tier_registry_generation: bool,
|
||||
supports_checkpoint_coverage: bool,
|
||||
) -> std::io::Result<Vec<u8>> {
|
||||
sign_ns_scanner_capability_with_durable_progress(
|
||||
challenge,
|
||||
server_epoch,
|
||||
supports_tier_registry_generation,
|
||||
supports_checkpoint_coverage,
|
||||
false,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn sign_ns_scanner_capability_with_durable_progress(
|
||||
challenge: Uuid,
|
||||
server_epoch: Uuid,
|
||||
supports_tier_registry_generation: bool,
|
||||
supports_checkpoint_coverage: bool,
|
||||
supports_durable_progress: bool,
|
||||
) -> std::io::Result<Vec<u8>> {
|
||||
generate_ns_scanner_capability_proof(
|
||||
&get_shared_secret()?,
|
||||
@@ -885,6 +947,7 @@ pub fn sign_ns_scanner_capability_with_checkpoint_coverage(
|
||||
server_epoch,
|
||||
supports_tier_registry_generation,
|
||||
supports_checkpoint_coverage,
|
||||
supports_durable_progress,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -909,6 +972,24 @@ pub fn verify_ns_scanner_capability_with_checkpoint_coverage(
|
||||
proof: &[u8],
|
||||
supports_tier_registry_generation: bool,
|
||||
supports_checkpoint_coverage: bool,
|
||||
) -> std::io::Result<()> {
|
||||
verify_ns_scanner_capability_with_durable_progress(
|
||||
challenge,
|
||||
server_epoch,
|
||||
proof,
|
||||
supports_tier_registry_generation,
|
||||
supports_checkpoint_coverage,
|
||||
false,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn verify_ns_scanner_capability_with_durable_progress(
|
||||
challenge: Uuid,
|
||||
server_epoch: Uuid,
|
||||
proof: &[u8],
|
||||
supports_tier_registry_generation: bool,
|
||||
supports_checkpoint_coverage: bool,
|
||||
supports_durable_progress: bool,
|
||||
) -> std::io::Result<()> {
|
||||
verify_ns_scanner_capability_proof(
|
||||
&get_shared_secret()?,
|
||||
@@ -917,6 +998,74 @@ pub fn verify_ns_scanner_capability_with_checkpoint_coverage(
|
||||
proof,
|
||||
supports_tier_registry_generation,
|
||||
supports_checkpoint_coverage,
|
||||
supports_durable_progress,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn sign_ns_scanner_capability_with_usage_capabilities(
|
||||
challenge: Uuid,
|
||||
server_epoch: Uuid,
|
||||
tier: bool,
|
||||
checkpoint: bool,
|
||||
progress: bool,
|
||||
observation: bool,
|
||||
) -> std::io::Result<Vec<u8>> {
|
||||
generate_ns_scanner_capability_proof_with_observation(
|
||||
&get_shared_secret()?,
|
||||
challenge,
|
||||
server_epoch,
|
||||
tier,
|
||||
checkpoint,
|
||||
progress,
|
||||
observation,
|
||||
)
|
||||
}
|
||||
fn ns_scanner_usage_watch_capability_mac(secret: &str, challenge: Uuid, server_epoch: Uuid) -> std::io::Result<HmacSha256> {
|
||||
if challenge.is_nil() || server_epoch.is_nil() {
|
||||
return Err(std::io::Error::other("Invalid namespace scanner usage watch capability scope"));
|
||||
}
|
||||
let mut mac =
|
||||
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
|
||||
mac.update(b"rustfs-ns-scanner-usage-watch-capability-v1\0");
|
||||
mac.update(&NS_SCANNER_PROTOCOL_VERSION.to_be_bytes());
|
||||
mac.update(challenge.as_bytes());
|
||||
mac.update(server_epoch.as_bytes());
|
||||
Ok(mac)
|
||||
}
|
||||
|
||||
/// Sign the independent worker capability to validate bucket watches before
|
||||
/// reusing or durably publishing a complete scanner cache.
|
||||
pub fn sign_ns_scanner_usage_watch_capability(challenge: Uuid, server_epoch: Uuid) -> std::io::Result<Vec<u8>> {
|
||||
Ok(ns_scanner_usage_watch_capability_mac(&get_shared_secret()?, challenge, server_epoch)?
|
||||
.finalize()
|
||||
.into_bytes()
|
||||
.to_vec())
|
||||
}
|
||||
|
||||
pub fn verify_ns_scanner_usage_watch_capability(challenge: Uuid, server_epoch: Uuid, proof: &[u8]) -> std::io::Result<()> {
|
||||
ns_scanner_usage_watch_capability_mac(&get_shared_secret()?, challenge, server_epoch)?
|
||||
.verify_slice(proof)
|
||||
.map_err(|_| std::io::Error::other("Invalid namespace scanner usage watch capability proof"))
|
||||
}
|
||||
|
||||
pub fn verify_ns_scanner_capability_with_usage_capabilities(
|
||||
challenge: Uuid,
|
||||
server_epoch: Uuid,
|
||||
proof: &[u8],
|
||||
tier: bool,
|
||||
checkpoint: bool,
|
||||
progress: bool,
|
||||
observation: bool,
|
||||
) -> std::io::Result<()> {
|
||||
verify_ns_scanner_capability_proof_with_observation(
|
||||
&get_shared_secret()?,
|
||||
challenge,
|
||||
server_epoch,
|
||||
proof,
|
||||
tier,
|
||||
checkpoint,
|
||||
progress,
|
||||
observation,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1094,6 +1243,50 @@ fn verify_replay_cache_capability_proof(
|
||||
.map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid RPC replay cache capability proof"))
|
||||
}
|
||||
|
||||
fn update_msgpack_manifest_capability_proof(mac: &mut HmacSha256, audience: &str, challenge: Uuid, boot_epoch: Uuid) {
|
||||
mac.update(RPC_MSGPACK_MANIFEST_CAPABILITY_PROOF_DOMAIN);
|
||||
for part in [
|
||||
audience.as_bytes(),
|
||||
b"|",
|
||||
challenge.as_bytes(),
|
||||
b"|",
|
||||
boot_epoch.as_bytes(),
|
||||
b"|",
|
||||
RPC_MSGPACK_MANIFEST_CAPABILITY_V1.as_bytes(),
|
||||
] {
|
||||
mac.update(part);
|
||||
}
|
||||
}
|
||||
|
||||
fn generate_msgpack_manifest_capability_proof(
|
||||
secret: &str,
|
||||
audience: &str,
|
||||
challenge: Uuid,
|
||||
boot_epoch: Uuid,
|
||||
) -> std::io::Result<String> {
|
||||
let mut mac =
|
||||
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
|
||||
update_msgpack_manifest_capability_proof(&mut mac, audience, challenge, boot_epoch);
|
||||
Ok(base64_simd::STANDARD.encode_to_string(mac.finalize().into_bytes()))
|
||||
}
|
||||
|
||||
fn verify_msgpack_manifest_capability_proof(
|
||||
secret: &str,
|
||||
audience: &str,
|
||||
challenge: Uuid,
|
||||
boot_epoch: Uuid,
|
||||
proof: &str,
|
||||
) -> std::io::Result<()> {
|
||||
let proof = base64_simd::STANDARD
|
||||
.decode_to_vec(proof)
|
||||
.map_err(|_| std::io::Error::other("Invalid RPC msgpack manifest capability proof"))?;
|
||||
let mut mac =
|
||||
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
|
||||
update_msgpack_manifest_capability_proof(&mut mac, audience, challenge, boot_epoch);
|
||||
mac.verify_slice(&proof)
|
||||
.map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid RPC msgpack manifest capability proof"))
|
||||
}
|
||||
|
||||
fn non_nil_uuid(value: &str, name: &str) -> std::io::Result<Uuid> {
|
||||
let value = Uuid::parse_str(value).map_err(|_| std::io::Error::other(format!("Invalid {name}")))?;
|
||||
(!value.is_nil())
|
||||
@@ -1193,8 +1386,24 @@ pub fn tonic_boot_epoch_challenge(headers: &HeaderMap) -> std::io::Result<Option
|
||||
.transpose()
|
||||
}
|
||||
|
||||
/// Whether the request opts into the optional msgpack manifest capability proof.
|
||||
pub fn tonic_msgpack_manifest_capability_requested(headers: &HeaderMap) -> bool {
|
||||
headers
|
||||
.get(RPC_MSGPACK_MANIFEST_CAPABILITY_REQUEST_HEADER)
|
||||
.is_some_and(|value| value == RPC_MSGPACK_MANIFEST_CAPABILITY_V1)
|
||||
}
|
||||
|
||||
/// Build the authenticated response headers for a client boot-epoch challenge.
|
||||
pub fn tonic_boot_epoch_response_headers(audience: &str, challenge: Uuid) -> std::io::Result<HeaderMap> {
|
||||
tonic_boot_epoch_response_headers_with_msgpack_capability(audience, challenge, false)
|
||||
}
|
||||
|
||||
/// Build authenticated boot-epoch headers and, when requested, the msgpack manifest capability.
|
||||
pub fn tonic_boot_epoch_response_headers_with_msgpack_capability(
|
||||
audience: &str,
|
||||
challenge: Uuid,
|
||||
include_msgpack_manifest: bool,
|
||||
) -> std::io::Result<HeaderMap> {
|
||||
let boot_epoch = tonic_rpc_boot_epoch();
|
||||
let secret = get_shared_secret()?;
|
||||
let proof = generate_boot_epoch_proof(&secret, audience, challenge, boot_epoch)?;
|
||||
@@ -1210,6 +1419,17 @@ pub fn tonic_boot_epoch_response_headers(audience: &str, challenge: Uuid) -> std
|
||||
RPC_REPLAY_CACHE_CAPABILITY_PROOF_HEADER,
|
||||
header_value(&capability_proof, RPC_REPLAY_CACHE_CAPABILITY_PROOF_HEADER)?,
|
||||
);
|
||||
if include_msgpack_manifest {
|
||||
let msgpack_capability_proof = generate_msgpack_manifest_capability_proof(&secret, audience, challenge, boot_epoch)?;
|
||||
headers.insert(
|
||||
RPC_MSGPACK_MANIFEST_CAPABILITY_HEADER,
|
||||
HeaderValue::from_static(RPC_MSGPACK_MANIFEST_CAPABILITY_V1),
|
||||
);
|
||||
headers.insert(
|
||||
RPC_MSGPACK_MANIFEST_CAPABILITY_PROOF_HEADER,
|
||||
header_value(&msgpack_capability_proof, RPC_MSGPACK_MANIFEST_CAPABILITY_PROOF_HEADER)?,
|
||||
);
|
||||
}
|
||||
Ok(headers)
|
||||
}
|
||||
|
||||
@@ -1243,6 +1463,12 @@ pub(crate) struct AuthenticatedPeerReplayCapabilities {
|
||||
pub(crate) dynamic_replay_cache: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub(crate) struct AuthenticatedPeerMsgpackCapabilities {
|
||||
pub(crate) boot_epoch: Uuid,
|
||||
pub(crate) msgpack_compat_manifest: bool,
|
||||
}
|
||||
|
||||
pub(crate) fn verify_tonic_peer_replay_capabilities_response(
|
||||
audience: &str,
|
||||
challenge: Uuid,
|
||||
@@ -1274,6 +1500,57 @@ pub(crate) fn verify_tonic_peer_replay_capabilities_response(
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn verify_tonic_peer_msgpack_capabilities_response(
|
||||
audience: &str,
|
||||
challenge: Uuid,
|
||||
headers: &HeaderMap,
|
||||
) -> std::io::Result<AuthenticatedPeerMsgpackCapabilities> {
|
||||
let secret = get_shared_secret()?;
|
||||
let boot_epoch = verify_tonic_boot_epoch_response_with_secret(&secret, audience, challenge, headers)?;
|
||||
verify_tonic_peer_msgpack_capability_for_epoch_with_secret(&secret, audience, challenge, boot_epoch, headers)
|
||||
}
|
||||
|
||||
pub(crate) fn verify_tonic_peer_msgpack_capability_for_authenticated_epoch(
|
||||
audience: &str,
|
||||
challenge: Uuid,
|
||||
boot_epoch: Uuid,
|
||||
headers: &HeaderMap,
|
||||
) -> std::io::Result<AuthenticatedPeerMsgpackCapabilities> {
|
||||
verify_tonic_peer_msgpack_capability_for_epoch_with_secret(&get_shared_secret()?, audience, challenge, boot_epoch, headers)
|
||||
}
|
||||
|
||||
fn verify_tonic_peer_msgpack_capability_for_epoch_with_secret(
|
||||
secret: &str,
|
||||
audience: &str,
|
||||
challenge: Uuid,
|
||||
boot_epoch: Uuid,
|
||||
headers: &HeaderMap,
|
||||
) -> std::io::Result<AuthenticatedPeerMsgpackCapabilities> {
|
||||
let capability = headers.get(RPC_MSGPACK_MANIFEST_CAPABILITY_HEADER);
|
||||
let proof = headers.get(RPC_MSGPACK_MANIFEST_CAPABILITY_PROOF_HEADER);
|
||||
if capability.is_none() && proof.is_none() {
|
||||
return Ok(AuthenticatedPeerMsgpackCapabilities {
|
||||
boot_epoch,
|
||||
msgpack_compat_manifest: false,
|
||||
});
|
||||
}
|
||||
let capability = capability
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing RPC msgpack manifest capability"))?;
|
||||
if capability != RPC_MSGPACK_MANIFEST_CAPABILITY_V1 {
|
||||
return Err(std::io::Error::other("Unsupported RPC msgpack manifest capability"));
|
||||
}
|
||||
let proof = proof
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing RPC msgpack manifest capability proof"))?;
|
||||
verify_msgpack_manifest_capability_proof(secret, audience, challenge, boot_epoch, proof)?;
|
||||
Ok(AuthenticatedPeerMsgpackCapabilities {
|
||||
boot_epoch,
|
||||
msgpack_compat_manifest: true,
|
||||
})
|
||||
}
|
||||
|
||||
fn valid_content_sha256(value: &str) -> bool {
|
||||
value == UNSIGNED_PAYLOAD
|
||||
|| (value.len() == 64
|
||||
@@ -1309,7 +1586,7 @@ fn tonic_rpc_metric_operation(path: &str) -> &'static str {
|
||||
Some("ReadMultiple") => INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
|
||||
Some("ReadVersion") => INTERNODE_OPERATION_GRPC_READ_VERSION,
|
||||
Some("BatchReadVersion") => INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
|
||||
Some("WriteAll") => INTERNODE_OPERATION_GRPC_WRITE_ALL,
|
||||
Some("WriteAll" | "WriteFormatControl") => INTERNODE_OPERATION_GRPC_WRITE_ALL,
|
||||
Some("CompareAndUpdateFile") => INTERNODE_OPERATION_GRPC_COMPARE_AND_UPDATE_FILE,
|
||||
Some("Lock") => INTERNODE_OPERATION_GRPC_LOCK,
|
||||
Some("UnLock") => INTERNODE_OPERATION_GRPC_UNLOCK,
|
||||
@@ -2202,13 +2479,119 @@ mod tests {
|
||||
let secret = "test-scanner-capability-secret";
|
||||
let challenge = Uuid::new_v4();
|
||||
let server_epoch = Uuid::new_v4();
|
||||
let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, false, false)
|
||||
let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, false, false, false)
|
||||
.expect("capability proof should be generated");
|
||||
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false, false).is_ok());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, Uuid::new_v4(), server_epoch, &proof, false, false).is_err());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, Uuid::new_v4(), &proof, false, false).is_err());
|
||||
assert!(verify_ns_scanner_capability_proof("different-secret", challenge, server_epoch, &proof, false, false).is_err());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false, false, false).is_ok());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, Uuid::new_v4(), server_epoch, &proof, false, false, false).is_err());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, Uuid::new_v4(), &proof, false, false, false).is_err());
|
||||
assert!(
|
||||
verify_ns_scanner_capability_proof("different-secret", challenge, server_epoch, &proof, false, false, false).is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn namespace_scanner_watch_capability_is_independent_and_rejects_replay() {
|
||||
let _ = rustfs_credentials::set_global_rpc_secret("usage-watch-capability-test-secret".into());
|
||||
let challenge = Uuid::new_v4();
|
||||
let epoch = Uuid::new_v4();
|
||||
let proof = sign_ns_scanner_usage_watch_capability(challenge, epoch).expect("watch capability");
|
||||
verify_ns_scanner_usage_watch_capability(challenge, epoch, &proof).expect("exact watch capability scope");
|
||||
assert!(verify_ns_scanner_usage_watch_capability(Uuid::new_v4(), epoch, &proof).is_err());
|
||||
assert!(verify_ns_scanner_usage_watch_capability(challenge, Uuid::new_v4(), &proof).is_err());
|
||||
for progress in [false, true] {
|
||||
for observation in [false, true] {
|
||||
let other =
|
||||
sign_ns_scanner_capability_with_usage_capabilities(challenge, epoch, true, true, progress, observation)
|
||||
.expect("independent older capabilities");
|
||||
assert!(verify_ns_scanner_usage_watch_capability(challenge, epoch, &other).is_err());
|
||||
assert!(
|
||||
verify_ns_scanner_capability_with_usage_capabilities(
|
||||
challenge,
|
||||
epoch,
|
||||
&proof,
|
||||
true,
|
||||
true,
|
||||
progress,
|
||||
observation
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn namespace_scanner_observation_and_progress_scopes_are_independent() {
|
||||
let challenge = Uuid::new_v4();
|
||||
let epoch = Uuid::new_v4();
|
||||
let secret = "independent-usage-capabilities";
|
||||
for progress in [false, true] {
|
||||
for observation in [false, true] {
|
||||
let proof = generate_ns_scanner_capability_proof_with_observation(
|
||||
secret,
|
||||
challenge,
|
||||
epoch,
|
||||
true,
|
||||
true,
|
||||
progress,
|
||||
observation,
|
||||
)
|
||||
.expect("proof");
|
||||
assert!(
|
||||
verify_ns_scanner_capability_proof_with_observation(
|
||||
secret,
|
||||
challenge,
|
||||
epoch,
|
||||
&proof,
|
||||
true,
|
||||
true,
|
||||
progress,
|
||||
observation
|
||||
)
|
||||
.is_ok()
|
||||
);
|
||||
assert!(
|
||||
verify_ns_scanner_capability_proof_with_observation(
|
||||
secret,
|
||||
challenge,
|
||||
epoch,
|
||||
&proof,
|
||||
true,
|
||||
true,
|
||||
!progress,
|
||||
observation
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
verify_ns_scanner_capability_proof_with_observation(
|
||||
secret,
|
||||
challenge,
|
||||
epoch,
|
||||
&proof,
|
||||
true,
|
||||
true,
|
||||
progress,
|
||||
!observation
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn namespace_scanner_capability_proof_binds_durable_progress_independently() {
|
||||
let challenge = Uuid::new_v4();
|
||||
let epoch = Uuid::new_v4();
|
||||
let secret = "durable-progress-test";
|
||||
for supported in [false, true] {
|
||||
let proof = generate_ns_scanner_capability_proof(secret, challenge, epoch, true, true, supported).expect("proof");
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, epoch, &proof, true, true, supported).is_ok());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, epoch, &proof, true, true, !supported).is_err());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, Uuid::new_v4(), epoch, &proof, true, true, supported).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2216,16 +2599,16 @@ mod tests {
|
||||
let secret = "checkpoint-capability-test-secret";
|
||||
let challenge = Uuid::new_v4();
|
||||
let server_epoch = Uuid::new_v4();
|
||||
let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, true, true)
|
||||
let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, true, true, false)
|
||||
.expect("checkpoint capability proof");
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, true, true).is_ok());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, true, false).is_err());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false, true).is_err());
|
||||
let legacy = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, true, false)
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, true, true, false).is_ok());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, true, false, false).is_err());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false, true, false).is_err());
|
||||
let legacy = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, true, false, false)
|
||||
.expect("legacy generation capability proof");
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &legacy, true, true).is_err());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &[], true, true).is_err());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, Uuid::new_v4(), server_epoch, &proof, true, true).is_err());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &legacy, true, true, false).is_err());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &[], true, true, false).is_err());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, Uuid::new_v4(), server_epoch, &proof, true, true, false).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2233,14 +2616,14 @@ mod tests {
|
||||
let secret = "test-scanner-capability-secret";
|
||||
let challenge = Uuid::new_v4();
|
||||
let server_epoch = Uuid::new_v4();
|
||||
let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, true, false)
|
||||
let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, true, false, false)
|
||||
.expect("generation capability proof should be generated");
|
||||
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, true, false).is_ok());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false, false).is_err());
|
||||
let legacy = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, false, false)
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, true, false, false).is_ok());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false, false, false).is_err());
|
||||
let legacy = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, false, false, false)
|
||||
.expect("legacy capability proof should be generated");
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &legacy, true, false).is_err());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &legacy, true, false, false).is_err());
|
||||
}
|
||||
|
||||
/// Security regression for GHSA-r5qv-rc46-hv8q (internode RPC fail-closed,
|
||||
@@ -2897,6 +3280,55 @@ mod tests {
|
||||
assert!(verify_tonic_peer_replay_capabilities_response("node-a:9000", challenge, &changed_capability).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn msgpack_manifest_capability_is_requested_explicitly_and_proved_for_one_epoch() {
|
||||
ensure_test_rpc_secret();
|
||||
let challenge = Uuid::new_v4();
|
||||
let mut request_headers = HeaderMap::new();
|
||||
assert!(!tonic_msgpack_manifest_capability_requested(&request_headers));
|
||||
request_headers.insert(
|
||||
RPC_MSGPACK_MANIFEST_CAPABILITY_REQUEST_HEADER,
|
||||
HeaderValue::from_static(RPC_MSGPACK_MANIFEST_CAPABILITY_V1),
|
||||
);
|
||||
assert!(tonic_msgpack_manifest_capability_requested(&request_headers));
|
||||
request_headers.insert(
|
||||
RPC_MSGPACK_MANIFEST_CAPABILITY_REQUEST_HEADER,
|
||||
HeaderValue::from_static("msgpack-compat-manifest-v2"),
|
||||
);
|
||||
assert!(!tonic_msgpack_manifest_capability_requested(&request_headers));
|
||||
|
||||
let legacy_headers = tonic_boot_epoch_response_headers("node-a:9000", challenge).expect("legacy capability headers");
|
||||
let legacy = verify_tonic_peer_msgpack_capabilities_response("node-a:9000", challenge, &legacy_headers)
|
||||
.expect("authenticated response without msgpack capability");
|
||||
assert_eq!(legacy.boot_epoch, tonic_rpc_boot_epoch());
|
||||
assert!(!legacy.msgpack_compat_manifest);
|
||||
|
||||
let headers = tonic_boot_epoch_response_headers_with_msgpack_capability("node-a:9000", challenge, true)
|
||||
.expect("msgpack capability headers");
|
||||
let capability = verify_tonic_peer_msgpack_capabilities_response("node-a:9000", challenge, &headers)
|
||||
.expect("matching msgpack capability proof");
|
||||
assert_eq!(capability.boot_epoch, tonic_rpc_boot_epoch());
|
||||
assert!(capability.msgpack_compat_manifest);
|
||||
assert!(verify_tonic_peer_msgpack_capabilities_response("node-b:9000", challenge, &headers).is_err());
|
||||
assert!(verify_tonic_peer_msgpack_capabilities_response("node-a:9000", Uuid::new_v4(), &headers).is_err());
|
||||
|
||||
let mut changed_capability = headers.clone();
|
||||
changed_capability.insert(
|
||||
RPC_MSGPACK_MANIFEST_CAPABILITY_HEADER,
|
||||
HeaderValue::from_static("msgpack-compat-manifest-v2"),
|
||||
);
|
||||
assert!(verify_tonic_peer_msgpack_capabilities_response("node-a:9000", challenge, &changed_capability).is_err());
|
||||
let mut forged_proof = headers;
|
||||
forged_proof.insert(RPC_MSGPACK_MANIFEST_CAPABILITY_PROOF_HEADER, HeaderValue::from_static("forged"));
|
||||
assert!(verify_tonic_peer_msgpack_capabilities_response("node-a:9000", challenge, &forged_proof).is_err());
|
||||
let mut missing_proof = forged_proof.clone();
|
||||
missing_proof.remove(RPC_MSGPACK_MANIFEST_CAPABILITY_PROOF_HEADER);
|
||||
assert!(verify_tonic_peer_msgpack_capabilities_response("node-a:9000", challenge, &missing_proof).is_err());
|
||||
let mut missing_value = forged_proof;
|
||||
missing_value.remove(RPC_MSGPACK_MANIFEST_CAPABILITY_HEADER);
|
||||
assert!(verify_tonic_peer_msgpack_capabilities_response("node-a:9000", challenge, &missing_value).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tonic_rpc_auth_failure_reason_maps_security_relevant_errors() {
|
||||
for (message, reason) in [
|
||||
@@ -3195,6 +3627,10 @@ mod tests {
|
||||
tonic_rpc_metric_operation("/node_service.NodeService/WriteAll"),
|
||||
INTERNODE_OPERATION_GRPC_WRITE_ALL
|
||||
);
|
||||
assert_eq!(
|
||||
tonic_rpc_metric_operation("/node_service.NodeService/WriteFormatControl"),
|
||||
INTERNODE_OPERATION_GRPC_WRITE_ALL
|
||||
);
|
||||
assert_eq!(
|
||||
tonic_rpc_metric_operation("/node_service.NodeService/CompareAndUpdateFile"),
|
||||
INTERNODE_OPERATION_GRPC_COMPARE_AND_UPDATE_FILE
|
||||
|
||||
@@ -14,14 +14,16 @@
|
||||
|
||||
use crate::cluster::rpc::{
|
||||
build_auth_headers, build_put_file_auth_trailer, verify_ns_scanner_capability_with_checkpoint_coverage,
|
||||
verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_capability,
|
||||
verify_ns_scanner_capability_with_durable_progress, verify_ns_scanner_capability_with_tier_registry_generation,
|
||||
verify_ns_scanner_capability_with_usage_capabilities, verify_put_file_capability,
|
||||
};
|
||||
use crate::disk::error::{Error, Result};
|
||||
use crate::disk::{FileReader, FileWriter};
|
||||
use crate::storage_api_contracts::internode::{
|
||||
NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CHECKPOINT_COVERAGE_QUERY,
|
||||
NS_SCANNER_CYCLE_QUERY, NS_SCANNER_LEADER_EPOCH_QUERY, NS_SCANNER_PROTOCOL_VERSION, NS_SCANNER_PROTOCOL_VERSION_QUERY,
|
||||
NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY, NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY,
|
||||
NS_SCANNER_COMPLETE_OBSERVATION_QUERY, NS_SCANNER_CYCLE_QUERY, NS_SCANNER_DURABLE_PROGRESS_QUERY,
|
||||
NS_SCANNER_LEADER_EPOCH_QUERY, NS_SCANNER_PROTOCOL_VERSION, NS_SCANNER_PROTOCOL_VERSION_QUERY, NS_SCANNER_REQUEST_ID_QUERY,
|
||||
NS_SCANNER_SERVER_EPOCH_QUERY, NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY,
|
||||
NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY, NsScannerCapabilityResponse, PUT_FILE_AUTH_QUERY, PUT_FILE_AUTH_V1,
|
||||
PUT_FILE_CAPABILITY_CHALLENGE_QUERY, PUT_FILE_CAPABILITY_QUERY, PUT_FILE_CAPABILITY_VERSION, PUT_FILE_NONCE_QUERY,
|
||||
PUT_FILE_SERVER_EPOCH_QUERY, PutFileCapabilityResponse, WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY,
|
||||
@@ -30,8 +32,8 @@ use crate::storage_api_contracts::internode::{
|
||||
use async_trait::async_trait;
|
||||
use http::{HeaderMap, HeaderValue, Method, header::CONTENT_TYPE};
|
||||
use rustfs_config::{
|
||||
DEFAULT_INTERNODE_DATA_TRANSPORT, ENV_RUSTFS_INTERNODE_DATA_TRANSPORT, INTERNODE_DATA_TRANSPORT_TCP,
|
||||
KNOWN_INTERNODE_DATA_TRANSPORT_BACKENDS,
|
||||
DEFAULT_INTERNODE_DATA_TRANSPORT, ENV_RUSTFS_INTERNODE_DATA_TRANSPORT, INTERNODE_DATA_TRANSPORT_NIXL,
|
||||
INTERNODE_DATA_TRANSPORT_TCP, KNOWN_INTERNODE_DATA_TRANSPORT_BACKENDS,
|
||||
};
|
||||
use rustfs_rio::{
|
||||
ChunkReaderBox, HttpChunkReader, HttpReader, HttpWriter, HttpWriterMemoryReservation, HttpWriterMemoryReserver,
|
||||
@@ -63,7 +65,7 @@ const NS_SCANNER_PATH: &str = "/rustfs/rpc/ns_scanner";
|
||||
const NS_SCANNER_MAX_CAPABILITY_RESPONSE_SIZE: usize = 1024;
|
||||
const PUT_FILE_MAX_CAPABILITY_RESPONSE_SIZE: usize = 1024;
|
||||
const PUT_FILE_LEGACY_CAPABILITY_TTL: Duration = Duration::from_secs(30);
|
||||
const PUT_FILE_V1_CAPABILITY_TTL: Duration = Duration::from_secs(30);
|
||||
pub(super) const PUT_FILE_V1_CAPABILITY_TTL: Duration = Duration::from_secs(30);
|
||||
const PUT_FILE_CAPABILITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
const CONTENT_TYPE_JSON: &str = "application/json";
|
||||
const CONTENT_TYPE_MSGPACK: &str = "application/msgpack";
|
||||
@@ -89,7 +91,7 @@ fn unsupported_transport_message(transport: &str) -> String {
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
enum PutFileCapabilityState {
|
||||
pub(super) enum PutFileCapabilityState {
|
||||
LegacyUntil(Instant),
|
||||
V1 { server_epoch: Uuid, revalidate_after: Instant },
|
||||
}
|
||||
@@ -118,8 +120,8 @@ struct PutFileCapabilityFlight {
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct PutFileCapabilityCacheState {
|
||||
cached: Option<PutFileCapabilityState>,
|
||||
pub(super) struct PutFileCapabilityCacheState {
|
||||
pub(super) cached: Option<PutFileCapabilityState>,
|
||||
generation: u64,
|
||||
in_flight: Option<PutFileCapabilityFlight>,
|
||||
rejected_server_epoch: Option<Uuid>,
|
||||
@@ -128,12 +130,12 @@ struct PutFileCapabilityCacheState {
|
||||
// The registry lock is released before taking an entry lock. Entry guards cover
|
||||
// only cache transitions, never a probe or await; poll-based writers must be
|
||||
// able to reject an epoch atomically with those transitions.
|
||||
type PutFileCapabilityCacheEntry = Arc<parking_lot::RwLock<PutFileCapabilityCacheState>>;
|
||||
pub(super) type PutFileCapabilityCacheEntry = Arc<parking_lot::RwLock<PutFileCapabilityCacheState>>;
|
||||
|
||||
static PUT_FILE_CAPABILITY_CACHE: LazyLock<parking_lot::RwLock<HashMap<String, PutFileCapabilityCacheEntry>>> =
|
||||
LazyLock::new(|| parking_lot::RwLock::new(HashMap::new()));
|
||||
|
||||
fn put_file_capability_cache_entry(endpoint: &str) -> PutFileCapabilityCacheEntry {
|
||||
pub(super) fn put_file_capability_cache_entry(endpoint: &str) -> PutFileCapabilityCacheEntry {
|
||||
if let Some(entry) = PUT_FILE_CAPABILITY_CACHE.read().get(endpoint).cloned() {
|
||||
return entry;
|
||||
}
|
||||
@@ -155,7 +157,7 @@ fn fresh_put_file_capability(state: Option<PutFileCapabilityState>, now: Instant
|
||||
}
|
||||
}
|
||||
|
||||
fn reject_put_file_server_epoch(endpoint: &str, server_epoch: Uuid) {
|
||||
pub(super) fn reject_put_file_server_epoch(endpoint: &str, server_epoch: Uuid) {
|
||||
let entry = PUT_FILE_CAPABILITY_CACHE.read().get(endpoint).cloned();
|
||||
if let Some(entry) = entry {
|
||||
let mut state = entry.write();
|
||||
@@ -277,6 +279,9 @@ pub struct NsScannerCapabilityRequest {
|
||||
pub endpoint: String,
|
||||
pub supports_tier_registry_generation: bool,
|
||||
pub supports_checkpoint_coverage: bool,
|
||||
pub supports_durable_progress: bool,
|
||||
pub supports_complete_observation: bool,
|
||||
pub supports_usage_watch: bool,
|
||||
}
|
||||
|
||||
/// Data-plane stream opener used by `RemoteDisk`.
|
||||
@@ -317,6 +322,9 @@ pub trait InternodeDataTransport: Send + Sync + std::fmt::Debug {
|
||||
proof: Vec::new(),
|
||||
supports_tier_registry_generation: None,
|
||||
supports_checkpoint_coverage: None,
|
||||
supports_durable_progress: None,
|
||||
supports_complete_observation: None,
|
||||
usage_watch_proof: None,
|
||||
})
|
||||
}
|
||||
// Interface facet nobody calls yet: every transport implements both, but no
|
||||
@@ -411,11 +419,53 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
|
||||
}
|
||||
|
||||
async fn probe_ns_scanner_capability(&self, request: NsScannerCapabilityRequest) -> Result<NsScannerCapabilityResponse> {
|
||||
let request = if request.supports_usage_watch {
|
||||
match self.probe_ns_scanner_capability_once(&request).await {
|
||||
Ok(response) => return Ok(response),
|
||||
Err(error) if ns_scanner_capability_error_allows_legacy(&error) => NsScannerCapabilityRequest {
|
||||
supports_usage_watch: false,
|
||||
..request
|
||||
},
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
} else {
|
||||
request
|
||||
};
|
||||
let request = if request.supports_complete_observation {
|
||||
match self.probe_ns_scanner_capability_once(&request).await {
|
||||
Ok(response) => return Ok(response),
|
||||
Err(error) if ns_scanner_capability_error_allows_legacy(&error) => NsScannerCapabilityRequest {
|
||||
supports_complete_observation: false,
|
||||
supports_usage_watch: false,
|
||||
..request
|
||||
},
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
} else {
|
||||
request
|
||||
};
|
||||
let request = if request.supports_durable_progress {
|
||||
match self.probe_ns_scanner_capability_once(&request).await {
|
||||
Ok(response) => return Ok(response),
|
||||
Err(error) if ns_scanner_capability_error_allows_legacy(&error) => NsScannerCapabilityRequest {
|
||||
supports_durable_progress: false,
|
||||
supports_complete_observation: false,
|
||||
supports_usage_watch: false,
|
||||
..request
|
||||
},
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
} else {
|
||||
request
|
||||
};
|
||||
let request = if request.supports_checkpoint_coverage {
|
||||
match self.probe_ns_scanner_capability_once(&request).await {
|
||||
Ok(response) => return Ok(response),
|
||||
Err(error) if ns_scanner_capability_error_allows_legacy(&error) => NsScannerCapabilityRequest {
|
||||
supports_checkpoint_coverage: false,
|
||||
supports_durable_progress: false,
|
||||
supports_complete_observation: false,
|
||||
supports_usage_watch: false,
|
||||
..request
|
||||
},
|
||||
Err(error) => return Err(error),
|
||||
@@ -435,6 +485,9 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
|
||||
endpoint: request.endpoint.clone(),
|
||||
supports_tier_registry_generation: false,
|
||||
supports_checkpoint_coverage: false,
|
||||
supports_durable_progress: false,
|
||||
supports_complete_observation: false,
|
||||
supports_usage_watch: false,
|
||||
};
|
||||
match self.probe_ns_scanner_capability_once(&legacy_request).await {
|
||||
Ok(mut response) => {
|
||||
@@ -492,13 +545,62 @@ impl TcpHttpInternodeDataTransport {
|
||||
if response.version != NS_SCANNER_PROTOCOL_VERSION || response.server_epoch.is_nil() {
|
||||
return Err(Error::other("incompatible remote namespace scanner capability response"));
|
||||
}
|
||||
if let Err(err) = verify_ns_scanner_capability_with_checkpoint_coverage(
|
||||
if request.supports_usage_watch {
|
||||
if let Some(proof) = response.usage_watch_proof.as_deref() {
|
||||
crate::cluster::rpc::verify_ns_scanner_usage_watch_capability(challenge, response.server_epoch, proof)
|
||||
.map_err(|_| Error::other("remote namespace scanner usage watch capability authentication failed"))?;
|
||||
}
|
||||
} else {
|
||||
response.usage_watch_proof = None;
|
||||
}
|
||||
if let Err(err) = verify_ns_scanner_capability_with_usage_capabilities(
|
||||
challenge,
|
||||
response.server_epoch,
|
||||
&response.proof,
|
||||
request.supports_tier_registry_generation,
|
||||
request.supports_checkpoint_coverage,
|
||||
request.supports_durable_progress,
|
||||
request.supports_complete_observation,
|
||||
) {
|
||||
if request.supports_complete_observation
|
||||
&& verify_ns_scanner_capability_with_durable_progress(
|
||||
challenge,
|
||||
response.server_epoch,
|
||||
&response.proof,
|
||||
request.supports_tier_registry_generation,
|
||||
request.supports_checkpoint_coverage,
|
||||
request.supports_durable_progress,
|
||||
)
|
||||
.is_ok()
|
||||
{
|
||||
normalize_ns_scanner_capability_response(
|
||||
&mut response,
|
||||
request.supports_tier_registry_generation,
|
||||
request.supports_checkpoint_coverage,
|
||||
request.supports_durable_progress,
|
||||
false,
|
||||
);
|
||||
return Ok(response);
|
||||
}
|
||||
if request.supports_durable_progress
|
||||
&& verify_ns_scanner_capability_with_checkpoint_coverage(
|
||||
challenge,
|
||||
response.server_epoch,
|
||||
&response.proof,
|
||||
request.supports_tier_registry_generation,
|
||||
request.supports_checkpoint_coverage,
|
||||
)
|
||||
.is_ok()
|
||||
{
|
||||
normalize_ns_scanner_capability_response(
|
||||
&mut response,
|
||||
request.supports_tier_registry_generation,
|
||||
request.supports_checkpoint_coverage,
|
||||
false,
|
||||
false,
|
||||
);
|
||||
return Ok(response);
|
||||
}
|
||||
if request.supports_checkpoint_coverage
|
||||
&& verify_ns_scanner_capability_with_tier_registry_generation(
|
||||
challenge,
|
||||
@@ -508,7 +610,13 @@ impl TcpHttpInternodeDataTransport {
|
||||
)
|
||||
.is_ok()
|
||||
{
|
||||
normalize_ns_scanner_capability_response(&mut response, request.supports_tier_registry_generation, false);
|
||||
normalize_ns_scanner_capability_response(
|
||||
&mut response,
|
||||
request.supports_tier_registry_generation,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
);
|
||||
return Ok(response);
|
||||
}
|
||||
// A permissive older peer can ignore the additive marker and
|
||||
@@ -518,6 +626,8 @@ impl TcpHttpInternodeDataTransport {
|
||||
if request.supports_tier_registry_generation && ns_scanner_capability_legacy_proof_is_valid(challenge, &response) {
|
||||
response.supports_tier_registry_generation = None;
|
||||
response.supports_checkpoint_coverage = None;
|
||||
response.supports_durable_progress = None;
|
||||
response.supports_complete_observation = None;
|
||||
return Ok(response);
|
||||
}
|
||||
return Err(Error::other(format!("remote namespace scanner capability authentication failed: {err}")));
|
||||
@@ -531,11 +641,13 @@ impl TcpHttpInternodeDataTransport {
|
||||
&mut response,
|
||||
request.supports_tier_registry_generation,
|
||||
request.supports_checkpoint_coverage,
|
||||
request.supports_durable_progress,
|
||||
request.supports_complete_observation,
|
||||
);
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
async fn put_file_auth_capability(&self, endpoint: &str) -> Result<Option<Uuid>> {
|
||||
pub(super) async fn put_file_auth_capability(&self, endpoint: &str) -> Result<Option<Uuid>> {
|
||||
resolve_put_file_auth_capability(endpoint, || async {
|
||||
tokio::time::timeout(PUT_FILE_CAPABILITY_PROBE_TIMEOUT, self.probe_put_file_auth(endpoint))
|
||||
.await
|
||||
@@ -665,7 +777,7 @@ fn verify_put_file_capability_response(challenge: Uuid, body: &[u8]) -> Result<U
|
||||
Ok(response.server_epoch)
|
||||
}
|
||||
|
||||
fn build_read_file_stream_url(request: &ReadStreamRequest) -> String {
|
||||
pub(super) fn build_read_file_stream_url(request: &ReadStreamRequest) -> String {
|
||||
format!(
|
||||
"{}{}?disk={}&volume={}&path={}&offset={}&length={}",
|
||||
request.endpoint,
|
||||
@@ -715,7 +827,7 @@ fn build_put_file_capability_url(endpoint: &str, challenge: Uuid) -> String {
|
||||
)
|
||||
}
|
||||
|
||||
struct PutFileAuthWriter<W> {
|
||||
pub(super) struct PutFileAuthWriter<W> {
|
||||
inner: W,
|
||||
url: String,
|
||||
nonce: Uuid,
|
||||
@@ -728,7 +840,7 @@ struct PutFileAuthWriter<W> {
|
||||
}
|
||||
|
||||
impl<W> PutFileAuthWriter<W> {
|
||||
fn new(inner: W, url: String, nonce: Uuid, endpoint: String, server_epoch: Uuid) -> Self {
|
||||
pub(super) fn new(inner: W, url: String, nonce: Uuid, endpoint: String, server_epoch: Uuid) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
url,
|
||||
@@ -865,9 +977,13 @@ fn normalize_ns_scanner_capability_response(
|
||||
response: &mut NsScannerCapabilityResponse,
|
||||
requested_generation_support: bool,
|
||||
requested_checkpoint_coverage: bool,
|
||||
requested_durable_progress: bool,
|
||||
requested_complete_observation: bool,
|
||||
) {
|
||||
response.supports_tier_registry_generation = requested_generation_support.then_some(true);
|
||||
response.supports_checkpoint_coverage = requested_checkpoint_coverage.then_some(true);
|
||||
response.supports_durable_progress = requested_durable_progress.then_some(true);
|
||||
response.supports_complete_observation = requested_complete_observation.then_some(true);
|
||||
}
|
||||
|
||||
fn ns_scanner_capability_legacy_proof_is_valid(challenge: Uuid, response: &NsScannerCapabilityResponse) -> bool {
|
||||
@@ -900,7 +1016,7 @@ fn build_ns_scanner_url(request: &NsScannerStreamRequest) -> String {
|
||||
|
||||
fn build_ns_scanner_capability_url(request: &NsScannerCapabilityRequest, challenge: Uuid) -> String {
|
||||
format!(
|
||||
"{}{}?{}={}&{}={}{}{}",
|
||||
"{}{}?{}={}&{}={}{}{}{}{}{}",
|
||||
request.endpoint,
|
||||
NS_SCANNER_PATH,
|
||||
NS_SCANNER_PROTOCOL_VERSION_QUERY,
|
||||
@@ -916,11 +1032,26 @@ fn build_ns_scanner_capability_url(request: &NsScannerCapabilityRequest, challen
|
||||
format!("&{}=true", NS_SCANNER_CHECKPOINT_COVERAGE_QUERY)
|
||||
} else {
|
||||
String::new()
|
||||
},
|
||||
if request.supports_durable_progress {
|
||||
format!("&{}=true", NS_SCANNER_DURABLE_PROGRESS_QUERY)
|
||||
} else {
|
||||
String::new()
|
||||
},
|
||||
if request.supports_complete_observation {
|
||||
format!("&{}=true", NS_SCANNER_COMPLETE_OBSERVATION_QUERY)
|
||||
} else {
|
||||
String::new()
|
||||
},
|
||||
if request.supports_usage_watch {
|
||||
"&ns_scanner_usage_watch=true".to_string()
|
||||
} else {
|
||||
String::new()
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
fn json_headers() -> HeaderMap {
|
||||
pub(super) fn json_headers() -> HeaderMap {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(CONTENT_TYPE, HeaderValue::from_static(CONTENT_TYPE_JSON));
|
||||
headers
|
||||
@@ -943,6 +1074,9 @@ fn build_internode_data_transport_result(
|
||||
{
|
||||
Ok(Arc::new(TcpHttpInternodeDataTransport))
|
||||
}
|
||||
Some(transport) if transport.eq_ignore_ascii_case(INTERNODE_DATA_TRANSPORT_NIXL) => {
|
||||
Ok(Arc::new(super::nixl_internode::NixlInternodeDataTransport::from_env()))
|
||||
}
|
||||
Some(transport) => Err(unsupported_transport_message(transport)),
|
||||
}
|
||||
}
|
||||
@@ -1294,6 +1428,9 @@ mod tests {
|
||||
endpoint: "http://node1:9000".to_string(),
|
||||
supports_tier_registry_generation: false,
|
||||
supports_checkpoint_coverage: false,
|
||||
supports_durable_progress: false,
|
||||
supports_complete_observation: false,
|
||||
supports_usage_watch: false,
|
||||
})
|
||||
.await
|
||||
.expect_err("legacy transport should report namespace scanner as unsupported");
|
||||
@@ -1320,6 +1457,106 @@ mod tests {
|
||||
assert!(matches!(open_err, Error::MethodNotAllowed));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn namespace_scanner_watch_capability_preserves_older_capabilities_and_rejects_stale_proof() {
|
||||
crate::runtime::sources::ensure_test_rpc_secret();
|
||||
for peer in ["current", "strict-old", "missing-watch", "stale-watch"] {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("watch capability listener");
|
||||
let address = listener.local_addr().expect("listener address");
|
||||
let epoch = Uuid::new_v4();
|
||||
let server = tokio::spawn(async move {
|
||||
let requests = if peer == "strict-old" { 2 } else { 1 };
|
||||
for index in 0..requests {
|
||||
let (mut socket, _) = listener.accept().await.expect("watch capability request");
|
||||
let mut bytes = Vec::new();
|
||||
while !bytes.windows(4).any(|window| window == b"\r\n\r\n") {
|
||||
let mut chunk = [0; 1024];
|
||||
let read = socket.read(&mut chunk).await.expect("request headers");
|
||||
assert!(read > 0 && bytes.len() < 16 * 1024);
|
||||
bytes.extend_from_slice(&chunk[..read]);
|
||||
}
|
||||
let headers = std::str::from_utf8(&bytes).expect("HTTP headers");
|
||||
let target = headers.split_whitespace().nth(1).expect("target");
|
||||
let url = reqwest::Url::parse(&format!("http://{address}{target}")).expect("request URL");
|
||||
let query: HashMap<_, _> = url.query_pairs().into_owned().collect();
|
||||
assert_eq!(query.contains_key("ns_scanner_usage_watch"), index == 0);
|
||||
assert!(query.contains_key(NS_SCANNER_DURABLE_PROGRESS_QUERY));
|
||||
assert!(query.contains_key(NS_SCANNER_COMPLETE_OBSERVATION_QUERY));
|
||||
if peer == "strict-old" && index == 0 {
|
||||
socket
|
||||
.write_all(b"HTTP/1.1 400 Bad Request\r\ncontent-length: 0\r\nconnection: close\r\n\r\n")
|
||||
.await
|
||||
.expect("old parser rejects only the new field");
|
||||
continue;
|
||||
}
|
||||
let challenge = query[NS_SCANNER_CAPABILITY_CHALLENGE_QUERY].parse().expect("challenge");
|
||||
let proof = crate::cluster::rpc::sign_ns_scanner_capability_with_usage_capabilities(
|
||||
challenge, epoch, true, true, true, true,
|
||||
)
|
||||
.expect("prior capability proof");
|
||||
let usage_watch_proof = match peer {
|
||||
"current" => Some(
|
||||
crate::cluster::rpc::sign_ns_scanner_usage_watch_capability(challenge, epoch).expect("watch proof"),
|
||||
),
|
||||
"stale-watch" => Some(
|
||||
crate::cluster::rpc::sign_ns_scanner_usage_watch_capability(Uuid::new_v4(), epoch)
|
||||
.expect("stale watch proof"),
|
||||
),
|
||||
_ => None,
|
||||
};
|
||||
let body = rmp_serde::to_vec_named(&NsScannerCapabilityResponse {
|
||||
version: NS_SCANNER_PROTOCOL_VERSION,
|
||||
server_epoch: epoch,
|
||||
proof,
|
||||
supports_tier_registry_generation: Some(true),
|
||||
supports_checkpoint_coverage: Some(true),
|
||||
supports_durable_progress: Some(true),
|
||||
supports_complete_observation: Some(true),
|
||||
usage_watch_proof,
|
||||
})
|
||||
.expect("capability body");
|
||||
socket
|
||||
.write_all(
|
||||
format!("HTTP/1.1 200 OK\r\ncontent-length: {}\r\nconnection: close\r\n\r\n", body.len()).as_bytes(),
|
||||
)
|
||||
.await
|
||||
.expect("response headers");
|
||||
socket.write_all(&body).await.expect("response body");
|
||||
}
|
||||
requests
|
||||
});
|
||||
let response = tokio::time::timeout(
|
||||
Duration::from_secs(10),
|
||||
TcpHttpInternodeDataTransport.probe_ns_scanner_capability(NsScannerCapabilityRequest {
|
||||
endpoint: format!("http://{address}"),
|
||||
supports_tier_registry_generation: true,
|
||||
supports_checkpoint_coverage: true,
|
||||
supports_durable_progress: true,
|
||||
supports_complete_observation: true,
|
||||
supports_usage_watch: true,
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.expect("bounded probe");
|
||||
if peer == "stale-watch" {
|
||||
assert!(
|
||||
response
|
||||
.expect_err("stale response must fail authentication")
|
||||
.to_string()
|
||||
.contains("authentication failed")
|
||||
);
|
||||
} else {
|
||||
let response = response.expect("verified capability result");
|
||||
assert_eq!(response.usage_watch_proof.is_some(), peer == "current");
|
||||
assert_eq!(response.supports_durable_progress, Some(true));
|
||||
assert_eq!(response.supports_complete_observation, Some(true));
|
||||
}
|
||||
assert_eq!(server.await.expect("capability server"), if peer == "strict-old" { 2 } else { 1 });
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn namespace_scanner_checkpoint_capability_authenticates_new_and_legacy_peers() {
|
||||
crate::runtime::sources::ensure_test_rpc_secret();
|
||||
@@ -1372,6 +1609,9 @@ mod tests {
|
||||
supports_tier_registry_generation: Some(true),
|
||||
// Strip a supported bit, or forge it for an older peer.
|
||||
supports_checkpoint_coverage: (peer != "current").then_some(true),
|
||||
supports_durable_progress: None,
|
||||
supports_complete_observation: None,
|
||||
usage_watch_proof: None,
|
||||
})
|
||||
.expect("capability response");
|
||||
socket
|
||||
@@ -1390,6 +1630,9 @@ mod tests {
|
||||
endpoint: format!("http://{address}"),
|
||||
supports_tier_registry_generation: true,
|
||||
supports_checkpoint_coverage: true,
|
||||
supports_durable_progress: false,
|
||||
supports_complete_observation: false,
|
||||
supports_usage_watch: false,
|
||||
}),
|
||||
)
|
||||
.await
|
||||
@@ -1411,6 +1654,206 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn namespace_scanner_durable_progress_capability_authenticates_new_and_legacy_peers() {
|
||||
crate::runtime::sources::ensure_test_rpc_secret();
|
||||
for peer in ["current", "strict-old", "permissive-old", "invalid-proof", "missing-proof"] {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("capability listener");
|
||||
let address = listener.local_addr().expect("capability listener address");
|
||||
let server_epoch = Uuid::new_v4();
|
||||
let server = tokio::spawn(async move {
|
||||
let requests = if peer == "strict-old" { 2 } else { 1 };
|
||||
for index in 0..requests {
|
||||
let (mut socket, _) = listener.accept().await.expect("capability request");
|
||||
let mut bytes = Vec::new();
|
||||
while !bytes.windows(4).any(|window| window == b"\r\n\r\n") {
|
||||
let mut chunk = [0; 1024];
|
||||
let read = socket.read(&mut chunk).await.expect("capability headers");
|
||||
assert!(read > 0 && bytes.len() < 16 * 1024, "bounded complete capability headers");
|
||||
bytes.extend_from_slice(&chunk[..read]);
|
||||
}
|
||||
let request = std::str::from_utf8(&bytes).expect("HTTP headers");
|
||||
let target = request.split_whitespace().nth(1).expect("HTTP request target");
|
||||
let url = reqwest::Url::parse(&format!("http://{address}{target}")).expect("capability URL");
|
||||
let query: HashMap<_, _> = url.query_pairs().into_owned().collect();
|
||||
assert_eq!(query.get(NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY).map(String::as_str), Some("true"));
|
||||
assert_eq!(query.contains_key(NS_SCANNER_DURABLE_PROGRESS_QUERY), index == 0);
|
||||
if peer == "strict-old" && index == 0 {
|
||||
socket
|
||||
.write_all(b"HTTP/1.1 400 Bad Request\r\ncontent-length: 0\r\nconnection: close\r\n\r\n")
|
||||
.await
|
||||
.expect("legacy parser rejection");
|
||||
continue;
|
||||
}
|
||||
let challenge = query[NS_SCANNER_CAPABILITY_CHALLENGE_QUERY].parse().expect("challenge UUID");
|
||||
let proof = match peer {
|
||||
"invalid-proof" => vec![0; 32],
|
||||
"missing-proof" => Vec::new(),
|
||||
_ => crate::cluster::rpc::sign_ns_scanner_capability_with_durable_progress(
|
||||
challenge,
|
||||
server_epoch,
|
||||
true,
|
||||
true,
|
||||
peer == "current",
|
||||
)
|
||||
.expect("server capability proof"),
|
||||
};
|
||||
let body = rmp_serde::to_vec_named(&NsScannerCapabilityResponse {
|
||||
version: NS_SCANNER_PROTOCOL_VERSION,
|
||||
server_epoch,
|
||||
proof,
|
||||
supports_tier_registry_generation: Some(true),
|
||||
// Strip a supported bit, or forge it for an older peer.
|
||||
supports_checkpoint_coverage: Some(true),
|
||||
supports_durable_progress: (peer != "current").then_some(true),
|
||||
supports_complete_observation: None,
|
||||
usage_watch_proof: None,
|
||||
})
|
||||
.expect("capability response");
|
||||
socket
|
||||
.write_all(
|
||||
format!("HTTP/1.1 200 OK\r\ncontent-length: {}\r\nconnection: close\r\n\r\n", body.len()).as_bytes(),
|
||||
)
|
||||
.await
|
||||
.expect("capability response headers");
|
||||
socket.write_all(&body).await.expect("capability response body");
|
||||
}
|
||||
requests
|
||||
});
|
||||
let response = tokio::time::timeout(
|
||||
Duration::from_secs(10),
|
||||
TcpHttpInternodeDataTransport.probe_ns_scanner_capability(NsScannerCapabilityRequest {
|
||||
endpoint: format!("http://{address}"),
|
||||
supports_tier_registry_generation: true,
|
||||
supports_checkpoint_coverage: true,
|
||||
supports_durable_progress: true,
|
||||
supports_complete_observation: false,
|
||||
supports_usage_watch: false,
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.expect("bounded capability probe");
|
||||
if peer.ends_with("proof") {
|
||||
assert!(
|
||||
response
|
||||
.expect_err("unverified capability must fail closed")
|
||||
.to_string()
|
||||
.contains("authentication failed")
|
||||
);
|
||||
} else {
|
||||
let response = response.expect("authenticated capability");
|
||||
assert_eq!(response.server_epoch, server_epoch);
|
||||
assert_eq!(response.supports_tier_registry_generation, Some(true));
|
||||
assert_eq!(response.supports_checkpoint_coverage, Some(true));
|
||||
assert_eq!(response.supports_durable_progress, (peer == "current").then_some(true));
|
||||
}
|
||||
assert_eq!(server.await.expect("capability server"), if peer == "strict-old" { 2 } else { 1 });
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn namespace_scanner_complete_observation_capability_authenticates_new_and_legacy_peers() {
|
||||
crate::runtime::sources::ensure_test_rpc_secret();
|
||||
for peer in ["current", "strict-old", "permissive-old", "invalid-proof", "missing-proof"] {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("capability listener");
|
||||
let address = listener.local_addr().expect("capability listener address");
|
||||
let server_epoch = Uuid::new_v4();
|
||||
let server = tokio::spawn(async move {
|
||||
let requests = if peer == "strict-old" { 2 } else { 1 };
|
||||
for index in 0..requests {
|
||||
let (mut socket, _) = listener.accept().await.expect("capability request");
|
||||
let mut bytes = Vec::new();
|
||||
while !bytes.windows(4).any(|window| window == b"\r\n\r\n") {
|
||||
let mut chunk = [0; 1024];
|
||||
let read = socket.read(&mut chunk).await.expect("capability headers");
|
||||
assert!(read > 0 && bytes.len() < 16 * 1024, "bounded complete capability headers");
|
||||
bytes.extend_from_slice(&chunk[..read]);
|
||||
}
|
||||
let request = std::str::from_utf8(&bytes).expect("HTTP headers");
|
||||
let target = request.split_whitespace().nth(1).expect("HTTP request target");
|
||||
let url = reqwest::Url::parse(&format!("http://{address}{target}")).expect("capability URL");
|
||||
let query: HashMap<_, _> = url.query_pairs().into_owned().collect();
|
||||
assert_eq!(query.get(NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY).map(String::as_str), Some("true"));
|
||||
assert_eq!(query.contains_key(NS_SCANNER_COMPLETE_OBSERVATION_QUERY), index == 0);
|
||||
if peer == "strict-old" && index == 0 {
|
||||
socket
|
||||
.write_all(b"HTTP/1.1 400 Bad Request\r\ncontent-length: 0\r\nconnection: close\r\n\r\n")
|
||||
.await
|
||||
.expect("legacy parser rejection");
|
||||
continue;
|
||||
}
|
||||
let challenge = query[NS_SCANNER_CAPABILITY_CHALLENGE_QUERY].parse().expect("challenge UUID");
|
||||
let proof = match peer {
|
||||
"invalid-proof" => vec![0; 32],
|
||||
"missing-proof" => Vec::new(),
|
||||
_ => crate::cluster::rpc::sign_ns_scanner_capability_with_usage_capabilities(
|
||||
challenge,
|
||||
server_epoch,
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
peer == "current",
|
||||
)
|
||||
.expect("server capability proof"),
|
||||
};
|
||||
let body = rmp_serde::to_vec_named(&NsScannerCapabilityResponse {
|
||||
version: NS_SCANNER_PROTOCOL_VERSION,
|
||||
server_epoch,
|
||||
proof,
|
||||
supports_tier_registry_generation: Some(true),
|
||||
// Strip a supported bit, or forge it for an older peer.
|
||||
supports_checkpoint_coverage: Some(true),
|
||||
supports_durable_progress: Some(true),
|
||||
supports_complete_observation: (peer != "current").then_some(true),
|
||||
usage_watch_proof: None,
|
||||
})
|
||||
.expect("capability response");
|
||||
socket
|
||||
.write_all(
|
||||
format!("HTTP/1.1 200 OK\r\ncontent-length: {}\r\nconnection: close\r\n\r\n", body.len()).as_bytes(),
|
||||
)
|
||||
.await
|
||||
.expect("capability response headers");
|
||||
socket.write_all(&body).await.expect("capability response body");
|
||||
}
|
||||
requests
|
||||
});
|
||||
let response = tokio::time::timeout(
|
||||
Duration::from_secs(10),
|
||||
TcpHttpInternodeDataTransport.probe_ns_scanner_capability(NsScannerCapabilityRequest {
|
||||
endpoint: format!("http://{address}"),
|
||||
supports_tier_registry_generation: true,
|
||||
supports_checkpoint_coverage: true,
|
||||
supports_durable_progress: true,
|
||||
supports_complete_observation: true,
|
||||
supports_usage_watch: false,
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.expect("bounded capability probe");
|
||||
if peer.ends_with("proof") {
|
||||
assert!(
|
||||
response
|
||||
.expect_err("unverified capability must fail closed")
|
||||
.to_string()
|
||||
.contains("authentication failed")
|
||||
);
|
||||
} else {
|
||||
let response = response.expect("authenticated capability");
|
||||
assert_eq!(response.server_epoch, server_epoch);
|
||||
assert_eq!(response.supports_tier_registry_generation, Some(true));
|
||||
assert_eq!(response.supports_checkpoint_coverage, Some(true));
|
||||
assert_eq!(response.supports_durable_progress, Some(true));
|
||||
assert_eq!(response.supports_complete_observation, (peer == "current").then_some(true));
|
||||
}
|
||||
assert_eq!(server.await.expect("capability server"), if peer == "strict-old" { 2 } else { 1 });
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn walk_dir_restores_explicit_remote_missing_error() {
|
||||
let _ = rustfs_credentials::set_global_rpc_secret("walk-dir-error-test-secret".to_string());
|
||||
@@ -2154,6 +2597,9 @@ mod tests {
|
||||
endpoint: "http://node1:9000".to_string(),
|
||||
supports_tier_registry_generation: false,
|
||||
supports_checkpoint_coverage: false,
|
||||
supports_durable_progress: false,
|
||||
supports_complete_observation: false,
|
||||
supports_usage_watch: false,
|
||||
},
|
||||
challenge,
|
||||
);
|
||||
@@ -2174,6 +2620,9 @@ mod tests {
|
||||
endpoint: "http://node1:9000".to_string(),
|
||||
supports_tier_registry_generation: true,
|
||||
supports_checkpoint_coverage: false,
|
||||
supports_durable_progress: false,
|
||||
supports_complete_observation: false,
|
||||
supports_usage_watch: false,
|
||||
},
|
||||
challenge,
|
||||
);
|
||||
@@ -2213,13 +2662,16 @@ mod tests {
|
||||
proof: Vec::new(),
|
||||
supports_tier_registry_generation: None,
|
||||
supports_checkpoint_coverage: None,
|
||||
supports_durable_progress: None,
|
||||
supports_complete_observation: None,
|
||||
usage_watch_proof: None,
|
||||
};
|
||||
|
||||
normalize_ns_scanner_capability_response(&mut response, true, true);
|
||||
normalize_ns_scanner_capability_response(&mut response, true, true, false, false);
|
||||
assert_eq!(response.supports_tier_registry_generation, Some(true));
|
||||
|
||||
response.supports_tier_registry_generation = Some(false);
|
||||
normalize_ns_scanner_capability_response(&mut response, false, false);
|
||||
normalize_ns_scanner_capability_response(&mut response, false, false, false, false);
|
||||
assert_eq!(response.supports_tier_registry_generation, None);
|
||||
}
|
||||
|
||||
@@ -2234,6 +2686,9 @@ mod tests {
|
||||
.expect("placeholder proof should be generated"),
|
||||
supports_tier_registry_generation: None,
|
||||
supports_checkpoint_coverage: None,
|
||||
supports_durable_progress: None,
|
||||
supports_complete_observation: None,
|
||||
usage_watch_proof: None,
|
||||
};
|
||||
// A proof bound to a different challenge cannot authorize the legacy
|
||||
// fallback, even though the response has the expected shape.
|
||||
@@ -2280,13 +2735,24 @@ mod tests {
|
||||
fn transport_config_known_backends_are_current_oss_values() {
|
||||
assert_eq!(
|
||||
KNOWN_INTERNODE_DATA_TRANSPORT_BACKENDS,
|
||||
&[DEFAULT_INTERNODE_DATA_TRANSPORT, INTERNODE_DATA_TRANSPORT_TCP]
|
||||
&[
|
||||
DEFAULT_INTERNODE_DATA_TRANSPORT,
|
||||
INTERNODE_DATA_TRANSPORT_TCP,
|
||||
INTERNODE_DATA_TRANSPORT_NIXL
|
||||
]
|
||||
);
|
||||
|
||||
for configured in KNOWN_INTERNODE_DATA_TRANSPORT_BACKENDS {
|
||||
let transport = build_internode_data_transport(Some(configured)).unwrap();
|
||||
// NIXL builds everywhere and serves reads over TCP/HTTP until a node
|
||||
// installs its receive pool; every other known name is the TCP transport.
|
||||
let expected = if *configured == INTERNODE_DATA_TRANSPORT_NIXL {
|
||||
INTERNODE_DATA_TRANSPORT_NIXL
|
||||
} else {
|
||||
DEFAULT_INTERNODE_DATA_TRANSPORT
|
||||
};
|
||||
|
||||
assert_eq!(transport.name(), DEFAULT_INTERNODE_DATA_TRANSPORT);
|
||||
assert_eq!(transport.name(), expected);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -20,6 +20,7 @@ pub(crate) mod context_propagation;
|
||||
pub(crate) mod http_auth;
|
||||
pub(crate) mod internode_data_transport;
|
||||
mod network_probe;
|
||||
pub(crate) mod nixl_internode;
|
||||
pub(crate) mod peer_rest_client;
|
||||
pub(crate) mod peer_s3_client;
|
||||
pub(crate) mod remote_disk;
|
||||
@@ -40,12 +41,16 @@ pub use http_auth::{
|
||||
gen_tonic_signature_headers, normalize_tonic_rpc_audience, rpc_auth_is_resource_exhausted, rpc_background_admission,
|
||||
set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, set_tonic_rolling_canonical_body_digest,
|
||||
set_tonic_rolling_mutation_body_digest, sign_ns_scanner_capability, sign_ns_scanner_capability_with_checkpoint_coverage,
|
||||
sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, sign_tonic_rpc_response_proof,
|
||||
tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_ns_scanner_capability,
|
||||
verify_ns_scanner_capability_with_checkpoint_coverage, verify_ns_scanner_capability_with_tier_registry_generation,
|
||||
verify_put_file_auth_trailer, verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response,
|
||||
verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned,
|
||||
verify_tonic_rpc_response_proof, verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
|
||||
sign_ns_scanner_capability_with_durable_progress, sign_ns_scanner_capability_with_tier_registry_generation,
|
||||
sign_ns_scanner_capability_with_usage_capabilities, sign_ns_scanner_usage_watch_capability, sign_put_file_capability,
|
||||
sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
|
||||
tonic_boot_epoch_response_headers_with_msgpack_capability, tonic_msgpack_manifest_capability_requested,
|
||||
tonic_rpc_auth_failure_reason, verify_ns_scanner_capability, verify_ns_scanner_capability_with_checkpoint_coverage,
|
||||
verify_ns_scanner_capability_with_durable_progress, verify_ns_scanner_capability_with_tier_registry_generation,
|
||||
verify_ns_scanner_capability_with_usage_capabilities, verify_ns_scanner_usage_watch_capability, verify_put_file_auth_trailer,
|
||||
verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
|
||||
verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned, verify_tonic_rpc_response_proof,
|
||||
verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;
|
||||
|
||||
@@ -0,0 +1,928 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Internode shard reads whose bytes move over NIXL/UCX.
|
||||
//!
|
||||
//! The reading node keeps a pool of receive buffers registered with its NIXL
|
||||
//! agent. For every chunk of a remote shard stream it sends a signed request
|
||||
//! over the existing internode HTTP control plane naming the shard range, its
|
||||
//! agent, one registered buffer and a digest of its agent metadata; the serving
|
||||
//! node reads the range into its own registered staging memory and writes it
|
||||
//! into that buffer with a NIXL WRITE, then answers with the byte count. The
|
||||
//! buffer reaches the decoder as a zero-copy `Bytes`. Several chunk requests
|
||||
//! run per stream so request latency overlaps the transfers.
|
||||
//!
|
||||
//! UCX addresses ports by LID or GID, so this works on InfiniBand rails that
|
||||
//! have no IP addresses. Writes, walks and any stream whose first chunk cannot
|
||||
//! be served this way use the TCP/HTTP transport.
|
||||
//!
|
||||
//! Buffer safety: the serving node may still write into a buffer whose request
|
||||
//! was abandoned (a dropped reader, chunks past the end of the file). Such
|
||||
//! buffers are held for [`ABANDONED_BUFFER_HOLD`] before they return to the
|
||||
//! pool, which covers the serving node's transfer timeout.
|
||||
|
||||
use super::internode_data_transport::{
|
||||
InternodeDataTransport, InternodeDataTransportCapabilities, NsScannerCapabilityRequest, NsScannerStreamRequest,
|
||||
ReadStreamRequest, TcpHttpInternodeDataTransport, WalkDirStreamRequest, WriteStreamRequest, build_read_file_stream_url,
|
||||
json_headers,
|
||||
};
|
||||
use crate::cluster::rpc::build_auth_headers;
|
||||
use crate::disk::error::Result;
|
||||
use crate::disk::{FileReader, FileWriter};
|
||||
use crate::storage_api_contracts::internode::NsScannerCapabilityResponse;
|
||||
use async_trait::async_trait;
|
||||
use bytes::Bytes;
|
||||
use futures::future::BoxFuture;
|
||||
use futures::stream::{FuturesOrdered, Stream};
|
||||
use http::{HeaderValue, Method};
|
||||
use rustfs_config::{DEFAULT_RUSTFS_NIXL_INTERNODE_DEPTH, ENV_RUSTFS_NIXL_INTERNODE_DEPTH};
|
||||
use rustfs_rio::{ChunkReader, ChunkReaderBox};
|
||||
use std::io;
|
||||
use std::pin::Pin;
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use std::task::{Context, Poll};
|
||||
use std::time::Duration;
|
||||
use tokio::io::{AsyncRead, ReadBuf};
|
||||
use tracing::debug;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Internode route of a NIXL chunk read.
|
||||
pub const READ_FILE_NIXL_PATH: &str = "/rustfs/rpc/read_file_nixl";
|
||||
/// Internode route of a NIXL shard write: the disk's node pulls the named buffers.
|
||||
pub const WRITE_FILE_NIXL_PATH: &str = "/rustfs/rpc/write_file_nixl";
|
||||
/// Comma-separated buffers a NIXL shard write pulls, in file order (signed).
|
||||
pub const NIXL_BUFFERS_QUERY: &str = "nixl_buffers";
|
||||
/// One-time nonce of a NIXL shard write (signed; recorded to refuse replays).
|
||||
pub const NIXL_NONCE_QUERY: &str = "nixl_nonce";
|
||||
/// Buffers a writer fills before it sends a segment; a part's shard file
|
||||
/// usually fits in one.
|
||||
const WRITE_SEGMENT_BUFFERS: usize = 4;
|
||||
/// The route the URL builder emits, replaced by [`READ_FILE_NIXL_PATH`].
|
||||
const READ_FILE_STREAM_PATH: &str = "/rustfs/rpc/read_file_stream";
|
||||
/// Query parameters of a NIXL chunk read (signed with the URL).
|
||||
pub const NIXL_AGENT_QUERY: &str = "nixl_agent";
|
||||
pub const NIXL_BUFFER_QUERY: &str = "nixl_buffer";
|
||||
pub const NIXL_METADATA_SHA256_QUERY: &str = "nixl_md";
|
||||
/// The reader's agent metadata, base64; must hash to the signed digest.
|
||||
pub const NIXL_METADATA_HEADER: &str = "x-rustfs-nixl-metadata";
|
||||
/// Bytes the serving node wrote into the buffer; fewer than asked means end of file.
|
||||
pub const NIXL_BYTES_HEADER: &str = "x-rustfs-nixl-bytes";
|
||||
|
||||
/// How long a buffer whose request was abandoned stays out of the pool.
|
||||
pub const ABANDONED_BUFFER_HOLD: Duration = Duration::from_secs(120);
|
||||
/// Request timeout when the caller sets no stall timeout.
|
||||
const DEFAULT_CHUNK_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
const EVENT_NIXL_INTERNODE_FALLBACK: &str = "nixl_internode_fallback";
|
||||
const LOG_COMPONENT_INTERNODE: &str = "internode";
|
||||
const LOG_SUBSYSTEM_NIXL: &str = "nixl";
|
||||
|
||||
/// Receive buffers registered with this node's NIXL agent.
|
||||
pub trait NixlRecvPool: Send + Sync + 'static {
|
||||
/// This node's agent name.
|
||||
fn agent_name(&self) -> &str;
|
||||
/// This node's agent metadata, base64.
|
||||
fn metadata_base64(&self) -> &str;
|
||||
/// Lowercase hex SHA-256 of the raw agent metadata.
|
||||
fn metadata_sha256_hex(&self) -> &str;
|
||||
/// Size of one receive buffer; one chunk request moves at most this much.
|
||||
fn chunk_bytes(&self) -> usize;
|
||||
/// One free buffer, waiting for one when the pool is busy.
|
||||
fn reserve(&self) -> BoxFuture<'static, io::Result<Box<dyn NixlRecvBuffer>>>;
|
||||
}
|
||||
|
||||
/// One registered buffer, back in the pool when dropped. Reads land in it;
|
||||
/// writes stage shard bytes in it for the disk's node to pull.
|
||||
pub trait NixlRecvBuffer: Send + Sync + 'static {
|
||||
/// `<dram|vram>:<device>:<address>:<length>` as the serving node parses it.
|
||||
fn descriptor(&self) -> String;
|
||||
/// The same for the first `len` bytes.
|
||||
fn descriptor_for(&self, len: usize) -> String;
|
||||
/// The whole buffer, writable.
|
||||
fn spare_mut(&mut self) -> &mut [u8];
|
||||
/// The first `len` bytes as an owned, zero-copy `Bytes`.
|
||||
fn into_bytes(self: Box<Self>, len: usize) -> io::Result<Bytes>;
|
||||
}
|
||||
|
||||
static RECV_POOL: OnceLock<Arc<dyn NixlRecvPool>> = OnceLock::new();
|
||||
|
||||
/// Installs this node's receive pool; only the first installation wins.
|
||||
pub fn install_nixl_recv_pool(pool: Arc<dyn NixlRecvPool>) -> bool {
|
||||
RECV_POOL.set(pool).is_ok()
|
||||
}
|
||||
|
||||
fn recv_pool() -> Option<Arc<dyn NixlRecvPool>> {
|
||||
RECV_POOL.get().cloned()
|
||||
}
|
||||
|
||||
/// Internode transport that reads shard streams over NIXL and delegates
|
||||
/// everything else to TCP/HTTP.
|
||||
#[derive(Debug)]
|
||||
pub struct NixlInternodeDataTransport {
|
||||
tcp: TcpHttpInternodeDataTransport,
|
||||
depth: usize,
|
||||
}
|
||||
|
||||
impl NixlInternodeDataTransport {
|
||||
pub fn from_env() -> Self {
|
||||
Self {
|
||||
tcp: TcpHttpInternodeDataTransport,
|
||||
depth: rustfs_utils::get_env_usize(ENV_RUSTFS_NIXL_INTERNODE_DEPTH, DEFAULT_RUSTFS_NIXL_INTERNODE_DEPTH).clamp(1, 64),
|
||||
}
|
||||
}
|
||||
|
||||
/// A NIXL chunk reader for `request` with its first chunk already landed,
|
||||
/// or `None` when this read should use TCP/HTTP.
|
||||
async fn open_nixl(&self, request: &ReadStreamRequest) -> Option<NixlChunkReader> {
|
||||
let pool = recv_pool()?;
|
||||
if request.length == 0 {
|
||||
// Unbounded reads need the file size; TCP/HTTP streams them.
|
||||
return None;
|
||||
}
|
||||
let mut reader = NixlChunkReader::new(pool, request.clone(), self.depth);
|
||||
match futures::future::poll_fn(|cx| reader.state_mut().poll_chunk(cx, usize::MAX)).await {
|
||||
Ok(Some(first)) => {
|
||||
reader.state_mut().current = first;
|
||||
Some(reader)
|
||||
}
|
||||
Ok(None) => Some(reader),
|
||||
Err(error) => {
|
||||
debug!(
|
||||
event = EVENT_NIXL_INTERNODE_FALLBACK,
|
||||
component = LOG_COMPONENT_INTERNODE,
|
||||
subsystem = LOG_SUBSYSTEM_NIXL,
|
||||
result = "fallback",
|
||||
endpoint = %request.endpoint,
|
||||
error = %error,
|
||||
"nixl internode read fell back to tcp"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl InternodeDataTransport for NixlInternodeDataTransport {
|
||||
async fn open_read(&self, request: ReadStreamRequest) -> Result<FileReader> {
|
||||
match self.open_nixl(&request).await {
|
||||
Some(reader) => Ok(Box::new(reader)),
|
||||
None => self.tcp.open_read(request).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn open_read_fresh(&self, request: ReadStreamRequest) -> Result<FileReader> {
|
||||
match self.open_nixl(&request).await {
|
||||
Some(reader) => Ok(Box::new(reader)),
|
||||
None => self.tcp.open_read_fresh(request).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn open_read_chunks(&self, request: ReadStreamRequest) -> Result<Option<ChunkReaderBox>> {
|
||||
match self.open_nixl(&request).await {
|
||||
Some(reader) => Ok(Some(Box::new(reader))),
|
||||
None => self.tcp.open_read_chunks(request).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn open_read_chunks_fresh(&self, request: ReadStreamRequest) -> Result<Option<ChunkReaderBox>> {
|
||||
match self.open_nixl(&request).await {
|
||||
Some(reader) => Ok(Some(Box::new(reader))),
|
||||
None => self.tcp.open_read_chunks_fresh(request).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn open_write(&self, request: WriteStreamRequest) -> Result<FileWriter> {
|
||||
match recv_pool() {
|
||||
Some(pool) => Ok(Box::new(NixlFileWriter::new(pool, request))),
|
||||
None => self.tcp.open_write(request).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn open_walk_dir(&self, request: WalkDirStreamRequest) -> Result<FileReader> {
|
||||
self.tcp.open_walk_dir(request).await
|
||||
}
|
||||
|
||||
async fn open_ns_scanner(&self, request: NsScannerStreamRequest) -> Result<FileReader> {
|
||||
self.tcp.open_ns_scanner(request).await
|
||||
}
|
||||
|
||||
async fn probe_ns_scanner(&self, request: NsScannerCapabilityRequest) -> Result<Uuid> {
|
||||
self.tcp.probe_ns_scanner(request).await
|
||||
}
|
||||
|
||||
async fn probe_ns_scanner_capability(&self, request: NsScannerCapabilityRequest) -> Result<NsScannerCapabilityResponse> {
|
||||
self.tcp.probe_ns_scanner_capability(request).await
|
||||
}
|
||||
|
||||
fn name(&self) -> &'static str {
|
||||
"nixl"
|
||||
}
|
||||
|
||||
fn capabilities(&self) -> InternodeDataTransportCapabilities {
|
||||
InternodeDataTransportCapabilities::tcp_http()
|
||||
}
|
||||
}
|
||||
|
||||
/// A receive buffer lent to one chunk request. Unless the request completed,
|
||||
/// dropping it holds the buffer out of the pool for [`ABANDONED_BUFFER_HOLD`]
|
||||
/// because the serving node may still be writing into it.
|
||||
struct LentBuffer {
|
||||
buffer: Option<Box<dyn NixlRecvBuffer>>,
|
||||
}
|
||||
|
||||
impl LentBuffer {
|
||||
fn complete(mut self, len: usize) -> io::Result<Bytes> {
|
||||
let buffer = self.buffer.take().expect("buffer present until completion");
|
||||
buffer.into_bytes(len)
|
||||
}
|
||||
|
||||
/// The peer is done with the buffer: back to the pool now.
|
||||
fn release(mut self) {
|
||||
drop(self.buffer.take());
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for LentBuffer {
|
||||
fn drop(&mut self) {
|
||||
if let Some(buffer) = self.buffer.take() {
|
||||
match tokio::runtime::Handle::try_current() {
|
||||
Ok(handle) => {
|
||||
handle.spawn(async move {
|
||||
tokio::time::sleep(ABANDONED_BUFFER_HOLD).await;
|
||||
drop(buffer);
|
||||
});
|
||||
}
|
||||
// No runtime to wait on: never hand the memory back.
|
||||
Err(_) => std::mem::forget(buffer),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetches one chunk: the requested length and the bytes that landed.
|
||||
async fn fetch_chunk(pool: Arc<dyn NixlRecvPool>, request: ReadStreamRequest, timeout: Duration) -> io::Result<(usize, Bytes)> {
|
||||
let buffer = LentBuffer {
|
||||
buffer: Some(pool.reserve().await?),
|
||||
};
|
||||
let descriptor = buffer.buffer.as_ref().map(|buffer| buffer.descriptor()).unwrap_or_default();
|
||||
let mut url = build_read_file_stream_url(&request).replacen(READ_FILE_STREAM_PATH, READ_FILE_NIXL_PATH, 1);
|
||||
url.push_str(&format!(
|
||||
"&{NIXL_AGENT_QUERY}={}&{NIXL_BUFFER_QUERY}={}&{NIXL_METADATA_SHA256_QUERY}={}",
|
||||
urlencoding::encode(pool.agent_name()),
|
||||
urlencoding::encode(&descriptor),
|
||||
pool.metadata_sha256_hex()
|
||||
));
|
||||
let mut headers = json_headers();
|
||||
headers.insert(
|
||||
NIXL_METADATA_HEADER,
|
||||
HeaderValue::from_str(pool.metadata_base64())
|
||||
.map_err(|_| io::Error::other("nixl agent metadata is not a header value"))?,
|
||||
);
|
||||
build_auth_headers(&url, &Method::GET, &mut headers)?;
|
||||
let (status, response_headers, _) = rustfs_rio::internode_http_request(&url, Method::GET, headers, Some(timeout)).await?;
|
||||
if !status.is_success() {
|
||||
debug!(
|
||||
component = LOG_COMPONENT_INTERNODE,
|
||||
subsystem = LOG_SUBSYSTEM_NIXL,
|
||||
result = "failed",
|
||||
status = status.as_u16(),
|
||||
"nixl internode read refused by peer"
|
||||
);
|
||||
return Err(io::Error::other("nixl internode read refused by peer"));
|
||||
}
|
||||
let landed = response_headers
|
||||
.get(NIXL_BYTES_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(|value| value.parse::<usize>().ok())
|
||||
.ok_or_else(|| io::Error::other("nixl internode read reply lacks the byte count"))?;
|
||||
if landed > request.length {
|
||||
return Err(io::Error::other("nixl internode read reports more bytes than requested"));
|
||||
}
|
||||
Ok((request.length, buffer.complete(landed)?))
|
||||
}
|
||||
|
||||
type ChunkFuture = BoxFuture<'static, io::Result<(usize, Bytes)>>;
|
||||
|
||||
struct ReaderState {
|
||||
pool: Arc<dyn NixlRecvPool>,
|
||||
base: ReadStreamRequest,
|
||||
next: usize,
|
||||
end: usize,
|
||||
chunk: usize,
|
||||
depth: usize,
|
||||
timeout: Duration,
|
||||
inflight: FuturesOrdered<ChunkFuture>,
|
||||
current: Bytes,
|
||||
done: bool,
|
||||
}
|
||||
|
||||
impl ReaderState {
|
||||
fn refill(&mut self) {
|
||||
while !self.done && self.inflight.len() < self.depth && self.next < self.end {
|
||||
let len = self.chunk.min(self.end - self.next);
|
||||
let mut request = self.base.clone();
|
||||
request.offset = self.next;
|
||||
request.length = len;
|
||||
self.next += len;
|
||||
self.inflight
|
||||
.push_back(Box::pin(fetch_chunk(Arc::clone(&self.pool), request, self.timeout)));
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_chunk(&mut self, cx: &mut Context<'_>, max: usize) -> Poll<io::Result<Option<Bytes>>> {
|
||||
loop {
|
||||
if !self.current.is_empty() {
|
||||
let take = self.current.len().min(max.max(1));
|
||||
return Poll::Ready(Ok(Some(self.current.split_to(take))));
|
||||
}
|
||||
if self.done {
|
||||
return Poll::Ready(Ok(None));
|
||||
}
|
||||
self.refill();
|
||||
match Pin::new(&mut self.inflight).poll_next(cx) {
|
||||
Poll::Ready(Some(Ok((asked, bytes)))) => {
|
||||
if bytes.len() < asked {
|
||||
// End of file: later chunks hold nothing; their buffers
|
||||
// are held back as they are dropped.
|
||||
self.done = true;
|
||||
self.inflight = FuturesOrdered::new();
|
||||
}
|
||||
if bytes.is_empty() {
|
||||
self.done = true;
|
||||
return Poll::Ready(Ok(None));
|
||||
}
|
||||
self.current = bytes;
|
||||
}
|
||||
Poll::Ready(Some(Err(error))) => {
|
||||
self.done = true;
|
||||
self.inflight = FuturesOrdered::new();
|
||||
return Poll::Ready(Err(error));
|
||||
}
|
||||
Poll::Ready(None) => {
|
||||
self.done = true;
|
||||
return Poll::Ready(Ok(None));
|
||||
}
|
||||
Poll::Pending => return Poll::Pending,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A remote shard stream read over NIXL in pipelined chunks.
|
||||
pub struct NixlChunkReader {
|
||||
// The mutex only makes the reader `Sync`; every access goes through `&mut self`.
|
||||
state: std::sync::Mutex<ReaderState>,
|
||||
}
|
||||
|
||||
impl NixlChunkReader {
|
||||
fn new(pool: Arc<dyn NixlRecvPool>, request: ReadStreamRequest, depth: usize) -> Self {
|
||||
let chunk = pool.chunk_bytes().max(1);
|
||||
let timeout = request.stall_timeout.unwrap_or(DEFAULT_CHUNK_TIMEOUT);
|
||||
Self {
|
||||
state: std::sync::Mutex::new(ReaderState {
|
||||
next: request.offset,
|
||||
end: request.offset.saturating_add(request.length),
|
||||
pool,
|
||||
base: request,
|
||||
chunk,
|
||||
depth,
|
||||
timeout,
|
||||
inflight: FuturesOrdered::new(),
|
||||
current: Bytes::new(),
|
||||
done: false,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
fn state_mut(&mut self) -> &mut ReaderState {
|
||||
self.state.get_mut().unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
}
|
||||
}
|
||||
|
||||
impl ChunkReader for NixlChunkReader {
|
||||
fn poll_read_chunk(self: Pin<&mut Self>, cx: &mut Context<'_>, max: usize) -> Poll<io::Result<Option<Bytes>>> {
|
||||
self.get_mut().state_mut().poll_chunk(cx, max)
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncRead for NixlChunkReader {
|
||||
fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
|
||||
if buf.remaining() == 0 {
|
||||
return Poll::Ready(Ok(()));
|
||||
}
|
||||
match self.get_mut().state_mut().poll_chunk(cx, buf.remaining()) {
|
||||
Poll::Ready(Ok(Some(bytes))) => {
|
||||
buf.put_slice(&bytes);
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
Poll::Ready(Ok(None)) => Poll::Ready(Ok(())),
|
||||
Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
|
||||
Poll::Pending => Poll::Pending,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Sends one segment of a shard write: the disk's node pulls `buffers` (in
|
||||
/// file order) and writes them at the end of the file, creating it unless
|
||||
/// `append`. The buffers return to the pool only after the node answered.
|
||||
async fn send_segment(
|
||||
pool: Arc<dyn NixlRecvPool>,
|
||||
request: WriteStreamRequest,
|
||||
buffers: Vec<(LentBuffer, usize)>,
|
||||
append: bool,
|
||||
last: bool,
|
||||
timeout: Duration,
|
||||
) -> io::Result<()> {
|
||||
let total: usize = buffers.iter().map(|(_, len)| *len).sum();
|
||||
let descriptors = buffers
|
||||
.iter()
|
||||
.map(|(buffer, len)| {
|
||||
buffer
|
||||
.buffer
|
||||
.as_ref()
|
||||
.map(|buffer| buffer.descriptor_for(*len))
|
||||
.unwrap_or_default()
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join(",");
|
||||
let url = format!(
|
||||
"{}{WRITE_FILE_NIXL_PATH}?disk={}&volume={}&path={}&append={append}&size={}&last={last}&{NIXL_NONCE_QUERY}={}&{NIXL_AGENT_QUERY}={}&{NIXL_BUFFERS_QUERY}={}&{NIXL_METADATA_SHA256_QUERY}={}",
|
||||
request.endpoint,
|
||||
urlencoding::encode(&request.disk),
|
||||
urlencoding::encode(&request.volume),
|
||||
urlencoding::encode(&request.path),
|
||||
request.size,
|
||||
Uuid::new_v4(),
|
||||
urlencoding::encode(pool.agent_name()),
|
||||
urlencoding::encode(&descriptors),
|
||||
pool.metadata_sha256_hex()
|
||||
);
|
||||
let mut headers = json_headers();
|
||||
headers.insert(
|
||||
NIXL_METADATA_HEADER,
|
||||
HeaderValue::from_str(pool.metadata_base64())
|
||||
.map_err(|_| io::Error::other("nixl agent metadata is not a header value"))?,
|
||||
);
|
||||
build_auth_headers(&url, &Method::PUT, &mut headers)?;
|
||||
let request_started = rustfs_io_metrics::put_stage_timer();
|
||||
let (status, response_headers, body) = rustfs_rio::internode_http_request(&url, Method::PUT, headers, Some(timeout)).await?;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from("nixl_write_segment_request", request_started);
|
||||
if !status.is_success() {
|
||||
debug!(
|
||||
component = LOG_COMPONENT_INTERNODE,
|
||||
subsystem = LOG_SUBSYSTEM_NIXL,
|
||||
result = "failed",
|
||||
status = status.as_u16(),
|
||||
body_len = body.len(),
|
||||
"nixl internode write refused by peer"
|
||||
);
|
||||
return Err(io::Error::other("nixl internode write refused by peer"));
|
||||
}
|
||||
let written = response_headers
|
||||
.get(NIXL_BYTES_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(|value| value.parse::<usize>().ok());
|
||||
if written != Some(total) {
|
||||
debug!(
|
||||
component = LOG_COMPONENT_INTERNODE,
|
||||
subsystem = LOG_SUBSYSTEM_NIXL,
|
||||
result = "failed",
|
||||
written = ?written,
|
||||
total,
|
||||
"nixl internode write stored a different byte count"
|
||||
);
|
||||
return Err(io::Error::other("nixl internode write stored a different byte count"));
|
||||
}
|
||||
for (buffer, _) in buffers {
|
||||
buffer.release();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
type SegmentFuture = BoxFuture<'static, io::Result<()>>;
|
||||
type ReserveFuture = BoxFuture<'static, io::Result<Box<dyn NixlRecvBuffer>>>;
|
||||
|
||||
struct WriterState {
|
||||
pool: Arc<dyn NixlRecvPool>,
|
||||
base: WriteStreamRequest,
|
||||
timeout: Duration,
|
||||
current: Option<(LentBuffer, usize)>,
|
||||
filled: Vec<(LentBuffer, usize)>,
|
||||
reserving: Option<ReserveFuture>,
|
||||
sending: Option<SegmentFuture>,
|
||||
segments_sent: usize,
|
||||
shut_down: bool,
|
||||
}
|
||||
|
||||
impl WriterState {
|
||||
fn start_segment(&mut self, last: bool) {
|
||||
let buffers = std::mem::take(&mut self.filled);
|
||||
let append = self.base.append || self.segments_sent > 0;
|
||||
self.segments_sent += 1;
|
||||
self.sending = Some(Box::pin(send_segment(
|
||||
Arc::clone(&self.pool),
|
||||
self.base.clone(),
|
||||
buffers,
|
||||
append,
|
||||
last,
|
||||
self.timeout,
|
||||
)));
|
||||
}
|
||||
|
||||
fn poll_sending(&mut self, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
match self.sending.as_mut() {
|
||||
None => Poll::Ready(Ok(())),
|
||||
Some(sending) => match sending.as_mut().poll(cx) {
|
||||
Poll::Pending => Poll::Pending,
|
||||
Poll::Ready(result) => {
|
||||
self.sending = None;
|
||||
Poll::Ready(result)
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_write(&mut self, cx: &mut Context<'_>, data: &[u8]) -> Poll<io::Result<usize>> {
|
||||
if self.shut_down {
|
||||
return Poll::Ready(Err(io::Error::other("nixl internode writer is shut down")));
|
||||
}
|
||||
if data.is_empty() {
|
||||
return Poll::Ready(Ok(0));
|
||||
}
|
||||
if let Err(error) = futures::ready!(self.poll_sending(cx)) {
|
||||
return Poll::Ready(Err(error));
|
||||
}
|
||||
while self.current.is_none() {
|
||||
let pool = Arc::clone(&self.pool);
|
||||
let reserving = self.reserving.get_or_insert_with(|| pool.reserve());
|
||||
match reserving.as_mut().poll(cx) {
|
||||
Poll::Ready(buffer) => {
|
||||
self.reserving = None;
|
||||
self.current = Some((LentBuffer { buffer: Some(buffer?) }, 0));
|
||||
}
|
||||
Poll::Pending if !self.filled.is_empty() => {
|
||||
// Send held buffers before waiting for capacity that their acknowledgement releases.
|
||||
self.reserving = None;
|
||||
self.start_segment(false);
|
||||
futures::ready!(self.poll_sending(cx))?;
|
||||
}
|
||||
Poll::Pending => return Poll::Pending,
|
||||
}
|
||||
}
|
||||
let (lent, filled) = self.current.as_mut().expect("buffer reserved above");
|
||||
let spare = lent.buffer.as_mut().expect("lent buffer present").spare_mut();
|
||||
let n = data.len().min(spare.len() - *filled);
|
||||
spare[*filled..*filled + n].copy_from_slice(&data[..n]);
|
||||
*filled += n;
|
||||
if *filled == spare.len() {
|
||||
let full = self.current.take().expect("current buffer");
|
||||
self.filled.push(full);
|
||||
if self.filled.len() >= WRITE_SEGMENT_BUFFERS {
|
||||
self.start_segment(false);
|
||||
}
|
||||
}
|
||||
Poll::Ready(Ok(n))
|
||||
}
|
||||
|
||||
fn poll_shutdown(&mut self, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
if !self.shut_down {
|
||||
if let Err(error) = futures::ready!(self.poll_sending(cx)) {
|
||||
return Poll::Ready(Err(error));
|
||||
}
|
||||
if let Some((buffer, filled)) = self.current.take() {
|
||||
if filled > 0 {
|
||||
self.filled.push((buffer, filled));
|
||||
} else {
|
||||
buffer.release();
|
||||
}
|
||||
}
|
||||
self.shut_down = true;
|
||||
if !self.filled.is_empty() || self.segments_sent == 0 {
|
||||
// The last segment; with nothing written at all it still creates the file.
|
||||
self.start_segment(true);
|
||||
}
|
||||
}
|
||||
self.poll_sending(cx)
|
||||
}
|
||||
}
|
||||
|
||||
/// A remote shard file written through NIXL: bytes are staged in registered
|
||||
/// buffers and the disk's node pulls them, a segment at a time.
|
||||
pub struct NixlFileWriter {
|
||||
// The mutex only makes the writer `Sync`; every access goes through `&mut self`.
|
||||
state: std::sync::Mutex<WriterState>,
|
||||
}
|
||||
|
||||
impl NixlFileWriter {
|
||||
fn new(pool: Arc<dyn NixlRecvPool>, request: WriteStreamRequest) -> Self {
|
||||
Self {
|
||||
state: std::sync::Mutex::new(WriterState {
|
||||
pool,
|
||||
base: request,
|
||||
timeout: DEFAULT_CHUNK_TIMEOUT,
|
||||
current: None,
|
||||
filled: Vec::new(),
|
||||
reserving: None,
|
||||
sending: None,
|
||||
segments_sent: 0,
|
||||
shut_down: false,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
fn state_mut(&mut self) -> &mut WriterState {
|
||||
self.state.get_mut().unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
}
|
||||
}
|
||||
|
||||
impl tokio::io::AsyncWrite for NixlFileWriter {
|
||||
fn poll_write(self: Pin<&mut Self>, cx: &mut Context<'_>, data: &[u8]) -> Poll<io::Result<usize>> {
|
||||
self.get_mut().state_mut().poll_write(cx, data)
|
||||
}
|
||||
|
||||
fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
// Bytes stay staged until a segment fills or the writer shuts down; a
|
||||
// flush only waits for a segment already on its way.
|
||||
self.get_mut().state_mut().poll_sending(cx)
|
||||
}
|
||||
|
||||
fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
self.get_mut().state_mut().poll_shutdown(cx)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
struct CountingBuffer {
|
||||
released: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl NixlRecvBuffer for CountingBuffer {
|
||||
fn descriptor(&self) -> String {
|
||||
"dram:0:0x1000:16".to_owned()
|
||||
}
|
||||
|
||||
fn descriptor_for(&self, len: usize) -> String {
|
||||
format!("dram:0:0x1000:{len}")
|
||||
}
|
||||
|
||||
fn spare_mut(&mut self) -> &mut [u8] {
|
||||
&mut []
|
||||
}
|
||||
|
||||
fn into_bytes(self: Box<Self>, len: usize) -> io::Result<Bytes> {
|
||||
Ok(Bytes::from(vec![7u8; len]))
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for CountingBuffer {
|
||||
fn drop(&mut self) {
|
||||
self.released.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
/// A buffer backed by a plain vector, for the writer's staging logic.
|
||||
struct VecBuffer(Vec<u8>);
|
||||
|
||||
impl NixlRecvBuffer for VecBuffer {
|
||||
fn descriptor(&self) -> String {
|
||||
self.descriptor_for(self.0.len())
|
||||
}
|
||||
|
||||
fn descriptor_for(&self, len: usize) -> String {
|
||||
format!("dram:0:0x2000:{len}")
|
||||
}
|
||||
|
||||
fn spare_mut(&mut self) -> &mut [u8] {
|
||||
&mut self.0
|
||||
}
|
||||
|
||||
fn into_bytes(self: Box<Self>, len: usize) -> io::Result<Bytes> {
|
||||
Ok(Bytes::copy_from_slice(&self.0[..len]))
|
||||
}
|
||||
}
|
||||
|
||||
struct VecPool;
|
||||
|
||||
impl NixlRecvPool for VecPool {
|
||||
fn agent_name(&self) -> &str {
|
||||
"test-agent"
|
||||
}
|
||||
|
||||
fn metadata_base64(&self) -> &str {
|
||||
""
|
||||
}
|
||||
|
||||
fn metadata_sha256_hex(&self) -> &str {
|
||||
""
|
||||
}
|
||||
|
||||
fn chunk_bytes(&self) -> usize {
|
||||
8
|
||||
}
|
||||
|
||||
fn reserve(&self) -> BoxFuture<'static, io::Result<Box<dyn NixlRecvBuffer>>> {
|
||||
Box::pin(async { Ok(Box::new(VecBuffer(vec![0; 8])) as Box<dyn NixlRecvBuffer>) })
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_writer_stages_bytes_in_order_until_a_segment_fills() {
|
||||
use tokio::io::AsyncWriteExt;
|
||||
let request = WriteStreamRequest {
|
||||
endpoint: "http://127.0.0.1:1".to_owned(),
|
||||
disk: "disk".to_owned(),
|
||||
volume: "volume".to_owned(),
|
||||
path: "path".to_owned(),
|
||||
append: false,
|
||||
size: 20,
|
||||
};
|
||||
let mut writer = NixlFileWriter::new(Arc::new(VecPool), request);
|
||||
let data: Vec<u8> = (0u8..20).collect();
|
||||
writer
|
||||
.write_all(&data)
|
||||
.await
|
||||
.expect("staging never sends below a full segment");
|
||||
let state = writer.state_mut();
|
||||
assert!(state.sending.is_none(), "two full 8-byte buffers stay below the four-buffer segment");
|
||||
let staged: Vec<u8> = state
|
||||
.filled
|
||||
.iter_mut()
|
||||
.chain(state.current.iter_mut())
|
||||
.flat_map(|(lent, len)| lent.buffer.as_mut().expect("buffer").spare_mut()[..*len].to_vec())
|
||||
.collect();
|
||||
assert_eq!(staged, data);
|
||||
assert_eq!(state.filled.len(), 2);
|
||||
assert_eq!(state.current.as_ref().map(|(_, len)| *len), Some(4));
|
||||
}
|
||||
|
||||
struct LimitedBuffer {
|
||||
bytes: VecBuffer,
|
||||
_permit: tokio::sync::OwnedSemaphorePermit,
|
||||
}
|
||||
|
||||
impl NixlRecvBuffer for LimitedBuffer {
|
||||
fn descriptor(&self) -> String {
|
||||
self.bytes.descriptor()
|
||||
}
|
||||
|
||||
fn descriptor_for(&self, len: usize) -> String {
|
||||
self.bytes.descriptor_for(len)
|
||||
}
|
||||
|
||||
fn spare_mut(&mut self) -> &mut [u8] {
|
||||
self.bytes.spare_mut()
|
||||
}
|
||||
|
||||
fn into_bytes(self: Box<Self>, len: usize) -> io::Result<Bytes> {
|
||||
Ok(Bytes::copy_from_slice(&self.bytes.0[..len]))
|
||||
}
|
||||
}
|
||||
|
||||
struct LimitedPool(Arc<tokio::sync::Semaphore>);
|
||||
|
||||
impl NixlRecvPool for LimitedPool {
|
||||
fn agent_name(&self) -> &str {
|
||||
"test-agent"
|
||||
}
|
||||
|
||||
fn metadata_base64(&self) -> &str {
|
||||
""
|
||||
}
|
||||
|
||||
fn metadata_sha256_hex(&self) -> &str {
|
||||
""
|
||||
}
|
||||
|
||||
fn chunk_bytes(&self) -> usize {
|
||||
8
|
||||
}
|
||||
|
||||
fn reserve(&self) -> ReserveFuture {
|
||||
let permits = Arc::clone(&self.0);
|
||||
Box::pin(async move {
|
||||
let permit = permits.acquire_owned().await.map_err(io::Error::other)?;
|
||||
Ok(Box::new(LimitedBuffer {
|
||||
bytes: VecBuffer(vec![0; 8]),
|
||||
_permit: permit,
|
||||
}) as Box<dyn NixlRecvBuffer>)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_writer_sends_staged_bytes_when_the_pool_is_full() {
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
|
||||
crate::runtime::sources::ensure_test_rpc_secret();
|
||||
for chunks in 1..=3 {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("segment listener");
|
||||
let address = listener.local_addr().expect("segment address");
|
||||
let server = tokio::spawn(async move {
|
||||
let mut received = 0;
|
||||
while received < 40 {
|
||||
let (mut socket, _) = listener.accept().await.expect("segment connection");
|
||||
let mut request = Vec::new();
|
||||
while !request.windows(4).any(|bytes| bytes == b"\r\n\r\n") {
|
||||
let mut bytes = [0; 1024];
|
||||
let n = socket.read(&mut bytes).await.expect("segment request");
|
||||
assert!(n > 0 && request.len() < 16384, "bounded request headers");
|
||||
request.extend_from_slice(&bytes[..n]);
|
||||
}
|
||||
let request = std::str::from_utf8(&request).expect("request text");
|
||||
let target = request.split_whitespace().nth(1).expect("request target");
|
||||
let url = url::Url::parse(&format!("http://{address}{target}")).expect("segment URL");
|
||||
let descriptors = url
|
||||
.query_pairs()
|
||||
.find(|(name, _)| name == NIXL_BUFFERS_QUERY)
|
||||
.expect("buffers")
|
||||
.1;
|
||||
let bytes: usize = descriptors
|
||||
.split(',')
|
||||
.map(|descriptor| {
|
||||
descriptor
|
||||
.rsplit(':')
|
||||
.next()
|
||||
.expect("buffer length")
|
||||
.parse::<usize>()
|
||||
.expect("numeric length")
|
||||
})
|
||||
.sum();
|
||||
assert!(bytes > 0 && bytes <= chunks * 8);
|
||||
received += bytes;
|
||||
let response = format!(
|
||||
"HTTP/1.1 200 OK\r\ncontent-length: 0\r\nconnection: close\r\nx-rustfs-nixl-bytes: {bytes}\r\n\r\n"
|
||||
);
|
||||
socket.write_all(response.as_bytes()).await.expect("segment acknowledgement");
|
||||
}
|
||||
received
|
||||
});
|
||||
let pool = Arc::new(LimitedPool(Arc::new(tokio::sync::Semaphore::new(chunks))));
|
||||
let request = WriteStreamRequest {
|
||||
endpoint: format!("http://{address}"),
|
||||
disk: "disk".to_owned(),
|
||||
volume: "volume".to_owned(),
|
||||
path: "path".to_owned(),
|
||||
append: false,
|
||||
size: 40,
|
||||
};
|
||||
let mut writer = NixlFileWriter::new(pool.clone(), request);
|
||||
let result = tokio::time::timeout(Duration::from_secs(5), async {
|
||||
writer.write_all(&[7; 40]).await?;
|
||||
writer.shutdown().await
|
||||
})
|
||||
.await;
|
||||
if result.is_err() {
|
||||
server.abort();
|
||||
}
|
||||
result
|
||||
.expect("a writer must dispatch before exhausting its pool")
|
||||
.expect("shard write");
|
||||
assert_eq!(server.await.expect("segment server"), 40);
|
||||
assert_eq!(pool.0.available_permits(), chunks, "acknowledged buffers return to the pool");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn abandoned_buffers_return_only_after_the_hold() {
|
||||
let released = Arc::new(AtomicUsize::new(0));
|
||||
let lent = LentBuffer {
|
||||
buffer: Some(Box::new(CountingBuffer {
|
||||
released: Arc::clone(&released),
|
||||
})),
|
||||
};
|
||||
drop(lent);
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(released.load(Ordering::SeqCst), 0, "an abandoned buffer must not return at once");
|
||||
tokio::time::sleep(ABANDONED_BUFFER_HOLD + Duration::from_secs(1)).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(released.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn completed_buffers_return_with_their_bytes() {
|
||||
let released = Arc::new(AtomicUsize::new(0));
|
||||
let lent = LentBuffer {
|
||||
buffer: Some(Box::new(CountingBuffer {
|
||||
released: Arc::clone(&released),
|
||||
})),
|
||||
};
|
||||
let bytes = lent.complete(5).expect("bytes");
|
||||
assert_eq!(bytes.len(), 5);
|
||||
assert_eq!(released.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
}
|
||||
@@ -97,6 +97,7 @@ const PEER_REST_RECOVERY_MAX_ATTEMPTS: u32 = 60;
|
||||
const PEER_REST_RECOVERY_MAX_BACKOFF: Duration = Duration::from_secs(30);
|
||||
const SCANNER_ACTIVITY_MAX_MESSAGE_SIZE: usize = 1024;
|
||||
const SCANNER_SCOPED_DIRTY_USAGE_STAGE_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
const SCANNER_USAGE_WATCH_PEER_CONCURRENCY: usize = 4;
|
||||
/// Reserve time for the acquire response's network/clock uncertainty. The
|
||||
/// server owns the real expiry; this local deadline is intentionally earlier
|
||||
/// so a coordinator never starts a bounded persistence operation at the edge
|
||||
@@ -298,6 +299,22 @@ fn scanner_scoped_dirty_usage_ack_payload(
|
||||
Ok(payload)
|
||||
}
|
||||
|
||||
/// Authentication binds a response to the complete current request before any
|
||||
/// returned watch can be consumed. Semantic validation still rejects a signed
|
||||
/// response for the wrong producer session, bucket, or token.
|
||||
fn validate_scanner_usage_watch_response(
|
||||
request: &rustfs_protos::proto_gen::node_service::ScannerUsageWatchRequest,
|
||||
response: &rustfs_protos::proto_gen::node_service::ScannerUsageWatchResponse,
|
||||
) -> Result<()> {
|
||||
use rustfs_protos::usage_watch::{
|
||||
canonical_usage_watch_request, canonical_usage_watch_response, validate_usage_watch_response,
|
||||
};
|
||||
let canonical = canonical_usage_watch_request(request).map_err(|error| Error::other(error.to_string()))?;
|
||||
let body = canonical_usage_watch_response(&canonical, response).map_err(|error| Error::other(error.to_string()))?;
|
||||
verify_tonic_rpc_response_proof(&body, response.response_proof.as_ref())?;
|
||||
validate_usage_watch_response(request, response).map_err(|error| Error::other(error.to_string()))
|
||||
}
|
||||
|
||||
fn scanner_scoped_dirty_usage_ack_response_matches(
|
||||
request: &ScannerScopedDirtyUsageAckRequest,
|
||||
response: &ScannerScopedDirtyUsageAckResponse,
|
||||
@@ -741,6 +758,9 @@ pub struct PeerRestClient {
|
||||
topology_member: String,
|
||||
offline: Arc<AtomicBool>,
|
||||
recovery_running: Arc<AtomicBool>,
|
||||
/// All clones used by admission, keepalive and root verification share
|
||||
/// this peer's bounded sender capacity.
|
||||
scanner_usage_watch_admission: Arc<tokio::sync::Semaphore>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
@@ -900,9 +920,20 @@ impl PeerRestClient {
|
||||
topology_member,
|
||||
offline: Arc::new(AtomicBool::new(false)),
|
||||
recovery_running: Arc::new(AtomicBool::new(false)),
|
||||
scanner_usage_watch_admission: Arc::new(tokio::sync::Semaphore::new(SCANNER_USAGE_WATCH_PEER_CONCURRENCY)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn acquire_usage_watch_send_permit(&self) -> Result<tokio::sync::OwnedSemaphorePermit> {
|
||||
timeout(
|
||||
SCANNER_SCOPED_DIRTY_USAGE_STAGE_TIMEOUT,
|
||||
self.scanner_usage_watch_admission.clone().acquire_owned(),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| Error::other("usage watch sender admission timed out"))?
|
||||
.map_err(|_| Error::other("usage watch sender admission is closed"))
|
||||
}
|
||||
|
||||
fn parse_topology_host(peer_host_port: &str, grid_host: &str) -> Result<XHost> {
|
||||
let url = url::Url::parse(grid_host).map_err(|_| Error::other("peer grid host is not a valid URL"))?;
|
||||
if !matches!(url.scheme(), "http" | "https")
|
||||
@@ -1225,12 +1256,23 @@ impl PeerRestClient {
|
||||
|
||||
impl PeerRestClient {
|
||||
pub async fn local_storage_info(&self) -> Result<rustfs_madmin::StorageInfo> {
|
||||
self.finalize_result(self.local_storage_info_inner().await).await
|
||||
self.local_storage_info_with_metrics(true).await
|
||||
}
|
||||
|
||||
async fn local_storage_info_inner(&self) -> Result<rustfs_madmin::StorageInfo> {
|
||||
/// Collect fresh local inventory from this peer, optionally including disk metrics.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns the existing transport, control-plane or inventory decoding error.
|
||||
pub async fn local_storage_info_with_metrics(&self, include_metrics: bool) -> Result<rustfs_madmin::StorageInfo> {
|
||||
self.finalize_result(self.local_storage_info_inner(include_metrics).await)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn local_storage_info_inner(&self, include_metrics: bool) -> Result<rustfs_madmin::StorageInfo> {
|
||||
let mut client = self.get_client().await?;
|
||||
let request = Request::new(LocalStorageInfoRequest { metrics: true });
|
||||
let request = Request::new(LocalStorageInfoRequest {
|
||||
metrics: include_metrics,
|
||||
});
|
||||
|
||||
let response = client.local_storage_info(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1639,6 +1681,51 @@ impl PeerRestClient {
|
||||
Err(Error::NotImplemented)
|
||||
}
|
||||
|
||||
/// Routes one authorized MRF operation without retrying ambiguous mutations.
|
||||
/// `None` means the selected node predates this protocol; it is never an empty result.
|
||||
pub async fn mrf_lifecycle(
|
||||
&self,
|
||||
topology: String,
|
||||
node_id: String,
|
||||
command: Vec<u8>,
|
||||
timeout: std::time::Duration,
|
||||
) -> Result<Option<Vec<u8>>> {
|
||||
use rustfs_protos::proto_gen::node_service::MrfLifecycleRequest;
|
||||
use rustfs_protos::{MRF_LIFECYCLE_MAX_COMMAND_BYTES, MRF_LIFECYCLE_MAX_MESSAGE_BYTES, MRF_LIFECYCLE_PROTOCOL_VERSION};
|
||||
if command.len() > MRF_LIFECYCLE_MAX_COMMAND_BYTES {
|
||||
return Err(Error::other("MRF command exceeds size limit"));
|
||||
}
|
||||
let payload = MrfLifecycleRequest {
|
||||
protocol_version: MRF_LIFECYCLE_PROTOCOL_VERSION,
|
||||
topology_fingerprint: topology,
|
||||
node_id,
|
||||
command: command.into(),
|
||||
challenge: Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
};
|
||||
let mut request = Request::new(payload.clone());
|
||||
request.set_timeout(timeout);
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
let mut client = self
|
||||
.get_client()
|
||||
.await?
|
||||
.max_decoding_message_size(MRF_LIFECYCLE_MAX_MESSAGE_BYTES)
|
||||
.max_encoding_message_size(MRF_LIFECYCLE_MAX_MESSAGE_BYTES);
|
||||
let response = match client.mrf_lifecycle(request).await {
|
||||
Ok(response) => response.into_inner(),
|
||||
// RUSTFS_COMPAT_TODO(backlog-3177): old peers lack MRF management. Remove after every supported peer implements this RPC.
|
||||
Err(status) if status.code() == tonic::Code::Unimplemented => return Ok(None),
|
||||
Err(status) => return Err(status.into()),
|
||||
};
|
||||
let canonical = rustfs_protos::canonical_mrf_lifecycle_response_body(&payload, &response)
|
||||
.map_err(|_| Error::other("MRF response is too large to authenticate"))?;
|
||||
verify_tonic_rpc_response_proof(&canonical, &response.response_proof)
|
||||
.map_err(|_| Error::other("MRF response authentication failed"))?;
|
||||
if response.protocol_version != MRF_LIFECYCLE_PROTOCOL_VERSION {
|
||||
return Err(Error::other("unsupported MRF response protocol"));
|
||||
}
|
||||
Ok(Some(response.result.to_vec()))
|
||||
}
|
||||
|
||||
pub async fn background_heal_status(&self) -> Result<Option<Vec<u8>>> {
|
||||
self.finalize_result(
|
||||
async {
|
||||
@@ -2438,6 +2525,129 @@ impl PeerRestClient {
|
||||
.await
|
||||
}
|
||||
|
||||
/// Independently negotiated bucket watches; an old peer returns an error
|
||||
/// and callers retain the global-generation fallback for that bucket.
|
||||
pub async fn scanner_usage_watch(
|
||||
&self,
|
||||
owner_id: Uuid,
|
||||
mut entries: Vec<crate::runtime::usage_watch::ScannerUsageWatchRequest>,
|
||||
) -> Result<Vec<crate::runtime::usage_watch::ScannerUsageWatchObservation>> {
|
||||
use crate::runtime::usage_watch::{
|
||||
SCANNER_USAGE_WATCH_COVERAGE_VERSION, SCANNER_USAGE_WATCH_LEASE_MS, ScannerUsageWatchKnown,
|
||||
ScannerUsageWatchObservation,
|
||||
};
|
||||
use rustfs_protos::proto_gen::node_service::{ScannerUsageWatchEntry, ScannerUsageWatchRequest};
|
||||
use rustfs_protos::usage_watch::*;
|
||||
entries.sort_by(|left, right| left.bucket.cmp(&right.bucket));
|
||||
if owner_id.is_nil() || entries.windows(2).any(|pair| pair[0].bucket == pair[1].bucket) {
|
||||
return Err(Error::other("usage watch request contains duplicate buckets or an invalid owner"));
|
||||
}
|
||||
// Local queue pressure is not a network failure. Keep admission errors
|
||||
// outside finalize_result so they cannot mark a healthy peer offline.
|
||||
let _send_permit = self.acquire_usage_watch_send_permit().await?;
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = super::client::scanner_control_time_out_client(
|
||||
&self.grid_host,
|
||||
TonicInterceptor::Signature(gen_tonic_signature_interceptor()),
|
||||
)
|
||||
.await?;
|
||||
let mut observations = Vec::with_capacity(entries.len());
|
||||
for batch in entries.chunks(USAGE_WATCH_MAX_ENTRIES) {
|
||||
let payload = ScannerUsageWatchRequest {
|
||||
challenge: Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
protocol_version: USAGE_WATCH_PROTOCOL_VERSION,
|
||||
owner_id: owner_id.to_string(),
|
||||
entries: batch
|
||||
.iter()
|
||||
.map(|entry| ScannerUsageWatchEntry {
|
||||
bucket: entry.bucket.clone(),
|
||||
bucket_incarnation: entry.bucket_incarnation.as_bytes().to_vec().into(),
|
||||
watch_id: entry.watch_id.map_or_else(Bytes::new, |id| id.as_bytes().to_vec().into()),
|
||||
release: entry.release,
|
||||
session_id: entry.session_id.map_or_else(Bytes::new, |id| id.as_bytes().to_vec().into()),
|
||||
})
|
||||
.collect(),
|
||||
};
|
||||
let canonical = canonical_usage_watch_request(&payload).map_err(|error| Error::other(error.to_string()))?;
|
||||
let mut request = Request::new(payload.clone());
|
||||
set_tonic_canonical_body_digest(&mut request, &canonical)?;
|
||||
let started = std::time::Instant::now();
|
||||
let response = timeout(SCANNER_SCOPED_DIRTY_USAGE_STAGE_TIMEOUT, client.scanner_usage_watch(request))
|
||||
.await
|
||||
.map_err(|_| Error::other("usage watch request timed out"))??
|
||||
.into_inner();
|
||||
let budget_used = u64::try_from(started.elapsed().as_millis())
|
||||
.unwrap_or(u64::MAX)
|
||||
.saturating_add(250);
|
||||
validate_scanner_usage_watch_response(&payload, &response)?;
|
||||
let session = Uuid::from_slice(response.session_id.as_ref())
|
||||
.map_err(|_| Error::other("invalid usage watch session"))?;
|
||||
if response.protocol_version != USAGE_WATCH_PROTOCOL_VERSION
|
||||
|| response.owner_id != payload.owner_id
|
||||
|| session.is_nil()
|
||||
|| response.entries.len() != payload.entries.len()
|
||||
{
|
||||
return Err(Error::other("usage watch response does not match request"));
|
||||
}
|
||||
for (requested, entry) in batch.iter().zip(response.entries) {
|
||||
if entry.bucket != requested.bucket {
|
||||
return Err(Error::other("usage watch response bucket order differs"));
|
||||
}
|
||||
let known = if entry.known {
|
||||
let token = Uuid::from_slice(entry.watch_id.as_ref())
|
||||
.map_err(|_| Error::other("invalid usage watch token"))?;
|
||||
if token.is_nil()
|
||||
|| entry.bucket_incarnation.as_ref() != requested.bucket_incarnation.as_bytes()
|
||||
|| requested.watch_id.is_some_and(|expected| expected != token)
|
||||
|| entry.revision == 0
|
||||
|| entry.revision == u64::MAX
|
||||
|| entry.inflight == u64::MAX
|
||||
|| entry.coverage_version != SCANNER_USAGE_WATCH_COVERAGE_VERSION
|
||||
|| entry.coverage_epoch == 0
|
||||
|| entry.lease_remaining_ms == 0
|
||||
|| entry.lease_remaining_ms > SCANNER_USAGE_WATCH_LEASE_MS
|
||||
{
|
||||
return Err(Error::other("usage watch response contains an invalid proof"));
|
||||
}
|
||||
let lease_remaining_ms = entry.lease_remaining_ms.saturating_sub(budget_used);
|
||||
(lease_remaining_ms != 0).then_some(ScannerUsageWatchKnown {
|
||||
owner_id,
|
||||
session_id: session,
|
||||
watch_id: token,
|
||||
bucket_incarnation: requested.bucket_incarnation,
|
||||
revision: entry.revision,
|
||||
inflight: entry.inflight,
|
||||
coverage_version: entry.coverage_version,
|
||||
coverage_epoch: entry.coverage_epoch,
|
||||
lease_remaining_ms,
|
||||
})
|
||||
} else {
|
||||
if !entry.watch_id.is_empty()
|
||||
|| !entry.bucket_incarnation.is_empty()
|
||||
|| entry.revision != 0
|
||||
|| entry.inflight != 0
|
||||
|| entry.coverage_version != 0
|
||||
|| entry.coverage_epoch != 0
|
||||
|| entry.lease_remaining_ms != 0
|
||||
{
|
||||
return Err(Error::other("Unknown usage watch entry contains proof fields"));
|
||||
}
|
||||
None
|
||||
};
|
||||
observations.push(ScannerUsageWatchObservation {
|
||||
bucket: entry.bucket,
|
||||
known,
|
||||
});
|
||||
}
|
||||
}
|
||||
Ok(observations)
|
||||
}
|
||||
.await,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn acknowledge_scanner_scoped_dirty_usage(
|
||||
&self,
|
||||
owner_id: String,
|
||||
@@ -3272,6 +3482,74 @@ mod tests {
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_watch_client_clones_share_receiver_sized_send_admission() {
|
||||
let receiver = Arc::new(tokio::sync::Semaphore::new(4));
|
||||
let occupied = (0..4)
|
||||
.map(|_| receiver.clone().try_acquire_owned().expect("four receiver slots"))
|
||||
.collect::<Vec<_>>();
|
||||
assert!(receiver.try_acquire().is_err(), "an uncoordinated fifth request is rejected");
|
||||
drop(occupied);
|
||||
|
||||
let client = test_peer_client();
|
||||
let active = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let peak = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let tasks = (0..12).map(|_| {
|
||||
// Simulate the shared client's independent admission, keeper and
|
||||
// publication callers. Each uses the actual production gate.
|
||||
let client = client.clone();
|
||||
let receiver = receiver.clone();
|
||||
let active = active.clone();
|
||||
let peak = peak.clone();
|
||||
tokio::spawn(async move {
|
||||
let _send = client
|
||||
.acquire_usage_watch_send_permit()
|
||||
.await
|
||||
.expect("bounded sender admission");
|
||||
let _receive = receiver
|
||||
.try_acquire_owned()
|
||||
.expect("coexisting client clones must never send more than the receiver's four slots");
|
||||
let count = active.fetch_add(1, Ordering::AcqRel) + 1;
|
||||
peak.fetch_max(count, Ordering::AcqRel);
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
active.fetch_sub(1, Ordering::AcqRel);
|
||||
})
|
||||
});
|
||||
for result in futures::future::join_all(tasks).await {
|
||||
result.expect("all queued callers finish");
|
||||
}
|
||||
assert_eq!(peak.load(Ordering::Acquire), 4);
|
||||
assert_eq!(active.load(Ordering::Acquire), 0);
|
||||
assert_eq!(client.scanner_usage_watch_admission.available_permits(), 4);
|
||||
assert_eq!(receiver.available_permits(), 4);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn usage_watch_sender_queue_timeout_does_not_mark_the_peer_offline() {
|
||||
let client = test_peer_client();
|
||||
let _occupied = client
|
||||
.scanner_usage_watch_admission
|
||||
.clone()
|
||||
.acquire_many_owned(4)
|
||||
.await
|
||||
.unwrap();
|
||||
let result = client
|
||||
.scanner_usage_watch(
|
||||
Uuid::from_u128(1),
|
||||
vec![crate::runtime::usage_watch::ScannerUsageWatchRequest {
|
||||
bucket: "bucket".into(),
|
||||
bucket_incarnation: Uuid::from_u128(2),
|
||||
watch_id: None,
|
||||
session_id: None,
|
||||
release: false,
|
||||
}],
|
||||
)
|
||||
.await;
|
||||
assert!(result.unwrap_err().to_string().contains("sender admission timed out"));
|
||||
assert!(!client.offline.load(Ordering::Acquire));
|
||||
assert!(!client.recovery_running.load(Ordering::Acquire));
|
||||
}
|
||||
|
||||
fn decode_test_scanner_activity(response: ScannerActivityResponse) -> Result<ScannerPeerActivity> {
|
||||
decode_scanner_activity_with_verifier(response, &[9; 16], |_canonical, proof| {
|
||||
(proof == b"proof")
|
||||
@@ -4347,6 +4625,112 @@ mod tests {
|
||||
assert!(!client.offline.load(Ordering::Acquire));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usage_watch_consumer_authenticates_the_current_request_and_rejects_replay() {
|
||||
use rustfs_protos::proto_gen::node_service::{
|
||||
ScannerUsageWatchEntry, ScannerUsageWatchRequest, ScannerUsageWatchResponse, ScannerUsageWatchResult,
|
||||
};
|
||||
use rustfs_protos::usage_watch::{canonical_usage_watch_request, canonical_usage_watch_response};
|
||||
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let request = ScannerUsageWatchRequest {
|
||||
challenge: Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
protocol_version: 1,
|
||||
owner_id: Uuid::new_v4().to_string(),
|
||||
entries: vec![ScannerUsageWatchEntry {
|
||||
bucket: "usage-watch".into(),
|
||||
bucket_incarnation: Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
watch_id: Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
session_id: Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
release: false,
|
||||
}],
|
||||
};
|
||||
let sign = |request: &ScannerUsageWatchRequest, response: &mut ScannerUsageWatchResponse| {
|
||||
let canonical = canonical_usage_watch_request(request).expect("valid request to sign");
|
||||
let body = canonical_usage_watch_response(&canonical, response).expect("bounded canonical watch response");
|
||||
response.response_proof = crate::cluster::rpc::sign_tonic_rpc_response_proof(&body)
|
||||
.expect("real response HMAC")
|
||||
.into();
|
||||
};
|
||||
let mut response = ScannerUsageWatchResponse {
|
||||
protocol_version: 1,
|
||||
owner_id: request.owner_id.clone(),
|
||||
session_id: request.entries[0].session_id.clone(),
|
||||
entries: vec![ScannerUsageWatchResult {
|
||||
bucket: request.entries[0].bucket.clone(),
|
||||
known: true,
|
||||
bucket_incarnation: request.entries[0].bucket_incarnation.clone(),
|
||||
watch_id: request.entries[0].watch_id.clone(),
|
||||
revision: 7,
|
||||
inflight: 0,
|
||||
coverage_version: 1,
|
||||
coverage_epoch: 3,
|
||||
lease_remaining_ms: 60_000,
|
||||
}],
|
||||
response_proof: Bytes::new(),
|
||||
};
|
||||
sign(&request, &mut response);
|
||||
validate_scanner_usage_watch_response(&request, &response).expect("current request accepts its signed response");
|
||||
|
||||
let mut rejected_replays = 0;
|
||||
for field in 0..8 {
|
||||
let mut current = request.clone();
|
||||
match field {
|
||||
0 => current.challenge = Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
1 => current.owner_id = Uuid::new_v4().to_string(),
|
||||
2 => current.entries[0].bucket.push('x'),
|
||||
3 => current.entries[0].bucket_incarnation = Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
4 => current.entries[0].watch_id = Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
5 => current.entries[0].session_id = Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
6 => current.entries[0].release = true,
|
||||
_ => current.protocol_version += 1,
|
||||
}
|
||||
assert!(
|
||||
validate_scanner_usage_watch_response(¤t, &response).is_err(),
|
||||
"a previously signed response must not authorize request mutation {field}"
|
||||
);
|
||||
rejected_replays += 1;
|
||||
}
|
||||
let mut tampered = response.clone();
|
||||
tampered.entries[0].revision += 1;
|
||||
assert!(validate_scanner_usage_watch_response(&request, &tampered).is_err());
|
||||
let mut unsigned = response.clone();
|
||||
unsigned.response_proof = Bytes::new();
|
||||
assert!(validate_scanner_usage_watch_response(&request, &unsigned).is_err());
|
||||
|
||||
let mut rejected_signed_mismatches = 0;
|
||||
for field in 0..5 {
|
||||
let mut mismatched = response.clone();
|
||||
match field {
|
||||
0 => mismatched.owner_id = Uuid::new_v4().to_string(),
|
||||
1 => mismatched.session_id = Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
2 => mismatched.entries[0].bucket_incarnation = Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
3 => mismatched.entries[0].watch_id = Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
_ => mismatched.entries[0].known = false,
|
||||
}
|
||||
sign(&request, &mut mismatched);
|
||||
assert!(
|
||||
validate_scanner_usage_watch_response(&request, &mismatched).is_err(),
|
||||
"an authentic HMAC cannot replace requested watch identity or authorize malformed Unknown: {field}"
|
||||
);
|
||||
rejected_signed_mismatches += 1;
|
||||
}
|
||||
println!(
|
||||
"usage_assertion: {}",
|
||||
serde_json::json!({
|
||||
"case_id": "USAGE-12", "assertion_id": "USAGE-12-A07", "status": "PASS",
|
||||
"evidence": {"current_response_authenticated": true, "replayed_requests_rejected": rejected_replays,
|
||||
"signed_identity_mismatches_rejected": rejected_signed_mismatches,
|
||||
"tampered_response_rejected": true, "missing_hmac_rejected": true}
|
||||
})
|
||||
);
|
||||
drop((request, response, tampered, unsigned));
|
||||
println!(
|
||||
"usage_cleanup: {}",
|
||||
serde_json::json!({"case_id": "USAGE-12", "status": "PASS", "scope": "process-local"})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn heal_control_capability_proof_must_authenticate_exact_ack() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -20,8 +20,8 @@ use async_trait::async_trait;
|
||||
use bytes::Bytes;
|
||||
use futures::{FutureExt, future::BoxFuture};
|
||||
use rustfs_lock::{
|
||||
BatchAcquireOutcome, LockCleanupOutcome, LockClient, LockError, LockInfo, LockRequest, LockResponse, LockStats, LockStatus,
|
||||
LockType, Result,
|
||||
BatchAcquireOutcome, CacheFenceStamp, LockCleanupOutcome, LockClient, LockError, LockInfo, LockRequest, LockResponse,
|
||||
LockStats, LockStatus, LockType, Result,
|
||||
types::{LockId, LockMetadata, LockPriority},
|
||||
};
|
||||
use rustfs_protos::proto_gen::node_service::{
|
||||
@@ -29,7 +29,7 @@ use rustfs_protos::proto_gen::node_service::{
|
||||
LockAcquireStatus, LockReleaseStatus, PingRequest,
|
||||
};
|
||||
use rustfs_protos::{
|
||||
ConnectionEvictionLogLevel, evict_failed_connection_with_log_level, models::PingBodyBuilder,
|
||||
ChannelRebuildReason, ConnectionEvictionLogLevel, evict_failed_connection_with_reason, models::PingBodyBuilder,
|
||||
proto_gen::node_service::node_service_client::NodeServiceClient,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
@@ -52,6 +52,10 @@ fn attach_lock_mutation_body_digest<T: rustfs_protos::CanonicalMutationBody>(req
|
||||
set_tonic_rolling_mutation_body_digest(request)
|
||||
}
|
||||
|
||||
fn decode_cache_fence_stamp(encoded: Option<&Bytes>) -> Option<CacheFenceStamp> {
|
||||
encoded.and_then(|encoded| CacheFenceStamp::decode(encoded).ok())
|
||||
}
|
||||
|
||||
/// Work to run if an RPC completes after its caller stops waiting.
|
||||
type LateCompletion<T> = Option<Box<dyn FnOnce(T) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send>>;
|
||||
type LateFailure = LateCompletion<()>;
|
||||
@@ -324,6 +328,7 @@ fn batch_acquire_outcome(lock_id: &LockId, lock_type: LockType, result: Option<&
|
||||
return BatchAcquireOutcome::Granted {
|
||||
lock_info,
|
||||
wait_time: Duration::ZERO,
|
||||
cache_fence_stamp: decode_cache_fence_stamp(result.cache_fence_stamp.as_ref()),
|
||||
};
|
||||
}
|
||||
if result.lock_info.is_some() {
|
||||
@@ -452,6 +457,7 @@ impl RemoteClient {
|
||||
deadlock_detection: false,
|
||||
suppress_contention_logs: false,
|
||||
refresh_interval: None,
|
||||
cache_fence_stamp_requested: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -498,7 +504,7 @@ impl RemoteClient {
|
||||
))
|
||||
}
|
||||
|
||||
async fn evict_connection(&self, op: &'static str, reason: &str, resource_summary: &str) {
|
||||
async fn evict_connection(&self, op: &'static str, reason: &str, resource_summary: &str, trigger: EvictionTrigger) {
|
||||
let log_level = if Self::is_scanner_leader_lock(resource_summary) {
|
||||
debug!(
|
||||
addr = %self.addr,
|
||||
@@ -518,7 +524,11 @@ impl RemoteClient {
|
||||
);
|
||||
ConnectionEvictionLogLevel::Warn
|
||||
};
|
||||
evict_failed_connection_with_log_level(&self.addr, log_level).await;
|
||||
let rebuild_reason = match trigger {
|
||||
EvictionTrigger::Timeout => ChannelRebuildReason::LockTimeout,
|
||||
EvictionTrigger::Transport => ChannelRebuildReason::LockTransport,
|
||||
};
|
||||
evict_failed_connection_with_reason(&self.addr, rebuild_reason, log_level).await;
|
||||
}
|
||||
|
||||
fn summarize_resources(requests: &[LockRequest]) -> String {
|
||||
@@ -772,7 +782,7 @@ impl RemoteClient {
|
||||
});
|
||||
if verdict == EvictionVerdict::Evict {
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_channel_eviction(&self.addr, trigger.as_str());
|
||||
self.evict_connection(op, reason, resource_summary).await;
|
||||
self.evict_connection(op, reason, resource_summary, trigger).await;
|
||||
return;
|
||||
}
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_channel_eviction_suppressed(&self.addr, verdict.as_str());
|
||||
@@ -1569,10 +1579,11 @@ impl LockClient for RemoteClient {
|
||||
|
||||
// Check if the lock acquisition was successful
|
||||
if resp.success {
|
||||
Ok(LockResponse::success(
|
||||
Self::build_lock_info(request, resp.lock_info),
|
||||
std::time::Duration::ZERO,
|
||||
))
|
||||
let lock_info = Self::build_lock_info(request, resp.lock_info);
|
||||
match decode_cache_fence_stamp(resp.cache_fence_stamp.as_ref()) {
|
||||
Some(stamp) => Ok(LockResponse::success_with_cache_fence_stamp(lock_info, std::time::Duration::ZERO, stamp)),
|
||||
None => Ok(LockResponse::success(lock_info, std::time::Duration::ZERO)),
|
||||
}
|
||||
} else {
|
||||
// Lock acquisition failed
|
||||
Ok(LockResponse::failure(
|
||||
@@ -1590,12 +1601,19 @@ impl LockClient for RemoteClient {
|
||||
.iter()
|
||||
.zip(outcomes)
|
||||
.map(|(request, outcome)| match outcome {
|
||||
BatchAcquireOutcome::Granted { lock_info, wait_time } => LockResponse::success(
|
||||
lock_info
|
||||
.map(|info| *info)
|
||||
.unwrap_or_else(|| Self::build_lock_info(request, None)),
|
||||
BatchAcquireOutcome::Granted {
|
||||
lock_info,
|
||||
wait_time,
|
||||
),
|
||||
cache_fence_stamp,
|
||||
} => {
|
||||
let lock_info = lock_info
|
||||
.map(|info| *info)
|
||||
.unwrap_or_else(|| Self::build_lock_info(request, None));
|
||||
match cache_fence_stamp {
|
||||
Some(stamp) => LockResponse::success_with_cache_fence_stamp(lock_info, wait_time, stamp),
|
||||
None => LockResponse::success(lock_info, wait_time),
|
||||
}
|
||||
}
|
||||
BatchAcquireOutcome::Ambiguous {
|
||||
error: error @ LockError::Timeout { .. },
|
||||
..
|
||||
@@ -2363,6 +2381,38 @@ mod tests {
|
||||
assert!(matches!(classify(None), BatchAcquireOutcome::Ambiguous { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cache_fence_stamp_decoder_accepts_current_wire_shape_and_rejects_malformed_values() {
|
||||
let mut encoded = Vec::with_capacity(27);
|
||||
encoded.push(1);
|
||||
encoded.extend_from_slice(uuid::Uuid::from_u128(7).as_bytes());
|
||||
encoded.extend_from_slice(&17_u16.to_le_bytes());
|
||||
encoded.extend_from_slice(&42_u64.to_le_bytes());
|
||||
let encoded = Bytes::from(encoded);
|
||||
|
||||
let stamp = decode_cache_fence_stamp(Some(&encoded)).expect("current stamp wire shape");
|
||||
assert_eq!(stamp.voter_boot_id(), uuid::Uuid::from_u128(7));
|
||||
assert_eq!(stamp.shard_index(), 17);
|
||||
assert_eq!(stamp.generation(), 42);
|
||||
assert!(decode_cache_fence_stamp(None).is_none(), "legacy responses remain valid without proof");
|
||||
assert!(decode_cache_fence_stamp(Some(&Bytes::from_static(b"malformed"))).is_none());
|
||||
|
||||
let request = test_lock_request(Duration::from_secs(30));
|
||||
let result = GenerallyLockResult {
|
||||
success: true,
|
||||
cache_fence_stamp: Some(encoded),
|
||||
..Default::default()
|
||||
};
|
||||
let BatchAcquireOutcome::Granted {
|
||||
cache_fence_stamp: Some(batch_stamp),
|
||||
..
|
||||
} = batch_acquire_outcome(&request.lock_id, request.lock_type, Some(&result))
|
||||
else {
|
||||
panic!("valid batch grant must retain its cache-fence stamp");
|
||||
};
|
||||
assert_eq!(batch_stamp, stamp);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn batch_success_crossing_deadline_in_one_poll_runs_late_cleanup_once() {
|
||||
use crate::cluster::rpc::client::NoOpInterceptor;
|
||||
|
||||
@@ -314,6 +314,16 @@ mod tests {
|
||||
assert_eq!(scanner_kvs.get(SCANNER_DELAY), "");
|
||||
assert_eq!(scanner_kvs.get(SCANNER_MAX_WAIT), "");
|
||||
assert_eq!(scanner_kvs.get(SCANNER_CYCLE_MAX_OBJECTS), "0");
|
||||
for key in [
|
||||
rustfs_config::SCANNER_BUCKET_QUANTUM_DURATION,
|
||||
rustfs_config::SCANNER_BUCKET_QUANTUM_OBJECTS,
|
||||
rustfs_config::SCANNER_BUCKET_QUANTUM_DIRECTORIES,
|
||||
rustfs_config::SCANNER_BUCKET_QUANTUM_ENTRIES,
|
||||
rustfs_config::SCANNER_BUCKET_QUANTUM_NAME_BYTES,
|
||||
] {
|
||||
assert!(rustfs_config::SCANNER_KEYS.contains(&key), "admin key {key} is registered");
|
||||
assert_eq!(scanner_kvs.get(key), "0", "quantum dimension {key} stays disabled until rollout");
|
||||
}
|
||||
|
||||
let heal_kvs = cfg
|
||||
.get_value(HEAL_SUB_SYS, DEFAULT_DELIMITER)
|
||||
|
||||
@@ -25,6 +25,12 @@ use rustfs_config::{
|
||||
SCANNER_MAX_CONCURRENT_SET_SCANS, SCANNER_MAX_WAIT, SCANNER_RECOVERY_BACKOFF_CAP, SCANNER_SPEED, SCANNER_START_DELAY,
|
||||
SCANNER_WALKERS_PER_DISK, SCANNER_YIELD_EVERY_N_OBJECTS,
|
||||
};
|
||||
use rustfs_config::{
|
||||
DEFAULT_SCANNER_BUCKET_QUANTUM_DIRECTORIES, DEFAULT_SCANNER_BUCKET_QUANTUM_DURATION_SECS,
|
||||
DEFAULT_SCANNER_BUCKET_QUANTUM_ENTRIES, DEFAULT_SCANNER_BUCKET_QUANTUM_NAME_BYTES, DEFAULT_SCANNER_BUCKET_QUANTUM_OBJECTS,
|
||||
SCANNER_BUCKET_QUANTUM_DIRECTORIES, SCANNER_BUCKET_QUANTUM_DURATION, SCANNER_BUCKET_QUANTUM_ENTRIES,
|
||||
SCANNER_BUCKET_QUANTUM_NAME_BYTES, SCANNER_BUCKET_QUANTUM_OBJECTS,
|
||||
};
|
||||
use std::sync::LazyLock;
|
||||
|
||||
pub static DEFAULT_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
@@ -69,6 +75,31 @@ pub static DEFAULT_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
value: DEFAULT_SCANNER_CYCLE_MAX_DIRECTORIES.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: SCANNER_BUCKET_QUANTUM_DURATION.to_owned(),
|
||||
value: DEFAULT_SCANNER_BUCKET_QUANTUM_DURATION_SECS.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: SCANNER_BUCKET_QUANTUM_OBJECTS.to_owned(),
|
||||
value: DEFAULT_SCANNER_BUCKET_QUANTUM_OBJECTS.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: SCANNER_BUCKET_QUANTUM_DIRECTORIES.to_owned(),
|
||||
value: DEFAULT_SCANNER_BUCKET_QUANTUM_DIRECTORIES.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: SCANNER_BUCKET_QUANTUM_ENTRIES.to_owned(),
|
||||
value: DEFAULT_SCANNER_BUCKET_QUANTUM_ENTRIES.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: SCANNER_BUCKET_QUANTUM_NAME_BYTES.to_owned(),
|
||||
value: DEFAULT_SCANNER_BUCKET_QUANTUM_NAME_BYTES.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: SCANNER_BITROT_CYCLE.to_owned(),
|
||||
value: DEFAULT_SCANNER_BITROT_CYCLE_SECS.to_string(),
|
||||
|
||||
@@ -101,7 +101,6 @@ const DEFAULT_RRS_STORAGE_CLASS: &str = "EC:1";
|
||||
const ZERO_SET_DRIVE_COUNT_ERROR: &str = "set drive count must be greater than zero";
|
||||
|
||||
pub static DEFAULT_INLINE_BLOCK: usize = 128 * 1024;
|
||||
const DEFAULT_INLINE_OBJECT_BUDGET: usize = 2 * DEFAULT_INLINE_BLOCK;
|
||||
|
||||
pub static DEFAULT_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
let kvs = vec![
|
||||
@@ -152,6 +151,7 @@ pub struct Config {
|
||||
inline_block: usize,
|
||||
initialized: bool,
|
||||
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
|
||||
// Preserve the explicit-limit marker when configurations round-trip through older readers.
|
||||
inline_block_explicit: bool,
|
||||
#[serde(skip)]
|
||||
standard_parities: Vec<PoolParity>,
|
||||
@@ -245,8 +245,10 @@ impl Config {
|
||||
return false;
|
||||
}
|
||||
|
||||
let shard_size = shard_size as usize;
|
||||
let inline_block = self.effective_inline_block(data_shards);
|
||||
let Ok(shard_size) = usize::try_from(shard_size) else {
|
||||
return false;
|
||||
};
|
||||
let inline_block = self.inline_block();
|
||||
|
||||
if versioned {
|
||||
shard_size <= inline_block / 8
|
||||
@@ -255,27 +257,7 @@ impl Config {
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the per-shard inline budget used by both write admission and
|
||||
/// legacy read fallback.
|
||||
///
|
||||
/// The default budget is scaled by the number of data shards so a wider EC
|
||||
/// layout does not silently increase the maximum inline object size. An
|
||||
/// explicitly configured `inline_block` remains a fixed per-shard limit for
|
||||
/// compatibility with deployments that opted into the historical policy.
|
||||
pub(crate) fn effective_inline_block(&self, data_shards: usize) -> usize {
|
||||
if data_shards == 0 {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if self.initialized && self.inline_block_explicit {
|
||||
self.inline_block
|
||||
} else {
|
||||
// Keep the historical two-data-shard object budget while preventing
|
||||
// wider EC layouts from multiplying the maximum inline object size.
|
||||
DEFAULT_INLINE_OBJECT_BUDGET.div_ceil(data_shards).min(DEFAULT_INLINE_BLOCK)
|
||||
}
|
||||
}
|
||||
|
||||
/// Per-shard inline data budget, defaulting to 128 KiB across EC layouts.
|
||||
pub fn inline_block(&self) -> usize {
|
||||
if !self.initialized {
|
||||
DEFAULT_INLINE_BLOCK
|
||||
@@ -567,7 +549,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn should_inline_scales_default_threshold_by_data_shards() {
|
||||
fn should_inline_uses_fixed_default_threshold_for_every_ec_layout() {
|
||||
let config = lookup_config_for_pools_with_env(&KVS::new(), &[3, 12], no_env_overrides())
|
||||
.expect("default inline policy should resolve for EC2+1 and EC8+4");
|
||||
|
||||
@@ -576,10 +558,14 @@ mod tests {
|
||||
("EC2+1 unversioned above", 128 * 1024 + 1, 2, false, false),
|
||||
("EC2+1 versioned exact", 16 * 1024, 2, true, true),
|
||||
("EC2+1 versioned above", 16 * 1024 + 1, 2, true, false),
|
||||
("EC8+4 unversioned exact", 32 * 1024, 8, false, true),
|
||||
("EC8+4 unversioned above", 32 * 1024 + 1, 8, false, false),
|
||||
("EC8+4 versioned exact", 4 * 1024, 8, true, true),
|
||||
("EC8+4 versioned above", 4 * 1024 + 1, 8, true, false),
|
||||
("EC8+4 unversioned exact", 128 * 1024, 8, false, true),
|
||||
("EC8+4 unversioned above", 128 * 1024 + 1, 8, false, false),
|
||||
("EC8+4 versioned exact", 16 * 1024, 8, true, true),
|
||||
("EC8+4 versioned above", 16 * 1024 + 1, 8, true, false),
|
||||
("EC12+4 unversioned exact", 128 * 1024, 12, false, true),
|
||||
("EC12+4 unversioned above", 128 * 1024 + 1, 12, false, false),
|
||||
("EC12+4 versioned exact", 16 * 1024, 12, true, true),
|
||||
("EC12+4 versioned above", 16 * 1024 + 1, 12, true, false),
|
||||
("negative", -1, 2, false, false),
|
||||
("zero data shards", 0, 0, false, false),
|
||||
] {
|
||||
@@ -626,37 +612,35 @@ mod tests {
|
||||
let shard_size = erasure.shard_file_size(object_size);
|
||||
let inline = config.should_inline(shard_size, data_shards, false);
|
||||
|
||||
// The effective policy is monotonic across object sizes. This
|
||||
// table covers the boundaries that previously exposed the
|
||||
// fixed-shard read-ahead mismatch, including the 1 MiB case.
|
||||
// The per-shard budget stays monotonic across object sizes.
|
||||
// PUT separately bounds inline body materialization.
|
||||
assert!(!inline || previous, "inline decision must not re-enable at {object_size} bytes");
|
||||
previous = inline;
|
||||
}
|
||||
|
||||
assert!(
|
||||
!config.should_inline(erasure.shard_file_size(1024 * 1024), data_shards, false),
|
||||
"1 MiB must use the non-inline path for EC{data_shards}+{parity_shards}"
|
||||
config.should_inline(erasure.shard_file_size(1024 * 1024), data_shards, false),
|
||||
"1 MiB fits the per-shard budget for EC{data_shards}+{parity_shards}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn effective_inline_block_scales_default_budget_and_preserves_explicit_limit() {
|
||||
fn inline_block_preserves_fixed_default_and_explicit_limits() {
|
||||
let config = Config::default();
|
||||
assert_eq!(config.effective_inline_block(8), 32 * 1024);
|
||||
assert_eq!(config.effective_inline_block(12), 21_846);
|
||||
assert_eq!(config.effective_inline_block(0), 0);
|
||||
assert_eq!(config.inline_block(), 128 * 1024);
|
||||
assert!(!config.should_inline(0, 0, false));
|
||||
|
||||
let explicit = lookup_config_for_pools_with_env(
|
||||
&KVS::new(),
|
||||
&[12],
|
||||
StorageClassEnvOverrides {
|
||||
inline_block: Some("128KiB".to_string()),
|
||||
inline_block: Some("32KiB".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.expect("explicit inline block should resolve");
|
||||
assert_eq!(explicit.effective_inline_block(12), 128 * 1024);
|
||||
assert_eq!(explicit.inline_block(), 32 * 1024);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -905,6 +889,21 @@ mod tests {
|
||||
assert!(decoded.should_inline(100 * 1024, 8, false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_scalar_inline_config_preserves_limit_without_explicit_marker() {
|
||||
let cfg = lookup_config_for_pools_with_env(&KVS::new(), &[12], no_env_overrides())
|
||||
.expect("default storage class should resolve");
|
||||
let mut legacy = serde_json::to_value(cfg).expect("serialize scalar config");
|
||||
legacy["inline_block"] = serde_json::json!(64 * 1024);
|
||||
legacy.as_object_mut().expect("config object").remove("inline_block_explicit");
|
||||
let decoded: Config = serde_json::from_value(legacy).expect("read historical scalar config");
|
||||
|
||||
assert!(decoded.should_inline(64 * 1024, 8, false));
|
||||
assert!(!decoded.should_inline(64 * 1024 + 1, 8, false));
|
||||
assert!(decoded.should_inline(8 * 1024, 8, true));
|
||||
assert!(!decoded.should_inline(8 * 1024 + 1, 8, true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lookup_config_reads_rrs_from_class_rrs_key() {
|
||||
// Regression: kvs.get(RRS) used RRS="REDUCED_REDUNDANCY" instead of
|
||||
|
||||
@@ -6400,7 +6400,7 @@ pub(crate) struct PoolMetaWriteState {
|
||||
pool_meta_absent: bool,
|
||||
bootstrap_authority: PoolMetaBootstrapAuthority,
|
||||
/// First-hand bootstrap authority this process holds for each pool
|
||||
/// (index = pool index): `Fresh` only for pools it formatted itself,
|
||||
/// (index = pool index): `Fresh` for pools it formatted or resumed from a complete durable plan,
|
||||
/// `LegacyAdoption` only for pools whose migration it verified. Empty when
|
||||
/// the caller tracks deployment-wide authority only.
|
||||
pool_bootstrap_authorities: Vec<PoolMetaBootstrapAuthority>,
|
||||
@@ -6478,8 +6478,7 @@ impl PoolMetaWriteState {
|
||||
}
|
||||
|
||||
/// Startup state for a process that loaded every pool format itself and
|
||||
/// remembers, per pool, whether it created (or adopted) that pool
|
||||
/// first-hand. Deployment-wide authority is the conjunction across pools:
|
||||
/// remembers, per pool, whether it created, durably resumed, or adopted that pool. Deployment-wide authority is the conjunction across pools:
|
||||
/// any pool this process merely read yields `None`, exactly as before.
|
||||
pub(crate) fn for_startup_with_pool_bootstrap_authorities(
|
||||
cluster_id: uuid::Uuid,
|
||||
@@ -6510,7 +6509,7 @@ impl PoolMetaWriteState {
|
||||
.is_some_and(|authority| authority.is_proven())
|
||||
}
|
||||
|
||||
/// Pools this process formatted or adopted first-hand during this startup.
|
||||
/// Pools this process formatted, durably resumed, or adopted during startup.
|
||||
pub(crate) fn attested_pool_indices(&self) -> Vec<usize> {
|
||||
self.pool_bootstrap_authorities
|
||||
.iter()
|
||||
@@ -6527,7 +6526,7 @@ impl PoolMetaWriteState {
|
||||
/// [`PoolMetaIdentityWriteScope::Pools`]), so a complete, agreeing pending
|
||||
/// set proves that every pool joined this bootstrap fresh. A missing,
|
||||
/// corrupt, or disagreeing replica keeps the writer fail-closed, and a
|
||||
/// restart without first-hand proof never reopens bootstrap on its own.
|
||||
/// restart requires a complete durable format plan to recover creator proof.
|
||||
fn pending_identity_attested_by_every_pool(&self) -> bool {
|
||||
self.elected_bootstrap_writer == Some(true)
|
||||
&& self.pool_bootstrap_authority_proven(0)
|
||||
@@ -6759,6 +6758,27 @@ impl PoolMetaWriteState {
|
||||
self.expected_cluster_id.is_some() && (self.identity_needs_repair || self.identity_initialized != Some(true))
|
||||
}
|
||||
|
||||
/// A committed predecessor authorizes repairing an established identity
|
||||
/// before the next metadata write. A pending fresh identity must remain
|
||||
/// pending until its initial metadata commit is recoverable.
|
||||
pub(crate) fn startup_identity_precedes_metadata_write(&self) -> bool {
|
||||
!self.pool_meta_absent && !self.identity_is_pending()
|
||||
}
|
||||
|
||||
pub(crate) fn ensure_startup_identity_initialized(&self) -> Result<()> {
|
||||
if self.expected_cluster_id.is_none()
|
||||
|| (self.identity_initialized == Some(true)
|
||||
&& !self.identity_needs_repair
|
||||
&& !self.identity_replicas_missing
|
||||
&& !self.identity_replicas_invalid)
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
Err(Error::other(
|
||||
"pool metadata bootstrap pending: waiting for the elected writer to initialize the identity on every pool",
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) fn ensure_write_safe(&self, operation: &str) -> Result<()> {
|
||||
let (Some(mut context), _) = self.write_block() else {
|
||||
return Ok(());
|
||||
@@ -10146,6 +10166,8 @@ impl PoolMeta {
|
||||
self.save_for_startup_observing(pools, &mut write_state).await.map(|_| ())
|
||||
}
|
||||
|
||||
/// Commits a complete startup snapshot while leaving identity finalization
|
||||
/// to the caller, after durable format-bootstrap authority is retired.
|
||||
pub(crate) async fn save_for_startup_observing<S>(
|
||||
&self,
|
||||
pools: Vec<Arc<S>>,
|
||||
@@ -10160,6 +10182,8 @@ impl PoolMeta {
|
||||
Ok(outcome.into_committed())
|
||||
}
|
||||
|
||||
/// Commits a complete startup snapshot under the writer fence. The caller
|
||||
/// retires format-bootstrap authority before finalizing initialized identity.
|
||||
pub(crate) async fn save_for_startup_guarded<S>(
|
||||
&self,
|
||||
pools: Vec<Arc<S>>,
|
||||
@@ -10362,7 +10386,10 @@ impl PoolMeta {
|
||||
selection.revision.epoch, identity.epoch
|
||||
)));
|
||||
}
|
||||
if !selection.absent && write_state.identity_requires_repair() {
|
||||
// Full-snapshot saves belong to startup. Their caller retires
|
||||
// durable format authority before publishing initialized identity.
|
||||
// Runtime scoped saves retain their existing identity repair.
|
||||
if indices.is_some() && !selection.absent && write_state.identity_requires_repair() {
|
||||
let initialized = write_state.identity_initialized != Some(false) || selection.revision.is_generation_protocol();
|
||||
persist_pool_meta_identity(
|
||||
pools.clone(),
|
||||
@@ -10640,7 +10667,10 @@ impl PoolMeta {
|
||||
confirmed
|
||||
};
|
||||
if confirmed.revision == revision && confirmed.canonical.as_ref() == Some(&durable) {
|
||||
persist_pool_meta_identity(pools, write_state, true, PoolMetaIdentityWriteScope::All, fence, transaction_arm).await?;
|
||||
if indices.is_some() {
|
||||
persist_pool_meta_identity(pools, write_state, true, PoolMetaIdentityWriteScope::All, fence, transaction_arm)
|
||||
.await?;
|
||||
}
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
startup_cas_test_observe(serde_json::json!({
|
||||
"kind": "confirmed", "object": POOL_META_NAME,
|
||||
|
||||
@@ -477,18 +477,18 @@ impl Sets {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn local_storage_info_snapshot(&self) -> rustfs_madmin::StorageInfo {
|
||||
pub(crate) async fn local_storage_info_snapshot(&self, include_metrics: bool) -> rustfs_madmin::StorageInfo {
|
||||
let mut futures = Vec::with_capacity(self.disk_set.len());
|
||||
|
||||
for set in self.disk_set.iter() {
|
||||
futures.push(set.local_storage_info_snapshot())
|
||||
futures.push(set.local_storage_info_snapshot(include_metrics))
|
||||
}
|
||||
|
||||
let results = join_all(futures).await;
|
||||
let mut disks = Vec::new();
|
||||
|
||||
for res in results.into_iter() {
|
||||
disks.extend_from_slice(&res.disks);
|
||||
disks.extend(res.disks);
|
||||
}
|
||||
|
||||
rustfs_madmin::StorageInfo {
|
||||
|
||||
@@ -56,6 +56,8 @@ pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR;
|
||||
const DATA_COMPRESSION_TOTAL_NAME: &str = ".compression.json";
|
||||
pub const DATA_USAGE_CACHE_NAME: &str = ".usage-cache.bin";
|
||||
const DATA_USAGE_CACHE_TTL_SECS: u64 = 30;
|
||||
const ADMIN_USAGE_INCARNATION_CONCURRENCY: usize = 4;
|
||||
const ADMIN_USAGE_INCARNATION_TIMEOUT: Duration = Duration::from_secs(2);
|
||||
// A scanner save invalidates only the saving process's snapshot cache; every
|
||||
// other node sees it only after reloading. A fail-closed usage lookup for a
|
||||
// bucket the cached snapshot does not mention at all may reload a copy this
|
||||
@@ -505,6 +507,7 @@ fn stale_data_usage_persist_reason_for_source(
|
||||
/// Store data usage info to backend storage
|
||||
#[instrument(skip(store))]
|
||||
pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc<ECStore>) -> Result<(), Error> {
|
||||
reject_admin_usage_projection(&data_usage_info)?;
|
||||
if data_usage_info.usage_snapshot_converged == Some(false) {
|
||||
return Err(Error::other(
|
||||
"nonconverged data usage observations cannot replace the quota-authoritative snapshot",
|
||||
@@ -534,6 +537,7 @@ async fn save_data_usage_in_backend(
|
||||
store: Arc<ECStore>,
|
||||
expected_publication_epoch: u64,
|
||||
) -> Result<(), Error> {
|
||||
reject_admin_usage_projection(&data_usage_info)?;
|
||||
let data =
|
||||
serde_json::to_vec(&data_usage_info).map_err(|e| Error::other(format!("Failed to serialize data usage info: {e}")))?;
|
||||
|
||||
@@ -663,8 +667,9 @@ fn remove_bucket_usage_from_info(data_usage_info: &mut DataUsageInfo, bucket: &s
|
||||
|
||||
let removed_usage = data_usage_info.buckets_usage.remove(bucket).is_some();
|
||||
let removed_size = data_usage_info.bucket_sizes.remove(bucket).is_some();
|
||||
let removed_observation = data_usage_info.usage_snapshot_bucket_observations.remove(bucket).is_some();
|
||||
|
||||
if !removed_usage && !removed_size {
|
||||
if !removed_usage && !removed_size && !removed_observation {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -816,7 +821,9 @@ fn map_data_usage_metadata_read_error(err: Error, object: &str) -> Error {
|
||||
}
|
||||
|
||||
fn data_usage_contains_bucket(data_usage_info: &DataUsageInfo, bucket: &str) -> bool {
|
||||
data_usage_info.buckets_usage.contains_key(bucket) || data_usage_info.bucket_sizes.contains_key(bucket)
|
||||
data_usage_info.buckets_usage.contains_key(bucket)
|
||||
|| data_usage_info.bucket_sizes.contains_key(bucket)
|
||||
|| data_usage_info.usage_snapshot_bucket_observations.contains_key(bucket)
|
||||
}
|
||||
|
||||
fn advance_data_usage_last_update(data_usage_info: &mut DataUsageInfo) {
|
||||
@@ -1137,14 +1144,26 @@ fn record_usage_snapshot_decode_failure(operation: &'static str, object: &str, e
|
||||
}
|
||||
|
||||
fn parse_usage_snapshot(buf: &[u8]) -> Result<DataUsageInfo, Error> {
|
||||
serde_json::from_slice(buf).map_err(|e| {
|
||||
let mut info: DataUsageInfo = serde_json::from_slice(buf).map_err(|e| {
|
||||
Error::other(format!(
|
||||
"Failed to deserialize data usage info: {:?} at line {} column {}",
|
||||
e.classify(),
|
||||
e.line(),
|
||||
e.column()
|
||||
))
|
||||
})
|
||||
})?;
|
||||
reject_admin_usage_projection(&info)?;
|
||||
for usage in info.buckets_usage.values_mut() {
|
||||
usage.normalize_usage_provenance();
|
||||
}
|
||||
Ok(info)
|
||||
}
|
||||
|
||||
fn reject_admin_usage_projection(info: &DataUsageInfo) -> Result<(), Error> {
|
||||
if info.usage_snapshot_admin_projection {
|
||||
return Err(Error::other("admin usage projections cannot be persisted or used as a storage baseline"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Decide which usage snapshot to serve: a primary usage object or its
|
||||
@@ -1327,6 +1346,9 @@ async fn load_data_usage_from_backend_with_baseline(store: Arc<ECStore>) -> Resu
|
||||
/// Fill quota-baseline gaps from an observed (nonconverged) snapshot without
|
||||
/// overriding any bucket the authoritative baseline already covers.
|
||||
fn backfill_degraded_baseline_from_observed(degraded_baseline: &mut HashMap<String, u64>, observed: &DataUsageInfo) {
|
||||
if observed.usage_snapshot_admin_projection {
|
||||
return;
|
||||
}
|
||||
for (bucket, usage) in &observed.buckets_usage {
|
||||
degraded_baseline.entry(bucket.clone()).or_insert(usage.size);
|
||||
}
|
||||
@@ -1419,6 +1441,7 @@ fn merge_partial_observation_for_admin(mut authoritative: DataUsageInfo, observe
|
||||
authoritative.usage_snapshot_converged = Some(false);
|
||||
authoritative.usage_snapshot_authoritative_baseline = observed.usage_snapshot_authoritative_baseline;
|
||||
authoritative.usage_snapshot_set_states = observed.usage_snapshot_set_states;
|
||||
authoritative.usage_snapshot_bucket_observations = observed.usage_snapshot_bucket_observations;
|
||||
authoritative.usage_snapshot_bootstrap_pending = false;
|
||||
authoritative.buckets_count = authoritative.buckets_usage.len() as u64;
|
||||
authoritative.bucket_sizes = authoritative
|
||||
@@ -1437,10 +1460,48 @@ async fn load_admin_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataU
|
||||
let (authoritative, source) = load_data_usage_snapshot(store.clone()).await?;
|
||||
// For the one-shot admin view a failed observed read degrades to "no
|
||||
// observation", same as before the read was fallible.
|
||||
let observed = load_observed_data_usage_snapshot(store).await.ok().flatten();
|
||||
let observed = load_observed_data_usage_snapshot(store.clone()).await.ok().flatten();
|
||||
let (selected, selected_is_current_format) =
|
||||
select_admin_data_usage_snapshot(authoritative, source.is_authoritative(), observed);
|
||||
Ok(normalize_loaded_data_usage(selected, selected_is_current_format).await.0)
|
||||
let mut response = normalize_loaded_data_usage(selected, selected_is_current_format).await.0;
|
||||
use futures::StreamExt as _;
|
||||
let deadline = tokio::time::Instant::now() + ADMIN_USAGE_INCARNATION_TIMEOUT;
|
||||
let mut incarnations = HashMap::new();
|
||||
{
|
||||
// Display-only identity reads must not serialize every bucket behind a
|
||||
// slow metadata lock while holding the shared admin refresh permit.
|
||||
// The common deadline bounds their total added latency; dropping this
|
||||
// stream cancels all unfinished reads without spawning detached work.
|
||||
// Own the names and store across awaits so this future remains Send
|
||||
// when composed into spawned metrics collectors.
|
||||
let buckets = response
|
||||
.usage_snapshot_bucket_observations
|
||||
.keys()
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
let mut identities = futures::stream::iter(buckets)
|
||||
.map(move |bucket| {
|
||||
let store = store.clone();
|
||||
async move {
|
||||
let identity = store.bucket_incarnation_id_from_disk(&bucket).await;
|
||||
(bucket, identity)
|
||||
}
|
||||
})
|
||||
.buffer_unordered(ADMIN_USAGE_INCARNATION_CONCURRENCY);
|
||||
while tokio::time::Instant::now() < deadline {
|
||||
let Ok(Some((bucket, identity))) = tokio::time::timeout_at(deadline, identities.next()).await else {
|
||||
break;
|
||||
};
|
||||
// Unavailable, timed-out or recreated buckets retain canonical values.
|
||||
if let Ok(incarnation) = identity
|
||||
&& !incarnation.is_nil()
|
||||
{
|
||||
incarnations.insert(bucket, incarnation.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
response.project_bucket_observations_for_admin(&incarnations);
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
fn discard_incomplete_bucket_usage(data_usage_info: &mut DataUsageInfo) {
|
||||
@@ -2082,7 +2143,9 @@ where
|
||||
fn apply_live_bucket_usage_to_response(data_usage_info: &mut DataUsageInfo, bucket: &str, usage: &BucketUsageInfo) {
|
||||
let inserted_bucket = !data_usage_info.buckets_usage.contains_key(bucket);
|
||||
data_usage_info.bucket_sizes.insert(bucket.to_string(), usage.size);
|
||||
data_usage_info.buckets_usage.insert(bucket.to_string(), usage.clone());
|
||||
let mut observed = usage.clone();
|
||||
observed.mark_observed();
|
||||
data_usage_info.buckets_usage.insert(bucket.to_string(), observed);
|
||||
if inserted_bucket {
|
||||
data_usage_info.usage_snapshot_complete = false;
|
||||
}
|
||||
@@ -2774,7 +2837,9 @@ async fn apply_bucket_usage_memory_overlay_if_authoritative(data_usage_info: &mu
|
||||
}
|
||||
|
||||
added_bucket |= !data_usage_info.buckets_usage.contains_key(bucket);
|
||||
data_usage_info.buckets_usage.insert(bucket.clone(), cached.usage.clone());
|
||||
let mut observed = cached.usage.clone();
|
||||
observed.mark_observed();
|
||||
data_usage_info.buckets_usage.insert(bucket.clone(), observed);
|
||||
data_usage_info.bucket_sizes.insert(bucket.clone(), cached.usage.size);
|
||||
changed = true;
|
||||
}
|
||||
@@ -3024,6 +3089,7 @@ pub async fn init_compression_total_memory_from_backend(store: Arc<ECStore>) {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
mod companion_projection;
|
||||
mod snapshot_projection;
|
||||
|
||||
use super::*;
|
||||
|
||||
@@ -0,0 +1,310 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
use rustfs_data_usage::{
|
||||
BucketUsageCertification, BucketUsageValueState, DATA_USAGE_BUCKET_OBSERVATION_VERSION, DataUsageBucketObservation,
|
||||
DataUsageBucketObservationSource, DataUsageSnapshotSetState,
|
||||
};
|
||||
|
||||
mod native_compatibility;
|
||||
|
||||
fn companion_snapshot(bucket: &str, incarnation: uuid::Uuid) -> DataUsageInfo {
|
||||
let completed_at = SystemTime::UNIX_EPOCH + Duration::from_secs(40);
|
||||
let mut snapshot = data_usage_info_for_test(bucket, 10, 100, completed_at + Duration::from_secs(10));
|
||||
snapshot.scanner_epoch = Some(2);
|
||||
snapshot.scanner_cycle = Some(9);
|
||||
snapshot.usage_snapshot_converged = Some(false);
|
||||
snapshot.usage_snapshot_pending_buckets = vec![bucket.to_string()];
|
||||
snapshot.usage_snapshot_pending_bucket_count = 1;
|
||||
let canonical = snapshot.buckets_usage.get_mut(bucket).expect("canonical bucket");
|
||||
canonical.bucket_incarnation = Some(incarnation.to_string());
|
||||
canonical.certify(BucketUsageCertification {
|
||||
epoch: 2,
|
||||
cycle: 3,
|
||||
certified_at: SystemTime::UNIX_EPOCH + Duration::from_secs(30),
|
||||
});
|
||||
canonical.mark_carried();
|
||||
snapshot.usage_snapshot_set_states = vec![DataUsageSnapshotSetState {
|
||||
pool_index: 0,
|
||||
set_index: 0,
|
||||
scanner_epoch: Some(2),
|
||||
scanner_cycle: Some(9),
|
||||
scan_plan_digest: Some([1; 32]),
|
||||
complete: true,
|
||||
..Default::default()
|
||||
}];
|
||||
let mut observed = BucketUsageInfo {
|
||||
size: 250,
|
||||
objects_count: 25,
|
||||
versions_count: 25,
|
||||
bucket_incarnation: Some(incarnation.to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
observed.mark_observed();
|
||||
snapshot.usage_snapshot_bucket_observations.insert(
|
||||
bucket.to_string(),
|
||||
DataUsageBucketObservation {
|
||||
version: DATA_USAGE_BUCKET_OBSERVATION_VERSION,
|
||||
usage: observed,
|
||||
snapshot_identity: snapshot.snapshot_identity(),
|
||||
completed_at,
|
||||
sources: vec![DataUsageBucketObservationSource {
|
||||
pool_index: 0,
|
||||
set_index: 0,
|
||||
bucket_incarnation: incarnation.to_string(),
|
||||
scanner_epoch: 2,
|
||||
scanner_cycle: 4,
|
||||
scan_plan_digest: [1; 32],
|
||||
started_plan: [2; 32],
|
||||
requested_plan: [3; 32],
|
||||
completed_at,
|
||||
}],
|
||||
},
|
||||
);
|
||||
snapshot
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn companion_cannot_backfill_quota_and_projection_is_rejected_at_storage_boundaries() {
|
||||
let incarnation = uuid::Uuid::from_u128(31);
|
||||
let mut snapshot = companion_snapshot("bucket", incarnation);
|
||||
let mut canonical_quota = HashMap::new();
|
||||
backfill_degraded_baseline_from_observed(&mut canonical_quota, &snapshot);
|
||||
assert_eq!(canonical_quota.get("bucket"), Some(&100), "quota reads only the unchanged main map");
|
||||
assert_eq!(
|
||||
snapshot.project_bucket_observations_for_admin(&HashMap::from([("bucket".to_string(), incarnation.to_string())])),
|
||||
1
|
||||
);
|
||||
let mut accidental_quota = HashMap::new();
|
||||
backfill_degraded_baseline_from_observed(&mut accidental_quota, &snapshot);
|
||||
assert!(accidental_quota.is_empty());
|
||||
assert!(reject_admin_usage_projection(&snapshot).is_err());
|
||||
assert!(certified_bucket_usage(&snapshot, "bucket", incarnation).is_none());
|
||||
let wire = serde_json::to_vec(&snapshot).expect("admin response remains serializable");
|
||||
assert!(parse_usage_snapshot(&wire).is_err(), "a serialized response is not a persisted snapshot");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn companion_only_observed_snapshot_does_not_create_a_legacy_quota_baseline() {
|
||||
let incarnation = uuid::Uuid::from_u128(31);
|
||||
let mut snapshot = companion_snapshot("bucket", incarnation);
|
||||
snapshot.buckets_usage.clear();
|
||||
snapshot.bucket_sizes.clear();
|
||||
snapshot.buckets_count = 0;
|
||||
snapshot.calculate_totals();
|
||||
snapshot.usage_snapshot_complete = false;
|
||||
snapshot.usage_snapshot_partial = true;
|
||||
snapshot.usage_snapshot_observational = true;
|
||||
snapshot.usage_snapshot_set_states[0].complete = false;
|
||||
let decoded = parse_usage_snapshot(&serde_json::to_vec(&snapshot).expect("encode companion-only snapshot"))
|
||||
.expect("read valid partial snapshot");
|
||||
assert!(decoded.is_valid_partial_snapshot());
|
||||
let mut baseline = HashMap::new();
|
||||
backfill_degraded_baseline_from_observed(&mut baseline, &decoded);
|
||||
assert!(baseline.is_empty(), "unknown usage must not become either zero or mixed quota usage");
|
||||
println!(
|
||||
"usage_assertion: {}",
|
||||
serde_json::json!({
|
||||
"case_id": "USAGE-08", "assertion_id": "USAGE-08-A02", "status": "PASS",
|
||||
"evidence": {"primary_buckets": decoded.buckets_usage.len(), "legacy_bucket_sizes": decoded.bucket_sizes.len(), "quota_baseline_buckets": baseline.len(), "companion_buckets": decoded.usage_snapshot_bucket_observations.len(), "partial": decoded.is_valid_partial_snapshot()}
|
||||
})
|
||||
);
|
||||
println!(
|
||||
"usage_cleanup: {}",
|
||||
serde_json::json!({"case_id": "USAGE-08", "status": "PASS", "scope": "process-local"})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bucket_removal_discards_companions_even_without_a_canonical_entry() {
|
||||
let mut snapshot = companion_snapshot("bucket", uuid::Uuid::from_u128(31));
|
||||
snapshot.buckets_usage.clear();
|
||||
snapshot.bucket_sizes.clear();
|
||||
assert!(data_usage_contains_bucket(&snapshot, "bucket"));
|
||||
assert!(remove_bucket_usage_from_info(&mut snapshot, "bucket"));
|
||||
assert!(!data_usage_contains_bucket(&snapshot, "bucket"));
|
||||
assert!(snapshot.usage_snapshot_bucket_observations.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn partial_selection_replaces_value_provenance_together() {
|
||||
let incarnation = uuid::Uuid::from_u128(31);
|
||||
let authoritative = companion_snapshot("bucket", incarnation);
|
||||
let mut observed = authoritative.clone();
|
||||
observed.usage_snapshot_authoritative_baseline = Some(authoritative.snapshot_identity());
|
||||
observed.scanner_cycle = Some(10);
|
||||
observed.last_update = Some(SystemTime::UNIX_EPOCH + Duration::from_secs(60));
|
||||
observed.usage_snapshot_complete = false;
|
||||
observed.usage_snapshot_partial = true;
|
||||
observed.usage_snapshot_observational = true;
|
||||
observed.usage_snapshot_bucket_observations.clear();
|
||||
let usage = observed.buckets_usage.get_mut("bucket").expect("observed main bucket");
|
||||
usage.size = 175;
|
||||
usage.mark_observed();
|
||||
let (selected, _) = select_admin_data_usage_snapshot(authoritative, true, Some(observed));
|
||||
assert_eq!(selected.buckets_usage["bucket"].size, 175);
|
||||
assert_eq!(selected.buckets_usage["bucket"].usage_value_state, Some(BucketUsageValueState::Observed));
|
||||
assert_eq!(selected.buckets_usage["bucket"].usage_certification, None);
|
||||
assert!(
|
||||
selected.usage_snapshot_bucket_observations.is_empty(),
|
||||
"old root companions cannot cross a baseline change"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn companion_admin_cache_isolated_from_quota_memory_and_persistence() {
|
||||
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
|
||||
use crate::config::com::save_config;
|
||||
use crate::storage_api_contracts::bucket::MakeBucketOptions;
|
||||
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = "usage-companion";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket for incarnation validation");
|
||||
let incarnation = store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("trusted incarnation");
|
||||
let canonical = companion_snapshot(bucket, incarnation);
|
||||
let bytes = serde_json::to_vec(&canonical).expect("encode canonical root with companion");
|
||||
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), bytes.clone())
|
||||
.await
|
||||
.expect("save canonical root");
|
||||
clear_usage_memory_cache_for_test().await;
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
let mut samples = Vec::new();
|
||||
let mut rejected_writes = 0;
|
||||
for _ in 0..2 {
|
||||
let response = load_admin_data_usage_from_backend_cached(store.clone())
|
||||
.await
|
||||
.expect("read admin cache");
|
||||
assert_eq!(response.buckets_usage[bucket].size, 250);
|
||||
assert!(response.usage_snapshot_admin_projection);
|
||||
replace_bucket_usage_memory_from_info(&response).await;
|
||||
assert!(memory_cache().read().await.is_empty(), "response cannot seed quota memory");
|
||||
let admin_bytes = response.buckets_usage[bucket].size;
|
||||
let rejected = store_data_usage_in_backend(response, store.clone()).await.is_err();
|
||||
assert!(rejected);
|
||||
rejected_writes += usize::from(rejected);
|
||||
let quota = load_data_usage_from_backend_cached(store.clone())
|
||||
.await
|
||||
.expect("read independent quota cache");
|
||||
assert_eq!(quota.buckets_usage[bucket].size, 100);
|
||||
assert!(!quota.usage_snapshot_admin_projection);
|
||||
samples.push((admin_bytes, quota.buckets_usage[bucket].size));
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
}
|
||||
let persisted = read_config(store, DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("read unchanged canonical root");
|
||||
assert_eq!(persisted, bytes);
|
||||
println!(
|
||||
"usage_assertion: {}",
|
||||
serde_json::json!({
|
||||
"case_id": "USAGE-18", "assertion_id": "USAGE-18-A03", "status": "PASS",
|
||||
"evidence": {"admin_and_quota_byte_samples": samples, "rejected_projected_writes": rejected_writes, "quota_memory_entries": memory_cache().read().await.len(), "persisted_canonical_unchanged": persisted == bytes}
|
||||
})
|
||||
);
|
||||
let paths = _dirs.iter().map(|dir| dir.path().to_path_buf()).collect::<Vec<_>>();
|
||||
drop(_dirs);
|
||||
assert!(paths.iter().all(|path| !path.exists()));
|
||||
println!(
|
||||
"usage_cleanup: {}",
|
||||
serde_json::json!({"case_id": "USAGE-18", "status": "PASS", "fixtures_removed": paths.iter().all(|path| !path.exists())})
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn blocked_companion_identity_keeps_canonical_without_delaying_healthy_buckets() {
|
||||
use crate::bucket::metadata_sys::{
|
||||
acquire_metadata_transaction_guard, init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks,
|
||||
};
|
||||
use crate::config::com::save_config;
|
||||
use crate::storage_api_contracts::bucket::MakeBucketOptions;
|
||||
|
||||
// Admin reads also feed spawned metrics collectors across crate boundaries.
|
||||
fn assert_send<T: Send>(future: T) -> T {
|
||||
future
|
||||
}
|
||||
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let blocked = "usage-blocked-identity";
|
||||
let healthy = "usage-healthy-identity";
|
||||
let mut snapshots = Vec::new();
|
||||
for bucket in [blocked, healthy] {
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create companion identity fixture");
|
||||
let incarnation = store.bucket_incarnation_id_from_disk(bucket).await.expect("real incarnation");
|
||||
snapshots.push(companion_snapshot(bucket, incarnation));
|
||||
}
|
||||
let mut canonical = snapshots.remove(0);
|
||||
let other = snapshots.remove(0);
|
||||
canonical.buckets_usage.extend(other.buckets_usage);
|
||||
canonical.bucket_sizes.extend(other.bucket_sizes);
|
||||
canonical
|
||||
.usage_snapshot_bucket_observations
|
||||
.extend(other.usage_snapshot_bucket_observations);
|
||||
canonical.usage_snapshot_pending_buckets.push(healthy.to_string());
|
||||
canonical.usage_snapshot_pending_bucket_count = 2;
|
||||
canonical.buckets_count = 2;
|
||||
canonical.calculate_totals();
|
||||
save_config(
|
||||
store.clone(),
|
||||
DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
serde_json::to_vec(&canonical).expect("encode both canonical values and companions"),
|
||||
)
|
||||
.await
|
||||
.expect("persist usage fixture");
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
let blocked_guard = acquire_metadata_transaction_guard(
|
||||
store.clone(),
|
||||
blocked,
|
||||
"admin_identity_timeout_test",
|
||||
rustfs_lock::LockType::Exclusive,
|
||||
)
|
||||
.await
|
||||
.expect("hold the real bucket metadata writer lock");
|
||||
let response = tokio::time::timeout(
|
||||
ADMIN_USAGE_INCARNATION_TIMEOUT + Duration::from_secs(3),
|
||||
assert_send(load_admin_data_usage_from_backend_cached(store.clone())),
|
||||
)
|
||||
.await
|
||||
.expect("optional projection must return before the blocked metadata lock's long timeout")
|
||||
.expect("canonical admin response stays available");
|
||||
assert_eq!(response.buckets_usage[blocked].size, 100);
|
||||
assert_eq!(response.buckets_usage[blocked].usage_value_state, Some(BucketUsageValueState::Carried));
|
||||
assert_eq!(response.buckets_usage[healthy].size, 250);
|
||||
assert_eq!(response.buckets_usage[healthy].usage_value_state, Some(BucketUsageValueState::Observed));
|
||||
assert!(response.usage_snapshot_admin_projection);
|
||||
assert!(
|
||||
ADMIN_DATA_USAGE_SNAPSHOT_REFRESH
|
||||
.get()
|
||||
.expect("admin refresh mutex")
|
||||
.try_lock()
|
||||
.is_ok()
|
||||
);
|
||||
drop(blocked_guard);
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
let recovered = load_admin_data_usage_from_backend_cached(store.clone())
|
||||
.await
|
||||
.expect("refresh after lock recovery");
|
||||
assert_eq!(
|
||||
recovered.buckets_usage[blocked].size, 250,
|
||||
"unavailable identity can recover on the next refresh"
|
||||
);
|
||||
assert_eq!(recovered.buckets_usage[healthy].size, 250);
|
||||
let quota = load_data_usage_from_backend_cached(store)
|
||||
.await
|
||||
.expect("separate canonical quota read");
|
||||
assert_eq!(quota.buckets_usage[blocked].size, 100);
|
||||
assert_eq!(quota.buckets_usage[healthy].size, 100);
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user