mirror of
https://github.com/rustfs/rustfs.git
synced 2026-10-30 07:46:36 +00:00
Compare commits
81 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| add363f084 | |||
| 9601a7c4bc | |||
| d16e3cf7b6 | |||
| 3e056ffb4d | |||
| 72b4f6182f | |||
| a9d3022f37 | |||
| c232f1552d | |||
| 7581a70412 | |||
| 8cb9296f20 | |||
| b1ba6cdfdc | |||
| 0bfc7612d4 | |||
| 4c997519e8 | |||
| 95c5d108d1 | |||
| f6ad79dd58 | |||
| 82947d3595 | |||
| aef67b6499 | |||
| 8e88419f3a | |||
| 0197a4b538 | |||
| 1f2dbe36e5 | |||
| 67fbafa13e | |||
| 1000860cb8 | |||
| b783f404ad | |||
| 11a991c777 | |||
| 3f2b6f690a | |||
| 88b626e7a0 | |||
| 8cf6187ff4 | |||
| a0a0cee9eb | |||
| 80fff4b5c5 | |||
| d91dfcedd2 | |||
| 79c4fedf8c | |||
| 73c846e09c | |||
| 6938c467d0 | |||
| 5323df7049 | |||
| 55b6fbd260 | |||
| dd677cc121 | |||
| 0c2f327974 | |||
| df42cf1ce8 | |||
| d290d46b34 | |||
| 0d297ed8cd | |||
| b8dffa2f1d | |||
| 62b6f5f3fc | |||
| be0986115c | |||
| 1bd91c8dad | |||
| dcb22bd4d5 | |||
| fa17718444 | |||
| 99abffc57a | |||
| fc061c7930 | |||
| d86bf1360f | |||
| f6097d1df2 | |||
| 7e5d32eecf | |||
| 72da0e21c4 | |||
| 78a0cf321c | |||
| cbe38a120f | |||
| db0828aa1c | |||
| 3ca889b099 | |||
| e6112c526f | |||
| c5967686f1 | |||
| 859de565ad | |||
| 1f7cc593e7 | |||
| 1d5a50f094 | |||
| 53245be9f4 | |||
| c47432e353 | |||
| f622c1bce0 | |||
| a99d8f4f89 | |||
| 4d15b81623 | |||
| 836d30192f | |||
| 860c61ee6d | |||
| f600af4704 | |||
| 9e747e8475 | |||
| 96fc067765 | |||
| e809b4899e | |||
| 584f03f300 | |||
| d52d6fade7 | |||
| 3623d6db2d | |||
| 484f1254ca | |||
| c9c16efbb6 | |||
| f2a59073ca | |||
| c1af94d1d1 | |||
| f84bbfe28f | |||
| 378a2ff7f8 | |||
| b88b53aa6a |
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=a67a347f26be186c598b5beaf68e6bcbe1855350a35e1423dd5b8a81bcba06a5
|
||||
sha256-linux=ad4fabe811c6cca1aba89fad34ccd404b5832a6a955875c2826f6e0fa1ea4f1f
|
||||
sha256-darwin=4fb92e88763982eabafd74d05eb3d269387624a447e427a8c6f84914e5b03bfe
|
||||
sha256-linux=bd0db7e846ce24f056a097655db8fe2265201913260d0e16feb38d99dcb954a7
|
||||
|
||||
@@ -1 +1 @@
|
||||
sha256=e4ee3898b840a8374cb02c64342cc15f8d7ecd1becaf4b4a2a4657290f9e7ff1
|
||||
sha256=4eac269f9a59b2f9be4e7a87ea6aa38454f5ebe10ede35c9b8d4218137d2cd35
|
||||
|
||||
@@ -1 +1 @@
|
||||
sha256=f065e7071aac9d48a7c1b301f328fa7cc14f11c4e7dc46da3e55057c4ea29bd5
|
||||
sha256=879f421f1115d44f930e229510a2e533cfd5ebd4effe38730b82ef61f4fd7a86
|
||||
|
||||
@@ -70,6 +70,11 @@ s3s-footprint-check: ## Check the s3s dependency footprint ratchet stays frozen
|
||||
@echo "📦 Checking s3s footprint ratchet..."
|
||||
./scripts/check_s3s_footprint.sh
|
||||
|
||||
.PHONY: test-count-ratchet-check
|
||||
test-count-ratchet-check: ## Check no workspace crate lost tests or gained #[ignore] against the committed baseline
|
||||
@echo "🧪 Checking per-crate test-count ratchet..."
|
||||
./scripts/check_test_count_ratchet.sh
|
||||
|
||||
.PHONY: fips-wording-check
|
||||
fips-wording-check: ## Check docs and crates/kms do not over-claim crypto capabilities
|
||||
@echo "📣 Checking cryptographic capability wording guard..."
|
||||
|
||||
@@ -20,11 +20,11 @@ planning-docs-check: ## Check that no planning-type documents are committed
|
||||
./scripts/check_no_planning_docs.sh
|
||||
|
||||
.PHONY: pre-commit
|
||||
pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
|
||||
pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check test-count-ratchet-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
|
||||
@echo "✅ All pre-commit checks passed!"
|
||||
|
||||
.PHONY: pre-pr
|
||||
pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check log-analyzer-rules-check offline-enrollment-e2e-check clippy-check test ## Run full pre-PR checks with clippy and tests
|
||||
pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check test-count-ratchet-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check log-analyzer-rules-check offline-enrollment-e2e-check clippy-check test ## Run full pre-PR checks with clippy and tests
|
||||
@echo "✅ All pre-PR checks passed!"
|
||||
|
||||
.PHONY: dev-check
|
||||
|
||||
@@ -50,7 +50,9 @@ script-tests: ## Run shell script tests
|
||||
./scripts/test_fuzz_runner.sh
|
||||
./scripts/test_python_bin.sh
|
||||
./scripts/check_embedded_secrets.sh --self-test
|
||||
./scripts/check_s3s_footprint.sh --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/codemods/s3_error_to_s3_types.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_e2e_binary.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_inspect_btf_prototypes.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_release_build_workflow.py
|
||||
|
||||
+23
-2
@@ -128,7 +128,7 @@ setup = 'ecstore-large-stack'
|
||||
# path on a real store; measured with RUST_MIN_STACK on 2026-10-15 they need
|
||||
# 2016-2040 KiB of the 2 MiB libtest default and overflowed on Linux CI.
|
||||
[[profile.default.scripts]]
|
||||
filter = '(package(rustfs) & test(/^(admin::handlers::replication::target_repair_tests::[a-z_]+|admin::handlers::site_replication::tests::durable_iam_state_apply_iam_item_rejects_a_stale_recreate_after_a_replicated_delete|admin::service::federated_identity::tests::(binding_keeps_explicit_iam_mapping_strict|markerless_legacy_oidc_session_crosses_auth_and_iam_deletion_boundaries)|storage::scanner_dirty_journal::tests::(clear_callback_keeps_unknown_coverage_until_all_pending_work_is_confirmed|owned_journals_keep_peer_records_after_interleaved_writes_and_clear))$/)) | (package(rustfs-heal) & ((binary(heal_b5_versioned_regression_test) & test(/^serial_tests::(test_heal_unversioned_bucket_e2e|test_historical_null_recursive_heal_restores_physical_versions|test_recursive_deep_heal_reports_stale_delete_marker_repair)$/)) | (binary(usage_observation_read_repair_test) & test(=usage_observation_read_repair_survives_snapshot_cleanup)))) | (package(rustfs-test-utils) & test(=data_usage_snapshot_tests::data_usage_snapshot_recovery_wires_primary_backup_and_negative_cache))'
|
||||
filter = '(package(rustfs) & test(/^(admin::handlers::replication::target_repair_tests::[a-z_]+|admin::handlers::plugin_attempt::v2_tests::delegation_v2_persists_and_vends_with_real_parent_intersection|admin::handlers::site_replication::tests::durable_iam_state_apply_iam_item_rejects_a_stale_recreate_after_a_replicated_delete|admin::service::federated_identity::tests::(binding_keeps_explicit_iam_mapping_strict|markerless_legacy_oidc_session_crosses_auth_and_iam_deletion_boundaries)|storage::scanner_dirty_journal::tests::(clear_callback_keeps_unknown_coverage_until_all_pending_work_is_confirmed|owned_journals_keep_peer_records_after_interleaved_writes_and_clear))$/)) | (package(rustfs-heal) & ((binary(heal_b5_versioned_regression_test) & test(/^serial_tests::(test_heal_unversioned_bucket_e2e|test_historical_null_recursive_heal_restores_physical_versions|test_recursive_deep_heal_reports_stale_delete_marker_repair)$/)) | (binary(usage_observation_read_repair_test) & test(=usage_observation_read_repair_survives_snapshot_cleanup)))) | (package(rustfs-test-utils) & test(=data_usage_snapshot_tests::data_usage_snapshot_recovery_wires_primary_backup_and_negative_cache))'
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
[[profile.default.scripts]]
|
||||
@@ -345,6 +345,12 @@ threads-required = "num-test-threads"
|
||||
filter = 'package(rustfs-ecstore) & test(=store::init::tests::dispatch_manifest_rollback_bounded_concurrency_reaches_tail_behind_slow_member)'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# These short-deadline real-disk progress probes need the test run's capacity
|
||||
# for scanner and object-lock handoffs; keep their assertions and deadlines.
|
||||
[[profile.default.overrides]]
|
||||
filter = '(package(rustfs-scanner) & (test(=scanner::tests::cycle_persist_failure::later_cycle_post_commit_failure_reloads_durable_state) | test(=scanner::tests::running_main_loop_catches_up_pause_cleared_after_startup_observe))) | (package(rustfs-ecstore) & test(=store::init::tests::restored_transitioned_delete_causally_enqueues_free_version))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ci profile — the strict CI gate (ci.yml `cargo nextest run --profile ci`)
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -413,13 +419,19 @@ threads-required = "num-test-threads"
|
||||
filter = 'package(rustfs-scanner) & (test(/^scanner::backlog::tests::native_/) | test(=scanner_folder::tests::mrf_ownership::mrf_ownership_manager_completion_preserves_scanner_pending) | test(/^scanner_io::tests::service_cohort::/))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# Keep short-deadline real-disk progress probes from competing with unrelated
|
||||
# tests for scanner and object-lock capacity. Assertions and deadlines stay on.
|
||||
[[profile.ci.overrides]]
|
||||
filter = '(package(rustfs-scanner) & (test(=scanner::tests::cycle_persist_failure::later_cycle_post_commit_failure_reloads_durable_state) | test(=scanner::tests::running_main_loop_catches_up_pause_cleared_after_startup_observe))) | (package(rustfs-ecstore) & test(=store::init::tests::restored_transitioned_delete_causally_enqueues_free_version))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
[[profile.ci.scripts]]
|
||||
filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(batch_transitioned_delete_uses_free_version_per_item|decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|dispatched_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|force_tier_remove_blocks_on_physical_free_version_hidden_by_other_pool|legacy_unknown_transition_delete_falls_back_for_single_batch_and_blocks_prefix|multi_pool_(recursive_prefix_rejects_legacy_or_hidden_merge_loser_before_delete|same_remote_tuple_(batch|single)_delete_waits_for_all_sources|same_tuple_recursive_prefix_uses_one_journal_owner|transitioned_delete_persists_one_free_version_per_remote_tuple)|recursive_prefix_partial_(pool|set)_failure_keeps_prepared_cleanup_owners|restored_transitioned_delete_uses_free_version_as_cleanup_owner|stable_transitioned_recursive_prefix_delete_uses_journal_owners|suspended_null_transition_delete_uses_free_version_as_sole_owner|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)|transitioned_delete_(free_version_replays_after_store_restart|local_quorum_failure_rolls_back_without_cleanup_owner|uses_free_version_as_cleanup_owner)|versioned_delete_marker_keeps_transitioned_source_and_remote_object|versioned_explicit_transition_delete_preserves_other_version_then_allows_bucket_delete))$/)'
|
||||
setup = 'ecstore-large-stack'
|
||||
|
||||
# These cross-crate real-storage fixtures need the same bounded stack as ECStore tests.
|
||||
[[profile.ci.scripts]]
|
||||
filter = '(package(rustfs) & test(/^(admin::handlers::replication::target_repair_tests::[a-z_]+|admin::handlers::site_replication::tests::durable_iam_state_apply_iam_item_rejects_a_stale_recreate_after_a_replicated_delete|admin::service::federated_identity::tests::(binding_keeps_explicit_iam_mapping_strict|markerless_legacy_oidc_session_crosses_auth_and_iam_deletion_boundaries)|storage::scanner_dirty_journal::tests::(clear_callback_keeps_unknown_coverage_until_all_pending_work_is_confirmed|owned_journals_keep_peer_records_after_interleaved_writes_and_clear))$/)) | (package(rustfs-heal) & ((binary(heal_b5_versioned_regression_test) & test(/^serial_tests::(test_heal_unversioned_bucket_e2e|test_historical_null_recursive_heal_restores_physical_versions|test_recursive_deep_heal_reports_stale_delete_marker_repair)$/)) | (binary(usage_observation_read_repair_test) & test(=usage_observation_read_repair_survives_snapshot_cleanup)))) | (package(rustfs-test-utils) & test(=data_usage_snapshot_tests::data_usage_snapshot_recovery_wires_primary_backup_and_negative_cache))'
|
||||
filter = '(package(rustfs) & test(/^(admin::handlers::replication::target_repair_tests::[a-z_]+|admin::handlers::plugin_attempt::v2_tests::delegation_v2_persists_and_vends_with_real_parent_intersection|admin::handlers::site_replication::tests::durable_iam_state_apply_iam_item_rejects_a_stale_recreate_after_a_replicated_delete|admin::service::federated_identity::tests::(binding_keeps_explicit_iam_mapping_strict|markerless_legacy_oidc_session_crosses_auth_and_iam_deletion_boundaries)|storage::scanner_dirty_journal::tests::(clear_callback_keeps_unknown_coverage_until_all_pending_work_is_confirmed|owned_journals_keep_peer_records_after_interleaved_writes_and_clear))$/)) | (package(rustfs-heal) & ((binary(heal_b5_versioned_regression_test) & test(/^serial_tests::(test_heal_unversioned_bucket_e2e|test_historical_null_recursive_heal_restores_physical_versions|test_recursive_deep_heal_reports_stale_delete_marker_repair)$/)) | (binary(usage_observation_read_repair_test) & test(=usage_observation_read_repair_survives_snapshot_cleanup)))) | (package(rustfs-test-utils) & test(=data_usage_snapshot_tests::data_usage_snapshot_recovery_wires_primary_backup_and_negative_cache))'
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
[[profile.ci.scripts]]
|
||||
@@ -672,6 +684,14 @@ test-group = 'ecstore-serial-flaky'
|
||||
# the PR-lane e2e coverage for scanner usage snapshots consumed by quota and
|
||||
# admin surfaces. It uses the same single-node, random-port, isolated-temp-dir
|
||||
# fixture as the existing smoke modules.
|
||||
#
|
||||
# Dual-stack differential (rustfs/backlog#2740): `dual_stack_differential::`
|
||||
# starts two single-node servers from the same binary, one per
|
||||
# RUSTFS_S3_STACK value, on random ports with isolated temp dirs, and runs one
|
||||
# SDK script covering every operation of `impl S3 for FS` against both. Its
|
||||
# comparator unit tests need no server. It meets the admission criteria
|
||||
# unchanged; the module clause admits a new differential test without another
|
||||
# filter edit.
|
||||
[profile.e2e-smoke]
|
||||
default-filter = """
|
||||
package(e2e_test) & (
|
||||
@@ -680,6 +700,7 @@ default-filter = """
|
||||
| test(/^reliant::lifecycle::/)
|
||||
| test(/^conditional_delete_objects_test::/)
|
||||
| test(/^reliant::tiering::/)
|
||||
| test(/^dual_stack_differential::/)
|
||||
| test(/^upgrade_compatibility_test::upgrade_write_readiness_tests::/)
|
||||
| test(/^on_demand_migration::(get_basic_test::(get_miss_pulls_inline_and_serves_locally_afterwards|head_miss_answers_from_the_source_without_persisting)|interaction_test::test_odm_admin_config_is_redacted_and_status_counts_match_the_source)$/)
|
||||
)
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
# Files that may still reference s3s (`s3s::` or `s3s_sigv4`) when Phase 1 of
|
||||
# the gateway migration ends (rustfs/backlog#2734 task T0.6, rustfs/backlog#2741).
|
||||
# Consumed only by `scripts/check_s3s_footprint.sh --mode allowlist`.
|
||||
#
|
||||
# One entry per line: a repository-relative file path or a git `:(glob)`
|
||||
# pathspec. `**` matches across directories, `*` stays within one path
|
||||
# component, and a directory entry matches everything under it. Blank lines
|
||||
# and lines starting with `#` are ignored; an entry may not contain whitespace,
|
||||
# so a note belongs on its own comment line.
|
||||
#
|
||||
# Every Rust file listed by `git grep -l '\bs3s\(::\|_sigv4\)' -- '*.rs'` that
|
||||
# no entry matches fails the allowlist mode. Task T1.9 switches CI to that
|
||||
# mode; until then `--dry-run` prints the files still outside this list, which
|
||||
# is the Phase 1 tracking list.
|
||||
#
|
||||
# Never add crates/** other than crates/e2e_test: the engine crates leave s3s
|
||||
# entirely, and the script rejects such an entry. Only the s3s-facing edge of
|
||||
# the rustfs binary and the e2e oracle keep it.
|
||||
|
||||
# rustfs binary: the s3s-facing edge that survives Phase 1
|
||||
rustfs/src/storage/ecfs.rs
|
||||
rustfs/src/storage/s3_compat/**
|
||||
rustfs/src/storage/access.rs
|
||||
rustfs/src/auth.rs
|
||||
rustfs/src/error.rs
|
||||
rustfs/src/storage_api.rs
|
||||
rustfs/src/app/storage_api.rs
|
||||
rustfs/src/admin/router.rs
|
||||
rustfs/src/app/metadata_route.rs
|
||||
rustfs/src/server/http.rs
|
||||
rustfs/src/server/layer.rs
|
||||
rustfs/src/server/hybrid.rs
|
||||
rustfs/src/protocols/client.rs
|
||||
|
||||
# e2e oracle: the test target that verifies S3 behaviour against s3s
|
||||
crates/e2e_test/src/fake_s3_target/**
|
||||
crates/e2e_test/src/**
|
||||
@@ -0,0 +1,17 @@
|
||||
# Exact baseline for the direct s3s footprint, consumed only by
|
||||
# scripts/check_s3s_footprint.sh (rustfs/backlog#1677; rustfs/backlog#2734 T0.4).
|
||||
#
|
||||
# Each value is the exact measurement of the tree, not a ceiling. CI fails when a
|
||||
# measured counter differs from its line here in either direction:
|
||||
# - a larger measurement means new code widened the s3s surface: route it
|
||||
# through the gateway abstractions instead; never raise a value here;
|
||||
# - a smaller measurement means the PR shrank the surface: lower the matching
|
||||
# value to the measured number in the same PR.
|
||||
#
|
||||
# files Rust files referencing s3s paths, repo-wide minus crates/e2e_test
|
||||
# s3_error_lines lines holding s3_error!, same scope, minus those the script
|
||||
# header's rule clears (rustfs_s3_types's macro); fails closed
|
||||
# ecstore_files Rust files referencing s3s paths under crates/ecstore/src
|
||||
files=65
|
||||
s3_error_lines=289
|
||||
ecstore_files=0
|
||||
@@ -0,0 +1,64 @@
|
||||
# Per-crate test-count baseline consumed by scripts/check_test_count_ratchet.sh
|
||||
# (rustfs/backlog#2734 section 1 rule 3; T0.3 in rustfs/backlog#2738).
|
||||
#
|
||||
# One row per workspace member: <package>, tests, ignored, tab-separated.
|
||||
# tests counts #[test], #[tokio::test], #[tokio::test(...)] and #[test_case(...)]
|
||||
# attribute lines under the package directory; ignored counts #[ignore] and
|
||||
# #[ignore = "..."]. Both columns are exact: CI fails when a crate's tests fall
|
||||
# below or rise above its row, or when its ignored count changes. Raise a row in
|
||||
# the PR that adds tests; lowering one is a reviewed statement that tests were
|
||||
# intentionally removed. Regenerate with:
|
||||
# scripts/check_test_count_ratchet.sh --update
|
||||
e2e_test 993 20
|
||||
rustfs 6052 64
|
||||
rustfs-audit 70 0
|
||||
rustfs-checksums 26 0
|
||||
rustfs-common 36 0
|
||||
rustfs-concurrency 54 0
|
||||
rustfs-config 43 0
|
||||
rustfs-connect-symbol-catalog 6 0
|
||||
rustfs-credentials 24 0
|
||||
rustfs-crypto 67 0
|
||||
rustfs-data-usage 56 0
|
||||
rustfs-ecstore 7372 2
|
||||
rustfs-extension-schema 23 0
|
||||
rustfs-filemeta 352 0
|
||||
rustfs-heal 1021 3
|
||||
rustfs-heal-contracts 8 0
|
||||
rustfs-iam 400 0
|
||||
rustfs-io-core 49 0
|
||||
rustfs-io-metrics 153 0
|
||||
rustfs-keystone 29 0
|
||||
rustfs-kms 715 25
|
||||
rustfs-license 2 0
|
||||
rustfs-lifecycle 155 0
|
||||
rustfs-lock 143 0
|
||||
rustfs-log-analyzer 90 0
|
||||
rustfs-madmin 210 0
|
||||
rustfs-notify 139 0
|
||||
rustfs-object-capacity 92 0
|
||||
rustfs-object-data-cache 140 0
|
||||
rustfs-obs 382 0
|
||||
rustfs-policy 734 0
|
||||
rustfs-protocols 748 1
|
||||
rustfs-protos 73 0
|
||||
rustfs-replication 209 0
|
||||
rustfs-rio 206 0
|
||||
rustfs-rio-v2 34 6
|
||||
rustfs-s3-client 69 0
|
||||
rustfs-s3-ops 8 0
|
||||
rustfs-s3-types 103 0
|
||||
rustfs-s3select-api 207 0
|
||||
rustfs-s3select-query 189 2
|
||||
rustfs-scanner 1260 20
|
||||
rustfs-scanner-metrics 90 0
|
||||
rustfs-security-governance 20 0
|
||||
rustfs-signer 31 0
|
||||
rustfs-storage-api 34 0
|
||||
rustfs-targets 603 26
|
||||
rustfs-test-utils 1 0
|
||||
rustfs-tls-runtime 43 0
|
||||
rustfs-trusted-proxies 68 0
|
||||
rustfs-utils 247 0
|
||||
rustfs-zip 30 0
|
||||
zstd 11 0
|
||||
@@ -86,6 +86,10 @@ runs:
|
||||
shell: bash
|
||||
run: ./scripts/check_s3s_footprint.sh
|
||||
|
||||
- name: Check per-crate test-count ratchet
|
||||
shell: bash
|
||||
run: ./scripts/check_test_count_ratchet.sh
|
||||
|
||||
- name: Check cryptographic capability wording
|
||||
shell: bash
|
||||
run: ./scripts/check_fips_wording.sh
|
||||
|
||||
@@ -16,7 +16,7 @@ Every other label in this repository is capacity RustFS pays for. ARC pods and l
|
||||
| `sm-standard-2` | RustFS | ARC pod, Ubuntu 24.04, minimal image: no `gh`, no C toolchain; Docker only on the `dind-` label | only a job that a hosted runner cannot run (see the placement rule) |
|
||||
| `sm-standard-4` | RustFS | ARC pod, Ubuntu 24.04, 14 CPUs and 28 GB (the label is not the core count), large disk | workspace-scale Cargo builds |
|
||||
| `ubicloud-standard-16-arm` | RustFS | Ubicloud VM, alias of `ubicloud-standard-16-arm-ubuntu-2404`: Ubuntu 24.04 arm64, 16 vCPU, 48 GB, 300 GB disk. Reached through the `build.yml` matrix, so `actionlint.yaml` does not list it | native aarch64 release builds (`linux-aarch64-musl`, `linux-aarch64-gnu`) |
|
||||
| `dind-sm-standard-2` | RustFS | ARC pod with Docker-in-Docker | image builds that need `buildx` and a registry push |
|
||||
| `dind-sm-standard-2` | RustFS | ARC pod with Docker-in-Docker | Docker workloads with a measured resource or environment requirement beyond the hosted runner |
|
||||
| `smoke-testing`, `pf-testing` | RustFS | long-lived lab machines | the packaged functional chain and performance suites |
|
||||
|
||||
The self-hosted pods and the hosted image are both Ubuntu 24.04, so a binary built on one runs on the other; `ci.yml` builds the debug binary on `sm-standard-4` and runs it on both pools.
|
||||
@@ -27,11 +27,15 @@ A job runs on `ubuntu-latest` unless it has one of these reasons, and the reason
|
||||
|
||||
1. **Disk.** It restores or produces the workspace `target/`: `cargo build` or `cargo test` with `--all-targets`, `cargo nextest run --all`, or any reader of the `ci-dev` / `ci-feat-*` caches. The workspace test build leaves about 64 GB in `target/`; a hosted runner has about 14 GB.
|
||||
2. **Critical-path CPU.** It is a build that several downstream lanes wait on and would blow its timeout at 4 vCPU (`build-rustfs-debug-binary` gates the e2e and s3-tests lanes).
|
||||
3. **Hardware.** Native aarch64, Docker-in-Docker, or the lab machines.
|
||||
3. **Hardware or environment.** Native aarch64 or access to the lab machines. Docker alone is not a reason: hosted Ubuntu ships Docker and supports the QEMU and Buildx setup actions.
|
||||
4. **Cache ownership.** It reads a `rust-cache` key that a self-hosted job writes (see the cache rule below).
|
||||
|
||||
Jobs that do only the following have no reason to be self-hosted: shell or Python scripts, `cargo fmt`, `cargo deny`, typos, actionlint, API calls (`gh`, dependency review, issue and stale bots, CLA checks), downloading an artifact and running a prebuilt binary, or a `cargo build -p rustfs` with its own cache key. `e2e-distributed.yml` and `e2e-upgrade.yml` build the server and the `e2e_test` harness on `ubuntu-latest` inside their budgets, so "it compiles something" alone is not a reason.
|
||||
|
||||
The functional chain's `prepare` and `complete-chain` jobs, the performance runner availability check, and runner hygiene use `ubuntu-latest`: they inspect APIs or artifacts, not the lab hosts. The actual functional and performance suites remain on their lab runners. Functional-chain health collection stays on the operator-managed `smoke-testing` runner as part of the automation environment. Docker image packaging also uses `ubuntu-latest` for both musl and glibc variants; it downloads release binaries rather than compiling the workspace, and still sets up QEMU and Buildx explicitly.
|
||||
|
||||
Moving these jobs off AKS reduces capacity demand, not necessarily the Azure bill immediately. VM costs fall only when nodes scale down or smaller capacity replaces them.
|
||||
|
||||
A hosted runner is also the only choice for a few things the pods cannot do: `mount -o loop` and `mount -t tmpfs` fail inside the ARC pods, and a container runtime cannot guarantee that `io_uring_setup` is allowed. `uring-integration` and `e2e-distributed` are on `ubuntu-latest` for that reason, not for cost.
|
||||
|
||||
## Cache rule
|
||||
|
||||
@@ -16,7 +16,9 @@ name: Continuous Integration
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main ]
|
||||
# gateway/integration receives the daily merge of main as a direct push
|
||||
# (rustfs/backlog#2734), so its pushes must prove the result like main's.
|
||||
branches: [ main, gateway/integration ]
|
||||
paths-ignore:
|
||||
- "**.md"
|
||||
- "docs/**"
|
||||
@@ -737,6 +739,8 @@ jobs:
|
||||
needs: [ build-rustfs-debug-binary ]
|
||||
runs-on: sm-standard-2
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
RUSTFS_S3_STACK: legacy
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -855,18 +859,105 @@ jobs:
|
||||
path: ${{ runner.temp }}/rustfs-e2e-*/rustfs.log
|
||||
retention-days: 3
|
||||
|
||||
# The same e2e-smoke suite against the same debug binary with the gateway
|
||||
# S3 stack selected (rustfs/backlog#2734). Report-only: it is outside the
|
||||
# Test and Lint aggregate and becomes required at T2.10. The archive and run
|
||||
# steps must stay identical to e2e-tests; scripts/ci_gate.py enforces that.
|
||||
e2e-tests-gateway:
|
||||
name: e2e-smoke (gateway, report-only until T2.10)
|
||||
needs: [ build-rustfs-debug-binary ]
|
||||
runs-on: sm-standard-2
|
||||
timeout-minutes: 30
|
||||
continue-on-error: true
|
||||
env:
|
||||
RUSTFS_S3_STACK: gateway
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: ci-dev
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install awscurl
|
||||
run: |
|
||||
python3 -m pip install --user --upgrade pip "awscurl==0.44"
|
||||
echo "AWSCURL_PATH=$HOME/.local/bin/awscurl" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Verify awscurl
|
||||
run: test -x "$AWSCURL_PATH"
|
||||
|
||||
- name: Download debug binary
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: rustfs-debug-binary
|
||||
path: target/debug
|
||||
|
||||
- name: Make binary executable
|
||||
run: chmod +x ./target/debug/rustfs
|
||||
|
||||
- name: Archive e2e smoke test binaries
|
||||
env:
|
||||
NEXTEST_ARCHIVE: ${{ runner.temp }}/rustfs-e2e-smoke.tar.zst
|
||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-smoke-list.json
|
||||
run: |
|
||||
cargo nextest archive --profile e2e-smoke -p e2e_test --archive-file "${NEXTEST_ARCHIVE}"
|
||||
cargo nextest list --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" --message-format json > "${NEXTEST_LISTING}"
|
||||
python3 ./scripts/check_test_wiring.py --check-profile e2e-smoke "${NEXTEST_LISTING}"
|
||||
./scripts/check_security_smoke_count.sh check "${NEXTEST_LISTING}"
|
||||
|
||||
- name: Run e2e smoke suite
|
||||
env:
|
||||
NEXTEST_ARCHIVE: ${{ runner.temp }}/rustfs-e2e-smoke.tar.zst
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-smoke-logs
|
||||
SAMPLER_INTERVAL_SECS: "15"
|
||||
run: |
|
||||
./scripts/ci/resource_sampler.sh start e2e-smoke
|
||||
trap './scripts/ci/resource_sampler.sh stop' EXIT
|
||||
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" \
|
||||
--status-level all --final-status-level all --failure-output final
|
||||
|
||||
- name: Upload e2e smoke diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-smoke-gateway-diagnostics-${{ github.run_number }}
|
||||
path: |
|
||||
${{ runner.temp }}/rustfs-e2e-smoke-logs/
|
||||
${{ runner.temp }}/rustfs-e2e-smoke-list.json
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Upload e2e smoke JUnit report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-smoke-gateway-junit-${{ github.run_number }}
|
||||
path: target/nextest/e2e-smoke/junit.xml
|
||||
if-no-files-found: warn
|
||||
|
||||
e2e-full:
|
||||
name: End-to-End Tests (full merge gate)
|
||||
# Merge gate only (backlog#1149 ci-5): the never-automated user-visible
|
||||
# suites — KMS, object_lock, multipart_auth, quota, checksum, encryption,
|
||||
# security-boundary, ... — via the e2e-full nextest profile. Too heavy for
|
||||
# every PR, so it is gated to main pushes, the merge queue, and manual
|
||||
# dispatch. protocols / the 7 cluster suites / replication / #[ignore] are
|
||||
# owned by other lanes (see .config/nextest.toml profile.e2e-full).
|
||||
# every PR, so it is gated to main and gateway/integration pushes, the merge
|
||||
# queue, and manual dispatch. protocols / the 7 cluster suites / replication
|
||||
# / #[ignore] are owned by other lanes (see .config/nextest.toml profile.e2e-full).
|
||||
if: >-
|
||||
github.event_name == 'workflow_dispatch' ||
|
||||
github.event_name == 'merge_group' ||
|
||||
(github.event_name == 'push' && github.ref == 'refs/heads/main')
|
||||
(github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/gateway/integration'))
|
||||
needs: [ build-rustfs-debug-binary ]
|
||||
runs-on: sm-standard-2
|
||||
timeout-minutes: 55
|
||||
|
||||
@@ -337,7 +337,7 @@ jobs:
|
||||
name: Build Docker Images
|
||||
needs: build-check
|
||||
if: needs.build-check.outputs.should_build == 'true'
|
||||
runs-on: dind-sm-standard-2
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
strategy:
|
||||
fail-fast: false
|
||||
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
check-ephemerality:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Check runner ephemerality
|
||||
runs-on: sm-standard-2
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
|
||||
@@ -42,7 +42,7 @@ concurrency:
|
||||
jobs:
|
||||
prepare:
|
||||
if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.event == 'schedule' }}
|
||||
runs-on: smoke-testing
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
outputs:
|
||||
manifest: ${{ steps.candidate.outputs.manifest }}
|
||||
@@ -188,7 +188,7 @@ jobs:
|
||||
complete-chain:
|
||||
needs: [prepare, upgrade, s3, kms, tier, storage, heal, pool, security, replication, fault-tolerance, table, performance]
|
||||
if: ${{ always() && needs.prepare.result != 'skipped' }}
|
||||
runs-on: smoke-testing
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
|
||||
@@ -90,7 +90,7 @@ env:
|
||||
jobs:
|
||||
runner-check:
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
runs-on: sm-standard-2
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
|
||||
+1
-1
@@ -155,7 +155,7 @@ default build (lifecycle:
|
||||
- ⚠️ PARTIALLY VIOLATED: serving-side `s3s` references remain in ecstore
|
||||
(bucket metadata/replication/lifecycle DTOs and error mapping). The
|
||||
count is ratcheted shrink-only by `scripts/check_s3s_footprint.sh`
|
||||
(`S3S_ECSTORE_FILES_BASELINE`; the `object_lock` module was converted to
|
||||
(the `ecstore_files` line of `.config/s3s-footprint-baseline.txt`; the `object_lock` module was converted to
|
||||
storage-level types as the first ratchet step). Target state: the
|
||||
baseline reaches zero and ecstore's `Cargo.toml` drops `s3s`.
|
||||
|
||||
|
||||
Generated
+80
-12
@@ -8545,6 +8545,15 @@ version = "1.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
|
||||
|
||||
[[package]]
|
||||
name = "quick-xml"
|
||||
version = "0.41.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quick-xml"
|
||||
version = "0.42.0"
|
||||
@@ -9409,7 +9418,7 @@ dependencies = [
|
||||
"proptest",
|
||||
"prost 0.14.4",
|
||||
"pyroscope",
|
||||
"quick-xml",
|
||||
"quick-xml 0.42.0",
|
||||
"quinn",
|
||||
"rand 0.10.3",
|
||||
"rcgen",
|
||||
@@ -9428,6 +9437,7 @@ dependencies = [
|
||||
"rustfs-ecstore",
|
||||
"rustfs-extension-schema",
|
||||
"rustfs-filemeta",
|
||||
"rustfs-gateway-types",
|
||||
"rustfs-heal",
|
||||
"rustfs-heal-contracts",
|
||||
"rustfs-iam",
|
||||
@@ -9706,6 +9716,7 @@ dependencies = [
|
||||
"rustfs-data-usage",
|
||||
"rustfs-erasure-codec",
|
||||
"rustfs-filemeta",
|
||||
"rustfs-gateway-types",
|
||||
"rustfs-heal-contracts",
|
||||
"rustfs-io-metrics",
|
||||
"rustfs-lifecycle",
|
||||
@@ -9811,6 +9822,48 @@ dependencies = [
|
||||
"xxhash-rust",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-gateway-stream"
|
||||
version = "0.2.0"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=21283e874c2381bab611e9347acbc5b92e9e2b3c#21283e874c2381bab611e9347acbc5b92e9e2b3c"
|
||||
dependencies = [
|
||||
"bitflags 2.13.2",
|
||||
"bytes",
|
||||
"http 1.5.0",
|
||||
"http-body",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-gateway-types"
|
||||
version = "0.32.0+aws.2026-10-02"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=21283e874c2381bab611e9347acbc5b92e9e2b3c#21283e874c2381bab611e9347acbc5b92e9e2b3c"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"crc-fast",
|
||||
"futures-core",
|
||||
"hex",
|
||||
"http 1.5.0",
|
||||
"md-5 0.11.0",
|
||||
"percent-encoding",
|
||||
"rustfs-gateway-stream",
|
||||
"rustfs-gateway-xml",
|
||||
"s3s",
|
||||
"sha1 0.11.0",
|
||||
"sha2 0.11.1",
|
||||
"time",
|
||||
"twox-hash",
|
||||
"unicode-normalization",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-gateway-xml"
|
||||
version = "0.4.0"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=21283e874c2381bab611e9347acbc5b92e9e2b3c#21283e874c2381bab611e9347acbc5b92e9e2b3c"
|
||||
dependencies = [
|
||||
"quick-xml 0.41.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-heal"
|
||||
version = "1.0.2"
|
||||
@@ -10081,10 +10134,10 @@ dependencies = [
|
||||
"metrics-util",
|
||||
"proptest",
|
||||
"rustfs-config",
|
||||
"rustfs-gateway-types",
|
||||
"rustfs-replication",
|
||||
"rustfs-scanner-metrics",
|
||||
"rustfs-storage-api",
|
||||
"s3s",
|
||||
"serial_test",
|
||||
"temp-env",
|
||||
"time",
|
||||
@@ -10147,7 +10200,6 @@ dependencies = [
|
||||
"reqwest",
|
||||
"rmp-serde",
|
||||
"rustfs-signer",
|
||||
"s3s",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sysinfo",
|
||||
@@ -10187,7 +10239,7 @@ dependencies = [
|
||||
"jiff",
|
||||
"metrics",
|
||||
"percent-encoding",
|
||||
"quick-xml",
|
||||
"quick-xml 0.42.0",
|
||||
"rayon",
|
||||
"rustc-hash",
|
||||
"rustfs-config",
|
||||
@@ -10368,10 +10420,13 @@ dependencies = [
|
||||
"rustfs-config",
|
||||
"rustfs-credentials",
|
||||
"rustfs-ecstore",
|
||||
"rustfs-gateway-stream",
|
||||
"rustfs-gateway-types",
|
||||
"rustfs-iam",
|
||||
"rustfs-keystone",
|
||||
"rustfs-policy",
|
||||
"rustfs-rio",
|
||||
"rustfs-s3-types",
|
||||
"rustfs-storage-api",
|
||||
"rustfs-test-utils",
|
||||
"rustfs-tls-runtime",
|
||||
@@ -10379,13 +10434,13 @@ dependencies = [
|
||||
"rustfs-trusted-proxies",
|
||||
"rustfs-utils",
|
||||
"rustls",
|
||||
"s3s",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha1 0.11.0",
|
||||
"sha2 0.11.1",
|
||||
"socket2",
|
||||
"subtle",
|
||||
"sync_wrapper",
|
||||
"tempfile",
|
||||
"thiserror 2.0.21",
|
||||
"time",
|
||||
@@ -10435,7 +10490,7 @@ dependencies = [
|
||||
"regex",
|
||||
"rmp",
|
||||
"rmp-serde",
|
||||
"s3s",
|
||||
"rustfs-gateway-types",
|
||||
"serde",
|
||||
"time",
|
||||
"tokio",
|
||||
@@ -10530,11 +10585,13 @@ dependencies = [
|
||||
"hyper-util",
|
||||
"lazy_static",
|
||||
"md-5 0.11.0",
|
||||
"quick-xml",
|
||||
"quick-xml 0.42.0",
|
||||
"rand 0.10.3",
|
||||
"rustfs-checksums",
|
||||
"rustfs-config",
|
||||
"rustfs-gateway-types",
|
||||
"rustfs-rio",
|
||||
"rustfs-s3-types",
|
||||
"rustfs-signer",
|
||||
"rustfs-storage-api",
|
||||
"rustfs-tls-runtime",
|
||||
@@ -10568,9 +10625,19 @@ dependencies = [
|
||||
name = "rustfs-s3-types"
|
||||
version = "1.0.2"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"hotpath",
|
||||
"http 1.5.0",
|
||||
"http-body",
|
||||
"http-body-util",
|
||||
"hyper",
|
||||
"hyper-util",
|
||||
"pin-project-lite",
|
||||
"s3s",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -10594,9 +10661,10 @@ dependencies = [
|
||||
"parking_lot 0.13.0",
|
||||
"rustfs-common",
|
||||
"rustfs-ecstore",
|
||||
"rustfs-gateway-types",
|
||||
"rustfs-s3-types",
|
||||
"rustfs-storage-api",
|
||||
"rustfs-test-utils",
|
||||
"s3s",
|
||||
"serde_json",
|
||||
"serial_test",
|
||||
"thiserror 2.0.21",
|
||||
@@ -10622,7 +10690,6 @@ dependencies = [
|
||||
"parking_lot 0.13.0",
|
||||
"rustfs-s3select-api",
|
||||
"rustfs-test-utils",
|
||||
"s3s",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tokio",
|
||||
@@ -10651,6 +10718,7 @@ dependencies = [
|
||||
"rustfs-data-usage",
|
||||
"rustfs-ecstore",
|
||||
"rustfs-filemeta",
|
||||
"rustfs-gateway-types",
|
||||
"rustfs-heal",
|
||||
"rustfs-heal-contracts",
|
||||
"rustfs-lifecycle",
|
||||
@@ -10707,9 +10775,10 @@ dependencies = [
|
||||
"bytes",
|
||||
"hotpath",
|
||||
"http 1.5.0",
|
||||
"http-body",
|
||||
"hyper",
|
||||
"rustfs-s3-types",
|
||||
"rustfs-utils",
|
||||
"s3s",
|
||||
"serde_urlencoded",
|
||||
"thiserror 2.0.21",
|
||||
"time",
|
||||
@@ -10767,7 +10836,6 @@ dependencies = [
|
||||
"rustfs-utils",
|
||||
"rustls",
|
||||
"rustls-native-certs",
|
||||
"s3s",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.11.1",
|
||||
@@ -11159,7 +11227,7 @@ dependencies = [
|
||||
"numeric_cast",
|
||||
"phf 0.14.0",
|
||||
"pin-project-lite",
|
||||
"quick-xml",
|
||||
"quick-xml 0.42.0",
|
||||
"regex",
|
||||
"s3s-multipart",
|
||||
"s3s-rfc2047",
|
||||
|
||||
+13
-1
@@ -48,7 +48,7 @@ members = [
|
||||
"crates/replication", # Replication contracts and wire formats
|
||||
"crates/concurrency", # Concurrency management for RustFS - timeout, locking, backpressure, and I/O scheduling
|
||||
"crates/s3-client", # S3 client for engine-side consumption of remote S3 endpoints (tiering, transition targets)
|
||||
"crates/s3-types", # S3 event type definitions
|
||||
"crates/s3-types", # S3 event and error type definitions
|
||||
"crates/s3-ops", # S3 operation definitions and mapping
|
||||
"crates/s3select-api", # S3 Select API interface
|
||||
"crates/s3select-query", # S3 Select query engine
|
||||
@@ -143,6 +143,13 @@ rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.2" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.2" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.2" }
|
||||
|
||||
# The gateway's S3 types: the DTOs and persisted bucket-configuration codecs the
|
||||
# engine crates move onto from s3s (rustfs/backlog#2745). The one pin of the
|
||||
# gateway in this workspace; no crate may enable a `compat-s3s*` feature except
|
||||
# the s3s-facing edge of the rustfs binary.
|
||||
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "21283e874c2381bab611e9347acbc5b92e9e2b3c", version = "0.32.0", default-features = false }
|
||||
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "21283e874c2381bab611e9347acbc5b92e9e2b3c", version = "0.2.0", default-features = false }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-channel = "2.5.0"
|
||||
async_zip = { default-features = false, version = "0.0.19" }
|
||||
@@ -326,6 +333,10 @@ rust-embed = { version = "8.13.0" }
|
||||
rustc-hash = { version = "2.1.3" }
|
||||
s3s = { git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fcac87e3b99fd5bd9e176b442c0", features = ["minio"] }
|
||||
s3s-sigv4 = { git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fcac87e3b99fd5bd9e176b442c0" }
|
||||
# The same s3s pin under the name the legacy edge of the gateway migration uses
|
||||
# (crates/s3-types, feature `compat-s3s`). Keep `rev` identical to `s3s` above.
|
||||
# DELETE BY T4.2 (rustfs/backlog#2784).
|
||||
legacy-s3s = { package = "s3s", git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fcac87e3b99fd5bd9e176b442c0", features = ["minio"] }
|
||||
serial_test = "4.0.1"
|
||||
shadow-rs = { default-features = false, version = "2.0.0" }
|
||||
siphasher = "1.0.4"
|
||||
@@ -334,6 +345,7 @@ compact_str = "0.10.0"
|
||||
snap = "1.1.2"
|
||||
starshard = { version = "2.4.0" }
|
||||
strum = { version = "0.28.0" }
|
||||
sync_wrapper = "1.0.2"
|
||||
sysinfo = "0.39.6"
|
||||
temp-env = "0.3.6"
|
||||
tempfile = "3.27.0"
|
||||
|
||||
@@ -26,7 +26,7 @@ rustfs-io-core = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
|
||||
# Async runtime
|
||||
tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread"] }
|
||||
tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread", "time"] }
|
||||
|
||||
# Logging
|
||||
tracing = { workspace = true }
|
||||
|
||||
@@ -100,6 +100,7 @@ when restoring `/rustfs/console` for a custom OEM build.
|
||||
## S3 API environment variables
|
||||
|
||||
- `RUSTFS_API_OBJECT_MAX_VERSIONS` caps the number of retained versions for a single object. It defaults to `9223372036854775807`, matching MinIO's practical-unlimited default. Set a positive integer to enforce a lower per-object metadata bound.
|
||||
- `RUSTFS_VEEAM_SOSAPI_ENABLE` serves Veeam SOSAPI `system.xml` and `capacity.xml` for an authenticated GET or HEAD of those exact bucket-root keys. It defaults to `true`. Explicit version reads use the normal object path. Capacity requests return `ServiceUnavailable` until the scanner publishes a complete snapshot certified for the current bucket incarnation and authoritative quota metadata can be read. Legacy or mixed-version snapshots without an incarnation proof cannot serve capacity until a compatible scanner publishes a certified snapshot. Capacity also returns `ServiceUnavailable` when cluster or disk observations are missing, stale, or degraded. Retry after scanner convergence and storage or metadata recovery. Capacity assumes shard paths have independent backing space; shared physical storage can make the estimate optimistic. Set the flag to `false` to restore the normal object path, including deletion of a real object previously stored at that key.
|
||||
- `MINIO_API_OBJECT_MAX_VERSIONS` is accepted as a compatibility alias when the canonical RustFS variable is not set.
|
||||
|
||||
## Distributed endpoint locality
|
||||
|
||||
@@ -222,6 +222,16 @@ pub const ENV_RUSTFS_TLS_PATH: &str = "RUSTFS_TLS_PATH";
|
||||
/// Environment variable for server KMS enable.
|
||||
pub const ENV_RUSTFS_KMS_ENABLE: &str = "RUSTFS_KMS_ENABLE";
|
||||
|
||||
/// Environment variable for Veeam SOSAPI virtual system objects.
|
||||
///
|
||||
/// When true (the default), authenticated GET and HEAD of the two Veeam
|
||||
/// SOSAPI keys return generated XML. When false, those keys use the normal
|
||||
/// object path so an operator can remove a real object stored there earlier.
|
||||
pub const ENV_VEEAM_SOSAPI_ENABLE: &str = "RUSTFS_VEEAM_SOSAPI_ENABLE";
|
||||
|
||||
/// Default Veeam SOSAPI enablement.
|
||||
pub const DEFAULT_VEEAM_SOSAPI_ENABLE: bool = true;
|
||||
|
||||
/// Default KMS enable for server-side encryption
|
||||
/// This is the default value for enabling KMS encryption for server-side encryption.
|
||||
/// Default value: false
|
||||
|
||||
@@ -466,6 +466,9 @@ pub struct BucketTargetUsageInfo {
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct BucketUsageInfo {
|
||||
pub size: u64,
|
||||
/// Bucket identity certified by every set contributing to this snapshot.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub bucket_incarnation: Option<String>,
|
||||
// Following five fields suffixed with V1 are here for backward compatibility
|
||||
// Total Size for objects that have not yet been replicated
|
||||
pub replication_pending_size_v1: u64,
|
||||
@@ -2316,6 +2319,9 @@ impl BucketUsageInfo {
|
||||
/// Add size summary to this bucket usage
|
||||
/// Merge another BucketUsageInfo into this one
|
||||
pub fn merge(&mut self, other: &BucketUsageInfo) {
|
||||
if self.bucket_incarnation != other.bucket_incarnation {
|
||||
self.bucket_incarnation = None;
|
||||
}
|
||||
self.size += other.size;
|
||||
self.objects_count += other.objects_count;
|
||||
self.versions_count += other.versions_count;
|
||||
@@ -3299,6 +3305,31 @@ mod tests {
|
||||
assert_eq!(usage1.versions_count, 15);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bucket_usage_merge_discards_incompatible_incarnation_proofs() {
|
||||
let id = "00000000-0000-4000-8000-000000000001";
|
||||
let mut same = BucketUsageInfo {
|
||||
bucket_incarnation: Some(id.into()),
|
||||
..Default::default()
|
||||
};
|
||||
same.merge(&BucketUsageInfo {
|
||||
bucket_incarnation: Some(id.into()),
|
||||
..Default::default()
|
||||
});
|
||||
assert_eq!(same.bucket_incarnation.as_deref(), Some(id));
|
||||
same.merge(&BucketUsageInfo::default());
|
||||
assert!(same.bucket_incarnation.is_none());
|
||||
let mut different = BucketUsageInfo {
|
||||
bucket_incarnation: Some(id.into()),
|
||||
..Default::default()
|
||||
};
|
||||
different.merge(&BucketUsageInfo {
|
||||
bucket_incarnation: Some("00000000-0000-4000-8000-000000000002".into()),
|
||||
..Default::default()
|
||||
});
|
||||
assert!(different.bucket_incarnation.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn size_summary_add_saturates_instead_of_overflowing() {
|
||||
// The scanner folds one summary per object into a per-prefix total, so a
|
||||
|
||||
@@ -17,7 +17,8 @@
|
||||
//! when PublicAccessBlock configuration is missing or explicitly set.
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use aws_sdk_s3::types::PublicAccessBlockConfiguration;
|
||||
use crate::sts_query_compat_test::{OPA_AUTH_TOKEN, OpaMock};
|
||||
use aws_sdk_s3::types::{PublicAccessBlockConfiguration, Tag, Tagging};
|
||||
use tracing::info;
|
||||
|
||||
async fn setup_public_bucket(
|
||||
@@ -69,6 +70,98 @@ async fn anonymous_get_object(
|
||||
local_http_client().get(&url).send().await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_plain_http_matches_secure_transport_false_bucket_policy() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
assert!(env.url.starts_with("http://"), "fixture must exercise plaintext HTTP: {}", env.url);
|
||||
|
||||
let bucket = "secure-transport-policy";
|
||||
let key = "plain-http.txt";
|
||||
let admin = env.create_s3_client();
|
||||
admin.create_bucket().bucket(bucket).send().await?;
|
||||
admin
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(aws_sdk_s3::primitives::ByteStream::from_static(b"plain http"))
|
||||
.send()
|
||||
.await?;
|
||||
admin
|
||||
.put_bucket_policy()
|
||||
.bucket(bucket)
|
||||
.policy(
|
||||
serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Principal": "*",
|
||||
"Action": "s3:GetObject",
|
||||
"Resource": format!("arn:aws:s3:::{bucket}/*"),
|
||||
"Condition": {"Bool": {"aws:SecureTransport": "false"}}
|
||||
}]
|
||||
})
|
||||
.to_string(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let response = anonymous_get_object(&env, bucket, key).await?;
|
||||
assert_eq!(response.status().as_u16(), 200);
|
||||
assert_eq!(response.bytes().await?.as_ref(), b"plain http");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Root credentials have no attached IAM policy, so these requests depend on
|
||||
/// the owner flag reaching policy evaluation unchanged.
|
||||
#[tokio::test]
|
||||
async fn test_root_requests_keep_policy_owner_flag() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let opa = OpaMock::start().await?;
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_with_env(
|
||||
vec![],
|
||||
&[
|
||||
("RUSTFS_POLICY_PLUGIN_URL", opa.url.as_str()),
|
||||
("RUSTFS_POLICY_PLUGIN_AUTH_TOKEN", OPA_AUTH_TOKEN),
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
|
||||
let bucket = "root-owner-flag";
|
||||
let key = "owner.txt";
|
||||
let client = env.create_s3_client();
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(aws_sdk_s3::primitives::ByteStream::from_static(b"owner access"))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
// Duplicate stored tags are ambiguous. An owner request must not enter the
|
||||
// OPA tag-loading path, which rejects this metadata before evaluation.
|
||||
client
|
||||
.put_bucket_tagging()
|
||||
.bucket(bucket)
|
||||
.tagging(
|
||||
Tagging::builder()
|
||||
.tag_set(Tag::builder().key("department").value("finance").build()?)
|
||||
.tag_set(Tag::builder().key("department").value("engineering").build()?)
|
||||
.build()?,
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
let object = client.get_object().bucket(bucket).key(key).send().await?;
|
||||
|
||||
assert_eq!(object.body.collect().await?.into_bytes().as_ref(), b"owner access");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Issue #2036: Anonymous GetObject should succeed when bucket policy allows it
|
||||
/// and no PublicAccessBlock configuration exists (ConfigNotFound).
|
||||
#[tokio::test]
|
||||
|
||||
@@ -1129,9 +1129,198 @@ async fn phase_quorum_matrix(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn list_parts_disk_loss_after_metadata_read(
|
||||
cluster: &RustFSTestClusterEnvironment,
|
||||
proxies: &[RpcFaultProxy],
|
||||
clients: &[Client],
|
||||
bucket: &str,
|
||||
) -> TestResult {
|
||||
let key = "list-parts-after-metadata-loss";
|
||||
let body = payload_for("list-parts-acknowledged", 4096);
|
||||
let upload = clients[0].create_multipart_upload().bucket(bucket).key(key).send().await?;
|
||||
let upload_id = upload.upload_id().ok_or("missing upload ID")?.to_owned();
|
||||
let part = clients[0]
|
||||
.upload_part()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.upload_id(&upload_id)
|
||||
.part_number(1)
|
||||
.body(ByteStream::from(body.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
let etag = part.e_tag().ok_or("missing part ETag")?.to_owned();
|
||||
let path = upload_path_scope(bucket, key, &upload_id)?;
|
||||
let root = Path::new(&cluster.nodes[0].data_dirs[0])
|
||||
.join(RUSTFS_META_MULTIPART_BUCKET)
|
||||
.join(&path);
|
||||
let data_dirs = std::fs::read_dir(&root)?
|
||||
.filter_map(Result::ok)
|
||||
.map(|entry| entry.path())
|
||||
.filter(|entry| entry.join("part.1").is_file() && entry.join("part.1.meta").is_file())
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(data_dirs.len(), 1, "the coordinator's actual part directory must be identified");
|
||||
let data_dir = &data_dirs[0];
|
||||
let dir_name = data_dir
|
||||
.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.ok_or("non-UTF8 data directory")?;
|
||||
let scoped_path = format!("{path}/{dir_name}");
|
||||
let held_dir = data_dir.with_extension("e2e-held");
|
||||
std::fs::rename(data_dir, &held_dir)?;
|
||||
let pauses = proxies[1..]
|
||||
.iter()
|
||||
.map(|proxy| proxy.pause(&["ListDir"], usize::MAX, Some(&scoped_path)))
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
let request = clients[0].list_parts().bucket(bucket).key(key).upload_id(&upload_id);
|
||||
let task = tokio::spawn(async move { request.send().await });
|
||||
// ListDir is dispatched only after the upload metadata read succeeded.
|
||||
// Require all three scoped remote requests; unrelated failures do not count.
|
||||
wait_for_faults(&pauses, &["ListDir"]).await?;
|
||||
let result = tokio::time::timeout(DEADLINE, task).await??;
|
||||
for pause in &pauses {
|
||||
pause.discard_pending();
|
||||
}
|
||||
drop(pauses);
|
||||
std::fs::rename(&held_dir, data_dir)?;
|
||||
let error = result.expect_err("one missing directory and three unavailable participants cannot prove an empty upload");
|
||||
let response = error.raw_response().expect("ListParts must return an S3 service response");
|
||||
assert_eq!(response.status().as_u16(), 503);
|
||||
assert_eq!(error.as_service_error().and_then(|err| err.meta().code()), Some("SlowDownRead"));
|
||||
assert!(response.headers().get("x-rustfs-error-cause").is_none());
|
||||
assert!(response.headers().get("retry-after").is_none());
|
||||
assert!(
|
||||
response
|
||||
.headers()
|
||||
.get("content-type")
|
||||
.is_some_and(|value| value.starts_with("application/xml"))
|
||||
);
|
||||
assert!(
|
||||
response
|
||||
.headers()
|
||||
.get("x-amz-request-id")
|
||||
.is_some_and(|value| !value.is_empty())
|
||||
);
|
||||
let listed = recover_after_fault("ListParts after directory restoration", || {
|
||||
clients[0].list_parts().bucket(bucket).key(key).upload_id(&upload_id).send()
|
||||
})
|
||||
.await??;
|
||||
assert_eq!(listed.parts().len(), 1);
|
||||
assert_eq!(listed.parts()[0].e_tag(), Some(etag.as_str()));
|
||||
clients[0]
|
||||
.complete_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.upload_id(&upload_id)
|
||||
.multipart_upload(
|
||||
CompletedMultipartUpload::builder()
|
||||
.parts(CompletedPart::builder().part_number(1).e_tag(&etag).build())
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
for client in clients {
|
||||
assert_object_bytes_after_fault(client, bucket, key, &body).await?;
|
||||
}
|
||||
eprintln!(
|
||||
"MPU list proof: metadata read completed, three scoped ListDir RPCs held, missing local directory rejected; restored part and all readers verified"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn concurrent_capped_parts_respect_limit_on_distinct_nodes(clients: &[Client], bucket: &str) -> TestResult {
|
||||
let key = "concurrent-capped-parts";
|
||||
let upload = clients[0]
|
||||
.create_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.customize()
|
||||
.mutate_request(|request| {
|
||||
let separator = if request.uri().contains('?') { "&" } else { "?" };
|
||||
let uri = format!("{}{separator}x-rustfs-max-total-object-size=10", request.uri());
|
||||
request
|
||||
.set_uri(uri)
|
||||
.expect("the signed multipart size query must be a valid URI");
|
||||
})
|
||||
.send()
|
||||
.await?;
|
||||
let upload_id = upload.upload_id().ok_or("missing capped upload ID")?.to_owned();
|
||||
let barrier = std::sync::Arc::new(tokio::sync::Barrier::new(3));
|
||||
let mut tasks = Vec::new();
|
||||
for (index, number, byte) in [(0_usize, 1_i32, 0x41_u8), (1, 2, 0x42)] {
|
||||
let client = clients[index].clone();
|
||||
let bucket = bucket.to_owned();
|
||||
let upload_id = upload_id.clone();
|
||||
let barrier = std::sync::Arc::clone(&barrier);
|
||||
tasks.push(tokio::spawn(async move {
|
||||
barrier.wait().await;
|
||||
let result = client
|
||||
.upload_part()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.upload_id(&upload_id)
|
||||
.part_number(number)
|
||||
.body(ByteStream::from(vec![byte; 6]))
|
||||
.send()
|
||||
.await;
|
||||
(number, byte, result)
|
||||
}));
|
||||
}
|
||||
barrier.wait().await;
|
||||
let mut winner = None;
|
||||
let mut rejected = 0;
|
||||
for task in tasks {
|
||||
let (number, byte, result) = tokio::time::timeout(DEADLINE, task).await??;
|
||||
match result {
|
||||
Ok(part) => {
|
||||
assert!(winner.is_none(), "both six-byte parts cannot fit the ten-byte limit");
|
||||
winner = Some((number, byte, part.e_tag().ok_or("missing accepted ETag")?.to_owned()));
|
||||
}
|
||||
Err(error) => {
|
||||
assert_eq!(
|
||||
error.as_service_error().and_then(|err| err.meta().code()),
|
||||
Some("EntityTooLarge"),
|
||||
"{error:?}"
|
||||
);
|
||||
rejected += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
let (number, byte, etag) = winner.ok_or("no part was admitted")?;
|
||||
assert_eq!(rejected, 1);
|
||||
let listed = clients[0]
|
||||
.list_parts()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.upload_id(&upload_id)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(listed.parts().len(), 1);
|
||||
assert_eq!(listed.parts()[0].part_number(), Some(number));
|
||||
assert_eq!(listed.parts()[0].e_tag(), Some(etag.as_str()));
|
||||
clients[0]
|
||||
.complete_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.upload_id(&upload_id)
|
||||
.multipart_upload(
|
||||
CompletedMultipartUpload::builder()
|
||||
.parts(CompletedPart::builder().part_number(number).e_tag(&etag).build())
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
for client in clients {
|
||||
assert_object_bytes_after_fault(client, bucket, key, &[byte; 6]).await?;
|
||||
}
|
||||
eprintln!("MPU size limit: two coordinators raced; exactly one six-byte part admitted under ten-byte limit");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_delay_preserves_write_quorum_and_rejects_quorum_minus_one() -> TestResult {
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
|
||||
list_parts_disk_loss_after_metadata_read(&cluster, &proxies, &clients, &bucket).await?;
|
||||
concurrent_capped_parts_respect_limit_on_distinct_nodes(&clients, &bucket).await?;
|
||||
phase_quorum_matrix(&cluster, &proxies, &clients, &bucket, false, 1, 3).await
|
||||
}
|
||||
|
||||
@@ -1214,6 +1403,22 @@ async fn mixed_version_failed_complete_requires_current_coordinator_for_abort()
|
||||
cluster.start_node(0).await?;
|
||||
crate::upgrade_compatibility_test::wait_for_upgrade_write_readiness(&clients, &bucket, "mixed-mpu-current-writer", DEADLINE)
|
||||
.await?;
|
||||
// Read-only ListParts remains available during the documented mutation pause.
|
||||
// Compare the actual old and current coordinators on the same legacy upload.
|
||||
for (version, client) in [("current", &clients[0]), ("previous", &clients[1])] {
|
||||
let listed = client
|
||||
.list_parts()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.upload_id(upload_id)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(listed.parts().len(), 1, "{version} coordinator must list the legacy part");
|
||||
assert_eq!(listed.parts()[0].part_number(), Some(1));
|
||||
assert_eq!(listed.parts()[0].size(), Some(9 * 1024 * 1024));
|
||||
assert_eq!(listed.parts()[0].e_tag(), part.e_tag());
|
||||
}
|
||||
eprintln!("MPU mixed version: old and current coordinators agree on legacy ListParts identity and size");
|
||||
let scope = upload_scope(upload_id)?;
|
||||
let observers = proxies[1..]
|
||||
.iter()
|
||||
@@ -1344,7 +1549,7 @@ async fn native_certified_partial_abort_retry_reclaims_survivors_below_metadata_
|
||||
capability_sha256: Option<[u8; 32]>,
|
||||
abort: Option<AbortRef>,
|
||||
}
|
||||
let (cluster, proxies, clients, bucket) = fixture(4, "EC:4").await?;
|
||||
let (mut cluster, proxies, clients, bucket) = fixture(4, "EC:4").await?;
|
||||
let key = "partial-abort";
|
||||
let committed = payload_for("committed-before-abort", 4096);
|
||||
clients[0]
|
||||
@@ -1475,6 +1680,48 @@ async fn native_certified_partial_abort_retry_reclaims_survivors_below_metadata_
|
||||
.raw_response()
|
||||
.and_then(|response| response.headers().get("x-amz-request-id"))
|
||||
);
|
||||
assert!(
|
||||
retained.values().all(|source| source.participant.0 != 3),
|
||||
"the selected offline node must have no surviving root"
|
||||
);
|
||||
cluster.stop_node(3)?;
|
||||
let offline = tokio::time::timeout(
|
||||
DEADLINE,
|
||||
clients[0]
|
||||
.abort_multipart_upload()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.upload_id(upload_id)
|
||||
.send(),
|
||||
)
|
||||
.await?
|
||||
.expect_err("an unavailable original participant cannot be replaced by the certificate");
|
||||
assert_unavailable(&offline, "storage_write_quorum", None);
|
||||
assert!(
|
||||
multipart_sources(&cluster, Some(&hash))? == retained,
|
||||
"an unavailable participant must not authorize deletion or change retained evidence"
|
||||
);
|
||||
cluster.start_node(3).await?;
|
||||
wait_for_storage_peer_recovery(&proxies, &clients[0], &bucket, upload_id, 4).await?;
|
||||
eprintln!("MPU certified residual: unavailable original returned storage_write_quorum; all surviving bytes retained");
|
||||
|
||||
// Kill the coordinator and a node owning a real surviving source. The
|
||||
// retained bytes and full-N certificate must survive both process restarts
|
||||
// before a fresh coordinator can authorize the retry.
|
||||
let survivor_node = retained.values().next().ok_or("missing surviving source")?.participant.0;
|
||||
let mut restarted = vec![0, survivor_node];
|
||||
restarted.sort_unstable();
|
||||
restarted.dedup();
|
||||
for node in restarted {
|
||||
cluster.stop_node(node)?;
|
||||
cluster.start_node(node).await?;
|
||||
wait_for_storage_peer_recovery(&proxies, &clients[0], &bucket, upload_id, 4).await?;
|
||||
assert!(
|
||||
multipart_sources(&cluster, Some(&hash))? == retained,
|
||||
"restart must preserve every surviving source and certificate byte"
|
||||
);
|
||||
}
|
||||
eprintln!("MPU certified residual: coordinator and surviving owner restarted; all retained bytes unchanged");
|
||||
let delete_observers = proxies
|
||||
.iter()
|
||||
.map(|proxy| proxy.pause(&["RecoverMultipartSource"], usize::MAX, Some(&path)))
|
||||
|
||||
@@ -47,6 +47,9 @@ struct UpgradeSeed {
|
||||
history_bucket: String,
|
||||
history_key: &'static str,
|
||||
history_body: Vec<u8>,
|
||||
multipart_upload: String,
|
||||
multipart_etag: String,
|
||||
multipart_body: Vec<u8>,
|
||||
versioned_bucket: String,
|
||||
versioned_key: &'static str,
|
||||
version1: String,
|
||||
@@ -241,6 +244,25 @@ async fn seed_history_and_iam(dist: &DistCluster) -> TestResult<UpgradeSeed> {
|
||||
let history_body = b"written by the previous 4-node release".to_vec();
|
||||
put_object(&root, &history_bucket, history_key, history_body.clone()).await?;
|
||||
|
||||
let multipart_body = vec![0x5a; 4096];
|
||||
let multipart = root
|
||||
.create_multipart_upload()
|
||||
.bucket(&history_bucket)
|
||||
.key("legacy-multipart.bin")
|
||||
.send()
|
||||
.await?;
|
||||
let multipart_upload = multipart.upload_id().ok_or("legacy multipart upload ID missing")?.to_owned();
|
||||
let part = root
|
||||
.upload_part()
|
||||
.bucket(&history_bucket)
|
||||
.key("legacy-multipart.bin")
|
||||
.upload_id(&multipart_upload)
|
||||
.part_number(1)
|
||||
.body(aws_sdk_s3::primitives::ByteStream::from(multipart_body.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
let multipart_etag = part.e_tag().ok_or("legacy multipart part ETag missing")?.to_owned();
|
||||
|
||||
let versioned_key = "versioned-history.txt";
|
||||
let version1_body = b"version-one-before-upgrade".to_vec();
|
||||
let version1 = root
|
||||
@@ -278,6 +300,9 @@ async fn seed_history_and_iam(dist: &DistCluster) -> TestResult<UpgradeSeed> {
|
||||
history_bucket,
|
||||
history_key,
|
||||
history_body,
|
||||
multipart_upload,
|
||||
multipart_etag,
|
||||
multipart_body,
|
||||
versioned_bucket,
|
||||
versioned_key,
|
||||
version1,
|
||||
@@ -304,6 +329,21 @@ async fn assert_history_and_iam(dist: &DistCluster, seed: &UpgradeSeed, context:
|
||||
)
|
||||
.await?;
|
||||
assert_object_bytes(&root_a, &seed.history_bucket, seed.history_key, &seed.history_body).await?;
|
||||
// During mixed-version operation this is read-only; mutation resumes only
|
||||
// after the whole fleet is upgraded.
|
||||
for client in [&root_a, &root_b] {
|
||||
let listed = client
|
||||
.list_parts()
|
||||
.bucket(&seed.history_bucket)
|
||||
.key("legacy-multipart.bin")
|
||||
.upload_id(&seed.multipart_upload)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(listed.parts().len(), 1, "{context}: legacy part listing");
|
||||
assert_eq!(listed.parts()[0].part_number(), Some(1));
|
||||
assert_eq!(listed.parts()[0].size(), Some(4096));
|
||||
assert_eq!(listed.parts()[0].e_tag(), Some(seed.multipart_etag.as_str()));
|
||||
}
|
||||
|
||||
let v1 = root_b
|
||||
.get_object()
|
||||
@@ -387,17 +427,146 @@ async fn assert_history_and_iam(dist: &DistCluster, seed: &UpgradeSeed, context:
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn compare_degraded_legacy_list_parts(dist: &mut DistCluster, seed: &UpgradeSeed, previous: &Path) -> TestResult {
|
||||
let decoded = String::from_utf8(base64_simd::URL_SAFE_NO_PAD.decode_to_vec(seed.multipart_upload.as_bytes())?)?;
|
||||
let (_, upload_dir) = decoded.split_once('.').ok_or("legacy upload ID lacks deployment delimiter")?;
|
||||
let hash = sha256_hex(format!("{}/legacy-multipart.bin", seed.history_bucket).as_bytes());
|
||||
let root = format!("{hash}/{upload_dir}");
|
||||
let mut held = Vec::new();
|
||||
let mut original_files = Vec::new();
|
||||
for (index, node) in dist.cluster.nodes.iter().enumerate() {
|
||||
let path = Path::new(&node.data_dirs[0])
|
||||
.join(crate::storage_api::multipart_fence_acceptance::RUSTFS_META_MULTIPART_BUCKET)
|
||||
.join(&root);
|
||||
let entries = std::fs::read_dir(path)?.collect::<Result<Vec<_>, _>>()?;
|
||||
let dirs = entries
|
||||
.into_iter()
|
||||
.map(|entry| entry.path())
|
||||
.filter(|entry| entry.join("part.1").is_file() && entry.join("part.1.meta").is_file())
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(dirs.len(), 1, "each configured disk must hold the original part before injection");
|
||||
let source_root = dirs[0].parent().ok_or("part directory has no upload root")?;
|
||||
for path in [
|
||||
source_root.join("xl.meta"),
|
||||
dirs[0].join("part.1"),
|
||||
dirs[0].join("part.1.meta"),
|
||||
] {
|
||||
original_files.push((path.clone(), std::fs::read(path)?));
|
||||
}
|
||||
if index < 3 {
|
||||
held.push((dirs[0].clone(), dirs[0].with_extension("held-for-mixed-test")));
|
||||
}
|
||||
}
|
||||
dist.cluster.stop_node(3)?;
|
||||
for (original, temporary) in &held {
|
||||
std::fs::rename(original, temporary)?;
|
||||
}
|
||||
|
||||
let clients = dist.clients()?;
|
||||
let current = tokio::time::timeout(
|
||||
CREDENTIAL_TIMEOUT,
|
||||
clients[0]
|
||||
.list_parts()
|
||||
.bucket(&seed.history_bucket)
|
||||
.key("legacy-multipart.bin")
|
||||
.upload_id(&seed.multipart_upload)
|
||||
.send(),
|
||||
)
|
||||
.await?;
|
||||
let legacy = tokio::time::timeout(
|
||||
CREDENTIAL_TIMEOUT,
|
||||
clients[1]
|
||||
.list_parts()
|
||||
.bucket(&seed.history_bucket)
|
||||
.key("legacy-multipart.bin")
|
||||
.upload_id(&seed.multipart_upload)
|
||||
.send(),
|
||||
)
|
||||
.await?;
|
||||
for (original, temporary) in &held {
|
||||
std::fs::rename(temporary, original)?;
|
||||
}
|
||||
// Three online disks retain metadata quorum but have no part directory;
|
||||
// the fourth disk is offline. A legacy empty response is not a proof that
|
||||
// the unavailable participant has no part.
|
||||
let legacy = legacy.expect("previous coordinator should expose its legacy empty-list behavior");
|
||||
assert!(legacy.parts().is_empty());
|
||||
let current = current.expect_err("current coordinator must not infer empty from missing directories and offline disks");
|
||||
assert_eq!(current.raw_response().expect("current S3 response").status().as_u16(), 503);
|
||||
assert_eq!(current.as_service_error().and_then(|error| error.meta().code()), Some("SlowDownRead"));
|
||||
dist.cluster.start_node_from_binary(3, previous).await?;
|
||||
wait_for_upgrade_write_readiness(&clients, &seed.history_bucket, "mixed-list-restored", CREDENTIAL_TIMEOUT).await?;
|
||||
for (path, before) in original_files {
|
||||
assert_eq!(
|
||||
std::fs::read(&path)?,
|
||||
before,
|
||||
"the fault must preserve every original source file: {path:?}"
|
||||
);
|
||||
}
|
||||
for client in [&clients[0], &clients[1]] {
|
||||
let listed = client
|
||||
.list_parts()
|
||||
.bucket(&seed.history_bucket)
|
||||
.key("legacy-multipart.bin")
|
||||
.upload_id(&seed.multipart_upload)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(listed.parts().len(), 1);
|
||||
assert_eq!(listed.parts()[0].e_tag(), Some(seed.multipart_etag.as_str()));
|
||||
}
|
||||
eprintln!(
|
||||
"MPU mixed degraded: one offline disk, three missing part directories; previous returned empty, current returned 503 SlowDownRead; original part restored on both"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn complete_legacy_multipart(dist: &DistCluster, seed: &UpgradeSeed) -> TestResult {
|
||||
// Native legacy-source upgrade needs every original participant, not just
|
||||
// the ordinary three-of-four write quorum proven by the earlier probes.
|
||||
let clients = dist.clients()?;
|
||||
crate::upgrade_compatibility_test::wait_for_drive_readmission(
|
||||
&clients,
|
||||
&seed.history_bucket,
|
||||
"legacy-multipart-all-drives",
|
||||
CREDENTIAL_TIMEOUT,
|
||||
)
|
||||
.await?;
|
||||
let root = &clients[0];
|
||||
root.complete_multipart_upload()
|
||||
.bucket(&seed.history_bucket)
|
||||
.key("legacy-multipart.bin")
|
||||
.upload_id(&seed.multipart_upload)
|
||||
.multipart_upload(
|
||||
aws_sdk_s3::types::CompletedMultipartUpload::builder()
|
||||
.parts(
|
||||
aws_sdk_s3::types::CompletedPart::builder()
|
||||
.part_number(1)
|
||||
.e_tag(&seed.multipart_etag)
|
||||
.build(),
|
||||
)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
for client in &clients {
|
||||
assert_object_bytes(client, &seed.history_bucket, "legacy-multipart.bin", &seed.multipart_body).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_direct_upgrade_preserves_history_and_iam_credentials() -> TestResult {
|
||||
init_logging();
|
||||
let previous = source_binary()?;
|
||||
let mut dist = DistCluster::new_stopped(DistLayout::FourNodeFourDisk).await?;
|
||||
dist.cluster.set_env("RUSTFS_STORAGE_CLASS_RRS", "EC:0");
|
||||
capture_upgrade_logs(&mut dist, "direct-upgrade")?;
|
||||
dist.start_from_binary(&previous).await?;
|
||||
|
||||
let seed = seed_history_and_iam(&dist).await?;
|
||||
dist.restart_with_current_binary().await?;
|
||||
assert_history_and_iam(&dist, &seed, "direct").await?;
|
||||
complete_legacy_multipart(&dist, &seed).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -419,6 +588,7 @@ async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> Te
|
||||
init_logging();
|
||||
let previous = source_binary()?;
|
||||
let mut dist = DistCluster::new_stopped(DistLayout::FourNodeFourDisk).await?;
|
||||
dist.cluster.set_env("RUSTFS_STORAGE_CLASS_RRS", "EC:0");
|
||||
capture_upgrade_logs(&mut dist, "rolling-upgrade")?;
|
||||
dist.start_from_binary(&previous).await?;
|
||||
|
||||
@@ -426,6 +596,7 @@ async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> Te
|
||||
|
||||
replace_node_and_wait_for_write_readiness(&mut dist, &seed, 0).await?;
|
||||
assert_history_and_iam(&dist, &seed, "one-current-node").await?;
|
||||
compare_degraded_legacy_list_parts(&mut dist, &seed, &previous).await?;
|
||||
|
||||
for node_idx in [1, 2] {
|
||||
replace_node_and_wait_for_write_readiness(&mut dist, &seed, node_idx).await?;
|
||||
@@ -434,5 +605,6 @@ async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> Te
|
||||
|
||||
replace_node_and_wait_for_write_readiness(&mut dist, &seed, 3).await?;
|
||||
assert_history_and_iam(&dist, &seed, "homogeneous-current").await?;
|
||||
complete_legacy_multipart(&dist, &seed).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -0,0 +1,512 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! What one stack answered, and how two answers are compared.
|
||||
//!
|
||||
//! An [`Observation`] keeps only what a client acts on: the HTTP status, the S3 error code, the
|
||||
//! allowlisted response headers and a few typed fields decoded from the response. The error
|
||||
//! message is kept for diagnostics and never compared: message wording is judged by the wire
|
||||
//! corpus, not here. The comparator is pure so it can be tested without a server.
|
||||
|
||||
use serde::Serialize;
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
/// Response headers compared exactly (after [`normalized_header_value`]).
|
||||
const HEADER_ALLOWLIST: [&str; 7] = [
|
||||
"content-type",
|
||||
"x-amz-version-id",
|
||||
"etag",
|
||||
"x-amz-delete-marker",
|
||||
"content-range",
|
||||
"accept-ranges",
|
||||
"x-amz-request-charged",
|
||||
];
|
||||
|
||||
/// Every response header with this prefix is compared as well.
|
||||
const HEADER_ALLOWLIST_PREFIX: &str = "x-amz-server-side-encryption";
|
||||
|
||||
/// Placeholder for a field one side did not send.
|
||||
const ABSENT: &str = "<absent>";
|
||||
|
||||
/// The lowercase header name if the comparator looks at this header.
|
||||
pub(crate) fn compared_header(name: &str) -> Option<String> {
|
||||
let name = name.to_ascii_lowercase();
|
||||
(HEADER_ALLOWLIST.contains(&name.as_str()) || name.starts_with(HEADER_ALLOWLIST_PREFIX)).then_some(name)
|
||||
}
|
||||
|
||||
/// A version ID is minted per process, so its value never matches across two servers; only
|
||||
/// whether it is the literal `null` (an unversioned write) carries meaning.
|
||||
fn normalized_header_value(name: &str, value: &str) -> String {
|
||||
if name == "x-amz-version-id" && value != "null" {
|
||||
"<version-id>".to_owned()
|
||||
} else {
|
||||
value.to_owned()
|
||||
}
|
||||
}
|
||||
|
||||
/// One request's outcome as a client sees it.
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub(crate) struct Observation {
|
||||
/// The S3 operation, named as in the `s3s::S3` trait (`GetObject`).
|
||||
pub op: &'static str,
|
||||
/// Which request of the script this was (`normal`, `negative`, or a setup label).
|
||||
pub case: &'static str,
|
||||
/// HTTP status; 0 when no response arrived.
|
||||
pub status: u16,
|
||||
/// The S3 error code, or the SDK's error kind when there was no service error.
|
||||
pub error_code: Option<String>,
|
||||
/// Allowlisted headers only, lowercase names, normalized values.
|
||||
pub headers: BTreeMap<String, String>,
|
||||
/// Typed values decoded from the response body, never free text.
|
||||
pub fields: BTreeMap<String, String>,
|
||||
/// Diagnostic only; never compared.
|
||||
#[serde(skip)]
|
||||
pub message: Option<String>,
|
||||
}
|
||||
|
||||
impl Observation {
|
||||
/// Builds an observation, keeping only the allowlisted headers.
|
||||
pub(crate) fn new<'h>(
|
||||
op: &'static str,
|
||||
case: &'static str,
|
||||
status: u16,
|
||||
error_code: Option<String>,
|
||||
headers: impl IntoIterator<Item = (&'h str, &'h str)>,
|
||||
) -> Self {
|
||||
let headers = headers
|
||||
.into_iter()
|
||||
.filter_map(|(name, value)| compared_header(name).map(|name| (name, value)))
|
||||
.map(|(name, value)| {
|
||||
let value = normalized_header_value(&name, value);
|
||||
(name, value)
|
||||
})
|
||||
.collect();
|
||||
Self {
|
||||
op,
|
||||
case,
|
||||
status,
|
||||
error_code,
|
||||
headers,
|
||||
fields: BTreeMap::new(),
|
||||
message: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn with_field(mut self, name: &str, value: impl Into<String>) -> Self {
|
||||
self.fields.insert(name.to_owned(), value.into());
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn is_success(&self) -> bool {
|
||||
(200..300).contains(&self.status)
|
||||
}
|
||||
}
|
||||
|
||||
/// One field on which the two stacks answered differently.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
|
||||
pub(crate) struct Difference {
|
||||
pub op: String,
|
||||
pub case: String,
|
||||
pub field: String,
|
||||
pub a: String,
|
||||
pub b: String,
|
||||
}
|
||||
|
||||
/// Every difference between two transcripts of the same script, matched by `(op, case)`.
|
||||
pub(crate) fn compare(a: &[Observation], b: &[Observation]) -> Vec<Difference> {
|
||||
let mut differences = Vec::new();
|
||||
let (index_a, duplicates_a) = index(a);
|
||||
let (index_b, duplicates_b) = index(b);
|
||||
for (op, case) in duplicates_a.union(&duplicates_b) {
|
||||
let count = |side: &[Observation]| {
|
||||
side.iter()
|
||||
.filter(|observation| (observation.op, observation.case) == (*op, *case))
|
||||
.count()
|
||||
.to_string()
|
||||
};
|
||||
differences.push(difference(op, case, "duplicate_case", count(a), count(b)));
|
||||
}
|
||||
let keys: BTreeSet<_> = index_a.keys().chain(index_b.keys()).copied().collect();
|
||||
for key @ (op, case) in keys {
|
||||
match (index_a.get(&key), index_b.get(&key)) {
|
||||
(Some(a), Some(b)) => compare_one(a, b, &mut differences),
|
||||
(a, b) => {
|
||||
let presence = |side: Option<&&Observation>| if side.is_some() { "present" } else { ABSENT }.to_owned();
|
||||
differences.push(difference(op, case, "observation", presence(a), presence(b)));
|
||||
}
|
||||
}
|
||||
}
|
||||
differences
|
||||
}
|
||||
|
||||
type Key = (&'static str, &'static str);
|
||||
|
||||
fn index(side: &[Observation]) -> (BTreeMap<Key, &Observation>, BTreeSet<Key>) {
|
||||
let mut index = BTreeMap::new();
|
||||
let mut duplicates = BTreeSet::new();
|
||||
for observation in side {
|
||||
let key = (observation.op, observation.case);
|
||||
if index.insert(key, observation).is_some() {
|
||||
duplicates.insert(key);
|
||||
}
|
||||
}
|
||||
(index, duplicates)
|
||||
}
|
||||
|
||||
fn difference(op: &str, case: &str, field: &str, a: String, b: String) -> Difference {
|
||||
Difference {
|
||||
op: op.to_owned(),
|
||||
case: case.to_owned(),
|
||||
field: field.to_owned(),
|
||||
a,
|
||||
b,
|
||||
}
|
||||
}
|
||||
|
||||
fn compare_one(a: &Observation, b: &Observation, differences: &mut Vec<Difference>) {
|
||||
let mut field = |name: String, value_a: Option<&str>, value_b: Option<&str>| {
|
||||
if value_a != value_b {
|
||||
let show = |value: Option<&str>| value.unwrap_or(ABSENT).to_owned();
|
||||
differences.push(difference(a.op, a.case, &name, show(value_a), show(value_b)));
|
||||
}
|
||||
};
|
||||
field(
|
||||
"status".to_owned(),
|
||||
Some(a.status.to_string().as_str()),
|
||||
Some(b.status.to_string().as_str()),
|
||||
);
|
||||
field("error_code".to_owned(), a.error_code.as_deref(), b.error_code.as_deref());
|
||||
let headers: BTreeSet<_> = a.headers.keys().chain(b.headers.keys()).collect();
|
||||
for name in headers {
|
||||
field(
|
||||
format!("header.{name}"),
|
||||
a.headers.get(name).map(String::as_str),
|
||||
b.headers.get(name).map(String::as_str),
|
||||
);
|
||||
}
|
||||
let fields: BTreeSet<_> = a.fields.keys().chain(b.fields.keys()).collect();
|
||||
for name in fields {
|
||||
field(
|
||||
format!("body.{name}"),
|
||||
a.fields.get(name).map(String::as_str),
|
||||
b.fields.get(name).map(String::as_str),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Operations observed on both sides.
|
||||
pub(crate) fn compared_ops(a: &[Observation], b: &[Observation]) -> BTreeSet<&'static str> {
|
||||
let ops_b: BTreeSet<_> = b.iter().map(|observation| observation.op).collect();
|
||||
a.iter()
|
||||
.map(|observation| observation.op)
|
||||
.filter(|op| ops_b.contains(op))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// `(op, case)` pairs observed on both sides.
|
||||
pub(crate) fn compared_cases(a: &[Observation], b: &[Observation]) -> usize {
|
||||
let cases_b: BTreeSet<_> = b.iter().map(|observation| (observation.op, observation.case)).collect();
|
||||
a.iter()
|
||||
.map(|observation| (observation.op, observation.case))
|
||||
.collect::<BTreeSet<_>>()
|
||||
.intersection(&cases_b)
|
||||
.count()
|
||||
}
|
||||
|
||||
/// The JSON report written for every run.
|
||||
#[derive(Debug, Serialize)]
|
||||
pub(crate) struct Report {
|
||||
pub schema: u32,
|
||||
pub run: String,
|
||||
pub stack_a: String,
|
||||
pub stack_b: String,
|
||||
pub region_a: Option<String>,
|
||||
pub region_b: Option<String>,
|
||||
pub compared_ops: usize,
|
||||
pub compared_cases: usize,
|
||||
pub differences: Vec<Difference>,
|
||||
}
|
||||
|
||||
impl Report {
|
||||
/// Writes the report as `<directory>/<run>.json` and returns its path.
|
||||
pub(crate) fn write_to(&self, directory: &Path) -> std::io::Result<PathBuf> {
|
||||
std::fs::create_dir_all(directory)?;
|
||||
let path = directory.join(format!("{}.json", self.run));
|
||||
let json = serde_json::to_vec_pretty(self).map_err(std::io::Error::other)?;
|
||||
std::fs::write(&path, json)?;
|
||||
Ok(path)
|
||||
}
|
||||
}
|
||||
|
||||
/// One line per difference, for a failing assertion.
|
||||
pub(crate) fn render(differences: &[Difference]) -> String {
|
||||
differences
|
||||
.iter()
|
||||
.map(|d| format!("({}, {}, {}, a={:?}, b={:?})", d.op, d.case, d.field, d.a, d.b))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn ok(op: &'static str) -> Observation {
|
||||
Observation::new(op, "normal", 200, None, [("content-type", "application/xml")])
|
||||
}
|
||||
|
||||
fn fields(differences: &[Difference]) -> Vec<&str> {
|
||||
differences.iter().map(|d| d.field.as_str()).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identical_transcripts_have_no_difference() {
|
||||
let a = vec![
|
||||
ok("HeadBucket"),
|
||||
ok("GetBucketLocation").with_field("location_constraint", "us-east-1"),
|
||||
];
|
||||
let b = a.clone();
|
||||
assert_eq!(compare(&a, &b), Vec::new());
|
||||
assert_eq!(compared_ops(&a, &b).len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_status_difference_is_reported() {
|
||||
let a = vec![ok("PutBucketAcl")];
|
||||
let b = vec![Observation::new(
|
||||
"PutBucketAcl",
|
||||
"normal",
|
||||
403,
|
||||
None,
|
||||
[("content-type", "application/xml")],
|
||||
)];
|
||||
assert_eq!(
|
||||
compare(&a, &b),
|
||||
vec![Difference {
|
||||
op: "PutBucketAcl".into(),
|
||||
case: "normal".into(),
|
||||
field: "status".into(),
|
||||
a: "200".into(),
|
||||
b: "403".into(),
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_error_code_difference_is_reported() {
|
||||
let a = vec![Observation::new("GetObject", "negative", 404, Some("NoSuchKey".into()), [])];
|
||||
let b = vec![Observation::new(
|
||||
"GetObject",
|
||||
"negative",
|
||||
404,
|
||||
Some("NoSuchBucket".into()),
|
||||
[],
|
||||
)];
|
||||
let differences = compare(&a, &b);
|
||||
assert_eq!(fields(&differences), ["error_code"]);
|
||||
assert_eq!((differences[0].a.as_str(), differences[0].b.as_str()), ("NoSuchKey", "NoSuchBucket"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_error_code_on_one_side_only_is_reported() {
|
||||
let a = vec![Observation::new("HeadBucket", "negative", 404, Some("NotFound".into()), [])];
|
||||
let b = vec![Observation::new("HeadBucket", "negative", 404, None, [])];
|
||||
let differences = compare(&a, &b);
|
||||
assert_eq!(fields(&differences), ["error_code"]);
|
||||
assert_eq!(differences[0].b, ABSENT);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_allowlisted_header_difference_is_reported() {
|
||||
let a = vec![Observation::new("HeadObject", "normal", 200, None, [("ETag", "\"aa\"")])];
|
||||
let b = vec![Observation::new("HeadObject", "normal", 200, None, [("etag", "\"bb\"")])];
|
||||
assert_eq!(fields(&compare(&a, &b)), ["header.etag"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_header_sent_by_one_side_only_is_reported() {
|
||||
let a = vec![Observation::new(
|
||||
"DeleteObject",
|
||||
"normal",
|
||||
204,
|
||||
None,
|
||||
[("x-amz-delete-marker", "true")],
|
||||
)];
|
||||
let b = vec![Observation::new("DeleteObject", "normal", 204, None, [])];
|
||||
let differences = compare(&a, &b);
|
||||
assert_eq!(fields(&differences), ["header.x-amz-delete-marker"]);
|
||||
assert_eq!((differences[0].a.as_str(), differences[0].b.as_str()), ("true", ABSENT));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_server_side_encryption_header_is_compared() {
|
||||
let header = "x-amz-server-side-encryption-customer-key-md5";
|
||||
let a = vec![Observation::new("PutObject", "sse-c", 200, None, [(header, "one")])];
|
||||
let b = vec![Observation::new("PutObject", "sse-c", 200, None, [(header, "two")])];
|
||||
assert_eq!(fields(&compare(&a, &b)), [format!("header.{header}")]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn headers_outside_the_allowlist_are_not_compared() {
|
||||
let a = vec![Observation::new(
|
||||
"ListBuckets",
|
||||
"normal",
|
||||
200,
|
||||
None,
|
||||
[
|
||||
("date", "Mon"),
|
||||
("x-amz-request-id", "1"),
|
||||
("server", "a"),
|
||||
("content-length", "10"),
|
||||
],
|
||||
)];
|
||||
let b = vec![Observation::new(
|
||||
"ListBuckets",
|
||||
"normal",
|
||||
200,
|
||||
None,
|
||||
[
|
||||
("date", "Tue"),
|
||||
("x-amz-request-id", "2"),
|
||||
("server", "b"),
|
||||
("content-length", "11"),
|
||||
],
|
||||
)];
|
||||
assert_eq!(compare(&a, &b), Vec::new());
|
||||
assert!(a[0].headers.is_empty(), "non-allowlisted headers were kept: {:?}", a[0].headers);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn distinct_version_ids_are_not_a_difference() {
|
||||
let a = vec![Observation::new(
|
||||
"PutObject",
|
||||
"normal",
|
||||
200,
|
||||
None,
|
||||
[("x-amz-version-id", "0b1c")],
|
||||
)];
|
||||
let b = vec![Observation::new(
|
||||
"PutObject",
|
||||
"normal",
|
||||
200,
|
||||
None,
|
||||
[("x-amz-version-id", "9f8e")],
|
||||
)];
|
||||
assert_eq!(compare(&a, &b), Vec::new());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_null_version_id_against_a_real_one_is_reported() {
|
||||
let a = vec![Observation::new(
|
||||
"PutObject",
|
||||
"normal",
|
||||
200,
|
||||
None,
|
||||
[("x-amz-version-id", "null")],
|
||||
)];
|
||||
let b = vec![Observation::new(
|
||||
"PutObject",
|
||||
"normal",
|
||||
200,
|
||||
None,
|
||||
[("x-amz-version-id", "9f8e")],
|
||||
)];
|
||||
assert_eq!(fields(&compare(&a, &b)), ["header.x-amz-version-id"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_body_field_difference_is_reported() {
|
||||
let a = vec![ok("GetBucketLocation").with_field("location_constraint", "us-east-1")];
|
||||
let b = vec![ok("GetBucketLocation").with_field("location_constraint", "eu-west-1")];
|
||||
let differences = compare(&a, &b);
|
||||
assert_eq!(fields(&differences), ["body.location_constraint"]);
|
||||
assert_eq!((differences[0].a.as_str(), differences[0].b.as_str()), ("us-east-1", "eu-west-1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_body_field_on_one_side_only_is_reported() {
|
||||
let a = vec![ok("ListObjectsV2").with_field("keys", "[]")];
|
||||
let b = vec![ok("ListObjectsV2")];
|
||||
assert_eq!(fields(&compare(&a, &b)), ["body.keys"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn error_messages_are_never_compared() {
|
||||
let mut a = Observation::new("GetObject", "negative", 404, Some("NoSuchKey".into()), []);
|
||||
let mut b = a.clone();
|
||||
a.message = Some("The specified key does not exist.".into());
|
||||
b.message = Some("Object not found".into());
|
||||
assert_eq!(compare(&[a], &[b]), Vec::new());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_case_observed_on_one_side_only_is_reported() {
|
||||
let a = vec![ok("HeadBucket"), ok("ListBuckets")];
|
||||
let b = vec![ok("HeadBucket")];
|
||||
let differences = compare(&a, &b);
|
||||
assert_eq!(fields(&differences), ["observation"]);
|
||||
assert_eq!(differences[0].op, "ListBuckets");
|
||||
assert_eq!(compared_ops(&a, &b).into_iter().collect::<Vec<_>>(), ["HeadBucket"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_case_recorded_twice_is_reported() {
|
||||
let a = vec![ok("HeadBucket"), ok("HeadBucket")];
|
||||
let b = vec![ok("HeadBucket")];
|
||||
assert_eq!(fields(&compare(&a, &b)), ["duplicate_case"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compared_ops_counts_operations_not_cases() {
|
||||
let negative = Observation::new("HeadBucket", "negative", 404, Some("NotFound".into()), []);
|
||||
let a = vec![ok("HeadBucket"), negative, ok("ListBuckets")];
|
||||
let b = a.clone();
|
||||
assert_eq!(compared_ops(&a, &b).len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_report_lists_each_difference_as_op_field_a_b() {
|
||||
let directory = std::env::temp_dir().join(format!("dual-stack-report-{}", uuid::Uuid::new_v4()));
|
||||
let report = Report {
|
||||
schema: 1,
|
||||
run: "unit".into(),
|
||||
stack_a: "legacy".into(),
|
||||
stack_b: "legacy".into(),
|
||||
region_a: None,
|
||||
region_b: Some("eu-west-1".into()),
|
||||
compared_ops: 1,
|
||||
compared_cases: 1,
|
||||
differences: compare(
|
||||
&[ok("GetBucketLocation").with_field("location_constraint", "us-east-1")],
|
||||
&[ok("GetBucketLocation").with_field("location_constraint", "eu-west-1")],
|
||||
),
|
||||
};
|
||||
let path = report.write_to(&directory).expect("write the report");
|
||||
let json: serde_json::Value = serde_json::from_slice(&std::fs::read(&path).expect("read the report")).expect("parse");
|
||||
let _ = std::fs::remove_dir_all(&directory);
|
||||
assert_eq!(path.file_name().and_then(|name| name.to_str()), Some("unit.json"));
|
||||
assert_eq!(json["compared_ops"], 1);
|
||||
assert_eq!(
|
||||
json["differences"],
|
||||
serde_json::json!([{
|
||||
"op": "GetBucketLocation",
|
||||
"case": "normal",
|
||||
"field": "body.location_constraint",
|
||||
"a": "us-east-1",
|
||||
"b": "eu-west-1",
|
||||
}])
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Dual-stack differential: one `rustfs` binary, two processes, one SDK script (rustfs/backlog#2740).
|
||||
//!
|
||||
//! Stack A always runs `RUSTFS_S3_STACK=legacy`. Stack B runs the value of `RUSTFS_E2E_STACK_B`
|
||||
//! (default `legacy` until the gateway stack is assembled), and `RUSTFS_E2E_STACK_B_REGION`, when
|
||||
//! set, starts B in another region so that a difference can be injected by hand. Both servers
|
||||
//! answer the script in `scenarios`, which covers every operation of `impl S3 for FS`; `compare`
|
||||
//! reports each `(op, field, a, b)` on which the answers differ, and every run writes its report
|
||||
//! to `<target>/dual-stack/<run>.json`. Two processes, not two embedded servers: bucket metadata
|
||||
//! is process-global, so two stacks in one process would read each other's buckets.
|
||||
|
||||
mod compare;
|
||||
mod scenarios;
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, workspace_root};
|
||||
use compare::{Difference, Observation, Report, compare, compared_cases, compared_ops, render};
|
||||
use std::path::PathBuf;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
type BoxError = Box<dyn std::error::Error + Send + Sync>;
|
||||
|
||||
const STACK_A: &str = "legacy";
|
||||
const ENV_STACK_B: &str = "RUSTFS_E2E_STACK_B";
|
||||
const ENV_STACK_B_REGION: &str = "RUSTFS_E2E_STACK_B_REGION";
|
||||
/// Becomes `gateway` once rustfs/backlog#2734 T2.5 assembles the gateway stack.
|
||||
const DEFAULT_STACK_B: &str = "legacy";
|
||||
/// The region the negative control gives stack A explicitly, so an inherited `RUSTFS_REGION`
|
||||
/// cannot erase the injected difference. Not `us-east-1`: that region answers GetBucketLocation
|
||||
/// with an empty LocationConstraint and re-creating an owned bucket with 200, so it would not
|
||||
/// report its own name and would add CreateBucket differences to the control.
|
||||
const CONTROL_REGION_A: &str = "us-west-2";
|
||||
const CONTROL_REGION_B: &str = "eu-west-1";
|
||||
/// `normal` cases the legacy stack must answer with a 2xx. The comparison alone cannot tell a
|
||||
/// script that exercises real behavior from one in which both stacks fail the same way. Five of
|
||||
/// the 73 fail on legacy by design of this script: PutBucketReplication (no registered remote
|
||||
/// target), GetBucketReplication (so no configuration), PutBucketNotificationConfiguration (no
|
||||
/// notification target configured), RestoreObject (the object is not archived) and
|
||||
/// GetObjectTorrent (legacy answers `NoSuchKey`).
|
||||
const MIN_NORMAL_SUCCESSES: usize = 68;
|
||||
|
||||
struct PairConfig {
|
||||
stack_b: String,
|
||||
region_a: Option<String>,
|
||||
region_b: Option<String>,
|
||||
}
|
||||
|
||||
impl PairConfig {
|
||||
fn from_env() -> Self {
|
||||
Self {
|
||||
stack_b: std::env::var(ENV_STACK_B).unwrap_or_else(|_| DEFAULT_STACK_B.to_owned()),
|
||||
region_a: None,
|
||||
region_b: std::env::var(ENV_STACK_B_REGION).ok(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Starts one server on `stack`, overriding its region only when asked.
|
||||
async fn start_stack(stack: &str, region: Option<&str>) -> Result<RustFSTestEnvironment, BoxError> {
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
let mut vars = vec![("RUSTFS_S3_STACK", stack)];
|
||||
if let Some(region) = region {
|
||||
vars.push(("RUSTFS_REGION", region));
|
||||
}
|
||||
env.start_rustfs_server_without_cleanup_with_env(&vars).await?;
|
||||
Ok(env)
|
||||
}
|
||||
|
||||
/// Starts stack A and stack B side by side, each with its own port and data directory.
|
||||
async fn spawn_pair(config: &PairConfig) -> Result<(RustFSTestEnvironment, RustFSTestEnvironment), BoxError> {
|
||||
tokio::try_join!(
|
||||
start_stack(STACK_A, config.region_a.as_deref()),
|
||||
start_stack(&config.stack_b, config.region_b.as_deref())
|
||||
)
|
||||
}
|
||||
|
||||
fn report_directory() -> PathBuf {
|
||||
let target = std::env::var_os("CARGO_TARGET_DIR").map_or_else(|| PathBuf::from("target"), PathBuf::from);
|
||||
let target = if target.is_absolute() {
|
||||
target
|
||||
} else {
|
||||
workspace_root().join(target)
|
||||
};
|
||||
target.join("dual-stack")
|
||||
}
|
||||
|
||||
struct Run {
|
||||
report: Report,
|
||||
path: PathBuf,
|
||||
a: Vec<Observation>,
|
||||
}
|
||||
|
||||
/// Runs the script against both stacks concurrently and writes the report.
|
||||
async fn run_differential(label: &str, config: &PairConfig) -> Run {
|
||||
let (server_a, server_b) = spawn_pair(config).await.expect("start stack A and stack B");
|
||||
let (a, b) = tokio::join!(Box::pin(scenarios::run(&server_a)), Box::pin(scenarios::run(&server_b)));
|
||||
let millis = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map_or(0, |elapsed| elapsed.as_millis());
|
||||
let report = Report {
|
||||
schema: 1,
|
||||
run: format!("{label}-{millis}-{}", std::process::id()),
|
||||
stack_a: STACK_A.to_owned(),
|
||||
stack_b: config.stack_b.clone(),
|
||||
region_a: config.region_a.clone(),
|
||||
region_b: config.region_b.clone(),
|
||||
compared_ops: compared_ops(&a, &b).len(),
|
||||
compared_cases: compared_cases(&a, &b),
|
||||
differences: compare(&a, &b),
|
||||
};
|
||||
let path = report.write_to(&report_directory()).expect("write the dual-stack report");
|
||||
Run { report, path, a }
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn both_stacks_answer_every_operation_identically() {
|
||||
init_logging();
|
||||
let config = PairConfig::from_env();
|
||||
let run = run_differential("self-comparison", &config).await;
|
||||
let report = &run.report;
|
||||
|
||||
let mut observed: Vec<_> = run.a.iter().map(|observation| observation.op).collect();
|
||||
observed.sort_unstable();
|
||||
observed.dedup();
|
||||
assert_eq!(
|
||||
observed,
|
||||
scenarios::OPERATIONS,
|
||||
"the script must drive exactly the operations of impl S3 for FS"
|
||||
);
|
||||
assert_eq!(report.compared_ops, scenarios::OPERATIONS.len(), "report: {}", run.path.display());
|
||||
assert!(
|
||||
report.differences.is_empty(),
|
||||
"stack {} and stack {} differ on {} of {} cases (report: {}):\n{}",
|
||||
report.stack_a,
|
||||
report.stack_b,
|
||||
report.differences.len(),
|
||||
report.compared_cases,
|
||||
run.path.display(),
|
||||
render(&report.differences)
|
||||
);
|
||||
|
||||
let not_4xx: Vec<_> = run
|
||||
.a
|
||||
.iter()
|
||||
.filter(|observation| observation.case == "negative" && !(400..500).contains(&observation.status))
|
||||
.collect();
|
||||
assert!(not_4xx.is_empty(), "negative cases must fail with a 4xx on {STACK_A}: {not_4xx:#?}");
|
||||
let successes = run
|
||||
.a
|
||||
.iter()
|
||||
.filter(|observation| observation.case == "normal" && observation.is_success())
|
||||
.count();
|
||||
assert!(
|
||||
successes >= MIN_NORMAL_SUCCESSES,
|
||||
"only {successes} normal cases succeeded on {STACK_A}: {:#?}",
|
||||
run.a
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn differential_detects_an_injected_difference() {
|
||||
init_logging();
|
||||
let config = PairConfig {
|
||||
stack_b: STACK_A.to_owned(),
|
||||
region_a: Some(CONTROL_REGION_A.to_owned()),
|
||||
region_b: Some(CONTROL_REGION_B.to_owned()),
|
||||
};
|
||||
let run = run_differential("injected-region", &config).await;
|
||||
let expected = Difference {
|
||||
op: "GetBucketLocation".to_owned(),
|
||||
case: "normal".to_owned(),
|
||||
field: "body.location_constraint".to_owned(),
|
||||
a: CONTROL_REGION_A.to_owned(),
|
||||
b: CONTROL_REGION_B.to_owned(),
|
||||
};
|
||||
assert!(
|
||||
run.report.differences.contains(&expected),
|
||||
"B started in {CONTROL_REGION_B} but GetBucketLocation was not reported (report: {}):\n{}",
|
||||
run.path.display(),
|
||||
render(&run.report.differences)
|
||||
);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1716,6 +1716,15 @@ mod tests {
|
||||
cluster.set_env("RUSTFS_HEAL_AUTO_HEAL_ENABLE", background_enabled.to_string());
|
||||
cluster.set_env("RUSTFS_HEAL_MRF_ENABLE", background_enabled.to_string());
|
||||
cluster.set_env("RUSTFS_SCANNER_ENABLED", background_enabled.to_string());
|
||||
let bucket = "heal-restart-during-rebuild";
|
||||
let outage_key_prefix = "cluster/written-while-node-down-";
|
||||
if !background_enabled {
|
||||
// With MRF disabled, an outage PUT still hands its missing shard to a
|
||||
// best-effort repair on the PUT coordinator. That repair retries until
|
||||
// the target rejoins, so suppress it to keep the explicit root heal
|
||||
// the only owner of the outage object.
|
||||
cluster.set_env("RUSTFS_E2E_SUPPRESS_RENAME_TAIL_HEAL_PREFIX", format!("{bucket}/{outage_key_prefix}"));
|
||||
}
|
||||
if background_enabled {
|
||||
// Only the scanner cadence is accelerated. Keep normal Heal
|
||||
// concurrency and every automatic recovery owner enabled.
|
||||
@@ -1748,7 +1757,6 @@ mod tests {
|
||||
cluster.start_with_binary(&server_binary).await?;
|
||||
let clients = cluster.create_all_clients()?;
|
||||
|
||||
let bucket = "heal-restart-during-rebuild";
|
||||
clients[0].create_bucket().bucket(bucket).send().await?;
|
||||
|
||||
let ReplacementDriveSelection {
|
||||
@@ -1876,7 +1884,7 @@ mod tests {
|
||||
let mut replacement_outage_erasure_index = None;
|
||||
let mut rejected_outage_keys = Vec::new();
|
||||
for attempt in 0..max_outage_write_attempts {
|
||||
let candidate_key = format!("cluster/written-while-node-down-{attempt:04}.bin");
|
||||
let candidate_key = format!("{outage_key_prefix}{attempt:04}.bin");
|
||||
let put_result = timeout(
|
||||
Duration::from_secs(30),
|
||||
clients[2]
|
||||
@@ -1986,25 +1994,6 @@ mod tests {
|
||||
}
|
||||
|
||||
let heal_body = r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
|
||||
if !background_enabled {
|
||||
// The PUT path may have admitted a direct Internal object repair while
|
||||
// node 1 was offline. Cancel the isolated bucket path before the target
|
||||
// returns; otherwise it could rebuild the outage object and invalidate
|
||||
// the explicit-root ownership assertion below.
|
||||
let cancel_outage_heal_path = format!("/rustfs/admin/v3/heal/{bucket}?forceStop=true");
|
||||
let (cancel_status, cancel_body) = admin_request(
|
||||
&cluster.nodes[0].url,
|
||||
Method::POST,
|
||||
&cancel_outage_heal_path,
|
||||
Some(heal_body.to_string()),
|
||||
&cluster.access_key,
|
||||
&cluster.secret_key,
|
||||
)
|
||||
.await?;
|
||||
if !cancel_status.is_success() {
|
||||
return Err(format!("cancel outage heal failed: {cancel_status} {cancel_body}").into());
|
||||
}
|
||||
}
|
||||
|
||||
// Keep the partial-repair checkpoint stable across readiness and admin
|
||||
// requests. Endpoint-blackhole tests must prove their own network stall.
|
||||
|
||||
@@ -31,6 +31,12 @@ pub mod chaos;
|
||||
#[cfg(test)]
|
||||
pub mod fake_s3_target;
|
||||
|
||||
// Dual-stack differential (rustfs/backlog#2740): one binary, two processes on
|
||||
// different RUSTFS_S3_STACK values, the same SDK script for every S3 operation,
|
||||
// compared on status, error code, allowlisted headers and typed body fields.
|
||||
#[cfg(test)]
|
||||
mod dual_stack_differential;
|
||||
|
||||
// On-demand migration (backlog#2147): shared two-server environment, admin
|
||||
// wrappers, and the harness self-test (backlog#2151). Behavior scenarios are
|
||||
// added by later ODM tasks.
|
||||
|
||||
@@ -38,6 +38,8 @@ mod tests {
|
||||
use tracing::info;
|
||||
|
||||
const PAGINATION_TIMEOUT: Duration = Duration::from_secs(120);
|
||||
// Bound queued durable commits on the single fixture disk under CI I/O pressure.
|
||||
const FIXTURE_PUT_CONCURRENCY: usize = 4;
|
||||
|
||||
// Fixture population can require thousands of durable PUTs. Bound the
|
||||
// listing traversal itself with one deadline shared by every page.
|
||||
@@ -564,7 +566,7 @@ mod tests {
|
||||
let object_count = 1002;
|
||||
// Finish every durable PUT before listing, with the same bounded fanout as the delimiter fixtures.
|
||||
stream::iter(0..object_count)
|
||||
.for_each_concurrent(16, |i| {
|
||||
.for_each_concurrent(FIXTURE_PUT_CONCURRENCY, |i| {
|
||||
let client = &client;
|
||||
async move {
|
||||
let key = format!("object{i:04}.txt");
|
||||
@@ -575,7 +577,7 @@ mod tests {
|
||||
.body(ByteStream::from_static(b"test content"))
|
||||
.send()
|
||||
.await
|
||||
.unwrap_or_else(|err| panic!("Failed to put fixture object {key}: {err}"));
|
||||
.unwrap_or_else(|err| panic!("Failed to put fixture object {key}: {err:?}"));
|
||||
}
|
||||
})
|
||||
.await;
|
||||
@@ -822,7 +824,7 @@ mod tests {
|
||||
.flat_map(|dir| (0..10).map(move |file| format!("dir-{dir:03}/file{file:02}.txt")))
|
||||
.collect();
|
||||
stream::iter(&all_keys)
|
||||
.for_each_concurrent(16, |key| {
|
||||
.for_each_concurrent(FIXTURE_PUT_CONCURRENCY, |key| {
|
||||
let client = &client;
|
||||
async move {
|
||||
client
|
||||
@@ -832,7 +834,7 @@ mod tests {
|
||||
.body(ByteStream::from_static(b"x"))
|
||||
.send()
|
||||
.await
|
||||
.unwrap_or_else(|err| panic!("Failed to put fixture object {key}: {err}"));
|
||||
.unwrap_or_else(|err| panic!("Failed to put fixture object {key}: {err:?}"));
|
||||
}
|
||||
})
|
||||
.await;
|
||||
@@ -1055,7 +1057,7 @@ mod tests {
|
||||
let files_per_dir = 100;
|
||||
let fixture_started = std::time::Instant::now();
|
||||
stream::iter((0..dir_count).flat_map(|d| (0..files_per_dir).map(move |f| (d, f))))
|
||||
.for_each_concurrent(16, |(d, f)| {
|
||||
.for_each_concurrent(FIXTURE_PUT_CONCURRENCY, |(d, f)| {
|
||||
let client = &client;
|
||||
async move {
|
||||
let key = format!("dir-{d:02}/file{f:03}.txt");
|
||||
@@ -1066,7 +1068,7 @@ mod tests {
|
||||
.body(ByteStream::from_static(b"x"))
|
||||
.send()
|
||||
.await
|
||||
.unwrap_or_else(|err| panic!("Failed to put fixture object {key}: {err}"));
|
||||
.unwrap_or_else(|err| panic!("Failed to put fixture object {key}: {err:?}"));
|
||||
}
|
||||
})
|
||||
.await;
|
||||
|
||||
@@ -32,7 +32,9 @@
|
||||
//! Advisory: <https://github.com/rustfs/rustfs/security/advisories/GHSA-3p3x-734c-h5vx>
|
||||
|
||||
use crate::common::rustfs_binary_path_with_features;
|
||||
use crate::common::{AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via};
|
||||
use crate::common::{
|
||||
AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, signed_s3_request,
|
||||
};
|
||||
use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, admin_request};
|
||||
use crate::protocols::test_env::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, ProtocolTestEnvironment};
|
||||
use anyhow::Result;
|
||||
@@ -774,6 +776,81 @@ async fn test_webdav_core_operations_direct() -> Result<()> {
|
||||
test_webdav_core_operations().await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_webdav_content_type_directory_marker() -> Result<()> {
|
||||
let env = ProtocolTestEnvironment::new().map_err(|e| anyhow::anyhow!("{}", e))?;
|
||||
let admin_base_url = format!("http://{}", S3_TEST_ADDRESS);
|
||||
let binary_path = rustfs_binary_path_with_features(Some("webdav"));
|
||||
let mut server_process = Command::new(&binary_path)
|
||||
.arg("--address")
|
||||
.arg(S3_TEST_ADDRESS)
|
||||
.env("RUSTFS_CONSOLE_ENABLE", "false")
|
||||
.env("RUSTFS_WEBDAV_ENABLE", "true")
|
||||
.env("RUSTFS_WEBDAV_ADDRESS", WEBDAV_ADDRESS)
|
||||
.env("RUSTFS_WEBDAV_TLS_ENABLED", "false")
|
||||
.arg(&env.temp_dir)
|
||||
.spawn()?;
|
||||
|
||||
let result = async {
|
||||
ProtocolTestEnvironment::wait_for_port_ready(WEBDAV_PORT, 30)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("{}", e))?;
|
||||
|
||||
let client = create_client();
|
||||
let bucket_name = "webdav-content-type-bucket";
|
||||
let directory_name = "content-type-directory";
|
||||
let webdav_base_url = format!("http://{}", WEBDAV_ADDRESS);
|
||||
let mkcol_bucket = client
|
||||
.request(
|
||||
reqwest::Method::from_bytes(b"MKCOL").expect("MKCOL is a valid HTTP method"),
|
||||
format!("{}/{}", webdav_base_url, bucket_name),
|
||||
)
|
||||
.header("Authorization", basic_auth_header())
|
||||
.send()
|
||||
.await?;
|
||||
anyhow::ensure!(
|
||||
mkcol_bucket.status().is_success() || mkcol_bucket.status().as_u16() == 201,
|
||||
"MKCOL bucket should succeed, got: {}",
|
||||
mkcol_bucket.status()
|
||||
);
|
||||
|
||||
let mkcol_directory = client
|
||||
.request(
|
||||
reqwest::Method::from_bytes(b"MKCOL").expect("MKCOL is a valid HTTP method"),
|
||||
format!("{}/{}/{}", webdav_base_url, bucket_name, directory_name),
|
||||
)
|
||||
.header("Authorization", basic_auth_header())
|
||||
.send()
|
||||
.await?;
|
||||
anyhow::ensure!(
|
||||
mkcol_directory.status().is_success() || mkcol_directory.status().as_u16() == 201,
|
||||
"MKCOL directory should succeed, got: {}",
|
||||
mkcol_directory.status()
|
||||
);
|
||||
|
||||
let head_url = format!("{}/{}/{}/", admin_base_url, bucket_name, directory_name);
|
||||
let head = signed_s3_request(http::Method::HEAD, &head_url, None, None, DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY)
|
||||
.await
|
||||
.map_err(|error| anyhow::anyhow!("{error}"))?;
|
||||
anyhow::ensure!(head.status().is_success(), "HEAD directory marker should succeed, got: {}", head.status());
|
||||
let content_type = head
|
||||
.headers()
|
||||
.get("content-type")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.unwrap_or_default();
|
||||
anyhow::ensure!(
|
||||
content_type == "application/x-directory",
|
||||
"WebDAV directory marker must preserve its content type, got: {content_type}"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
.await;
|
||||
|
||||
let _ = server_process.kill().await;
|
||||
let _ = server_process.wait().await;
|
||||
result
|
||||
}
|
||||
|
||||
/// Extract the numeric value of the first occurrence of a DAV property in a
|
||||
/// PROPFIND multistatus body. Namespace-prefix agnostic; a self-closing
|
||||
/// (unreported) property yields None.
|
||||
|
||||
@@ -261,6 +261,61 @@ mod tests {
|
||||
archive
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn snowball_auto_extract_rejects_veeam_virtual_members() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_VEEAM_SOSAPI_ENABLE", "true")])
|
||||
.await?;
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "snowball-veeam-reserved";
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
let prefix = ".system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c";
|
||||
for (member, destination, use_prefix) in [
|
||||
(format!("{prefix}/system.xml"), format!("{prefix}/system.xml"), false),
|
||||
("capacity.xml".to_string(), format!("{prefix}/capacity.xml"), true),
|
||||
] {
|
||||
let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
|
||||
append_typed_entry(&mut builder, &member, tokio_tar::EntryType::Regular, b"blocked").await?;
|
||||
let archive = builder.into_inner().await?.into_inner();
|
||||
let mut request = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("fixture.tar")
|
||||
.metadata("Snowball-Auto-Extract", "true")
|
||||
.body(ByteStream::from(archive));
|
||||
if use_prefix {
|
||||
request = request
|
||||
.metadata("Minio-Snowball-Prefix", prefix)
|
||||
.metadata("Minio-Snowball-Ignore-Errors", "true");
|
||||
}
|
||||
let error = request.send().await.expect_err("virtual archive members must be rejected");
|
||||
assert_eq!(error.code(), Some("MethodNotAllowed"));
|
||||
let listed = client.list_objects_v2().bucket(bucket).prefix(&destination).send().await?;
|
||||
assert!(listed.contents().is_empty(), "rejected virtual member must not be stored");
|
||||
}
|
||||
let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
|
||||
append_typed_entry(&mut builder, "ordinary.xml", tokio_tar::EntryType::Regular, b"ordinary").await?;
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("fixture.tar")
|
||||
.metadata("Snowball-Auto-Extract", "true")
|
||||
.metadata("Minio-Snowball-Prefix", prefix)
|
||||
.body(ByteStream::from(builder.into_inner().await?.into_inner()))
|
||||
.send()
|
||||
.await?;
|
||||
let ordinary = client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(format!("{prefix}/ordinary.xml"))
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(ordinary.body.collect().await?.into_bytes().as_ref(), b"ordinary");
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn snowball_auto_extract_supports_minio_prefix_and_directory_markers() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -13,9 +13,13 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::common::{RustFSTestClusterEnvironment, init_logging};
|
||||
use crate::storage_api::multipart_fence_acceptance::RUSTFS_META_MULTIPART_BUCKET;
|
||||
use aws_sdk_s3::error::SdkError;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use tokio::time::{Duration, sleep};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use tokio::time::{Duration, Instant, sleep};
|
||||
use tracing::info;
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -84,6 +88,126 @@ async fn wait_for_cleanup_on_all_nodes(
|
||||
Err("stale multipart upload was not cleaned up on all nodes within timeout".into())
|
||||
}
|
||||
|
||||
type PhysicalTree = BTreeMap<PathBuf, Option<Vec<u8>>>;
|
||||
|
||||
fn physical_tree(root: &Path) -> Result<PhysicalTree, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let mut result = BTreeMap::new();
|
||||
let mut pending = vec![root.to_path_buf()];
|
||||
while let Some(directory) = pending.pop() {
|
||||
for entry in std::fs::read_dir(directory)? {
|
||||
let entry = entry?;
|
||||
let path = entry.path();
|
||||
let relative = path.strip_prefix(root)?.to_path_buf();
|
||||
let kind = entry.file_type()?;
|
||||
if kind.is_dir() {
|
||||
result.insert(relative, None);
|
||||
pending.push(path);
|
||||
} else if kind.is_file() {
|
||||
result.insert(relative, Some(std::fs::read(path)?));
|
||||
} else {
|
||||
return Err(format!("unexpected staging entry type: {relative:?}").into());
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
async fn assert_low_quorum_residuals_are_diagnosed_and_preserved(
|
||||
cluster: &RustFSTestClusterEnvironment,
|
||||
clients: &[aws_sdk_s3::Client],
|
||||
log_paths: &[PathBuf],
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let mut residuals = Vec::new();
|
||||
for copies in [1_usize, 2] {
|
||||
let key = format!("multipart/partial-{copies}-{}", Uuid::new_v4().simple());
|
||||
let upload = clients[0]
|
||||
.create_multipart_upload()
|
||||
.bucket(CLEANUP_BUCKET)
|
||||
.key(&key)
|
||||
.send()
|
||||
.await?;
|
||||
let upload_id = upload.upload_id().ok_or("create response missing upload ID")?;
|
||||
clients[0]
|
||||
.upload_part()
|
||||
.bucket(CLEANUP_BUCKET)
|
||||
.key(&key)
|
||||
.upload_id(upload_id)
|
||||
.part_number(1)
|
||||
.body(ByteStream::from(vec![0x41; 4096]))
|
||||
.send()
|
||||
.await?;
|
||||
let decoded = String::from_utf8(base64_simd::URL_SAFE_NO_PAD.decode_to_vec(upload_id.as_bytes())?)?;
|
||||
let (_, upload_dir) = decoded.split_once('.').ok_or("upload ID lacks deployment delimiter")?;
|
||||
let digest = Sha256::digest(format!("{CLEANUP_BUCKET}/{key}").as_bytes());
|
||||
let hash = hex_simd::encode_to_string(digest, hex_simd::AsciiCase::Lower);
|
||||
let relative = format!("{hash}/{upload_dir}");
|
||||
let roots = cluster
|
||||
.nodes
|
||||
.iter()
|
||||
.map(|node| {
|
||||
Path::new(&node.data_dirs[0])
|
||||
.join(RUSTFS_META_MULTIPART_BUCKET)
|
||||
.join(&relative)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(roots.len(), 4);
|
||||
let mut preserved = Vec::new();
|
||||
for root in &roots {
|
||||
assert!(
|
||||
root.join("xl.meta").is_file(),
|
||||
"all four original metadata files must exist before injection"
|
||||
);
|
||||
preserved.push(physical_tree(root)?);
|
||||
}
|
||||
for root in roots.iter().skip(copies) {
|
||||
std::fs::remove_dir_all(root)?;
|
||||
}
|
||||
residuals.push((relative, roots, preserved, copies));
|
||||
}
|
||||
|
||||
let deadline = Instant::now() + Duration::from_secs(45);
|
||||
loop {
|
||||
let logs = log_paths
|
||||
.iter()
|
||||
.filter_map(|path| std::fs::read_to_string(path).ok())
|
||||
.collect::<Vec<_>>();
|
||||
let diagnosed = residuals.iter().all(|(path, _, _, _)| {
|
||||
logs.iter().any(|log| {
|
||||
log.lines()
|
||||
.any(|line| line.contains(path) && line.contains("multipart_cleanup_lock_or_recheck_failed"))
|
||||
})
|
||||
});
|
||||
if diagnosed {
|
||||
break;
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
return Err("stale low-quorum residuals were not diagnosed before the deadline".into());
|
||||
}
|
||||
sleep(Duration::from_millis(200)).await;
|
||||
}
|
||||
for (path, roots, preserved, copies) in &residuals {
|
||||
for slot in 0..*copies {
|
||||
assert_eq!(
|
||||
physical_tree(&roots[slot])?,
|
||||
preserved[slot],
|
||||
"retained source evidence changed: {path}, slot {slot}"
|
||||
);
|
||||
}
|
||||
for root in roots.iter().skip(*copies) {
|
||||
assert!(!root.exists(), "cleanup must not recreate missing source roots: {path}");
|
||||
}
|
||||
}
|
||||
for path in log_paths {
|
||||
let log = std::fs::read_to_string(path)?;
|
||||
assert!(
|
||||
!log.contains("op=\"upload_part\"") && !log.contains("\"op\":\"upload_part\""),
|
||||
"background cleanup must not be logged as a client UploadPart"
|
||||
);
|
||||
}
|
||||
eprintln!("Stale MPU low-quorum: EC:2 one/two-copy roots diagnosed; full surviving file inventories preserved");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_stale_multipart_cleanup_removes_incomplete_upload_across_cluster()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
@@ -92,6 +216,22 @@ async fn test_stale_multipart_cleanup_removes_incomplete_upload_across_cluster()
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
|
||||
cluster.set_env("RUSTFS_API_STALE_UPLOADS_EXPIRY", "5s");
|
||||
cluster.set_env("RUSTFS_API_STALE_UPLOADS_CLEANUP_INTERVAL", "1s");
|
||||
cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", "EC:2");
|
||||
cluster.set_env(
|
||||
"RUST_LOG",
|
||||
concat!("warn,rustfs_ecstore", "::bucket::lifecycle::bucket_lifecycle_ops=debug"),
|
||||
);
|
||||
let log_dir = std::env::var_os("RUSTFS_E2E_LOG_DIR")
|
||||
.map(PathBuf::from)
|
||||
.map(|root| root.join(format!("stale-multipart-{}", Uuid::new_v4().simple())))
|
||||
.unwrap_or_else(|| PathBuf::from(&cluster.temp_dir));
|
||||
std::fs::create_dir_all(&log_dir)?;
|
||||
let log_paths = (0..cluster.nodes.len())
|
||||
.map(|index| log_dir.join(format!("stale-node-{index}.log")))
|
||||
.collect::<Vec<_>>();
|
||||
for (index, path) in log_paths.iter().enumerate() {
|
||||
cluster.set_node_capture_log_path(index, path.to_string_lossy().into_owned())?;
|
||||
}
|
||||
cluster.start().await?;
|
||||
cluster.create_test_bucket(CLEANUP_BUCKET).await?;
|
||||
|
||||
@@ -137,6 +277,7 @@ async fn test_stale_multipart_cleanup_removes_incomplete_upload_across_cluster()
|
||||
);
|
||||
|
||||
wait_for_cleanup_on_all_nodes(&clients, CLEANUP_BUCKET, &key, &upload_id).await?;
|
||||
assert_low_quorum_residuals_are_diagnosed_and_preserved(&cluster, &clients, &log_paths).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1066,7 +1066,7 @@ pub(crate) async fn wait_for_upgrade_write_readiness(
|
||||
/// reduced-redundancy class then fails with 503 `ServiceUnavailable` until the
|
||||
/// serving node writes through every drive, on the previous release and on
|
||||
/// the current build alike.
|
||||
async fn wait_for_drive_readmission(clients: &[Client], bucket: &str, context: &str, budget: Duration) -> TestResult {
|
||||
pub(crate) async fn wait_for_drive_readmission(clients: &[Client], bucket: &str, context: &str, budget: Duration) -> TestResult {
|
||||
wait_for_upgrade_writes(
|
||||
clients,
|
||||
bucket,
|
||||
|
||||
@@ -600,6 +600,154 @@ mod tests {
|
||||
info!("✅ PASSED: Veeam backup workflow simulation completed successfully");
|
||||
}
|
||||
|
||||
/// Veeam B&R discovers SOSAPI by GET/HEAD of two well-known keys.
|
||||
#[tokio::test]
|
||||
async fn test_veeam_sosapi_system_and_capacity() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
use crate::common::local_http_client;
|
||||
|
||||
const SYSTEM_KEY: &str = ".system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml";
|
||||
const CAPACITY_KEY: &str = ".system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/capacity.xml";
|
||||
|
||||
init_logging();
|
||||
info!("🧪 TEST: Veeam SOSAPI system.xml and capacity.xml");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_with_env(vec![], crate::common::FAST_DATA_USAGE_SCANNER_ENV)
|
||||
.await?;
|
||||
let client = create_s3_client(&env);
|
||||
let bucket = "veeam-sosapi";
|
||||
create_bucket(&client, bucket).await?;
|
||||
|
||||
let system = client.get_object().bucket(bucket).key(SYSTEM_KEY).send().await?;
|
||||
let system_etag = system.e_tag().map(str::to_string);
|
||||
let system_len = system.content_length();
|
||||
assert_eq!(system.content_type(), Some("application/xml"));
|
||||
let system_body = system.body.collect().await?.into_bytes();
|
||||
let system_xml = std::str::from_utf8(&system_body)?;
|
||||
assert!(system_xml.contains("<ProtocolVersion>\"1.0\"</ProtocolVersion>"), "{system_xml}");
|
||||
assert!(
|
||||
system_xml.contains("<ModelName>\"RustFS ") && system_xml.contains("\"</ModelName>"),
|
||||
"{system_xml}"
|
||||
);
|
||||
assert!(system_xml.contains("<CapacityInfo>true</CapacityInfo>"));
|
||||
assert!(system_xml.contains("<UploadSessions>false</UploadSessions>"));
|
||||
assert!(system_xml.contains("<IAMSTS>false</IAMSTS>"));
|
||||
assert!(!system_xml.contains("SystemRecommendations"));
|
||||
assert!(!system_xml.contains("APIEndpoints"));
|
||||
assert!(!system_xml.contains("""));
|
||||
|
||||
let head = client.head_object().bucket(bucket).key(SYSTEM_KEY).send().await?;
|
||||
assert_eq!(head.content_type(), Some("application/xml"));
|
||||
assert_eq!(head.content_length(), system_len);
|
||||
assert_eq!(head.e_tag().map(str::to_string), system_etag);
|
||||
assert!(head.last_modified().is_some());
|
||||
|
||||
let ranged = client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(SYSTEM_KEY)
|
||||
.range("bytes=0-3")
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(ranged.content_length(), Some(4));
|
||||
let ranged_header = ranged.content_range().unwrap_or("");
|
||||
assert!(ranged_header.starts_with("bytes 0-3/"), "{ranged_header}");
|
||||
let ranged_body = ranged.body.collect().await?.into_bytes();
|
||||
assert_eq!(&ranged_body[..], &system_body[..4]);
|
||||
|
||||
let capacity = tokio::time::timeout(std::time::Duration::from_secs(180), async {
|
||||
loop {
|
||||
match client.get_object().bucket(bucket).key(CAPACITY_KEY).send().await {
|
||||
Ok(capacity) => break Ok(capacity),
|
||||
Err(err) if err.code() == Some("ServiceUnavailable") => {
|
||||
tokio::time::sleep(std::time::Duration::from_secs(2)).await;
|
||||
}
|
||||
Err(err) => break Err(err),
|
||||
}
|
||||
}
|
||||
})
|
||||
.await??;
|
||||
assert_eq!(capacity.content_type(), Some("application/xml"));
|
||||
let capacity_body = capacity.body.collect().await?.into_bytes();
|
||||
let capacity_xml = std::str::from_utf8(&capacity_body)?;
|
||||
let capacity_value = xml_u64(capacity_xml, "Capacity")?;
|
||||
let available_value = xml_u64(capacity_xml, "Available")?;
|
||||
let used_value = xml_u64(capacity_xml, "Used")?;
|
||||
assert!(used_value <= capacity_value, "{capacity_xml}");
|
||||
assert!(available_value <= capacity_value.saturating_sub(used_value), "{capacity_xml}");
|
||||
|
||||
let put_err = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(SYSTEM_KEY)
|
||||
.body(ByteStream::from_static(b"overwrite"))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("SOSAPI put must fail");
|
||||
assert_eq!(put_err.code(), Some("MethodNotAllowed"));
|
||||
let delete_err = client
|
||||
.delete_object()
|
||||
.bucket(bucket)
|
||||
.key(CAPACITY_KEY)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("SOSAPI delete must fail");
|
||||
assert_eq!(delete_err.code(), Some("MethodNotAllowed"));
|
||||
|
||||
let normal_key = "Veeam/Backup/job.dat";
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(normal_key)
|
||||
.body(ByteStream::from_static(b"backup-bytes"))
|
||||
.send()
|
||||
.await?;
|
||||
let normal = client.get_object().bucket(bucket).key(normal_key).send().await?;
|
||||
let normal_body = normal.body.collect().await?.into_bytes();
|
||||
assert_eq!(&normal_body[..], b"backup-bytes");
|
||||
|
||||
let anon = local_http_client()
|
||||
.get(format!("{}/{bucket}/{SYSTEM_KEY}", env.url))
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(anon.status(), reqwest::StatusCode::FORBIDDEN);
|
||||
let anon_body = anon.text().await?;
|
||||
assert!(!anon_body.contains("ProtocolVersion"), "{anon_body}");
|
||||
|
||||
let now = time::OffsetDateTime::now_utc();
|
||||
let scope_date = format!("{:04}{:02}{:02}", now.year(), u8::from(now.month()), now.day());
|
||||
let amz_date = format!("{scope_date}T{:02}{:02}{:02}Z", now.hour(), now.minute(), now.second());
|
||||
let bad = local_http_client()
|
||||
.get(format!("{}/{bucket}/{SYSTEM_KEY}", env.url))
|
||||
.header("x-amz-date", &amz_date)
|
||||
.header("x-amz-content-sha256", "UNSIGNED-PAYLOAD")
|
||||
.header(
|
||||
"authorization",
|
||||
format!(
|
||||
"AWS4-HMAC-SHA256 Credential=bad/{scope_date}/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=0000000000000000000000000000000000000000000000000000000000000000"
|
||||
),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(bad.status(), reqwest::StatusCode::FORBIDDEN);
|
||||
let bad_body = bad.text().await?;
|
||||
assert!(!bad_body.contains("ProtocolVersion"), "{bad_body}");
|
||||
|
||||
info!("✅ PASSED: Veeam SOSAPI system.xml and capacity.xml");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn xml_u64(xml: &str, element: &str) -> Result<u64, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let start = format!("<{element}>");
|
||||
let end = format!("</{element}>");
|
||||
let start_idx = xml.find(&start).ok_or_else(|| format!("missing {element} in {xml}"))? + start.len();
|
||||
let end_idx = xml[start_idx..]
|
||||
.find(&end)
|
||||
.ok_or_else(|| format!("unclosed {element} in {xml}"))?
|
||||
+ start_idx;
|
||||
Ok(xml[start_idx..end_idx].parse()?)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_terraform_put_after_delete() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -137,6 +137,7 @@ rustfs-lifecycle.workspace = true
|
||||
rustfs-s3-client = { workspace = true }
|
||||
rustfs-s3-types = { workspace = true }
|
||||
rustfs-data-usage.workspace = true
|
||||
rustfs-gateway-types.workspace = true
|
||||
rustfs-object-capacity.workspace = true
|
||||
rustfs-object-data-cache = { workspace = true, features = ["runtime-memory"] }
|
||||
arc-swap.workspace = true
|
||||
@@ -155,7 +156,6 @@ serde = { workspace = true, features = ["derive"] }
|
||||
time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }
|
||||
bytesize.workspace = true
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
http.workspace = true
|
||||
opentelemetry.workspace = true
|
||||
http-body = { workspace = true }
|
||||
@@ -244,6 +244,7 @@ windows-sys = { workspace = true, features = [
|
||||
windows-sys = { workspace = true, features = ["Win32_System_Ioctl"] }
|
||||
|
||||
[dev-dependencies]
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
aws-smithy-async.workspace = true
|
||||
rustfs-s3-ops.workspace = true
|
||||
tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util", "fs"] }
|
||||
|
||||
@@ -62,6 +62,8 @@ pub mod bucket {
|
||||
}
|
||||
|
||||
pub mod bucket_lifecycle_ops {
|
||||
pub use rustfs_lifecycle::{RestoreOutputLocation, RestoreRequest, RestoreS3Location, RestoreTag, RestoreTagging};
|
||||
|
||||
pub use crate::bucket::lifecycle::bucket_lifecycle_ops::{
|
||||
ExpiryState, LifecycleOps, ManualTransitionCancelCheck, ManualTransitionProgressSink,
|
||||
ManualTransitionQueueSnapshot, ManualTransitionRunExecution, ManualTransitionRunOptions,
|
||||
@@ -133,7 +135,7 @@ pub mod bucket {
|
||||
pub use crate::bucket::lifecycle::lifecycle::{
|
||||
Event, ExpirationOptions, IlmAction, LIFECYCLE_MALFORMED_XML_ERROR_KIND, Lifecycle, LifecycleCalculate,
|
||||
ObjectOpts, RuleValidate, TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, expected_expiry_time,
|
||||
object_opts_from_object_info,
|
||||
format_lifecycle_date, object_opts_from_object_info, parse_lifecycle_date,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -225,7 +227,7 @@ pub mod bucket {
|
||||
}
|
||||
|
||||
pub mod object_lock {
|
||||
pub use crate::bucket::object_lock::{ObjectLockApi, ObjectLockStatusExt};
|
||||
pub use crate::bucket::object_lock::ObjectLockApi;
|
||||
|
||||
pub mod types {
|
||||
pub use crate::bucket::object_lock::types::{
|
||||
@@ -252,7 +254,7 @@ pub mod bucket {
|
||||
pub mod quota {
|
||||
pub use crate::bucket::quota::{
|
||||
BucketQuota, QUOTA_RESERVATION_PROTOCOL_V1, QUOTA_RESERVATION_PROTOCOL_V2, QuotaCheckResult, QuotaError,
|
||||
QuotaMigrationSkipReason, QuotaOperation, QuotaProtocolMigration,
|
||||
QuotaMigrationSkipReason, QuotaOperation, QuotaProtocolMigration, QuotaType,
|
||||
};
|
||||
|
||||
pub mod checker {
|
||||
@@ -320,9 +322,9 @@ pub mod bucket {
|
||||
|
||||
pub mod utils {
|
||||
pub use crate::bucket::utils::{
|
||||
check_bucket_and_object_names, check_list_objs_args, check_object_name_for_length_and_slash,
|
||||
check_valid_bucket_name_strict, deserialize, has_bad_path_component, is_meta_bucketname, is_valid_object_prefix,
|
||||
serialize,
|
||||
BucketConfigXml, BucketConfigXmlError, check_bucket_and_object_names, check_list_objs_args,
|
||||
check_object_name_for_length_and_slash, check_valid_bucket_name_strict, deserialize, has_bad_path_component,
|
||||
is_meta_bucketname, is_valid_object_prefix, serialize,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -349,7 +351,7 @@ pub mod capacity {
|
||||
}
|
||||
|
||||
pub mod object_api_utils {
|
||||
pub use crate::object_api::object_api_utils::{ObjReaderFn, PutObjReader, get_raw_etag, new_getobjectreader, to_s3s_etag};
|
||||
pub use crate::object_api::object_api_utils::{ObjReaderFn, PutObjReader, get_raw_etag, new_getobjectreader};
|
||||
}
|
||||
|
||||
pub mod cluster {
|
||||
@@ -371,15 +373,24 @@ pub mod compression {
|
||||
|
||||
pub mod config {
|
||||
pub mod com {
|
||||
/// Preserve empty existing objects so callers can distinguish corruption from absence.
|
||||
pub async fn read_config_preserve_empty_with_metadata_opts(
|
||||
api: std::sync::Arc<crate::store::ECStore>,
|
||||
file: &str,
|
||||
opts: &crate::object_api::ObjectOptions,
|
||||
) -> crate::error::Result<(Vec<u8>, crate::object_api::ObjectInfo)> {
|
||||
crate::config::com::read_config_preserve_empty_with_metadata_opts(api, file, opts).await
|
||||
}
|
||||
|
||||
pub use crate::config::com::{
|
||||
COMMA_SEPARATED_LISTS, CONFIG_PREFIX, ENV_CONFIG_RECOVER_ON_CORRUPTION, STORAGE_CLASS_SUB_SYS,
|
||||
ServerConfigCorruptError, ServerConfigSaveResult, ServerConfigSnapshot, delete_config, delete_config_no_lock,
|
||||
is_server_config_corrupt_error, lookup_configs, read_config, read_config_no_lock, read_config_with_metadata,
|
||||
read_config_without_migrate, read_config_without_migrate_no_lock, read_existing_server_config_no_lock,
|
||||
read_server_config_snapshot, save_config, save_config_no_lock, save_config_with_opts, save_config_with_opts_quiet,
|
||||
save_server_config, save_server_config_no_lock, save_server_config_snapshot,
|
||||
save_server_config_snapshot_with_generation, server_config_path, try_migrate_server_config,
|
||||
with_config_object_read_lock, with_config_object_write_lock, with_server_config_read_lock,
|
||||
read_server_config_snapshot, read_server_config_with_generation, save_config, save_config_no_lock,
|
||||
save_config_with_opts, save_config_with_opts_quiet, save_server_config, save_server_config_no_lock,
|
||||
save_server_config_snapshot, save_server_config_snapshot_with_generation, server_config_path,
|
||||
try_migrate_server_config, with_config_object_read_lock, with_config_object_write_lock, with_server_config_read_lock,
|
||||
with_server_config_write_lock,
|
||||
};
|
||||
}
|
||||
@@ -416,9 +427,10 @@ pub mod data_usage {
|
||||
DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage,
|
||||
init_compression_total_memory_from_backend, invalidate_admin_data_usage_snapshot_cache,
|
||||
invalidate_data_usage_snapshot_cache, live_bucket_usage_computations, load_admin_data_usage_from_backend_cached,
|
||||
load_compression_total_from_memory, load_data_usage_from_backend, load_data_usage_from_backend_cached, quota_object_size,
|
||||
record_bucket_delete_marker_memory, record_bucket_object_delete_memory, record_bucket_object_version_write_memory,
|
||||
record_bucket_object_write_memory, record_bucket_object_write_unknown_previous_memory, record_compression_total_memory,
|
||||
load_bucket_usage_from_backend_cached, load_compression_total_from_memory, load_data_usage_from_backend,
|
||||
load_data_usage_from_backend_cached, quota_object_size, record_bucket_delete_marker_memory,
|
||||
record_bucket_object_delete_memory, record_bucket_object_version_write_memory, record_bucket_object_write_memory,
|
||||
record_bucket_object_write_unknown_previous_memory, record_compression_total_memory,
|
||||
refresh_bucket_usage_from_object_layer, refresh_versioned_bucket_usage_from_object_layer,
|
||||
remove_bucket_usage_from_backend, replace_bucket_usage_memory_from_info, store_compression_total_in_backend,
|
||||
store_data_usage_in_backend,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -17,7 +17,7 @@ use crate::object_api::ObjectInfo;
|
||||
pub use rustfs_lifecycle::{
|
||||
Event, ExpirationOptions, IlmAction, LIFECYCLE_MALFORMED_XML_ERROR_KIND, Lifecycle, LifecycleCalculate, ObjectOpts,
|
||||
RuleValidate, TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, abort_incomplete_multipart_upload_due,
|
||||
expected_expiry_time, expiration_action_has_valid_target,
|
||||
expected_expiry_time, expiration_action_has_valid_target, format_lifecycle_date, parse_lifecycle_date,
|
||||
};
|
||||
|
||||
pub fn object_opts_from_object_info(oi: &ObjectInfo) -> ObjectOpts {
|
||||
|
||||
@@ -14,7 +14,8 @@
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration};
|
||||
use rustfs_gateway_types::persistence::PersistedLifecycleConfiguration;
|
||||
use rustfs_gateway_types::persistence::PersistedObjectLockConfiguration as ObjectLockConfiguration;
|
||||
use time::OffsetDateTime;
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -24,7 +25,7 @@ use crate::error::{Error, Result};
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct LifecycleExpiryConfigs {
|
||||
pub(crate) lifecycle: Option<Arc<BucketLifecycleConfiguration>>,
|
||||
pub(crate) lifecycle: Option<Arc<PersistedLifecycleConfiguration>>,
|
||||
pub(crate) object_lock: Option<Arc<ObjectLockConfiguration>>,
|
||||
pub(crate) bucket_incarnation_id: Uuid,
|
||||
pub(crate) table_bucket_enabled: bool,
|
||||
@@ -63,14 +64,7 @@ pub(crate) async fn get_expiry_configs_for_options(
|
||||
bucket: &str,
|
||||
opts: &crate::object_api::ObjectOptions,
|
||||
) -> Result<LifecycleExpiryConfigs> {
|
||||
let bucket_incarnation_id = match opts
|
||||
.object_lock_config_snapshot
|
||||
.as_ref()
|
||||
.and_then(|snapshot| snapshot.validated_bucket_incarnation_for(api.id, bucket, opts.expected_bucket_incarnation_id))
|
||||
{
|
||||
Some(incarnation_id) => incarnation_id,
|
||||
None => metadata_sys::get_bucket_incarnation_id_for_options_in(&api.ctx, bucket, opts).await?,
|
||||
};
|
||||
let bucket_incarnation_id = metadata_sys::get_bucket_incarnation_id_for_options_in(&api.ctx, bucket, opts).await?;
|
||||
let metadata = get_authoritative_metadata(api, bucket, bucket_incarnation_id).await?;
|
||||
let table_bucket_enabled = metadata.table_bucket_enabled();
|
||||
|
||||
@@ -107,7 +101,7 @@ pub(crate) async fn get_expiry_configs_for_options(
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn get_lifecycle_config(bucket: &str) -> Result<(BucketLifecycleConfiguration, OffsetDateTime)> {
|
||||
pub(crate) async fn get_lifecycle_config(bucket: &str) -> Result<(PersistedLifecycleConfiguration, OffsetDateTime)> {
|
||||
metadata_sys::get_lifecycle_config(bucket).await
|
||||
}
|
||||
|
||||
@@ -117,15 +111,15 @@ mod tests {
|
||||
use crate::bucket::metadata::BucketMetadata;
|
||||
use crate::bucket::metadata_sys::{self, test_support::isolated_store_over_temp_disks};
|
||||
use crate::storage_api_contracts::bucket::MakeBucketOptions;
|
||||
use s3s::dto::{ExpirationStatus, LifecycleExpiration, LifecycleRule};
|
||||
use rustfs_gateway_types::persistence::{PersistedLifecycleExpiration, PersistedLifecycleRule};
|
||||
use serial_test::serial;
|
||||
|
||||
fn lifecycle_config() -> BucketLifecycleConfiguration {
|
||||
BucketLifecycleConfiguration {
|
||||
fn lifecycle_config() -> PersistedLifecycleConfiguration {
|
||||
PersistedLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
expiration: Some(LifecycleExpiration {
|
||||
rules: vec![PersistedLifecycleRule {
|
||||
status: "Enabled".to_owned(),
|
||||
expiration: Some(PersistedLifecycleExpiration {
|
||||
days: Some(1),
|
||||
..Default::default()
|
||||
}),
|
||||
|
||||
@@ -22,7 +22,7 @@ pub(crate) fn is_object_locked_by_metadata(user_defined: &HashMap<String, String
|
||||
}
|
||||
|
||||
pub(crate) fn check_object_lock_for_deletion_with_config(
|
||||
config: Option<&s3s::dto::ObjectLockConfiguration>,
|
||||
config: Option<&rustfs_gateway_types::persistence::PersistedObjectLockConfiguration>,
|
||||
obj_info: &ObjectInfo,
|
||||
bypass_governance: bool,
|
||||
) -> crate::error::Result<Option<ObjectLockBlockReason>> {
|
||||
|
||||
@@ -82,12 +82,13 @@ mod tests {
|
||||
use crate::bucket::replication::{DeleteReplicationConfigSnapshot, ReplicationObjectBridge};
|
||||
use crate::object_api::{ObjectInfo, ObjectOptions};
|
||||
use crate::storage_api_contracts::object::ObjectToDelete;
|
||||
use rustfs_scanner_metrics::metrics::IlmAction;
|
||||
use s3s::dto::{
|
||||
BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, DeleteReplication,
|
||||
DeleteReplicationStatus, Destination, ReplicationConfiguration, ReplicationRule, ReplicationRuleStatus,
|
||||
VersioningConfiguration,
|
||||
use rustfs_gateway_types::dto::Status;
|
||||
use rustfs_gateway_types::persistence::PersistedVersioningConfiguration as VersioningConfiguration;
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedOptionalReplicationStatus, PersistedReplicationConfiguration, PersistedReplicationDestination,
|
||||
PersistedReplicationRule, PersistedReplicationStatus,
|
||||
};
|
||||
use rustfs_scanner_metrics::metrics::IlmAction;
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::*;
|
||||
@@ -160,27 +161,27 @@ mod tests {
|
||||
for purge_enabled in [false, true] {
|
||||
let snapshot = DeleteReplicationConfigSnapshot::from_configs_for_test(
|
||||
VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
status: Some("Enabled".to_owned()),
|
||||
..Default::default()
|
||||
},
|
||||
Some(ReplicationConfiguration {
|
||||
Some(PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![ReplicationRule {
|
||||
delete_marker_replication: Some(DeleteMarkerReplication {
|
||||
rules: vec![PersistedReplicationRule {
|
||||
delete_marker_replication: Some(PersistedOptionalReplicationStatus {
|
||||
status: Some(if marker_enabled {
|
||||
DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)
|
||||
Status::ENABLED.to_string()
|
||||
} else {
|
||||
DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::DISABLED)
|
||||
Status::DISABLED.to_string()
|
||||
}),
|
||||
}),
|
||||
delete_replication: Some(DeleteReplication {
|
||||
delete_replication: Some(PersistedReplicationStatus {
|
||||
status: if purge_enabled {
|
||||
DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED)
|
||||
Status::ENABLED.to_string()
|
||||
} else {
|
||||
DeleteReplicationStatus::from_static(DeleteReplicationStatus::DISABLED)
|
||||
Status::DISABLED.to_string()
|
||||
},
|
||||
}),
|
||||
destination: Destination {
|
||||
destination: PersistedReplicationDestination {
|
||||
bucket: "arn:rustfs:replication:target".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
@@ -190,7 +191,7 @@ mod tests {
|
||||
prefix: Some(String::new()),
|
||||
priority: Some(1),
|
||||
source_selection_criteria: None,
|
||||
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
|
||||
status: Status::ENABLED.to_string(),
|
||||
}],
|
||||
}),
|
||||
);
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use s3s::dto::BucketLifecycleConfiguration;
|
||||
use rustfs_gateway_types::persistence::PersistedLifecycleConfiguration;
|
||||
use tokio::sync::RwLock;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
@@ -52,7 +52,7 @@ pub(crate) fn deployment_id() -> Option<String> {
|
||||
sources::deployment_id()
|
||||
}
|
||||
|
||||
pub(crate) async fn bucket_lifecycle_config(bucket: &str) -> Option<BucketLifecycleConfiguration> {
|
||||
pub(crate) async fn bucket_lifecycle_config(bucket: &str) -> Option<PersistedLifecycleConfiguration> {
|
||||
sources::bucket_lifecycle_config(bucket).await
|
||||
}
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
use super::msgp_decode::{read_msgp_ext8_time, skip_msgp_value, write_msgp_time};
|
||||
use super::object_lock::{ObjectLockApi, ObjectLockStatusExt};
|
||||
use super::object_lock::ObjectLockApi;
|
||||
use super::versioning::VersioningApi;
|
||||
use super::{quota::BucketQuota, target::BucketTargets};
|
||||
use crate::bucket::replication::invalid_replication_config_status_field;
|
||||
@@ -25,12 +25,17 @@ use crate::object_api::WriteCompletion;
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use crate::store::ECStore;
|
||||
use byteorder::{BigEndian, ByteOrder, LittleEndian};
|
||||
use rustfs_policy::policy::BucketPolicy;
|
||||
use s3s::dto::{
|
||||
AccelerateConfiguration, BucketLifecycleConfiguration, BucketLoggingStatus, BucketVersioningStatus, CORSConfiguration,
|
||||
NotificationConfiguration, ObjectLockConfiguration, PublicAccessBlockConfiguration, ReplicationConfiguration,
|
||||
RequestPaymentConfiguration, ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration, WebsiteConfiguration,
|
||||
use rustfs_gateway_types::cors_tagging::{PersistedCorsConfiguration as CORSConfiguration, PersistedTagging as Tagging};
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedAccelerateConfiguration as AccelerateConfiguration,
|
||||
PersistedBucketEncryptionConfiguration as ServerSideEncryptionConfiguration,
|
||||
PersistedBucketLoggingStatus as BucketLoggingStatus, PersistedLifecycleConfiguration,
|
||||
PersistedNotificationConfiguration as NotificationConfiguration, PersistedObjectLockConfiguration as ObjectLockConfiguration,
|
||||
PersistedPublicAccessBlockConfiguration as PublicAccessBlockConfiguration, PersistedReplicationConfiguration,
|
||||
PersistedRequestPaymentConfiguration as RequestPaymentConfiguration,
|
||||
PersistedVersioningConfiguration as VersioningConfiguration, PersistedWebsiteConfiguration as WebsiteConfiguration,
|
||||
};
|
||||
use rustfs_policy::policy::BucketPolicy;
|
||||
use serde::Serializer;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashMap;
|
||||
@@ -40,23 +45,11 @@ use time::{Date, OffsetDateTime, PrimitiveDateTime, Time as CivilTime, UtcOffset
|
||||
use tracing::error;
|
||||
use uuid::Uuid;
|
||||
|
||||
// The serving-layer DTO impls for the storage-level Object Lock traits live
|
||||
// here because this module owns the persisted `ObjectLockConfiguration`
|
||||
// during the s3s ratchet migration (rustfs/backlog#1842).
|
||||
// The persisted Object Lock configuration answers the storage-level trait
|
||||
// here because this module owns the persisted `ObjectLockConfiguration`.
|
||||
impl ObjectLockApi for ObjectLockConfiguration {
|
||||
fn enabled(&self) -> bool {
|
||||
self.object_lock_enabled
|
||||
.as_ref()
|
||||
.is_some_and(|v| v.as_str() == s3s::dto::ObjectLockEnabled::ENABLED)
|
||||
}
|
||||
}
|
||||
|
||||
impl ObjectLockStatusExt for s3s::dto::ObjectLockLegalHoldStatus {
|
||||
fn valid(&self) -> bool {
|
||||
matches!(
|
||||
self.as_str(),
|
||||
s3s::dto::ObjectLockLegalHoldStatus::ON | s3s::dto::ObjectLockLegalHoldStatus::OFF
|
||||
)
|
||||
self.object_lock_enabled.as_deref() == Some("Enabled")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -439,6 +432,8 @@ pub struct BucketMetadata {
|
||||
pub bucket_incarnation_id: Uuid,
|
||||
pub(crate) bucket_incarnation_sidecar: bool,
|
||||
pub(crate) bucket_creation_committed: bool,
|
||||
// A legacy default is not proof of an interrupted creation.
|
||||
pub(crate) bucket_creation_commit_record_present: bool,
|
||||
pub policy_config_json: Vec<u8>,
|
||||
pub notification_config_xml: Vec<u8>,
|
||||
pub lifecycle_config_xml: Vec<u8>,
|
||||
@@ -487,13 +482,13 @@ pub struct BucketMetadata {
|
||||
|
||||
pub policy_config: Option<BucketPolicy>,
|
||||
pub notification_config: Option<NotificationConfiguration>,
|
||||
pub lifecycle_config: Option<BucketLifecycleConfiguration>,
|
||||
pub lifecycle_config: Option<PersistedLifecycleConfiguration>,
|
||||
pub object_lock_config: Option<ObjectLockConfiguration>,
|
||||
pub versioning_config: Option<VersioningConfiguration>,
|
||||
pub sse_config: Option<ServerSideEncryptionConfiguration>,
|
||||
pub tagging_config: Option<Tagging>,
|
||||
pub quota_config: Option<BucketQuota>,
|
||||
pub replication_config: Option<ReplicationConfiguration>,
|
||||
pub replication_config: Option<PersistedReplicationConfiguration>,
|
||||
pub bucket_target_config: Option<BucketTargets>,
|
||||
pub bucket_target_config_meta: Option<HashMap<String, String>>,
|
||||
pub cors_config: Option<CORSConfiguration>,
|
||||
@@ -514,6 +509,7 @@ impl Default for BucketMetadata {
|
||||
bucket_incarnation_id: Uuid::nil(),
|
||||
bucket_incarnation_sidecar: false,
|
||||
bucket_creation_committed: false,
|
||||
bucket_creation_commit_record_present: false,
|
||||
policy_config_json: Default::default(),
|
||||
notification_config_xml: Default::default(),
|
||||
lifecycle_config_xml: Default::default(),
|
||||
@@ -752,7 +748,10 @@ impl BucketMetadata {
|
||||
"TableBucketConfigUpdatedAt" => self.table_bucket_config_updated_at = read_msgp_time_value(rd)?,
|
||||
"DurabilityConfigUpdatedAt" => self.durability_config_updated_at = read_msgp_time_value(rd)?,
|
||||
"OnDemandMigrationConfigUpdatedAt" => self.on_demand_migration_config_updated_at = read_msgp_time_value(rd)?,
|
||||
"BucketCreationCommitted" => self.bucket_creation_committed = read_msgp_bool(rd)?,
|
||||
"BucketCreationCommitted" => {
|
||||
self.bucket_creation_committed = read_msgp_bool(rd)?;
|
||||
self.bucket_creation_commit_record_present = true;
|
||||
}
|
||||
other => {
|
||||
tracing::debug!(field = %other, "BucketMetadata decode_from: skipping unknown field");
|
||||
skip_msgp_value(rd)?;
|
||||
@@ -765,8 +764,10 @@ impl BucketMetadata {
|
||||
|
||||
/// Encode to msgp bytes. Field order follows MinIO BucketMetadata for compatibility.
|
||||
pub fn encode_to<W: Write>(&self, wr: &mut W) -> Result<()> {
|
||||
// Map size: MinIO fields (25) + RustFS extensions (22)
|
||||
let map_len: u32 = 47;
|
||||
// Preserve the absence of a legacy commit record across config rewrites.
|
||||
let commit_record_present = self.bucket_creation_commit_record_present || self.bucket_creation_committed;
|
||||
// Map size: MinIO fields (25) + RustFS extensions (21) + optional commit record.
|
||||
let map_len: u32 = 46 + u32::from(commit_record_present);
|
||||
rmp::encode::write_map_len(wr, map_len)?;
|
||||
|
||||
// MinIO field order (same as Go struct)
|
||||
@@ -847,8 +848,10 @@ impl BucketMetadata {
|
||||
write_msgp_time(wr, self.durability_config_updated_at)?;
|
||||
rmp::encode::write_str(wr, "OnDemandMigrationConfigUpdatedAt")?;
|
||||
write_msgp_time(wr, self.on_demand_migration_config_updated_at)?;
|
||||
rmp::encode::write_str(wr, "BucketCreationCommitted")?;
|
||||
rmp::encode::write_bool(wr, self.bucket_creation_committed)?;
|
||||
if commit_record_present {
|
||||
rmp::encode::write_str(wr, "BucketCreationCommitted")?;
|
||||
rmp::encode::write_bool(wr, self.bucket_creation_committed)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1018,9 +1021,11 @@ impl BucketMetadata {
|
||||
None
|
||||
} else {
|
||||
let config = deserialize::<VersioningConfiguration>(&data)?;
|
||||
if config.status.as_ref().is_some_and(|status| {
|
||||
!matches!(status.as_str(), BucketVersioningStatus::ENABLED | BucketVersioningStatus::SUSPENDED)
|
||||
}) {
|
||||
if config
|
||||
.status
|
||||
.as_ref()
|
||||
.is_some_and(|status| !matches!(status.as_str(), "Enabled" | "Suspended"))
|
||||
{
|
||||
return Err(Error::other("bucket versioning configuration has an invalid status"));
|
||||
}
|
||||
Some(config)
|
||||
@@ -1033,7 +1038,7 @@ impl BucketMetadata {
|
||||
let config = if data.is_empty() {
|
||||
None
|
||||
} else {
|
||||
let config = deserialize::<ReplicationConfiguration>(&data)?;
|
||||
let config = deserialize::<PersistedReplicationConfiguration>(&data)?;
|
||||
if let Some(field) = invalid_replication_config_status_field(&config) {
|
||||
return Err(Error::other(format!("replication field {field} has an invalid status")));
|
||||
}
|
||||
@@ -1227,8 +1232,8 @@ impl BucketMetadata {
|
||||
}
|
||||
self.lifecycle_config = None;
|
||||
if !self.lifecycle_config_xml.is_empty()
|
||||
&& let Err(e) =
|
||||
deserialize::<BucketLifecycleConfiguration>(&self.lifecycle_config_xml).map(|c| self.lifecycle_config = Some(c))
|
||||
&& let Err(e) = deserialize::<PersistedLifecycleConfiguration>(&self.lifecycle_config_xml)
|
||||
.map(|c| self.lifecycle_config = Some(c))
|
||||
{
|
||||
tracing::warn!(
|
||||
event = "bucket_metadata_parse_failed",
|
||||
@@ -1314,8 +1319,8 @@ impl BucketMetadata {
|
||||
}
|
||||
self.replication_config = None;
|
||||
if !self.replication_config_xml.is_empty()
|
||||
&& let Err(e) =
|
||||
deserialize::<ReplicationConfiguration>(&self.replication_config_xml).map(|c| self.replication_config = Some(c))
|
||||
&& let Err(e) = deserialize::<PersistedReplicationConfiguration>(&self.replication_config_xml)
|
||||
.map(|c| self.replication_config = Some(c))
|
||||
{
|
||||
tracing::warn!(
|
||||
event = "bucket_metadata_parse_failed",
|
||||
@@ -1691,6 +1696,13 @@ mod test {
|
||||
let mut bm = BucketMetadata::unmarshal(&body[4..]).expect("unmarshal recovered blob");
|
||||
assert_eq!(bm.name, "interop");
|
||||
assert!(!bm.bucket_creation_committed, "legacy metadata must default to uncommitted");
|
||||
assert!(!bm.bucket_creation_commit_record_present, "legacy metadata has no creation-commit record");
|
||||
let rewritten = BucketMetadata::unmarshal(&bm.marshal_msg().expect("rewrite legacy metadata"))
|
||||
.expect("read rewritten legacy metadata");
|
||||
assert!(
|
||||
!rewritten.bucket_creation_commit_record_present,
|
||||
"rewriting legacy metadata must not fabricate a creation intent"
|
||||
);
|
||||
bm.parse_all_configs().expect("parse recovered configs");
|
||||
assert!(bm.lifecycle_config.is_some());
|
||||
}
|
||||
@@ -1708,6 +1720,7 @@ mod test {
|
||||
|
||||
assert_eq!(bm.name, new.name);
|
||||
assert!(new.bucket_creation_committed);
|
||||
assert!(new.bucket_creation_commit_record_present);
|
||||
assert!(!bm.bucket_incarnation_id.is_nil());
|
||||
assert_eq!(bm.bucket_incarnation_id, new.bucket_incarnation_id);
|
||||
}
|
||||
@@ -1812,17 +1825,15 @@ mod test {
|
||||
|
||||
#[test]
|
||||
fn object_locking_requires_lock_metadata_not_plain_versioning() {
|
||||
use s3s::dto::ObjectLockEnabled;
|
||||
|
||||
let mut bm = BucketMetadata::new("test-bucket");
|
||||
bm.versioning_config = Some(VersioningConfiguration {
|
||||
status: Some(s3s::dto::BucketVersioningStatus::from_static("Enabled")),
|
||||
status: Some("Enabled".to_owned()),
|
||||
..Default::default()
|
||||
});
|
||||
assert!(!bm.object_locking());
|
||||
|
||||
bm.object_lock_config = Some(ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
object_lock_enabled: Some("Enabled".to_owned()),
|
||||
..Default::default()
|
||||
});
|
||||
assert!(bm.object_locking());
|
||||
|
||||
@@ -30,14 +30,18 @@ use crate::storage_api_contracts::heal::HealOperations as _;
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
use crate::store::{ECStore, await_bucket_namespace_operation};
|
||||
use futures::future::join_all;
|
||||
use rustfs_gateway_types::cors_tagging::{PersistedCorsConfiguration as CORSConfiguration, PersistedTagging as Tagging};
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedAccelerateConfiguration as AccelerateConfiguration,
|
||||
PersistedBucketEncryptionConfiguration as ServerSideEncryptionConfiguration,
|
||||
PersistedBucketLoggingStatus as BucketLoggingStatus, PersistedLifecycleConfiguration,
|
||||
PersistedNotificationConfiguration as NotificationConfiguration, PersistedObjectLockConfiguration as ObjectLockConfiguration,
|
||||
PersistedPublicAccessBlockConfiguration as PublicAccessBlockConfiguration, PersistedReplicationConfiguration,
|
||||
PersistedRequestPaymentConfiguration as RequestPaymentConfiguration,
|
||||
PersistedVersioningConfiguration as VersioningConfiguration, PersistedWebsiteConfiguration as WebsiteConfiguration,
|
||||
};
|
||||
use rustfs_heal_contracts::heal_channel::HealOpts;
|
||||
use rustfs_policy::policy::BucketPolicy;
|
||||
use s3s::dto::ReplicationConfiguration;
|
||||
use s3s::dto::{
|
||||
AccelerateConfiguration, BucketLifecycleConfiguration, BucketLoggingStatus, CORSConfiguration, NotificationConfiguration,
|
||||
ObjectLockConfiguration, ObjectLockEnabled, ObjectLockRetentionMode, PublicAccessBlockConfiguration,
|
||||
RequestPaymentConfiguration, ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration, WebsiteConfiguration,
|
||||
};
|
||||
use std::collections::HashSet;
|
||||
use std::time::Duration;
|
||||
use std::{
|
||||
@@ -168,7 +172,7 @@ pub(crate) fn object_lock_config_state_from_authoritative_metadata(bm: &BucketMe
|
||||
if bm.lock_enabled {
|
||||
return Ok(ObjectLockConfigState::Configured {
|
||||
config: ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
object_lock_enabled: Some("Enabled".to_owned()),
|
||||
rule: None,
|
||||
},
|
||||
updated_at: bm.object_lock_config_updated_at,
|
||||
@@ -212,17 +216,10 @@ pub(crate) fn configured_object_lock_state_for_tests(
|
||||
) -> ObjectLockConfigState {
|
||||
ObjectLockConfigState::Configured {
|
||||
config: ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
rule: Some(s3s::dto::ObjectLockRule {
|
||||
default_retention: Some(s3s::dto::DefaultRetention {
|
||||
mode: Some(s3s::dto::ObjectLockRetentionMode::from_static(match mode {
|
||||
crate::bucket::object_lock::types::RetentionMode::Governance => {
|
||||
s3s::dto::ObjectLockRetentionMode::GOVERNANCE
|
||||
}
|
||||
crate::bucket::object_lock::types::RetentionMode::Compliance => {
|
||||
s3s::dto::ObjectLockRetentionMode::COMPLIANCE
|
||||
}
|
||||
})),
|
||||
object_lock_enabled: Some("Enabled".to_owned()),
|
||||
rule: Some(rustfs_gateway_types::persistence::PersistedObjectLockRule {
|
||||
default_retention: Some(rustfs_gateway_types::persistence::PersistedDefaultRetention {
|
||||
mode: Some(mode.as_str().to_owned()),
|
||||
days: Some(days),
|
||||
years: None,
|
||||
}),
|
||||
@@ -233,7 +230,7 @@ pub(crate) fn configured_object_lock_state_for_tests(
|
||||
}
|
||||
|
||||
fn validate_authoritative_object_lock_config(config: &ObjectLockConfiguration) -> Result<()> {
|
||||
if config.object_lock_enabled.as_ref().map(ObjectLockEnabled::as_str) != Some(ObjectLockEnabled::ENABLED) {
|
||||
if config.object_lock_enabled.as_deref() != Some("Enabled") {
|
||||
return Err(Error::other("persisted bucket Object Lock enabled state is invalid"));
|
||||
}
|
||||
|
||||
@@ -246,7 +243,7 @@ fn validate_authoritative_object_lock_config(config: &ObjectLockConfiguration) -
|
||||
if !retention
|
||||
.mode
|
||||
.as_ref()
|
||||
.is_some_and(|mode| matches!(mode.as_str(), ObjectLockRetentionMode::COMPLIANCE | ObjectLockRetentionMode::GOVERNANCE))
|
||||
.is_some_and(|mode| matches!(mode.as_str(), "COMPLIANCE" | "GOVERNANCE"))
|
||||
{
|
||||
return Err(Error::other("persisted bucket Object Lock retention mode is invalid"));
|
||||
}
|
||||
@@ -564,12 +561,39 @@ pub(crate) async fn validate_bucket_reconstruction_in(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Validate against disk while retaining an already-held Object Lock fence.
|
||||
/// Resolve the authoritative bucket incarnation id for `opts`:
|
||||
/// 1. When namespace locking is enabled and `opts.object_lock_config_snapshot` is
|
||||
/// present, validate and return the snapshot's incarnation under its held
|
||||
/// lifecycle and metadata-transaction fences.
|
||||
/// 2. When namespace locking is enabled and `opts.bucket_lifecycle_lock_fence` is
|
||||
/// present and not lost, reuse the store's `BucketFenceRegistry` memo if a
|
||||
/// registered lifecycle read guard is currently held without lock loss.
|
||||
/// 3. Otherwise, acquire the bucket metadata transaction read lock (unless
|
||||
/// already held on `opts`) and read the incarnation from disk.
|
||||
pub(crate) async fn get_bucket_incarnation_id_for_options_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<Uuid> {
|
||||
if !ctx.lock_manager().is_disabled() {
|
||||
if let Some(snapshot) = opts.object_lock_config_snapshot.as_ref() {
|
||||
let store = object_store_in(ctx).await?;
|
||||
return snapshot
|
||||
.validated_bucket_incarnation_for(store.id, bucket, opts.expected_bucket_incarnation_id)
|
||||
.ok_or_else(|| {
|
||||
Error::other("Object Lock snapshot does not hold a valid metadata transaction fence for this bucket")
|
||||
});
|
||||
}
|
||||
if opts
|
||||
.bucket_lifecycle_lock_fence
|
||||
.as_ref()
|
||||
.is_some_and(|fence| !fence.is_lock_lost())
|
||||
&& let Some(store) = object_store_if_initialized_in(ctx).await
|
||||
&& let Some(incarnation) = store.bucket_fence_registry.validated_incarnation(bucket)
|
||||
{
|
||||
return Ok(incarnation);
|
||||
}
|
||||
}
|
||||
let sys = bucket_metadata_sys_of(ctx)?;
|
||||
let sys = sys.read().await.clone();
|
||||
let guard = acquire_bucket_metadata_transaction_read_lock_for_options_in(ctx, bucket, opts).await?;
|
||||
@@ -1368,7 +1392,7 @@ pub async fn get_public_access_block_config(bucket: &str) -> Result<(PublicAcces
|
||||
bucket_meta_sys.get_public_access_block_config(bucket).await
|
||||
}
|
||||
|
||||
pub async fn get_lifecycle_config(bucket: &str) -> Result<(BucketLifecycleConfiguration, OffsetDateTime)> {
|
||||
pub async fn get_lifecycle_config(bucket: &str) -> Result<(PersistedLifecycleConfiguration, OffsetDateTime)> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||
|
||||
@@ -1587,7 +1611,7 @@ pub(crate) async fn get_cached_quota_config_and_incarnation_in(
|
||||
bucket_meta_sys.cached_quota_config_and_incarnation(bucket).await
|
||||
}
|
||||
|
||||
pub async fn get_replication_config(bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
|
||||
pub async fn get_replication_config(bucket: &str) -> Result<(PersistedReplicationConfiguration, OffsetDateTime)> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||
|
||||
@@ -1597,7 +1621,7 @@ pub async fn get_replication_config(bucket: &str) -> Result<(ReplicationConfigur
|
||||
pub(crate) async fn get_replication_config_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
|
||||
) -> Result<(PersistedReplicationConfiguration, OffsetDateTime)> {
|
||||
let bucket_meta_sys_lock = bucket_metadata_sys_of(ctx)?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||
|
||||
@@ -1723,9 +1747,27 @@ struct MetadataPublishGuard {
|
||||
_guard: tokio::sync::OwnedMutexGuard<MetadataPublishLockState>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct PendingBucketCreationProof {
|
||||
pub(crate) generation: Uuid,
|
||||
pub(crate) intent: Vec<u8>,
|
||||
pub(crate) versioning_requested: bool,
|
||||
pub(crate) witnesses: Vec<crate::disk::os::NativeBucketCreationWitness>,
|
||||
}
|
||||
|
||||
impl PendingBucketCreationProof {
|
||||
pub(crate) fn matches_metadata(&self, metadata: &BucketMetadata) -> Result<bool> {
|
||||
Ok(!metadata.bucket_creation_committed
|
||||
&& metadata.bucket_incarnation_sidecar
|
||||
&& metadata.bucket_incarnation_id == self.generation
|
||||
&& metadata.marshal_msg()? == self.intent)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BucketMetadataSys {
|
||||
metadata_map: Arc<RwLock<HashMap<String, Arc<BucketMetadata>>>>,
|
||||
pending_bucket_creations: Arc<RwLock<HashMap<String, PendingBucketCreationProof>>>,
|
||||
/// Serializes metadata-map commits and their derived cache updates for one
|
||||
/// bucket. Namespace locks, when present, are acquired before this lock.
|
||||
metadata_publish_locks: Arc<MetadataPublishLockRegistry>,
|
||||
@@ -1769,6 +1811,7 @@ impl BucketMetadataSys {
|
||||
pub fn new(api: Arc<ECStore>) -> Self {
|
||||
Self {
|
||||
metadata_map: Arc::new(RwLock::new(HashMap::new())),
|
||||
pending_bucket_creations: Arc::new(RwLock::new(HashMap::new())),
|
||||
metadata_publish_locks: Arc::new(MetadataPublishLockRegistry {
|
||||
locks: StdMutex::new(HashMap::new()),
|
||||
}),
|
||||
@@ -1799,6 +1842,81 @@ impl BucketMetadataSys {
|
||||
.expect("bucket metadata object store should still be live")
|
||||
}
|
||||
|
||||
pub(crate) async fn pending_bucket_creation(&self, bucket: &str) -> Option<PendingBucketCreationProof> {
|
||||
self.pending_bucket_creations.read().await.get(bucket).cloned()
|
||||
}
|
||||
|
||||
pub(crate) async fn grant_pending_bucket_creation(&self, bucket: &str, proof: PendingBucketCreationProof) {
|
||||
self.pending_bucket_creations.write().await.insert(bucket.to_owned(), proof);
|
||||
}
|
||||
|
||||
pub(crate) async fn revoke_pending_bucket_creation(&self, bucket: &str) {
|
||||
self.pending_bucket_creations.write().await.remove(bucket);
|
||||
}
|
||||
|
||||
pub(crate) async fn consume_pending_bucket_creation(
|
||||
&self,
|
||||
bucket: &str,
|
||||
metadata: &BucketMetadata,
|
||||
witnesses: &[crate::disk::os::NativeBucketCreationWitness],
|
||||
) -> Result<bool> {
|
||||
let mut pending = self.pending_bucket_creations.write().await;
|
||||
let Some(proof) = pending.get(bucket) else {
|
||||
return Ok(false);
|
||||
};
|
||||
if !proof.matches_metadata(metadata)?
|
||||
|| proof.witnesses.len() != witnesses.len()
|
||||
|| !proof
|
||||
.witnesses
|
||||
.iter()
|
||||
.zip(witnesses)
|
||||
.all(|(old, current)| old.matches(current))
|
||||
|| !crate::disk::os::consume_native_creation_witnesses(witnesses)
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
pending.remove(bucket);
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn refresh_bucket_for_test(&self, bucket: &str) -> Result<()> {
|
||||
let expected = self.metadata_map.read().await.get(bucket).cloned();
|
||||
let lock = self.object_store().new_ns_lock(bucket, bucket).await?;
|
||||
let guard = lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
|
||||
self.load_bucket_under_namespace(bucket, MetadataLoadMode::Refresh, expected.as_ref(), &guard)
|
||||
.await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn publish_persisted_metadata_for_test(&self, bucket: &str) -> Result<()> {
|
||||
let expected = self.metadata_map.read().await.get(bucket).cloned();
|
||||
let lock = self.object_store().new_ns_lock(bucket, bucket).await?;
|
||||
let guard = lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
|
||||
let (metadata, persisted) = await_bucket_namespace_operation(
|
||||
Some(&guard),
|
||||
bucket,
|
||||
"test cached metadata load",
|
||||
load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true),
|
||||
)
|
||||
.await?;
|
||||
self.publish_if_unchanged(bucket, expected.as_ref(), metadata, persisted, &guard)
|
||||
.await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn migrate_bucket_creation_commit_for_test(&self, bucket: &str) -> Result<()> {
|
||||
self.migrate_bucket_creation_commit(bucket).await.map(|_| ())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn legacy_migration_reports_missing_bucket_for_test(&self, bucket: &str) -> Result<bool> {
|
||||
Ok(matches!(
|
||||
self.migrate_legacy_metadata(bucket).await?,
|
||||
BucketMetadataAuthority::MissingBucket
|
||||
))
|
||||
}
|
||||
|
||||
fn object_store_if_live(&self) -> Option<Arc<ECStore>> {
|
||||
self.api.upgrade()
|
||||
}
|
||||
@@ -1989,6 +2107,7 @@ impl BucketMetadataSys {
|
||||
expected: Option<&Arc<BucketMetadata>>,
|
||||
namespace_guard: &rustfs_lock::NamespaceLockGuard,
|
||||
) -> Result<()> {
|
||||
self.revoke_pending_bucket_creation(bucket).await;
|
||||
if !await_bucket_namespace_operation(
|
||||
Some(namespace_guard),
|
||||
bucket,
|
||||
@@ -2072,6 +2191,7 @@ impl BucketMetadataSys {
|
||||
persisted: bool,
|
||||
namespace_guard: &rustfs_lock::NamespaceLockGuard,
|
||||
) -> Result<()> {
|
||||
self.revoke_pending_bucket_creation(bucket).await;
|
||||
if !persisted {
|
||||
let _publish_guard = self
|
||||
.lock_metadata_publish(bucket, namespace_guard, "refreshed bucket metadata absence publish")
|
||||
@@ -2133,6 +2253,7 @@ impl BucketMetadataSys {
|
||||
}
|
||||
|
||||
pub async fn set(&self, bucket: String, bm: Arc<BucketMetadata>) {
|
||||
self.revoke_pending_bucket_creation(&bucket).await;
|
||||
if !is_meta_bucketname(&bucket) {
|
||||
let publish_lock = self.metadata_publish_lock(&bucket);
|
||||
let _publish_guard = publish_lock.lock().await;
|
||||
@@ -2154,6 +2275,7 @@ impl BucketMetadataSys {
|
||||
if is_meta_bucketname(bucket) {
|
||||
return false;
|
||||
}
|
||||
self.revoke_pending_bucket_creation(bucket).await;
|
||||
let publish_lock = self.metadata_publish_lock(bucket);
|
||||
let _publish_guard = publish_lock.lock().await;
|
||||
let mut map = self.metadata_map.write().await;
|
||||
@@ -2171,6 +2293,7 @@ impl BucketMetadataSys {
|
||||
}
|
||||
|
||||
async fn _reset(&mut self) {
|
||||
self.pending_bucket_creations.write().await.clear();
|
||||
let mut map = self.metadata_map.write().await;
|
||||
map.clear();
|
||||
drop(map);
|
||||
@@ -2309,6 +2432,7 @@ impl BucketMetadataSys {
|
||||
/// server's metadata never leaks into the ambient (first) instance.
|
||||
pub(crate) async fn persist_and_set(&self, bm: BucketMetadata) -> Result<()> {
|
||||
let mut bm = bm;
|
||||
self.revoke_pending_bucket_creation(&bm.name).await;
|
||||
bm.save_with_store(self.object_store()).await?;
|
||||
|
||||
self.set(bm.name.clone(), Arc::new(bm)).await;
|
||||
@@ -2317,6 +2441,7 @@ impl BucketMetadataSys {
|
||||
}
|
||||
|
||||
async fn persist_new_and_set(&self, mut bm: BucketMetadata) -> Result<()> {
|
||||
self.revoke_pending_bucket_creation(&bm.name).await;
|
||||
bm.bucket_creation_committed = true;
|
||||
bm.save_with_store_committed(self.object_store()).await?;
|
||||
save_bucket_incarnation(self.object_store(), &bm.name, bm.bucket_incarnation_id).await?;
|
||||
@@ -2326,7 +2451,9 @@ impl BucketMetadataSys {
|
||||
}
|
||||
|
||||
async fn persist_new_bucket_metadata_intent(&self, mut bm: BucketMetadata) -> Result<()> {
|
||||
self.revoke_pending_bucket_creation(&bm.name).await;
|
||||
bm.bucket_creation_committed = false;
|
||||
bm.bucket_creation_commit_record_present = true;
|
||||
bm.save_with_store(self.object_store()).await?;
|
||||
save_bucket_incarnation(self.object_store(), &bm.name, bm.bucket_incarnation_id).await?;
|
||||
bm.bucket_incarnation_sidecar = true;
|
||||
@@ -2336,6 +2463,7 @@ impl BucketMetadataSys {
|
||||
}
|
||||
|
||||
async fn commit_bucket_metadata(&self, mut bm: BucketMetadata) -> Result<()> {
|
||||
self.revoke_pending_bucket_creation(&bm.name).await;
|
||||
bm.parse_all_configs()?;
|
||||
bm.bucket_incarnation_sidecar = true;
|
||||
// The caller holds the full bucket creation fence (lifecycle, metadata
|
||||
@@ -2384,6 +2512,7 @@ impl BucketMetadataSys {
|
||||
bucket: &str,
|
||||
namespace_guard: &rustfs_lock::NamespaceLockGuard,
|
||||
) -> Result<()> {
|
||||
self.revoke_pending_bucket_creation(bucket).await;
|
||||
let expected = self.metadata_map.read().await.get(bucket).cloned();
|
||||
if !self
|
||||
.bucket_exists(bucket, namespace_guard, "peer bucket metadata existence check")
|
||||
@@ -2440,6 +2569,7 @@ impl BucketMetadataSys {
|
||||
#[cfg(test)]
|
||||
self.lazy_disk_loads.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
|
||||
self.revoke_pending_bucket_creation(bucket).await;
|
||||
let lock = self.object_store().new_ns_lock(bucket, bucket).await?;
|
||||
let guard = lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
|
||||
#[cfg(test)]
|
||||
@@ -2765,6 +2895,7 @@ impl BucketMetadataSys {
|
||||
/// revalidated at write quorum, so a stale snapshot can neither revert an
|
||||
/// acknowledged configuration update nor outlive a delete/recreate.
|
||||
async fn migrate_bucket_creation_commit(&self, bucket: &str) -> Result<BucketMetadataAuthority> {
|
||||
self.revoke_pending_bucket_creation(bucket).await;
|
||||
let transaction_lock = self
|
||||
.object_store()
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
|
||||
@@ -2842,6 +2973,7 @@ impl BucketMetadataSys {
|
||||
&self,
|
||||
bucket: &str,
|
||||
) -> Result<BucketMetadataAuthority> {
|
||||
self.revoke_pending_bucket_creation(bucket).await;
|
||||
#[cfg(test)]
|
||||
if self.object_lock_disk_read_errors.write().await.remove(bucket) {
|
||||
return Err(Error::other(format!("injected Object Lock metadata disk read failure: {bucket}")));
|
||||
@@ -3017,7 +3149,7 @@ impl BucketMetadataSys {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_lifecycle_config(&self, bucket: &str) -> Result<(BucketLifecycleConfiguration, OffsetDateTime)> {
|
||||
pub async fn get_lifecycle_config(&self, bucket: &str) -> Result<(PersistedLifecycleConfiguration, OffsetDateTime)> {
|
||||
let (bm, _) = self.get_config(bucket).await?;
|
||||
|
||||
match ConfigState::of(&bm.lifecycle_config_xml, &bm.lifecycle_config).require(bucket, BUCKET_LIFECYCLE_CONFIG)? {
|
||||
@@ -3140,7 +3272,7 @@ impl BucketMetadataSys {
|
||||
}))
|
||||
}
|
||||
|
||||
pub async fn get_replication_config(&self, bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
|
||||
pub async fn get_replication_config(&self, bucket: &str) -> Result<(PersistedReplicationConfiguration, OffsetDateTime)> {
|
||||
let (bm, _) = self.get_config(bucket).await?;
|
||||
|
||||
if !bm.replication_config_xml.is_empty() && bm.replication_config.is_none() {
|
||||
@@ -3234,6 +3366,7 @@ mod tests {
|
||||
};
|
||||
use crate::bucket::target::{BucketTarget, BucketTargetType, Credentials};
|
||||
use crate::config::com::read_config;
|
||||
use crate::disk::{DiskAPI as _, RUSTFS_META_TMP_BUCKET};
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions};
|
||||
use byteorder::{ByteOrder as _, LittleEndian};
|
||||
use serial_test::serial;
|
||||
@@ -3288,6 +3421,180 @@ mod tests {
|
||||
),
|
||||
];
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(storage_class_env)]
|
||||
async fn persistence_entry_failures_revoke_pending_creation_before_cache_publish() {
|
||||
let (dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let sys = bucket_metadata_sys_of(&store.ctx).expect("instance metadata system");
|
||||
let sys = sys.read().await.clone();
|
||||
let options = MakeBucketOptions {
|
||||
lock_enabled: true,
|
||||
..Default::default()
|
||||
};
|
||||
for entry in ["save", "persist", "persist-new", "commit"] {
|
||||
let bucket = format!("revoke-{entry}-{}", Uuid::new_v4());
|
||||
store.fail_next_bucket_creation_after_intent_for_test(&bucket);
|
||||
store
|
||||
.make_bucket(&bucket, &options)
|
||||
.await
|
||||
.expect_err("stop the real producer before physical publication");
|
||||
sys.pending_bucket_creation(&bucket)
|
||||
.await
|
||||
.expect("the producer must leave a real live creation permission");
|
||||
let (intent, persisted) = get_config_from_disk_with_presence_in(&store.ctx, &bucket)
|
||||
.await
|
||||
.expect("read the original producer intent");
|
||||
assert!(persisted, "{entry}: the complete original intent must be persisted");
|
||||
let original_intent = intent.marshal_msg().expect("encode the original intent");
|
||||
|
||||
// Keep the creation fence order used by the real producer.
|
||||
let lifecycle_guard = store
|
||||
.acquire_bucket_lifecycle_write_lock(&bucket)
|
||||
.await
|
||||
.expect("hold the bucket lifecycle fence");
|
||||
let transaction_guard = acquire_bucket_metadata_transaction_lock_in(&store.ctx, &bucket)
|
||||
.await
|
||||
.expect("hold the metadata transaction fence");
|
||||
let namespace_lock = store
|
||||
.new_ns_lock(&bucket, &bucket)
|
||||
.await
|
||||
.expect("create the exact bucket namespace lock");
|
||||
let namespace_guard = namespace_lock
|
||||
.get_write_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await
|
||||
.expect("hold the exact bucket namespace fence");
|
||||
|
||||
// A file at each native staging root causes actual IO to fail before
|
||||
// metadata can reach the cache-publishing set call. Preserve the
|
||||
// original staging directories so no fixture data is discarded.
|
||||
for dir in &dirs {
|
||||
let staging = dir.path().join(RUSTFS_META_TMP_BUCKET);
|
||||
let saved = dir.path().join(".rustfs.sys/revocation-saved-tmp");
|
||||
std::fs::rename(&staging, &saved).expect("preserve the native staging directory");
|
||||
std::fs::write(&staging, b"block metadata staging with a non-directory")
|
||||
.expect("install an actual non-directory IO obstruction");
|
||||
}
|
||||
let result = match entry {
|
||||
"save" => sys.save(intent).await,
|
||||
"persist" => sys.persist_and_set(intent).await,
|
||||
"persist-new" => sys.persist_new_and_set(intent).await,
|
||||
"commit" => sys.commit_bucket_metadata(intent).await,
|
||||
_ => unreachable!(),
|
||||
};
|
||||
for dir in &dirs {
|
||||
let staging = dir.path().join(RUSTFS_META_TMP_BUCKET);
|
||||
let saved = dir.path().join(".rustfs.sys/revocation-saved-tmp");
|
||||
std::fs::remove_file(&staging).expect("remove only the fixture's staging obstruction");
|
||||
std::fs::rename(&saved, &staging).expect("restore the complete original staging directory");
|
||||
}
|
||||
drop((namespace_guard, transaction_guard, lifecycle_guard));
|
||||
result.expect_err("native persistence must fail before its downstream cache set");
|
||||
assert!(
|
||||
sys.pending_bucket_creation(&bucket).await.is_none(),
|
||||
"{entry}: the entry revoker must retire live permission even when persistence fails"
|
||||
);
|
||||
let (after_failure, present) = get_config_from_disk_with_presence_in(&store.ctx, &bucket)
|
||||
.await
|
||||
.expect("read the intent after the failed persistence entry");
|
||||
assert!(present, "{entry}: a failed persistence entry must retain the original metadata");
|
||||
assert_eq!(
|
||||
after_failure
|
||||
.marshal_msg()
|
||||
.expect("encode the original intent after persistence failure"),
|
||||
original_intent,
|
||||
"{entry}: native IO failure must not change the original generation or complete intent"
|
||||
);
|
||||
let error = store
|
||||
.make_bucket(&bucket, &options)
|
||||
.await
|
||||
.expect_err("a failed ordinary persistence entry cannot preserve creation permission");
|
||||
assert!(error.to_string().contains("administrator recovery"), "{entry}: {error}");
|
||||
let (after_retry, _) = get_config_from_disk_with_presence_in(&store.ctx, &bucket)
|
||||
.await
|
||||
.expect("read the original intent after the rejected retry");
|
||||
assert_eq!(
|
||||
after_retry.marshal_msg().expect("encode the intent after the rejected retry"),
|
||||
original_intent,
|
||||
"{entry}: the rejected retry must preserve the complete original intent"
|
||||
);
|
||||
for disk in store.pools[0].disk_set[0].disks.read().await.iter() {
|
||||
assert!(
|
||||
matches!(
|
||||
disk.as_ref().expect("native disk").stat_volume(&bucket).await,
|
||||
Err(crate::disk::error::DiskError::VolumeNotFound)
|
||||
),
|
||||
"{entry}: failed persistence and the rejected retry must not publish a physical volume"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(storage_class_env)]
|
||||
async fn metadata_reset_revokes_real_pending_creation_without_changing_persisted_intent() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("revoke-reset-{}", Uuid::new_v4());
|
||||
let options = MakeBucketOptions {
|
||||
lock_enabled: true,
|
||||
..Default::default()
|
||||
};
|
||||
store.fail_next_bucket_creation_after_intent_for_test(&bucket);
|
||||
store
|
||||
.make_bucket(&bucket, &options)
|
||||
.await
|
||||
.expect_err("stop the real producer before physical publication");
|
||||
let sys = bucket_metadata_sys_of(&store.ctx).expect("instance metadata system");
|
||||
let mut sys = sys.read().await.clone();
|
||||
sys.pending_bucket_creation(&bucket)
|
||||
.await
|
||||
.expect("the real producer must leave live creation permission before reset");
|
||||
let (intent, persisted) = get_config_from_disk_with_presence_in(&store.ctx, &bucket)
|
||||
.await
|
||||
.expect("read the original intent before cache reset");
|
||||
assert!(persisted, "cache reset starts from a complete real persisted intent");
|
||||
let original_intent = intent.marshal_msg().expect("encode the original reset intent");
|
||||
sys._reset().await;
|
||||
assert!(
|
||||
sys.pending_bucket_creation(&bucket).await.is_none(),
|
||||
"reset must revoke the producer-issued permission shared with the instance metadata system"
|
||||
);
|
||||
let (after_reset, present) = get_config_from_disk_with_presence_in(&store.ctx, &bucket)
|
||||
.await
|
||||
.expect("read the original intent after cache reset");
|
||||
assert!(present, "cache reset must retain the persisted intent");
|
||||
assert_eq!(
|
||||
after_reset.marshal_msg().expect("encode the intent after cache reset"),
|
||||
original_intent,
|
||||
"cache reset must leave the original generation and complete persisted intent unchanged"
|
||||
);
|
||||
let error = store
|
||||
.make_bucket(&bucket, &options)
|
||||
.await
|
||||
.expect_err("reset cannot regrant the producer's old creation permission");
|
||||
assert!(error.to_string().contains("administrator recovery"), "{error}");
|
||||
let (after_retry, _) = get_config_from_disk_with_presence_in(&store.ctx, &bucket)
|
||||
.await
|
||||
.expect("read the intent after the reset-induced retry rejection");
|
||||
assert_eq!(
|
||||
after_retry
|
||||
.marshal_msg()
|
||||
.expect("encode the intent after the rejected reset retry"),
|
||||
original_intent,
|
||||
"the reset-induced rejection must preserve the original complete intent"
|
||||
);
|
||||
for disk in store.pools[0].disk_set[0].disks.read().await.iter() {
|
||||
assert!(
|
||||
matches!(
|
||||
disk.as_ref().expect("native disk").stat_volume(&bucket).await,
|
||||
Err(crate::disk::error::DiskError::VolumeNotFound)
|
||||
),
|
||||
"cache reset and the rejected retry must not publish a physical volume"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn g_d3_003_new_writer_replication_loads_without_fail_closed_state() {
|
||||
let (dirs, store) = isolated_store_over_temp_disks().await;
|
||||
@@ -3476,7 +3783,7 @@ mod tests {
|
||||
let mut versioned = BucketMetadata::new("authoritative-versioned");
|
||||
versioned.versioning_config_xml = b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec();
|
||||
versioned.versioning_config = Some(VersioningConfiguration {
|
||||
status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::ENABLED)),
|
||||
status: Some("Enabled".to_owned()),
|
||||
..Default::default()
|
||||
});
|
||||
sys.set("authoritative-versioned".to_string(), Arc::new(versioned)).await;
|
||||
@@ -3599,25 +3906,24 @@ mod tests {
|
||||
else {
|
||||
panic!("legacy lock-enabled metadata must remain configured");
|
||||
};
|
||||
assert_eq!(
|
||||
config.object_lock_enabled.as_ref().map(|value| value.as_str()),
|
||||
Some(ObjectLockEnabled::ENABLED)
|
||||
);
|
||||
assert_eq!(config.object_lock_enabled.as_deref(), Some("Enabled"));
|
||||
assert!(config.rule.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authoritative_object_lock_state_rejects_invalid_retention_periods() {
|
||||
use s3s::dto::{DefaultRetention, ObjectLockRule};
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedDefaultRetention as DefaultRetention, PersistedObjectLockRule as ObjectLockRule,
|
||||
};
|
||||
|
||||
for (days, years) in [(Some(0), None), (Some(-1), None), (Some(1), Some(1)), (None, None)] {
|
||||
let mut metadata = BucketMetadata::new("invalid-object-lock");
|
||||
metadata.object_lock_config = Some(ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
object_lock_enabled: Some("Enabled".to_owned()),
|
||||
rule: Some(ObjectLockRule {
|
||||
default_retention: Some(DefaultRetention {
|
||||
days,
|
||||
mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE)),
|
||||
mode: Some("COMPLIANCE".to_owned()),
|
||||
years,
|
||||
}),
|
||||
}),
|
||||
@@ -3632,9 +3938,11 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn authoritative_object_lock_state_rejects_invalid_enabled_mode_and_rule() {
|
||||
use s3s::dto::{DefaultRetention, ObjectLockRule};
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedDefaultRetention as DefaultRetention, PersistedObjectLockRule as ObjectLockRule,
|
||||
};
|
||||
|
||||
for enabled in [None, Some(ObjectLockEnabled::from("Disabled".to_string()))] {
|
||||
for enabled in [None, Some("Disabled".to_string())] {
|
||||
let mut metadata = BucketMetadata::new("invalid-object-lock-enabled");
|
||||
metadata.lock_enabled = true;
|
||||
metadata.object_lock_config = Some(ObjectLockConfiguration {
|
||||
@@ -3646,11 +3954,11 @@ mod tests {
|
||||
|
||||
let mut invalid_mode = BucketMetadata::new("invalid-object-lock-mode");
|
||||
invalid_mode.object_lock_config = Some(ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
object_lock_enabled: Some("Enabled".to_owned()),
|
||||
rule: Some(ObjectLockRule {
|
||||
default_retention: Some(DefaultRetention {
|
||||
days: Some(1),
|
||||
mode: Some(ObjectLockRetentionMode::from("invalid".to_string())),
|
||||
mode: Some("invalid".to_string()),
|
||||
years: None,
|
||||
}),
|
||||
}),
|
||||
@@ -3659,7 +3967,7 @@ mod tests {
|
||||
|
||||
let mut missing_retention = BucketMetadata::new("missing-object-lock-retention");
|
||||
missing_retention.object_lock_config = Some(ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
object_lock_enabled: Some("Enabled".to_owned()),
|
||||
rule: Some(ObjectLockRule { default_retention: None }),
|
||||
});
|
||||
assert!(object_lock_config_state_from_authoritative_metadata(&missing_retention).is_err());
|
||||
@@ -3669,7 +3977,7 @@ mod tests {
|
||||
fn object_lock_ever_enabled_marker_survives_config_removal() {
|
||||
let mut metadata = BucketMetadata::new("lock-config-removed");
|
||||
let config = ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
object_lock_enabled: Some("Enabled".to_owned()),
|
||||
rule: None,
|
||||
};
|
||||
metadata
|
||||
@@ -3687,10 +3995,7 @@ mod tests {
|
||||
else {
|
||||
panic!("an ever-enabled bucket must remain Object Lock configured");
|
||||
};
|
||||
assert_eq!(
|
||||
config.object_lock_enabled.as_ref().map(ObjectLockEnabled::as_str),
|
||||
Some(ObjectLockEnabled::ENABLED)
|
||||
);
|
||||
assert_eq!(config.object_lock_enabled.as_deref(), Some("Enabled"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -3706,7 +4011,7 @@ mod tests {
|
||||
|
||||
let node = Arc::new(RwLock::new(BucketMetadataSys::new(ecstore.clone())));
|
||||
let config = ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
object_lock_enabled: Some("Enabled".to_owned()),
|
||||
rule: None,
|
||||
};
|
||||
update_with_sys(
|
||||
@@ -4687,7 +4992,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn update_config_with_persists_tagging_rewrite_across_disk_reload() {
|
||||
use crate::bucket::metadata::BUCKET_TAGGING_CONFIG;
|
||||
use s3s::dto::Tag;
|
||||
use rustfs_gateway_types::cors_tagging::PersistedTag as Tag;
|
||||
|
||||
let (dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||
|
||||
@@ -4851,7 +5156,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn concurrent_update_config_with_calls_do_not_lose_writes() {
|
||||
use crate::bucket::metadata::BUCKET_TAGGING_CONFIG;
|
||||
use s3s::dto::Tag;
|
||||
use rustfs_gateway_types::cors_tagging::PersistedTag as Tag;
|
||||
|
||||
let (dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||
let sys = Arc::new(RwLock::new(BucketMetadataSys::new(ecstore)));
|
||||
@@ -4951,7 +5256,7 @@ mod tests {
|
||||
let mut persisted = BucketMetadata::new("reload-bucket");
|
||||
persisted.policy_config_json = b"persisted-marker".to_vec();
|
||||
let lock_config = ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
object_lock_enabled: Some("Enabled".to_owned()),
|
||||
rule: None,
|
||||
};
|
||||
persisted
|
||||
@@ -5230,7 +5535,7 @@ mod tests {
|
||||
#[serial]
|
||||
async fn bucket_delete_waits_for_config_mutation_fence() {
|
||||
use crate::bucket::metadata::BUCKET_TAGGING_CONFIG;
|
||||
use s3s::dto::Tag;
|
||||
use rustfs_gateway_types::cors_tagging::PersistedTag as Tag;
|
||||
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
|
||||
@@ -17,15 +17,8 @@ pub mod objectlock_sys;
|
||||
pub mod types;
|
||||
|
||||
/// Whether a bucket Object Lock configuration has locking enabled. The
|
||||
/// serving-layer `ObjectLockConfiguration` DTO implements this in
|
||||
/// the bucket-metadata module, which owns the persisted configuration type
|
||||
/// during the s3s ratchet migration (rustfs/backlog#1842).
|
||||
/// persisted `ObjectLockConfiguration` implements this in the bucket-metadata
|
||||
/// module, which owns the persisted configuration type.
|
||||
pub trait ObjectLockApi {
|
||||
fn enabled(&self) -> bool;
|
||||
}
|
||||
|
||||
/// Whether a legal-hold status value is one of the two valid wire values.
|
||||
/// Implemented for the serving-layer DTO in the bucket-metadata module.
|
||||
pub trait ObjectLockStatusExt {
|
||||
fn valid(&self) -> bool;
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub use rustfs_replication::config_cache_serde;
|
||||
pub use rustfs_replication::{
|
||||
ObjectOpts, OperatorRuleContract, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_READ_ONLY_HISTORICAL_FIELDS,
|
||||
REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS, REPLICATION_CAPABILITY_CONTRACT_VERSION,
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use s3s::dto::ReplicationConfiguration;
|
||||
use rustfs_gateway_types::persistence::PersistedReplicationConfiguration;
|
||||
|
||||
use super::replication_config_boundary::ReplicationConfigurationExt as _;
|
||||
use super::replication_filemeta_boundary::{
|
||||
@@ -42,7 +42,7 @@ impl ReplicationLifecycleBridge {
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn new_config(config: ReplicationConfiguration) -> ReplicationLifecycleConfig {
|
||||
pub(crate) fn new_config(config: PersistedReplicationConfiguration) -> ReplicationLifecycleConfig {
|
||||
ReplicationConfig::new(Some(config), None)
|
||||
}
|
||||
|
||||
@@ -118,16 +118,17 @@ impl ReplicationLifecycleBridge {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use s3s::dto::{Destination, ReplicationRule, ReplicationRuleStatus};
|
||||
use rustfs_gateway_types::dto::Status;
|
||||
use rustfs_gateway_types::persistence::{PersistedReplicationDestination, PersistedReplicationRule};
|
||||
|
||||
use super::super::replication_filemeta_boundary::{ReplicateTargetDecision, VersionPurgeStatusType};
|
||||
use super::*;
|
||||
|
||||
fn replication_rule() -> ReplicationRule {
|
||||
ReplicationRule {
|
||||
fn replication_rule() -> PersistedReplicationRule {
|
||||
PersistedReplicationRule {
|
||||
delete_marker_replication: None,
|
||||
delete_replication: None,
|
||||
destination: Destination {
|
||||
destination: PersistedReplicationDestination {
|
||||
bucket: "arn:aws:s3:::target-bucket".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
@@ -137,13 +138,13 @@ mod tests {
|
||||
prefix: Some(String::new()),
|
||||
priority: Some(1),
|
||||
source_selection_criteria: None,
|
||||
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
|
||||
status: Status::ENABLED.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn has_pending_version_purge_preserves_replication_active_rule_behavior() {
|
||||
let config = ReplicationLifecycleBridge::new_config(ReplicationConfiguration {
|
||||
let config = ReplicationLifecycleBridge::new_config(PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![replication_rule()],
|
||||
});
|
||||
|
||||
@@ -17,8 +17,8 @@ use std::sync::Arc;
|
||||
use crate::bucket::{metadata::BucketMetadata, metadata_sys};
|
||||
use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
|
||||
use crate::runtime::instance::InstanceContext;
|
||||
use rustfs_gateway_types::persistence::PersistedReplicationConfiguration;
|
||||
use rustfs_utils::path::path_join_buf;
|
||||
use s3s::dto::ReplicationConfiguration;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
use super::replication_error_boundary::{Error, Result};
|
||||
@@ -36,11 +36,11 @@ impl ReplicationMetadataStore {
|
||||
pub(crate) const FORCE_DELETE_REPLICATION_FILE: &'static str = "config/replication/force-delete.bin";
|
||||
pub(crate) const FORCE_DELETE_REPLICATION_TRANSACTION_LOCK: &'static str = "config/replication/force-delete.bin.transaction";
|
||||
|
||||
pub(crate) async fn replication_config(bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
|
||||
pub(crate) async fn replication_config(bucket: &str) -> Result<(PersistedReplicationConfiguration, OffsetDateTime)> {
|
||||
metadata_sys::get_replication_config(bucket).await
|
||||
}
|
||||
|
||||
pub(crate) async fn optional_replication_config(bucket: &str) -> Result<Option<ReplicationConfiguration>> {
|
||||
pub(crate) async fn optional_replication_config(bucket: &str) -> Result<Option<PersistedReplicationConfiguration>> {
|
||||
let config = match Self::replication_config(bucket).await {
|
||||
Ok((config, _)) => Some(config),
|
||||
Err(err) => {
|
||||
|
||||
@@ -16,7 +16,9 @@ use std::{collections::HashMap, fmt, sync::Arc};
|
||||
|
||||
use super::replication_filemeta_boundary::metadata_keys;
|
||||
use crate::bucket::metadata::BucketMetadata;
|
||||
use s3s::dto::{BucketVersioningStatus, ReplicationConfiguration, ReplicationRuleStatus, VersioningConfiguration};
|
||||
use rustfs_gateway_types::dto::Status;
|
||||
use rustfs_gateway_types::persistence::PersistedReplicationConfiguration;
|
||||
use rustfs_gateway_types::persistence::PersistedVersioningConfiguration as VersioningConfiguration;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use time::OffsetDateTime;
|
||||
use tracing::error;
|
||||
@@ -39,7 +41,7 @@ use super::replication_target_boundary::{BucketTargets, ReplicationTargetStore};
|
||||
use super::replication_versioning_boundary::ReplicationVersioningStore;
|
||||
use super::runtime_boundary as runtime_sources;
|
||||
|
||||
pub(crate) async fn get_replication_config(bucket: &str) -> Result<Option<ReplicationConfiguration>> {
|
||||
pub(crate) async fn get_replication_config(bucket: &str) -> Result<Option<PersistedReplicationConfiguration>> {
|
||||
let config = ReplicationMetadataStore::optional_replication_config(bucket).await?;
|
||||
validate_delete_replication_config(&VersioningConfiguration::default(), config.as_ref())?;
|
||||
Ok(config)
|
||||
@@ -64,7 +66,7 @@ impl DeleteReplicationConfigSnapshot {
|
||||
#[cfg(test)]
|
||||
pub(crate) fn from_configs_for_test(
|
||||
versioning: VersioningConfiguration,
|
||||
replication: Option<ReplicationConfiguration>,
|
||||
replication: Option<PersistedReplicationConfiguration>,
|
||||
) -> Self {
|
||||
let metadata = replication.map(|config| {
|
||||
let mut metadata = BucketMetadata::new("test-bucket");
|
||||
@@ -78,7 +80,7 @@ impl DeleteReplicationConfigSnapshot {
|
||||
&self.versioning
|
||||
}
|
||||
|
||||
pub fn replication_config(&self) -> Option<&ReplicationConfiguration> {
|
||||
pub fn replication_config(&self) -> Option<&PersistedReplicationConfiguration> {
|
||||
self.metadata
|
||||
.as_ref()
|
||||
.and_then(|metadata| metadata.replication_config.as_ref())
|
||||
@@ -101,7 +103,7 @@ impl DeleteReplicationConfigSnapshot {
|
||||
pub(crate) fn active_delete_marker_rules_require_tags(&self, object: &str) -> bool {
|
||||
self.replication_config().is_some_and(|config| {
|
||||
config.rules.iter().any(|rule| {
|
||||
if rule.status == ReplicationRuleStatus::from_static(ReplicationRuleStatus::DISABLED) {
|
||||
if rule.status == Status::DISABLED.as_str() {
|
||||
return false;
|
||||
}
|
||||
if !object.starts_with(rule.prefix()) {
|
||||
@@ -122,12 +124,12 @@ impl DeleteReplicationConfigSnapshot {
|
||||
|
||||
fn validate_delete_replication_config(
|
||||
versioning: &VersioningConfiguration,
|
||||
config: Option<&ReplicationConfiguration>,
|
||||
config: Option<&PersistedReplicationConfiguration>,
|
||||
) -> Result<()> {
|
||||
if versioning
|
||||
.status
|
||||
.as_ref()
|
||||
.is_some_and(|status| !matches!(status.as_str(), BucketVersioningStatus::ENABLED | BucketVersioningStatus::SUSPENDED))
|
||||
.is_some_and(|status| !matches!(status.as_str(), "Enabled" | "Suspended"))
|
||||
{
|
||||
return Err(super::replication_error_boundary::Error::other(
|
||||
"bucket versioning configuration has an invalid status",
|
||||
@@ -144,7 +146,7 @@ fn validate_delete_replication_config(
|
||||
let role = config.role.trim();
|
||||
let mut role_destination = None;
|
||||
for rule in &config.rules {
|
||||
if rule.status.as_str() == ReplicationRuleStatus::ENABLED {
|
||||
if rule.status.as_str() == Status::ENABLED.as_str() {
|
||||
let destination = rule.destination.bucket.trim();
|
||||
if role.is_empty() && destination.is_empty() {
|
||||
return Err(super::replication_error_boundary::Error::other(
|
||||
@@ -169,7 +171,7 @@ fn validate_delete_replication_config(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn replication_config_from_metadata(metadata: &BucketMetadata) -> Result<Option<&ReplicationConfiguration>> {
|
||||
fn replication_config_from_metadata(metadata: &BucketMetadata) -> Result<Option<&PersistedReplicationConfiguration>> {
|
||||
if !metadata.replication_config_xml.is_empty() && metadata.replication_config.is_none() {
|
||||
return Err(super::replication_error_boundary::Error::other(
|
||||
"persisted bucket replication configuration is invalid",
|
||||
@@ -250,12 +252,15 @@ pub(crate) async fn load_delete_replication_config_in(
|
||||
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub struct ReplicationConfig {
|
||||
pub config: Option<ReplicationConfiguration>,
|
||||
/// Carried in the scanner's persisted data usage cache, so its serde form
|
||||
/// is the one the s3s DTO wrote there (see `config_cache_serde` in the replication crate).
|
||||
#[serde(with = "super::replication_config_boundary::config_cache_serde")]
|
||||
pub config: Option<PersistedReplicationConfiguration>,
|
||||
pub remotes: Option<BucketTargets>,
|
||||
}
|
||||
|
||||
impl ReplicationConfig {
|
||||
pub fn new(config: Option<ReplicationConfiguration>, remotes: Option<BucketTargets>) -> Self {
|
||||
pub fn new(config: Option<PersistedReplicationConfiguration>, remotes: Option<BucketTargets>) -> Self {
|
||||
Self { config, remotes }
|
||||
}
|
||||
|
||||
@@ -468,7 +473,7 @@ fn check_replicate_delete_with_config(
|
||||
oi: &ObjectInfo,
|
||||
del_opts: &ObjectOptions,
|
||||
source_error: bool,
|
||||
config: Option<&ReplicationConfiguration>,
|
||||
config: Option<&PersistedReplicationConfiguration>,
|
||||
existing_delete_marker: bool,
|
||||
trust_persisted_replica_status: bool,
|
||||
) -> ReplicateDecision {
|
||||
@@ -587,7 +592,7 @@ pub(crate) async fn must_replicate(bucket: &str, object: &str, mopts: MustReplic
|
||||
}
|
||||
|
||||
fn metadata_target_should_replicate(
|
||||
cfg: &ReplicationConfiguration,
|
||||
cfg: &PersistedReplicationConfiguration,
|
||||
opts: &ObjectOpts,
|
||||
mopts: &MustReplicateOptions,
|
||||
arn: &str,
|
||||
@@ -597,20 +602,21 @@ fn metadata_target_should_replicate(
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use s3s::dto::{
|
||||
DeleteMarkerReplication, DeleteMarkerReplicationStatus, DeleteReplication, DeleteReplicationStatus, Destination,
|
||||
ReplicaModifications, ReplicationRule, ReplicationRuleFilter, ReplicationRuleStatus, SourceSelectionCriteria, Tag,
|
||||
use rustfs_gateway_types::dto::Status;
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedOptionalReplicationStatus, PersistedReplicationDestination, PersistedReplicationFilter,
|
||||
PersistedReplicationRule, PersistedReplicationStatus, PersistedReplicationTag, PersistedSourceSelectionCriteria,
|
||||
};
|
||||
|
||||
use super::super::replication_filemeta_boundary::VersionPurgeStatusType;
|
||||
use super::super::replication_target_boundary::BucketTarget;
|
||||
use super::*;
|
||||
|
||||
fn replication_rule() -> ReplicationRule {
|
||||
ReplicationRule {
|
||||
fn replication_rule() -> PersistedReplicationRule {
|
||||
PersistedReplicationRule {
|
||||
delete_marker_replication: None,
|
||||
delete_replication: None,
|
||||
destination: Destination {
|
||||
destination: PersistedReplicationDestination {
|
||||
bucket: "arn:aws:s3:::target-bucket".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
@@ -620,7 +626,7 @@ mod tests {
|
||||
prefix: Some(String::new()),
|
||||
priority: Some(1),
|
||||
source_selection_criteria: None,
|
||||
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
|
||||
status: Status::ENABLED.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -629,11 +635,11 @@ mod tests {
|
||||
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
|
||||
let mut rule = replication_rule();
|
||||
rule.destination.bucket = arn.to_string();
|
||||
rule.filter = Some(ReplicationRuleFilter {
|
||||
rule.filter = Some(PersistedReplicationFilter {
|
||||
prefix: Some("admitted/".to_string()),
|
||||
..Default::default()
|
||||
});
|
||||
let cfg = ReplicationConfiguration {
|
||||
let cfg = PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![rule],
|
||||
};
|
||||
@@ -671,7 +677,7 @@ mod tests {
|
||||
assert!(!empty.replicate(&ObjectOpts::default()));
|
||||
|
||||
let config = ReplicationConfig::new(
|
||||
Some(ReplicationConfiguration {
|
||||
Some(PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![replication_rule()],
|
||||
}),
|
||||
@@ -725,7 +731,7 @@ mod tests {
|
||||
fn delete_snapshot_rejects_enabled_rules_without_a_destination() {
|
||||
let mut rule = replication_rule();
|
||||
rule.destination.bucket.clear();
|
||||
let config = ReplicationConfiguration {
|
||||
let config = PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![rule],
|
||||
};
|
||||
@@ -741,7 +747,7 @@ mod tests {
|
||||
let first = replication_rule();
|
||||
let mut second = replication_rule();
|
||||
second.destination.bucket = "arn:aws:s3:::other-target".to_string();
|
||||
let config = ReplicationConfiguration {
|
||||
let config = PersistedReplicationConfiguration {
|
||||
role: "arn:aws:s3:::role-target".to_string(),
|
||||
rules: vec![first, second],
|
||||
};
|
||||
@@ -752,47 +758,47 @@ mod tests {
|
||||
#[test]
|
||||
fn delete_snapshot_rejects_unknown_status_values() {
|
||||
let invalid_versioning = VersioningConfiguration {
|
||||
status: Some("Enabld".to_string().into()),
|
||||
status: Some("Enabld".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(validate_delete_replication_config(&invalid_versioning, None).is_err());
|
||||
|
||||
let mut invalid_rule = replication_rule();
|
||||
invalid_rule.status = "Enabld".to_string().into();
|
||||
let config = ReplicationConfiguration {
|
||||
invalid_rule.status = "Enabld".to_string();
|
||||
let config = PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![invalid_rule],
|
||||
};
|
||||
assert!(validate_delete_replication_config(&VersioningConfiguration::default(), Some(&config)).is_err());
|
||||
|
||||
let mut invalid_delete = replication_rule();
|
||||
invalid_delete.delete_replication = Some(DeleteReplication {
|
||||
status: "Enabld".to_string().into(),
|
||||
invalid_delete.delete_replication = Some(PersistedReplicationStatus {
|
||||
status: "Enabld".to_string(),
|
||||
});
|
||||
let config = ReplicationConfiguration {
|
||||
let config = PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![invalid_delete],
|
||||
};
|
||||
assert!(validate_delete_replication_config(&VersioningConfiguration::default(), Some(&config)).is_err());
|
||||
|
||||
let mut invalid_delete_marker = replication_rule();
|
||||
invalid_delete_marker.delete_marker_replication = Some(DeleteMarkerReplication {
|
||||
status: Some("Enabld".to_string().into()),
|
||||
invalid_delete_marker.delete_marker_replication = Some(PersistedOptionalReplicationStatus {
|
||||
status: Some("Enabld".to_string()),
|
||||
});
|
||||
let config = ReplicationConfiguration {
|
||||
let config = PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![invalid_delete_marker],
|
||||
};
|
||||
assert!(validate_delete_replication_config(&VersioningConfiguration::default(), Some(&config)).is_err());
|
||||
|
||||
let mut invalid_replica_modifications = replication_rule();
|
||||
invalid_replica_modifications.source_selection_criteria = Some(SourceSelectionCriteria {
|
||||
replica_modifications: Some(ReplicaModifications {
|
||||
status: "Enabld".to_string().into(),
|
||||
invalid_replica_modifications.source_selection_criteria = Some(PersistedSourceSelectionCriteria {
|
||||
replica_modifications: Some(PersistedReplicationStatus {
|
||||
status: "Enabld".to_string(),
|
||||
}),
|
||||
sse_kms_encrypted_objects: None,
|
||||
});
|
||||
let config = ReplicationConfiguration {
|
||||
let config = PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![invalid_replica_modifications],
|
||||
};
|
||||
@@ -804,11 +810,11 @@ mod tests {
|
||||
let mut metadata = BucketMetadata::new("bucket");
|
||||
metadata.versioning_config_xml = b"configured".to_vec();
|
||||
metadata.versioning_config = Some(VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
status: Some("Enabled".to_owned()),
|
||||
..Default::default()
|
||||
});
|
||||
metadata.replication_config_xml = b"configured".to_vec();
|
||||
metadata.replication_config = Some(ReplicationConfiguration {
|
||||
metadata.replication_config = Some(PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![replication_rule()],
|
||||
});
|
||||
@@ -848,7 +854,7 @@ mod tests {
|
||||
);
|
||||
|
||||
let mut inconsistent = BucketMetadata::new("bucket");
|
||||
inconsistent.replication_config = Some(ReplicationConfiguration {
|
||||
inconsistent.replication_config = Some(PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![replication_rule()],
|
||||
});
|
||||
@@ -863,11 +869,11 @@ mod tests {
|
||||
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
|
||||
let mut rule = replication_rule();
|
||||
rule.destination.bucket = arn.to_string();
|
||||
rule.delete_marker_replication = Some(DeleteMarkerReplication {
|
||||
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)),
|
||||
rule.delete_marker_replication = Some(PersistedOptionalReplicationStatus {
|
||||
status: Some(Status::ENABLED.to_string()),
|
||||
});
|
||||
let mut metadata = BucketMetadata::new("bucket");
|
||||
metadata.replication_config = Some(ReplicationConfiguration {
|
||||
metadata.replication_config = Some(PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![rule],
|
||||
});
|
||||
@@ -898,27 +904,27 @@ mod tests {
|
||||
fn delete_marker_source_read_is_required_only_for_tag_filtered_rules() {
|
||||
let mut prefix_rule = replication_rule();
|
||||
prefix_rule.prefix = Some("logs/".to_string());
|
||||
prefix_rule.delete_marker_replication = Some(DeleteMarkerReplication {
|
||||
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)),
|
||||
prefix_rule.delete_marker_replication = Some(PersistedOptionalReplicationStatus {
|
||||
status: Some(Status::ENABLED.to_string()),
|
||||
});
|
||||
|
||||
let prefix_snapshot = DeleteReplicationConfigSnapshot::from_configs_for_test(
|
||||
VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
status: Some("Enabled".to_owned()),
|
||||
..Default::default()
|
||||
},
|
||||
Some(ReplicationConfiguration {
|
||||
Some(PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![prefix_rule],
|
||||
}),
|
||||
);
|
||||
|
||||
let mut tag_rule = replication_rule();
|
||||
tag_rule.delete_marker_replication = Some(DeleteMarkerReplication {
|
||||
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)),
|
||||
tag_rule.delete_marker_replication = Some(PersistedOptionalReplicationStatus {
|
||||
status: Some(Status::ENABLED.to_string()),
|
||||
});
|
||||
tag_rule.filter = Some(ReplicationRuleFilter {
|
||||
tag: Some(Tag {
|
||||
tag_rule.filter = Some(PersistedReplicationFilter {
|
||||
tag: Some(PersistedReplicationTag {
|
||||
key: Some("class".to_string()),
|
||||
value: Some("audit".to_string()),
|
||||
}),
|
||||
@@ -926,10 +932,10 @@ mod tests {
|
||||
});
|
||||
let tag_snapshot = DeleteReplicationConfigSnapshot::from_configs_for_test(
|
||||
VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
status: Some("Enabled".to_owned()),
|
||||
..Default::default()
|
||||
},
|
||||
Some(ReplicationConfiguration {
|
||||
Some(PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![tag_rule],
|
||||
}),
|
||||
@@ -944,14 +950,14 @@ mod tests {
|
||||
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
|
||||
let mut rule = replication_rule();
|
||||
rule.destination.bucket = arn.to_string();
|
||||
rule.delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
rule.delete_replication = Some(PersistedReplicationStatus {
|
||||
status: Status::ENABLED.to_string(),
|
||||
});
|
||||
rule.delete_marker_replication = Some(DeleteMarkerReplication {
|
||||
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::DISABLED)),
|
||||
rule.delete_marker_replication = Some(PersistedOptionalReplicationStatus {
|
||||
status: Some(Status::DISABLED.to_string()),
|
||||
});
|
||||
let config = ReplicationConfig::new(
|
||||
Some(ReplicationConfiguration {
|
||||
Some(PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![rule],
|
||||
}),
|
||||
@@ -985,22 +991,22 @@ mod tests {
|
||||
#[test]
|
||||
fn live_delete_does_not_trust_persisted_replica_status() {
|
||||
let mut rule = replication_rule();
|
||||
rule.delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
rule.delete_replication = Some(PersistedReplicationStatus {
|
||||
status: Status::ENABLED.to_string(),
|
||||
});
|
||||
rule.source_selection_criteria = Some(SourceSelectionCriteria {
|
||||
replica_modifications: Some(ReplicaModifications {
|
||||
status: s3s::dto::ReplicaModificationsStatus::from_static(s3s::dto::ReplicaModificationsStatus::DISABLED),
|
||||
rule.source_selection_criteria = Some(PersistedSourceSelectionCriteria {
|
||||
replica_modifications: Some(PersistedReplicationStatus {
|
||||
status: Status::DISABLED.to_string(),
|
||||
}),
|
||||
sse_kms_encrypted_objects: None,
|
||||
});
|
||||
let replication = ReplicationConfiguration {
|
||||
let replication = PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![rule],
|
||||
};
|
||||
let snapshot = DeleteReplicationConfigSnapshot::from_configs_for_test(
|
||||
VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
status: Some("Enabled".to_owned()),
|
||||
..Default::default()
|
||||
},
|
||||
Some(replication.clone()),
|
||||
@@ -1036,14 +1042,14 @@ mod tests {
|
||||
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
|
||||
let mut rule = replication_rule();
|
||||
rule.destination.bucket = arn.to_string();
|
||||
rule.delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
rule.delete_replication = Some(PersistedReplicationStatus {
|
||||
status: Status::ENABLED.to_string(),
|
||||
});
|
||||
rule.delete_marker_replication = Some(DeleteMarkerReplication {
|
||||
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::DISABLED)),
|
||||
rule.delete_marker_replication = Some(PersistedOptionalReplicationStatus {
|
||||
status: Some(Status::DISABLED.to_string()),
|
||||
});
|
||||
let config = ReplicationConfig::new(
|
||||
Some(ReplicationConfiguration {
|
||||
Some(PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![rule],
|
||||
}),
|
||||
|
||||
@@ -4794,9 +4794,10 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn heal_queue_marks_missing_versioning_state_as_missed() {
|
||||
use super::super::replication_target_boundary::BucketTargets;
|
||||
use s3s::dto::{
|
||||
DeleteReplication, DeleteReplicationStatus, Destination, ReplicationConfiguration, ReplicationRule,
|
||||
ReplicationRuleStatus,
|
||||
use rustfs_gateway_types::dto::Status;
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedReplicationConfiguration, PersistedReplicationDestination, PersistedReplicationRule,
|
||||
PersistedReplicationStatus,
|
||||
};
|
||||
|
||||
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
|
||||
@@ -4811,14 +4812,14 @@ mod tests {
|
||||
..Default::default()
|
||||
},
|
||||
ReplicationConfig::new(
|
||||
Some(ReplicationConfiguration {
|
||||
Some(PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![ReplicationRule {
|
||||
rules: vec![PersistedReplicationRule {
|
||||
delete_marker_replication: None,
|
||||
delete_replication: Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
delete_replication: Some(PersistedReplicationStatus {
|
||||
status: Status::ENABLED.to_string(),
|
||||
}),
|
||||
destination: Destination {
|
||||
destination: PersistedReplicationDestination {
|
||||
bucket: arn.to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
@@ -4828,7 +4829,7 @@ mod tests {
|
||||
prefix: Some(String::new()),
|
||||
priority: Some(1),
|
||||
source_selection_criteria: None,
|
||||
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
|
||||
status: Status::ENABLED.to_string(),
|
||||
}],
|
||||
}),
|
||||
Some(BucketTargets::default()),
|
||||
|
||||
@@ -85,6 +85,8 @@ use http_body_util::StreamBody;
|
||||
use metrics::counter;
|
||||
#[cfg(test)]
|
||||
use rmp_serde;
|
||||
#[cfg(test)]
|
||||
use rustfs_gateway_types::persistence::PersistedReplicationConfiguration;
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_utils::http::{
|
||||
AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_MODE, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_TAGGING_DIRECTIVE,
|
||||
@@ -92,8 +94,6 @@ use rustfs_utils::http::{
|
||||
insert_str, replication_target_versions,
|
||||
};
|
||||
use rustfs_utils::{DEFAULT_SIP_HASH_KEY, get_env_usize, sip_hash};
|
||||
#[cfg(test)]
|
||||
use s3s::dto::ReplicationConfiguration;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::fmt::Display;
|
||||
use std::pin::Pin;
|
||||
@@ -6500,9 +6500,10 @@ mod tests {
|
||||
|
||||
use super::super::replication_target_boundary::{BucketTarget, BucketTargets};
|
||||
use super::*;
|
||||
use s3s::dto::{
|
||||
BucketVersioningStatus, DeleteReplication, DeleteReplicationStatus, Destination, ExcludedPrefix, ReplicationRule,
|
||||
ReplicationRuleStatus, VersioningConfiguration,
|
||||
use rustfs_gateway_types::dto::Status;
|
||||
use rustfs_gateway_types::persistence::PersistedVersioningConfiguration as VersioningConfiguration;
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedReplicationDestination, PersistedReplicationRule, PersistedReplicationStatus,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use time::OffsetDateTime;
|
||||
@@ -8568,7 +8569,7 @@ mod tests {
|
||||
..Default::default()
|
||||
};
|
||||
let rcfg = ReplicationConfig::new(
|
||||
Some(ReplicationConfiguration {
|
||||
Some(PersistedReplicationConfiguration {
|
||||
role: role.to_string(),
|
||||
rules: vec![],
|
||||
}),
|
||||
@@ -8594,7 +8595,7 @@ mod tests {
|
||||
ReplicationVersioningStore::install_prefix_state_test_config(
|
||||
&bucket,
|
||||
VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
status: Some("Enabled".to_owned()),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
@@ -8627,22 +8628,20 @@ mod tests {
|
||||
ReplicationVersioningStore::install_prefix_state_test_config(
|
||||
&bucket,
|
||||
VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
excluded_prefixes: Some(vec![ExcludedPrefix {
|
||||
prefix: Some("archive/".to_string()),
|
||||
}]),
|
||||
status: Some("Enabled".to_owned()),
|
||||
excluded_prefixes: Some(vec![Some("archive/".to_string())]),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let rcfg = ReplicationConfig::new(
|
||||
Some(ReplicationConfiguration {
|
||||
Some(PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![ReplicationRule {
|
||||
rules: vec![PersistedReplicationRule {
|
||||
delete_marker_replication: None,
|
||||
delete_replication: Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
delete_replication: Some(PersistedReplicationStatus {
|
||||
status: Status::ENABLED.to_string(),
|
||||
}),
|
||||
destination: Destination {
|
||||
destination: PersistedReplicationDestination {
|
||||
bucket: arn.to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
@@ -8652,7 +8651,7 @@ mod tests {
|
||||
prefix: Some(String::new()),
|
||||
priority: Some(1),
|
||||
source_selection_criteria: None,
|
||||
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
|
||||
status: Status::ENABLED.to_string(),
|
||||
}],
|
||||
}),
|
||||
Some(BucketTargets {
|
||||
@@ -8688,14 +8687,14 @@ mod tests {
|
||||
let admitted_arn = "arn:rustfs:replication:us-east-1:target:admitted";
|
||||
let current_role = "arn:rustfs:replication:us-east-1:target:current";
|
||||
let rcfg = ReplicationConfig::new(
|
||||
Some(ReplicationConfiguration {
|
||||
Some(PersistedReplicationConfiguration {
|
||||
role: current_role.to_string(),
|
||||
rules: vec![ReplicationRule {
|
||||
rules: vec![PersistedReplicationRule {
|
||||
delete_marker_replication: None,
|
||||
delete_replication: Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::DISABLED),
|
||||
delete_replication: Some(PersistedReplicationStatus {
|
||||
status: Status::DISABLED.to_string(),
|
||||
}),
|
||||
destination: Destination {
|
||||
destination: PersistedReplicationDestination {
|
||||
bucket: current_role.to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
@@ -8705,7 +8704,7 @@ mod tests {
|
||||
prefix: Some(String::new()),
|
||||
priority: Some(1),
|
||||
source_selection_criteria: None,
|
||||
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
|
||||
status: Status::ENABLED.to_string(),
|
||||
}],
|
||||
}),
|
||||
Some(BucketTargets {
|
||||
@@ -8753,12 +8752,12 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn heal_rejects_semantically_invalid_replication_config() {
|
||||
let rcfg = ReplicationConfig::new(
|
||||
Some(ReplicationConfiguration {
|
||||
Some(PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![ReplicationRule {
|
||||
rules: vec![PersistedReplicationRule {
|
||||
delete_marker_replication: None,
|
||||
delete_replication: None,
|
||||
destination: Destination {
|
||||
destination: PersistedReplicationDestination {
|
||||
bucket: "arn:rustfs:replication:us-east-1:target:bucket".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
@@ -8768,7 +8767,7 @@ mod tests {
|
||||
prefix: Some(String::new()),
|
||||
priority: Some(1),
|
||||
source_selection_criteria: None,
|
||||
status: ReplicationRuleStatus::from_static("Enabld"),
|
||||
status: "Enabld".to_owned(),
|
||||
}],
|
||||
}),
|
||||
Some(BucketTargets::default()),
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use s3s::dto::ReplicationConfiguration;
|
||||
use rustfs_gateway_types::persistence::PersistedReplicationConfiguration;
|
||||
|
||||
use super::replication_config_boundary::{ObjectOpts, ReplicationConfigurationExt};
|
||||
use super::replication_filemeta_boundary::ReplicationType;
|
||||
@@ -20,7 +20,7 @@ use super::replication_filemeta_boundary::ReplicationType;
|
||||
pub(crate) struct ReplicationTargetConfigBridge;
|
||||
|
||||
impl ReplicationTargetConfigBridge {
|
||||
pub(crate) fn target_is_used_by_rules(config: &ReplicationConfiguration, arn: &str) -> bool {
|
||||
pub(crate) fn target_is_used_by_rules(config: &PersistedReplicationConfiguration, arn: &str) -> bool {
|
||||
config
|
||||
.filter_target_arns(&ObjectOpts {
|
||||
op_type: ReplicationType::All,
|
||||
@@ -37,7 +37,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn target_config_bridge_matches_role_target() {
|
||||
let config = ReplicationConfiguration {
|
||||
let config = PersistedReplicationConfiguration {
|
||||
role: " arn:target ".to_string(),
|
||||
rules: Vec::new(),
|
||||
};
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
use super::replication_error_boundary::Result;
|
||||
use crate::bucket::{versioning::VersioningApi as _, versioning_sys::BucketVersioningSys};
|
||||
#[cfg(test)]
|
||||
use s3s::dto::VersioningConfiguration;
|
||||
use rustfs_gateway_types::persistence::PersistedVersioningConfiguration as VersioningConfiguration;
|
||||
#[cfg(test)]
|
||||
use std::{collections::HashMap, sync::LazyLock, sync::Mutex};
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use s3s::dto::Tag;
|
||||
use rustfs_gateway_types::cors_tagging::PersistedTag as Tag;
|
||||
use std::cmp::Ordering;
|
||||
use std::collections::HashMap;
|
||||
use url::form_urlencoded;
|
||||
|
||||
@@ -15,9 +15,23 @@
|
||||
use crate::disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
|
||||
use crate::error::{Error, Result, StorageError};
|
||||
use regex::Regex;
|
||||
use rustfs_gateway_types::cors_tagging::{
|
||||
CorsTaggingCodecError, PersistedCorsConfiguration, PersistedTagging, parse_cors, parse_tagging, serialize_cors,
|
||||
serialize_tagging,
|
||||
};
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedAccelerateConfiguration, PersistedBucketEncryptionConfiguration, PersistedBucketLoggingStatus,
|
||||
PersistedLifecycleConfiguration, PersistedNotificationConfiguration, PersistedObjectLockConfiguration,
|
||||
PersistedPublicAccessBlockConfiguration, PersistedReplicationConfiguration, PersistedRequestPaymentConfiguration,
|
||||
PersistedVersioningConfiguration, PersistedWebsiteConfiguration, PersistenceCodecError, parse_accelerate,
|
||||
parse_bucket_encryption, parse_bucket_logging, parse_lifecycle_legacy_dates, parse_notification, parse_object_lock,
|
||||
parse_public_access_block, parse_replication, parse_request_payment, parse_versioning, parse_website, serialize_accelerate,
|
||||
serialize_bucket_encryption, serialize_bucket_logging, serialize_lifecycle_legacy_dates, serialize_notification,
|
||||
serialize_object_lock, serialize_public_access_block, serialize_replication, serialize_request_payment, serialize_versioning,
|
||||
serialize_website,
|
||||
};
|
||||
use rustfs_utils::WindowsFileNameError;
|
||||
use rustfs_utils::path::SLASH_SEPARATOR;
|
||||
use s3s::xml;
|
||||
use tracing::instrument;
|
||||
|
||||
pub fn is_meta_bucketname(name: &str) -> bool {
|
||||
@@ -74,37 +88,173 @@ pub fn check_valid_bucket_name_strict(bucket_name: &str) -> Result<()> {
|
||||
check_bucket_name_common(bucket_name, true)
|
||||
}
|
||||
|
||||
// RUSTFS_COMPAT_TODO(s3gate-metadata-xml): the s3s codec reads persisted XML during migration. Remove after every supported writer uses the gateway codec and every retained metadata object and backup archive is verified or rewritten.
|
||||
pub fn deserialize<T>(input: &[u8]) -> xml::DeResult<T>
|
||||
where
|
||||
T: for<'xml> xml::Deserialize<'xml>,
|
||||
{
|
||||
let mut d = xml::Deserializer::new(input);
|
||||
let ans = T::deserialize(&mut d)?;
|
||||
d.expect_eof()?;
|
||||
Ok(ans)
|
||||
/// A bucket configuration persisted as XML in the bucket metadata, together
|
||||
/// with the codec that reads and writes its stored bytes.
|
||||
///
|
||||
/// Every family reads and writes its stored bytes through the gateway
|
||||
/// persistence codecs (`rustfs_gateway_types::persistence` and
|
||||
/// `rustfs_gateway_types::cors_tagging`, rustfs/backlog#2745). The codecs keep
|
||||
/// the persisted bytes the `bucket_config_goldens` pin, so a document the old
|
||||
/// s3s codec wrote reads back and re-encodes unchanged.
|
||||
pub trait BucketConfigXml: Sized {
|
||||
/// Decodes the stored bytes of one configuration document.
|
||||
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError>;
|
||||
|
||||
/// Encodes the value as RustFS stores it.
|
||||
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError>;
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "xml serialize helper with no caller in this port; the live sibling is deserialize (backlog#1823)"
|
||||
)]
|
||||
pub fn serialize_content<T: xml::SerializeContent>(val: &T) -> xml::SerResult<String> {
|
||||
let mut buf = Vec::with_capacity(256);
|
||||
{
|
||||
let mut ser = xml::Serializer::new(&mut buf);
|
||||
val.serialize_content(&mut ser)?;
|
||||
}
|
||||
Ok(String::from_utf8(buf).unwrap())
|
||||
/// Why a persisted bucket configuration could not be decoded or encoded.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum BucketConfigXmlError {
|
||||
/// A gateway persistence codec refused the bytes or the value.
|
||||
#[error(transparent)]
|
||||
Persisted(#[from] PersistenceCodecError),
|
||||
/// The gateway CORS or Tagging codec refused the bytes or the value.
|
||||
#[error(transparent)]
|
||||
CorsTagging(#[from] CorsTaggingCodecError),
|
||||
}
|
||||
|
||||
pub fn serialize<T: xml::Serialize>(val: &T) -> xml::SerResult<Vec<u8>> {
|
||||
let mut buf = Vec::with_capacity(256);
|
||||
{
|
||||
let mut ser = xml::Serializer::new(&mut buf);
|
||||
val.serialize(&mut ser)?;
|
||||
/// Decodes one persisted bucket configuration document.
|
||||
pub fn deserialize<T: BucketConfigXml>(input: &[u8]) -> std::result::Result<T, BucketConfigXmlError> {
|
||||
T::decode_config_xml(input)
|
||||
}
|
||||
|
||||
/// Encodes one bucket configuration as RustFS persists it.
|
||||
pub fn serialize<T: BucketConfigXml>(val: &T) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
|
||||
val.encode_config_xml()
|
||||
}
|
||||
|
||||
impl BucketConfigXml for PersistedReplicationConfiguration {
|
||||
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
|
||||
Ok(parse_replication(input)?)
|
||||
}
|
||||
|
||||
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
|
||||
Ok(serialize_replication(self)?)
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketConfigXml for PersistedLifecycleConfiguration {
|
||||
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
|
||||
Ok(parse_lifecycle_legacy_dates(input)?)
|
||||
}
|
||||
|
||||
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
|
||||
Ok(serialize_lifecycle_legacy_dates(self)?)
|
||||
}
|
||||
}
|
||||
|
||||
// The gateway writers of these families are infallible: a value that exists is
|
||||
// always encodable, so only the readers can refuse bytes.
|
||||
impl BucketConfigXml for PersistedAccelerateConfiguration {
|
||||
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
|
||||
Ok(parse_accelerate(input)?)
|
||||
}
|
||||
|
||||
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
|
||||
Ok(serialize_accelerate(self))
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketConfigXml for PersistedBucketLoggingStatus {
|
||||
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
|
||||
Ok(parse_bucket_logging(input)?)
|
||||
}
|
||||
|
||||
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
|
||||
Ok(serialize_bucket_logging(self))
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketConfigXml for PersistedCorsConfiguration {
|
||||
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
|
||||
Ok(parse_cors(input)?)
|
||||
}
|
||||
|
||||
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
|
||||
Ok(serialize_cors(self))
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketConfigXml for PersistedNotificationConfiguration {
|
||||
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
|
||||
Ok(parse_notification(input)?)
|
||||
}
|
||||
|
||||
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
|
||||
Ok(serialize_notification(self))
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketConfigXml for PersistedObjectLockConfiguration {
|
||||
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
|
||||
Ok(parse_object_lock(input)?)
|
||||
}
|
||||
|
||||
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
|
||||
Ok(serialize_object_lock(self))
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketConfigXml for PersistedPublicAccessBlockConfiguration {
|
||||
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
|
||||
Ok(parse_public_access_block(input)?)
|
||||
}
|
||||
|
||||
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
|
||||
Ok(serialize_public_access_block(self))
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketConfigXml for PersistedRequestPaymentConfiguration {
|
||||
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
|
||||
Ok(parse_request_payment(input)?)
|
||||
}
|
||||
|
||||
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
|
||||
Ok(serialize_request_payment(self))
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketConfigXml for PersistedBucketEncryptionConfiguration {
|
||||
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
|
||||
Ok(parse_bucket_encryption(input)?)
|
||||
}
|
||||
|
||||
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
|
||||
Ok(serialize_bucket_encryption(self))
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketConfigXml for PersistedTagging {
|
||||
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
|
||||
Ok(parse_tagging(input)?)
|
||||
}
|
||||
|
||||
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
|
||||
Ok(serialize_tagging(self))
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketConfigXml for PersistedVersioningConfiguration {
|
||||
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
|
||||
Ok(parse_versioning(input)?)
|
||||
}
|
||||
|
||||
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
|
||||
Ok(serialize_versioning(self))
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketConfigXml for PersistedWebsiteConfiguration {
|
||||
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
|
||||
Ok(parse_website(input)?)
|
||||
}
|
||||
|
||||
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
|
||||
Ok(serialize_website(self))
|
||||
}
|
||||
Ok(buf)
|
||||
}
|
||||
|
||||
pub fn has_bad_path_component(path: &str) -> bool {
|
||||
@@ -434,6 +584,40 @@ mod tests {
|
||||
use super::*;
|
||||
use proptest::prelude::*;
|
||||
|
||||
/// Every bucket-configuration family reads and writes through its own
|
||||
/// gateway codec (rustfs/backlog#2745): the family's golden document
|
||||
/// decodes and re-encodes to the same bytes, and a document of another
|
||||
/// family is refused instead of decoding into an empty configuration.
|
||||
#[test]
|
||||
fn every_bucket_config_family_uses_its_own_gateway_codec() {
|
||||
fn assert_family<T: BucketConfigXml>(family: &str, own: &[u8], foreign: &[u8]) {
|
||||
let value = deserialize::<T>(own).unwrap_or_else(|err| panic!("{family}: own document must decode: {err}"));
|
||||
let encoded = serialize(&value).unwrap_or_else(|err| panic!("{family}: value must encode: {err}"));
|
||||
assert_eq!(encoded, own, "{family}: re-encoding must keep the persisted bytes");
|
||||
assert!(deserialize::<T>(foreign).is_err(), "{family}: a foreign root element must be refused");
|
||||
}
|
||||
|
||||
macro_rules! golden {
|
||||
($family:literal) => {
|
||||
include_bytes!(concat!("../../tests/fixtures/bucket-config-goldens/", $family, "/minio.out.xml"))
|
||||
};
|
||||
}
|
||||
let tagging = golden!("tagging");
|
||||
let versioning = golden!("versioning");
|
||||
|
||||
assert_family::<PersistedAccelerateConfiguration>("accelerate", golden!("accelerate"), tagging);
|
||||
assert_family::<PersistedBucketLoggingStatus>("logging", golden!("logging"), tagging);
|
||||
assert_family::<PersistedCorsConfiguration>("cors", golden!("cors"), tagging);
|
||||
assert_family::<PersistedNotificationConfiguration>("notification", golden!("notification"), tagging);
|
||||
assert_family::<PersistedObjectLockConfiguration>("object_lock", golden!("object_lock"), tagging);
|
||||
assert_family::<PersistedPublicAccessBlockConfiguration>("public_access_block", golden!("public_access_block"), tagging);
|
||||
assert_family::<PersistedRequestPaymentConfiguration>("request_payment", golden!("request_payment"), tagging);
|
||||
assert_family::<PersistedBucketEncryptionConfiguration>("encryption", golden!("encryption"), tagging);
|
||||
assert_family::<PersistedTagging>("tagging", tagging, versioning);
|
||||
assert_family::<PersistedVersioningConfiguration>("versioning", versioning, tagging);
|
||||
assert_family::<PersistedWebsiteConfiguration>("website", golden!("website"), tagging);
|
||||
}
|
||||
|
||||
/// rustfs#5785: keys whose path segments exceed the on-disk NAME_MAX
|
||||
/// budget must be rejected up front as ObjectNameInvalid (4xx), not leak
|
||||
/// ENAMETOOLONG as InternalError 500 from the disk layer.
|
||||
|
||||
@@ -12,8 +12,8 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use rustfs_gateway_types::persistence::PersistedVersioningConfiguration as VersioningConfiguration;
|
||||
use rustfs_utils::string::match_simple;
|
||||
use s3s::dto::{BucketVersioningStatus, VersioningConfiguration};
|
||||
|
||||
pub trait VersioningApi {
|
||||
fn enabled(&self) -> bool;
|
||||
@@ -28,10 +28,10 @@ pub trait VersioningApi {
|
||||
|
||||
impl VersioningApi for VersioningConfiguration {
|
||||
fn enabled(&self) -> bool {
|
||||
self.status == Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED))
|
||||
self.status.as_deref() == Some("Enabled")
|
||||
}
|
||||
fn prefix_enabled(&self, prefix: &str) -> bool {
|
||||
if self.status != Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)) {
|
||||
if self.status.as_deref() != Some("Enabled") {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -44,12 +44,10 @@ impl VersioningApi for VersioningConfiguration {
|
||||
}
|
||||
|
||||
if let Some(ref excluded_prefixes) = self.excluded_prefixes {
|
||||
for p in excluded_prefixes.iter() {
|
||||
if let Some(ref sprefix) = p.prefix {
|
||||
let pattern = format!("{sprefix}*");
|
||||
if match_simple(&pattern, prefix) {
|
||||
return false;
|
||||
}
|
||||
for sprefix in excluded_prefixes.iter().flatten() {
|
||||
let pattern = format!("{sprefix}*");
|
||||
if match_simple(&pattern, prefix) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -58,11 +56,11 @@ impl VersioningApi for VersioningConfiguration {
|
||||
}
|
||||
|
||||
fn prefix_suspended(&self, prefix: &str) -> bool {
|
||||
if self.status == Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED)) {
|
||||
if self.status.as_deref() == Some("Suspended") {
|
||||
return true;
|
||||
}
|
||||
|
||||
if self.status == Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)) {
|
||||
if self.status.as_deref() == Some("Enabled") {
|
||||
if prefix.is_empty() {
|
||||
return false;
|
||||
}
|
||||
@@ -75,12 +73,10 @@ impl VersioningApi for VersioningConfiguration {
|
||||
}
|
||||
|
||||
if let Some(ref excluded_prefixes) = self.excluded_prefixes {
|
||||
for p in excluded_prefixes.iter() {
|
||||
if let Some(ref sprefix) = p.prefix {
|
||||
let pattern = format!("{sprefix}*");
|
||||
if match_simple(&pattern, prefix) {
|
||||
return true;
|
||||
}
|
||||
for sprefix in excluded_prefixes.iter().flatten() {
|
||||
let pattern = format!("{sprefix}*");
|
||||
if match_simple(&pattern, prefix) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -92,14 +88,12 @@ impl VersioningApi for VersioningConfiguration {
|
||||
self.prefix_enabled(prefix) || self.prefix_suspended(prefix)
|
||||
}
|
||||
fn suspended(&self) -> bool {
|
||||
self.status == Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED))
|
||||
self.status.as_deref() == Some("Suspended")
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use s3s::dto::{BucketVersioningStatus, ExcludedPrefix};
|
||||
|
||||
use super::*;
|
||||
|
||||
struct LegacyVersioning;
|
||||
@@ -134,10 +128,8 @@ mod tests {
|
||||
#[test]
|
||||
fn delete_state_treats_excluded_prefixes_as_unversioned() {
|
||||
let config = VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
excluded_prefixes: Some(vec![ExcludedPrefix {
|
||||
prefix: Some("archive/".to_string()),
|
||||
}]),
|
||||
status: Some("Enabled".to_owned()),
|
||||
excluded_prefixes: Some(vec![Some("archive/".to_string())]),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -146,7 +138,7 @@ mod tests {
|
||||
assert_eq!(config.delete_state("live/object"), (true, false));
|
||||
|
||||
let suspended = VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED)),
|
||||
status: Some("Suspended".to_owned()),
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(suspended.delete_state("archive/object"), (false, true));
|
||||
|
||||
@@ -17,7 +17,7 @@ use super::metadata::unreadable_config_refusal;
|
||||
use super::{metadata_sys::get_bucket_metadata_sys, versioning::VersioningApi};
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::Result;
|
||||
use s3s::dto::VersioningConfiguration;
|
||||
use rustfs_gateway_types::persistence::PersistedVersioningConfiguration as VersioningConfiguration;
|
||||
use tracing::{error, warn};
|
||||
|
||||
pub struct BucketVersioningSys {}
|
||||
@@ -163,7 +163,7 @@ mod tests {
|
||||
|
||||
fn enabled() -> VersioningConfiguration {
|
||||
VersioningConfiguration {
|
||||
status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::ENABLED)),
|
||||
status: Some("Enabled".to_owned()),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1965,6 +1965,15 @@ impl PeerRestClient {
|
||||
Ok((self.topology_member.clone(), epoch))
|
||||
}
|
||||
|
||||
pub async fn probe_restore_copyback_capability(&self, topology_fingerprint: String) -> Result<(String, Uuid)> {
|
||||
let probe = rustfs_protos::restore_copyback_capability_probe(Uuid::new_v4().as_bytes());
|
||||
let result = self
|
||||
.heal_control(rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION, topology_fingerprint, probe)
|
||||
.await?;
|
||||
let epoch = decode_remote_version_state_capability(&self.topology_member, &result)?;
|
||||
Ok((self.topology_member.clone(), epoch))
|
||||
}
|
||||
|
||||
pub async fn probe_cross_pool_fence(&self, topology_fingerprint: String) -> Result<(String, u32, Uuid)> {
|
||||
let mut probe = rustfs_protos::CROSS_POOL_FENCE_CAPABILITY_PROBE_PREFIX.to_vec();
|
||||
probe.extend_from_slice(Uuid::new_v4().as_bytes());
|
||||
|
||||
@@ -488,6 +488,18 @@ where
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
/// Read config bytes and metadata while preserving an existing empty payload.
|
||||
pub(crate) async fn read_config_preserve_empty_with_metadata_opts<S>(
|
||||
api: Arc<S>,
|
||||
file: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<(Vec<u8>, ObjectInfo)>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
read_config_with_metadata_inner(api, file, opts, true, None).await
|
||||
}
|
||||
|
||||
pub async fn read_config_no_lock<S>(api: Arc<S>, file: &str) -> Result<Vec<u8>>
|
||||
where
|
||||
S: ObjectIO<
|
||||
@@ -2605,6 +2617,15 @@ where
|
||||
save_server_config_snapshot(api, cfg, &snapshot).await.map(|_| ())
|
||||
}
|
||||
|
||||
/// Read config and generation from the same object without taking the transaction write lock.
|
||||
pub async fn read_server_config_with_generation(api: Arc<ECStore>) -> Result<(Config, Option<Uuid>)> {
|
||||
match read_config_with_metadata_inner(api.clone(), &server_config_path(), &ObjectOptions::default(), false, None).await {
|
||||
Ok((raw, info)) => Ok((decode_persisted_server_config(&raw)?.merge(), info.data_dir.filter(|id| !id.is_nil()))),
|
||||
Err(Error::ConfigNotFound) => read_config_without_migrate(api).await.map(|config| (config, None)),
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
|
||||
/// One serialized read of the persisted server config used as the seed and
|
||||
/// compare-and-swap fence for an admin update.
|
||||
#[derive(Debug)]
|
||||
|
||||
@@ -84,12 +84,13 @@ use rand::RngExt as _;
|
||||
use rmp_serde::Deserializer;
|
||||
use rmp_serde::Serializer;
|
||||
use rustfs_filemeta::{FileInfo, FileInfoVersions, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams};
|
||||
use rustfs_gateway_types::persistence::PersistedObjectLockConfiguration as ObjectLockConfiguration;
|
||||
use rustfs_gateway_types::persistence::{PersistedLifecycleConfiguration, PersistedReplicationConfiguration};
|
||||
use rustfs_heal_contracts::heal_channel::HealOpts;
|
||||
use rustfs_utils::crypto::{hex_sha256, is_sha256_checksum};
|
||||
use rustfs_utils::path::{
|
||||
decode_dir_object, encode_dir_object, path_join, path_to_bucket_object, path_to_bucket_object_with_base_path,
|
||||
};
|
||||
use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration, ReplicationConfiguration};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
@@ -12883,7 +12884,7 @@ pub(crate) async fn should_skip_lifecycle_for_data_movement(
|
||||
store: Arc<ECStore>,
|
||||
bucket: &str,
|
||||
version: &rustfs_filemeta::FileInfo,
|
||||
lifecycle_config: Option<&BucketLifecycleConfiguration>,
|
||||
lifecycle_config: Option<&PersistedLifecycleConfiguration>,
|
||||
object_lock_config: Option<&ObjectLockConfiguration>,
|
||||
apply_actions: bool,
|
||||
event_source: &LcEventSrc,
|
||||
@@ -16812,9 +16813,9 @@ impl ECStore {
|
||||
generation: OffsetDateTime,
|
||||
bucket: String,
|
||||
set: Arc<SetDisks>,
|
||||
lifecycle_config: Option<BucketLifecycleConfiguration>,
|
||||
lifecycle_config: Option<PersistedLifecycleConfiguration>,
|
||||
object_lock_config: Option<ObjectLockConfiguration>,
|
||||
replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
|
||||
replication_config: Option<(PersistedReplicationConfiguration, OffsetDateTime)>,
|
||||
expected_bucket_incarnation_id: Option<uuid::Uuid>,
|
||||
source_changed_exhaustions: Arc<AtomicUsize>,
|
||||
entry_budget: Arc<Semaphore>,
|
||||
@@ -16986,9 +16987,9 @@ impl ECStore {
|
||||
mut deferred_entry: DeferredDecommissionEntry,
|
||||
bucket: String,
|
||||
set: Arc<SetDisks>,
|
||||
lifecycle_config: Option<BucketLifecycleConfiguration>,
|
||||
lifecycle_config: Option<PersistedLifecycleConfiguration>,
|
||||
object_lock_config: Option<ObjectLockConfiguration>,
|
||||
replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
|
||||
replication_config: Option<(PersistedReplicationConfiguration, OffsetDateTime)>,
|
||||
expected_bucket_incarnation_id: Option<uuid::Uuid>,
|
||||
source_changed_exhaustions: Arc<AtomicUsize>,
|
||||
entry_budget: Arc<Semaphore>,
|
||||
@@ -17095,9 +17096,9 @@ impl ECStore {
|
||||
generation: OffsetDateTime,
|
||||
set: Arc<SetDisks>,
|
||||
bi: DecomBucketInfo,
|
||||
lifecycle_config: Option<BucketLifecycleConfiguration>,
|
||||
lifecycle_config: Option<PersistedLifecycleConfiguration>,
|
||||
object_lock_config: Option<ObjectLockConfiguration>,
|
||||
replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
|
||||
replication_config: Option<(PersistedReplicationConfiguration, OffsetDateTime)>,
|
||||
expected_bucket_incarnation_id: Option<uuid::Uuid>,
|
||||
source_changed_exhaustions: Arc<AtomicUsize>,
|
||||
entry_budget: Arc<Semaphore>,
|
||||
@@ -17772,9 +17773,9 @@ impl ECStore {
|
||||
entry: MetaCacheEntry,
|
||||
bucket: String,
|
||||
set: Arc<SetDisks>,
|
||||
lifecycle_config: Option<BucketLifecycleConfiguration>,
|
||||
lifecycle_config: Option<PersistedLifecycleConfiguration>,
|
||||
object_lock_config: Option<ObjectLockConfiguration>,
|
||||
replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
|
||||
replication_config: Option<(PersistedReplicationConfiguration, OffsetDateTime)>,
|
||||
expected_bucket_incarnation_id: Option<uuid::Uuid>,
|
||||
source_changed_exhaustions: Arc<AtomicUsize>,
|
||||
attempts: &mut usize,
|
||||
@@ -17958,9 +17959,9 @@ impl ECStore {
|
||||
entry: MetaCacheEntry,
|
||||
bucket: String,
|
||||
set: Arc<SetDisks>,
|
||||
lifecycle_config: Option<BucketLifecycleConfiguration>,
|
||||
lifecycle_config: Option<PersistedLifecycleConfiguration>,
|
||||
object_lock_config: Option<ObjectLockConfiguration>,
|
||||
replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
|
||||
replication_config: Option<(PersistedReplicationConfiguration, OffsetDateTime)>,
|
||||
expected_bucket_incarnation_id: Option<uuid::Uuid>,
|
||||
entry_attempt: usize,
|
||||
scanner_backlog_handoff_attempt: usize,
|
||||
@@ -19246,7 +19247,7 @@ impl ECStore {
|
||||
entry: MetaCacheEntry,
|
||||
bucket: String,
|
||||
set: Arc<SetDisks>,
|
||||
lifecycle_config: Option<BucketLifecycleConfiguration>,
|
||||
lifecycle_config: Option<PersistedLifecycleConfiguration>,
|
||||
) -> Result<()> {
|
||||
let expected_bucket_incarnation_id = if is_meta_bucketname(&bucket) {
|
||||
None
|
||||
|
||||
@@ -65,6 +65,38 @@ use uuid::Uuid;
|
||||
|
||||
const LIST_MULTIPART_SETS_CONCURRENCY: usize = 4;
|
||||
|
||||
#[cfg(test)]
|
||||
type TestSetLockers = HashMap<usize, Vec<Arc<dyn rustfs_lock::LockClient>>>;
|
||||
|
||||
#[cfg(test)]
|
||||
static TEST_SET_LOCKERS: std::sync::LazyLock<std::sync::Mutex<TestSetLockers>> =
|
||||
std::sync::LazyLock::new(|| std::sync::Mutex::new(HashMap::new()));
|
||||
|
||||
/// Keep a fixture's real locker domain scoped to its owning instance.
|
||||
#[cfg(test)]
|
||||
pub(crate) struct ScopedSetLockers(Arc<InstanceContext>);
|
||||
|
||||
#[cfg(test)]
|
||||
impl ScopedSetLockers {
|
||||
pub(crate) fn install(ctx: Arc<InstanceContext>, lockers: Vec<Arc<dyn rustfs_lock::LockClient>>) -> Self {
|
||||
let key = Arc::as_ptr(&ctx) as usize;
|
||||
let mut domains = TEST_SET_LOCKERS.lock().expect("test set lockers mutex");
|
||||
assert!(!domains.contains_key(&key), "the instance already owns a test locker domain");
|
||||
domains.insert(key, lockers);
|
||||
Self(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for ScopedSetLockers {
|
||||
fn drop(&mut self) {
|
||||
TEST_SET_LOCKERS
|
||||
.lock()
|
||||
.expect("test set lockers mutex")
|
||||
.remove(&(Arc::as_ptr(&self.0) as usize));
|
||||
}
|
||||
}
|
||||
|
||||
fn is_idempotent_delete_prefix_error(err: &Error) -> bool {
|
||||
is_err_object_not_found(err) || is_err_strict_volume_not_found(err)
|
||||
}
|
||||
@@ -266,6 +298,14 @@ impl Sets {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
let pool_lockers = TEST_SET_LOCKERS
|
||||
.lock()
|
||||
.expect("test set lockers mutex")
|
||||
.get(&(Arc::as_ptr(&instance_ctx) as usize))
|
||||
.cloned()
|
||||
.unwrap_or_else(|| pool_lockers.clone());
|
||||
|
||||
let set_disks = SetDisks::new_with_instance_ctx(
|
||||
runtime_sources::local_node_name().await,
|
||||
Arc::new(RwLock::new(set_drive)),
|
||||
|
||||
@@ -1493,6 +1493,48 @@ pub async fn load_data_usage_from_backend_cached(store: Arc<ECStore>) -> Result<
|
||||
project_data_usage_from_backend_cached(store, DataUsageInfo::clone).await
|
||||
}
|
||||
|
||||
/// Read one bucket's persisted usage without cloning the complete snapshot.
|
||||
pub async fn load_bucket_usage_from_backend_cached(
|
||||
store: Arc<ECStore>,
|
||||
bucket: &str,
|
||||
incarnation: uuid::Uuid,
|
||||
) -> Result<Option<u64>, Error> {
|
||||
project_data_usage_from_backend_cached(store, |info| certified_bucket_usage(info, bucket, incarnation)).await
|
||||
}
|
||||
|
||||
fn certified_bucket_usage(info: &DataUsageInfo, bucket: &str, incarnation: uuid::Uuid) -> Option<u64> {
|
||||
if incarnation.is_nil()
|
||||
|| !info.is_complete_bucket_usage_snapshot()
|
||||
|| info.usage_snapshot_observational
|
||||
|| info.scanner_cycle.is_none()
|
||||
|| info.scanner_epoch.is_none()
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let pending_count = u64::try_from(info.usage_snapshot_pending_buckets.len()).ok()?;
|
||||
if pending_count != info.usage_snapshot_pending_bucket_count {
|
||||
return None;
|
||||
}
|
||||
match info.usage_snapshot_converged {
|
||||
Some(true) if pending_count == 0 => {}
|
||||
Some(false)
|
||||
if pending_count > 0
|
||||
&& info.usage_snapshot_pending_buckets.windows(2).all(|pair| pair[0] < pair[1])
|
||||
&& info
|
||||
.usage_snapshot_pending_buckets
|
||||
.binary_search_by(|pending| pending.as_str().cmp(bucket))
|
||||
.is_err() => {}
|
||||
_ => return None,
|
||||
}
|
||||
let usage = info.buckets_usage.get(bucket)?;
|
||||
let certified = usage
|
||||
.bucket_incarnation
|
||||
.as_deref()
|
||||
.and_then(|id| uuid::Uuid::parse_str(id).ok())
|
||||
.filter(|id| !id.is_nil())?;
|
||||
(certified == incarnation).then_some(usage.size)
|
||||
}
|
||||
|
||||
// Project under the cache guard so scalar readers never copy the snapshot's
|
||||
// bucket maps. The owned API shares the same TTL, refresh and invalidation path.
|
||||
async fn project_data_usage_from_backend_cached<T>(
|
||||
|
||||
@@ -129,6 +129,41 @@ fn projected_publication_moves_bucket_maps_into_cache() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn certified_bucket_usage_rejects_unproven_and_pending_data() {
|
||||
let id = uuid::Uuid::from_u128(7);
|
||||
let mut info = data_usage_info_for_test("known", 1, 42, SystemTime::UNIX_EPOCH);
|
||||
info.usage_snapshot_converged = Some(true);
|
||||
info.scanner_cycle = Some(1);
|
||||
info.scanner_epoch = Some(1);
|
||||
info.buckets_usage.get_mut("known").unwrap().bucket_incarnation = Some(id.to_string());
|
||||
assert_eq!(certified_bucket_usage(&info, "known", id), Some(42));
|
||||
assert_eq!(certified_bucket_usage(&info, "known", uuid::Uuid::from_u128(8)), None);
|
||||
assert_eq!(certified_bucket_usage(&info, "known", uuid::Uuid::nil()), None);
|
||||
let mut legacy = info.clone();
|
||||
legacy.buckets_usage.get_mut("known").unwrap().bucket_incarnation = None;
|
||||
assert_eq!(certified_bucket_usage(&legacy, "known", id), None);
|
||||
legacy.buckets_usage.get_mut("known").unwrap().bucket_incarnation = Some("corrupt".into());
|
||||
assert_eq!(certified_bucket_usage(&legacy, "known", id), None);
|
||||
let mut pending = info.clone();
|
||||
pending.usage_snapshot_converged = Some(false);
|
||||
pending.usage_snapshot_pending_buckets = vec!["other".into()];
|
||||
pending.usage_snapshot_pending_bucket_count = 1;
|
||||
assert_eq!(certified_bucket_usage(&pending, "known", id), Some(42));
|
||||
pending.usage_snapshot_pending_buckets = vec!["known".into()];
|
||||
assert_eq!(certified_bucket_usage(&pending, "known", id), None);
|
||||
pending.usage_snapshot_pending_bucket_count = 0;
|
||||
assert_eq!(certified_bucket_usage(&pending, "known", id), None);
|
||||
pending.usage_snapshot_pending_buckets = vec!["z".into(), "a".into()];
|
||||
pending.usage_snapshot_pending_bucket_count = 2;
|
||||
assert_eq!(certified_bucket_usage(&pending, "known", id), None);
|
||||
let mut observational = info.clone();
|
||||
observational.usage_snapshot_observational = true;
|
||||
assert_eq!(certified_bucket_usage(&observational, "known", id), None);
|
||||
info.usage_snapshot_converged = None;
|
||||
assert_eq!(certified_bucket_usage(&info, "known", id), None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn persisted_scalar_projection_shares_cache_refresh_and_invalidation() {
|
||||
@@ -140,6 +175,13 @@ async fn persisted_scalar_projection_shares_cache_refresh_and_invalidation() {
|
||||
info.buckets_usage.insert("empty".to_string(), BucketUsageInfo::default());
|
||||
info.bucket_sizes.insert("empty".to_string(), 0);
|
||||
info.buckets_count = 2;
|
||||
info.usage_snapshot_converged = Some(true);
|
||||
info.scanner_cycle = Some(1);
|
||||
info.scanner_epoch = Some(1);
|
||||
let incarnation = uuid::Uuid::from_u128(7);
|
||||
for usage in info.buckets_usage.values_mut() {
|
||||
usage.bucket_incarnation = Some(incarnation.to_string());
|
||||
}
|
||||
save_config(
|
||||
store.clone(),
|
||||
DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
@@ -151,11 +193,29 @@ async fn persisted_scalar_projection_shares_cache_refresh_and_invalidation() {
|
||||
|
||||
for (bucket, expected) in [("known", Some(42)), ("empty", Some(0)), ("missing", None)] {
|
||||
assert_eq!(persisted_bucket_usage(store.clone(), bucket).await, expected);
|
||||
assert_eq!(
|
||||
load_bucket_usage_from_backend_cached(store.clone(), bucket, incarnation)
|
||||
.await
|
||||
.expect("project bucket usage"),
|
||||
expected
|
||||
);
|
||||
let owned = load_data_usage_from_backend_cached(store.clone())
|
||||
.await
|
||||
.expect("owned cached snapshot");
|
||||
assert_eq!(expected, owned.buckets_usage.get(bucket).map(|usage| usage.size));
|
||||
}
|
||||
assert_eq!(
|
||||
load_bucket_usage_from_backend_cached(store.clone(), "known", uuid::Uuid::from_u128(8))
|
||||
.await
|
||||
.unwrap(),
|
||||
None
|
||||
);
|
||||
assert_eq!(
|
||||
load_bucket_usage_from_backend_cached(store.clone(), "known", uuid::Uuid::nil())
|
||||
.await
|
||||
.unwrap(),
|
||||
None
|
||||
);
|
||||
info.buckets_usage.get_mut("known").expect("known bucket").size = 84;
|
||||
info.bucket_sizes.insert("known".to_string(), 84);
|
||||
save_config(
|
||||
|
||||
@@ -2776,6 +2776,15 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
})
|
||||
}
|
||||
|
||||
async fn bucket_creation_witness(&self, volume: &str) -> Result<Option<super::os::NativeBucketCreationWitness>> {
|
||||
self.track_disk_health_with_op(
|
||||
"bucket_creation_witness",
|
||||
|| async { self.disk.bucket_creation_witness(volume).await },
|
||||
Duration::ZERO,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn make_volume(&self, volume: &str) -> Result<()> {
|
||||
// Scoped heal must drain directory creation before releasing its lifecycle owner.
|
||||
let timeout = if crate::store::bucket_heal_scope(volume).is_some() {
|
||||
|
||||
@@ -117,6 +117,9 @@ pub const O_APPEND: FileMode = 0x00400;
|
||||
// create_new: bool,
|
||||
|
||||
pub async fn open_file(path: impl AsRef<Path>, mode: FileMode) -> io::Result<File> {
|
||||
if mode & O_CREATE != 0 {
|
||||
super::os::record_native_publication(path.as_ref());
|
||||
}
|
||||
let base_opts = match mode & (O_RDONLY | O_WRONLY | O_RDWR) {
|
||||
O_RDONLY => get_readonly_options(),
|
||||
O_WRONLY => get_writeonly_options(),
|
||||
@@ -169,6 +172,7 @@ pub fn create_dir_all_std(path: &Path) -> io::Result<()> {
|
||||
}
|
||||
|
||||
pub async fn make_dir_all(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
super::os::record_native_publication(path.as_ref());
|
||||
let path = path.as_ref().to_path_buf();
|
||||
tokio::task::spawn_blocking(move || create_dir_all_std(&path)).await?
|
||||
}
|
||||
@@ -222,15 +226,18 @@ pub fn remove_all_std(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
|
||||
#[cfg(test)]
|
||||
pub async fn mkdir(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
super::os::record_native_publication(path.as_ref());
|
||||
fs::create_dir(path.as_ref()).await
|
||||
}
|
||||
|
||||
pub async fn rename(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()> {
|
||||
super::os::record_native_publication(to.as_ref());
|
||||
fs::rename(from, to).await
|
||||
}
|
||||
|
||||
#[cfg(any(not(windows), test))]
|
||||
pub fn rename_std(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()> {
|
||||
super::os::record_native_publication(to.as_ref());
|
||||
std::fs::rename(from, to)
|
||||
}
|
||||
|
||||
|
||||
@@ -520,6 +520,7 @@ fn snapshot_part_transaction_file(src: &Path, backup: &Path, absent: &Path) -> s
|
||||
}
|
||||
|
||||
fn restore_part_transaction_file(current: &Path, backup: &Path, absent: &Path, restore: &Path) -> std::io::Result<()> {
|
||||
os::record_native_publication(current);
|
||||
match std::fs::symlink_metadata(backup) {
|
||||
Ok(metadata) if metadata.is_file() => {
|
||||
remove_file_if_exists(restore)?;
|
||||
@@ -572,6 +573,7 @@ async fn write_delete_rollback_files(
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<Option<crate::disk::DeleteRecoveryInspection>> {
|
||||
let backup_dir = object_dir.join(rollback_dir.to_string());
|
||||
os::record_native_publication(&backup_dir);
|
||||
let mut removed_entries = Vec::with_capacity(remove_before_publish.len());
|
||||
for &name in remove_before_publish {
|
||||
let path = backup_dir.join(name);
|
||||
@@ -2667,6 +2669,9 @@ impl AsyncWrite for DirectWriter {
|
||||
fn open_direct_writer(file_path: &Path, state: &DirectIoWriteState) -> Result<Option<DirectWriter>> {
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
|
||||
#[cfg(test)]
|
||||
os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::DirectWriteOpen, file_path);
|
||||
os::record_native_publication(file_path);
|
||||
let open_result = std::fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.write(true)
|
||||
@@ -3830,6 +3835,9 @@ impl StdBackend {
|
||||
}
|
||||
|
||||
async fn open_file(&self, path: impl AsRef<Path>, mode: usize, skip_parent: impl AsRef<Path>) -> Result<File> {
|
||||
if mode == O_CREATE | O_WRONLY | O_TRUNC {
|
||||
os::record_native_publication(path.as_ref());
|
||||
}
|
||||
let mut skip_parent = skip_parent.as_ref();
|
||||
if skip_parent.as_os_str().is_empty() {
|
||||
skip_parent = self.io_root();
|
||||
@@ -4470,6 +4478,7 @@ impl LocalIoBackend for StdBackend {
|
||||
if is_direct_io_write_enabled() && self.direct_io_write.supported.load(Ordering::Relaxed) {
|
||||
let write_state = self.direct_io_write.clone();
|
||||
let direct_path = file_path.clone();
|
||||
os::record_native_publication(&direct_path);
|
||||
let direct = tokio::task::spawn_blocking(move || open_direct_writer(&direct_path, &write_state))
|
||||
.await
|
||||
.map_err(|err| DiskError::other(format!("O_DIRECT open task failed: {err}")))??;
|
||||
@@ -5722,6 +5731,7 @@ async fn build_local_io_backend(root: PathBuf) -> Arc<dyn LocalIoBackend> {
|
||||
pub struct LocalDisk {
|
||||
pub root: PathBuf,
|
||||
publication_root: os::PublicationRoot,
|
||||
publication_history: Option<os::NativePublicationRegistration>,
|
||||
multipart_proof_authority: OnceLock<crate::disk::multipart_source::ProofVerificationContext>,
|
||||
/// I/O root pinned to the mount instance that was opened while the disk
|
||||
/// was initialized. On Linux this is `/proc/self/fd/<dirfd>/.`; resolving
|
||||
@@ -6114,6 +6124,10 @@ impl LocalDisk {
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let io_root = Self::open_mount_lease(&root)?;
|
||||
|
||||
let publication_history = os::register_native_publication_root(&root, &io_root);
|
||||
|
||||
#[cfg(all(test, target_os = "linux"))]
|
||||
os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::BeforeDataUsageLayout, &root);
|
||||
ensure_data_usage_layout(&io_root)
|
||||
.await
|
||||
.map_err(|e| e.narrow_to_disk().unwrap_or_else(DiskError::other))
|
||||
@@ -6252,6 +6266,7 @@ impl LocalDisk {
|
||||
let mut disk = Self {
|
||||
root: root.clone(),
|
||||
publication_root,
|
||||
publication_history,
|
||||
multipart_proof_authority: OnceLock::new(),
|
||||
io_root: io_root.clone(),
|
||||
#[cfg(target_os = "linux")]
|
||||
@@ -8450,6 +8465,23 @@ impl LocalDisk {
|
||||
return Err(std::io::Error::other("metadata read worker cancelled"));
|
||||
}
|
||||
let _io_permit = io_permit;
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
{
|
||||
let opened = root.open_readonly(&path);
|
||||
if opened.is_ok() {
|
||||
os::prepared_publication_test_hooks::run(
|
||||
os::prepared_publication_test_hooks::Stage::MetadataReadAfterOpen,
|
||||
&path,
|
||||
);
|
||||
}
|
||||
let result = read_fn(opened);
|
||||
os::prepared_publication_test_hooks::run(
|
||||
os::prepared_publication_test_hooks::Stage::MetadataReadAfterRead,
|
||||
&path,
|
||||
);
|
||||
Ok(result)
|
||||
}
|
||||
#[cfg(not(all(any(test, feature = "test-util"), not(windows))))]
|
||||
Ok(read_fn(root.open_readonly(&path)))
|
||||
})
|
||||
.await
|
||||
@@ -8911,6 +8943,7 @@ impl LocalDisk {
|
||||
|| should_fail_after_delete_data_staged(path))
|
||||
{
|
||||
let rollback_path = object_dir.join(rollback_dir.to_string());
|
||||
os::record_native_publication(&rollback_path);
|
||||
if let Err(err) = fs::create_dir_all(&rollback_path).await {
|
||||
let err: DiskError = to_file_error(err).into();
|
||||
if reserved_version_delete {
|
||||
@@ -9471,6 +9504,7 @@ impl LocalDisk {
|
||||
};
|
||||
|
||||
let path = file_path.to_path_buf();
|
||||
os::record_native_publication(&path);
|
||||
let create_parent = if let Some(parent) = path.parent()
|
||||
&& parent != skip_parent
|
||||
{
|
||||
@@ -9485,6 +9519,7 @@ impl LocalDisk {
|
||||
if is_cancelled() {
|
||||
return Err(std::io::Error::other("blocking namespace operation cancelled"));
|
||||
}
|
||||
os::record_native_publication(&path);
|
||||
if create_parent && let Some(parent) = path.parent() {
|
||||
super::fs::create_dir_all_std(parent)?;
|
||||
}
|
||||
@@ -11313,7 +11348,7 @@ pub(crate) async fn batch_shard_pread(requests: Vec<(std::path::PathBuf, usize,
|
||||
let submitted = metrics.as_ref().map(|_| std::time::Instant::now());
|
||||
tokio::task::spawn_blocking(move || {
|
||||
use rustfs_io_metrics::LocalBatchReadStage;
|
||||
use std::os::unix::fs::FileExt;
|
||||
use std::os::unix::fs::{FileExt, OpenOptionsExt};
|
||||
|
||||
let execution_started = metrics.as_ref().map(|_| std::time::Instant::now());
|
||||
if let (Some(metrics), Some(submitted)) = (&metrics, submitted) {
|
||||
@@ -11324,17 +11359,24 @@ pub(crate) async fn batch_shard_pread(requests: Vec<(std::path::PathBuf, usize,
|
||||
for (file_path, offset, length) in requests {
|
||||
let mut bytes_read = 0;
|
||||
let r = (|| -> Result<Bytes> {
|
||||
let meta = measure_local_batch_stage(metrics.as_deref(), LocalBatchReadStage::Metadata, || {
|
||||
std::fs::metadata(&file_path).map_err(DiskError::from)
|
||||
})?;
|
||||
let end = offset.checked_add(length).ok_or(DiskError::FileCorrupt)?;
|
||||
if meta.len() < u64::try_from(end).unwrap_or(u64::MAX) {
|
||||
let end = u64::try_from(end).map_err(|_| DiskError::FileCorrupt)?;
|
||||
// Bind validation and reads to one descriptor. Nonblocking open
|
||||
// lets corrupt non-regular entries fail without waiting on a FIFO.
|
||||
let file = measure_local_batch_stage(metrics.as_deref(), LocalBatchReadStage::Open, || {
|
||||
std::fs::OpenOptions::new()
|
||||
.read(true)
|
||||
.custom_flags(libc::O_NONBLOCK)
|
||||
.open(&file_path)
|
||||
.map_err(DiskError::from)
|
||||
})?;
|
||||
let meta = measure_local_batch_stage(metrics.as_deref(), LocalBatchReadStage::Metadata, || {
|
||||
file.metadata().map_err(DiskError::from)
|
||||
})?;
|
||||
if !meta.is_file() || meta.len() < end {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
|
||||
let file = measure_local_batch_stage(metrics.as_deref(), LocalBatchReadStage::Open, || {
|
||||
std::fs::File::open(&file_path).map_err(DiskError::from)
|
||||
})?;
|
||||
let mut buf = vec![0u8; length];
|
||||
if length > 0 {
|
||||
measure_local_batch_stage(metrics.as_deref(), LocalBatchReadStage::Read, || {
|
||||
@@ -11479,6 +11521,9 @@ impl DiskAPI for LocalDisk {
|
||||
}
|
||||
|
||||
async fn close(&self) -> Result<()> {
|
||||
if let Some(history) = self.publication_history.as_ref() {
|
||||
history.invalidate();
|
||||
}
|
||||
// This disk instance is being retired (e.g. replaced by renew_disk on
|
||||
// reconnect). Drop its cached descriptors so a replacement instance's
|
||||
// invalidations are never defeated by this one continuing to serve stale
|
||||
@@ -11655,6 +11700,7 @@ impl DiskAPI for LocalDisk {
|
||||
let file_path = self.io_get_object_path(volume, path)?;
|
||||
let path = path.to_string();
|
||||
let sync_metadata = effective_durability(volume).syncs_commit_metadata();
|
||||
os::record_native_publication(&file_path);
|
||||
return Ok(tokio::task::spawn_blocking(move || {
|
||||
commit_allowed()?;
|
||||
// A persistent directory lock bounds metadata growth. Removing
|
||||
@@ -12067,6 +12113,7 @@ impl DiskAPI for LocalDisk {
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
os::record_native_publication(&transaction_path);
|
||||
let retained_owner = source_owner.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let _owner = retained_owner;
|
||||
@@ -12145,6 +12192,7 @@ impl DiskAPI for LocalDisk {
|
||||
}
|
||||
let durability = effective_durability(volume);
|
||||
|
||||
os::record_native_publication(¤t_data_path);
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let _owner = source_owner;
|
||||
match std::fs::symlink_metadata(&transaction_path) {
|
||||
@@ -12859,12 +12907,27 @@ impl DiskAPI for LocalDisk {
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn bucket_creation_witness(&self, volume: &str) -> Result<Option<os::NativeBucketCreationWitness>> {
|
||||
let Some(history) = self.publication_history.as_ref() else {
|
||||
return Ok(None);
|
||||
};
|
||||
let Some(disk_id) = self.get_disk_id().await? else {
|
||||
return Ok(None);
|
||||
};
|
||||
Ok(history.witness(volume, disk_id, Arc::clone(&self.io_backend)))
|
||||
}
|
||||
|
||||
async fn make_volume(&self, volume: &str) -> Result<()> {
|
||||
if !Self::is_valid_volname(volume) {
|
||||
return Err(Error::other("Invalid arguments specified"));
|
||||
}
|
||||
|
||||
let volume_dir = self.io_get_bucket_path(volume)?;
|
||||
#[cfg(all(test, not(windows)))]
|
||||
os::prepared_publication_test_hooks::run(
|
||||
os::prepared_publication_test_hooks::Stage::CreateDirectory,
|
||||
&self.root.join(volume),
|
||||
);
|
||||
|
||||
// Volume creation is a mutation boundary, so it must observe the live
|
||||
// filesystem rather than a potentially stale existence-cache entry.
|
||||
@@ -13656,6 +13719,254 @@ mod test {
|
||||
use tokio::io::{AsyncReadExt, AsyncWrite, AsyncWriteExt, ReadBuf};
|
||||
use tracing_subscriber::fmt::MakeWriter;
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn owned_direct_child_open_attempt_consumes_native_history() {
|
||||
let dir = tempfile::tempdir().expect("native disk directory");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 disk path")).expect("disk endpoint");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("native local disk");
|
||||
let bucket = format!("owned-open-attempt-{}", Uuid::new_v4());
|
||||
let witness = disk
|
||||
.publication_history
|
||||
.as_ref()
|
||||
.expect("native registration")
|
||||
.witness(&bucket, Uuid::new_v4(), Arc::clone(&disk.io_backend))
|
||||
.expect("fresh history");
|
||||
|
||||
disk.write_all_public(&bucket, "direct-child", Bytes::from_static(b"payload"))
|
||||
.await
|
||||
.expect_err("the direct child has no bucket parent");
|
||||
assert!(
|
||||
!os::validate_native_creation_witnesses(&[witness]),
|
||||
"even a failed O_CREATE in the real owned executor must permanently consume native history"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn borrowed_open_cancellation_retains_started_history_before_execution() {
|
||||
use std::sync::mpsc;
|
||||
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.max_blocking_threads(1)
|
||||
.build()
|
||||
.expect("single blocking worker runtime");
|
||||
runtime.block_on(async {
|
||||
let dir = tempfile::tempdir().expect("native disk directory");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 disk path")).expect("disk endpoint");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("native local disk");
|
||||
let history = disk.publication_history.as_ref().expect("native registration");
|
||||
let bucket = format!("borrowed-queued-open-{}", Uuid::new_v4());
|
||||
let disk_id = Uuid::new_v4();
|
||||
let witness = history
|
||||
.witness(&bucket, disk_id, Arc::clone(&disk.io_backend))
|
||||
.expect("fresh native history");
|
||||
let path = disk.io_root().join(&bucket);
|
||||
let (entered_tx, entered_rx) = mpsc::channel();
|
||||
let (release_tx, release_rx) = mpsc::channel();
|
||||
let blocker = tokio::task::spawn_blocking(move || {
|
||||
entered_tx.send(()).expect("signal occupied blocking worker");
|
||||
let _ = release_rx.recv();
|
||||
});
|
||||
entered_rx
|
||||
.recv_timeout(Duration::from_secs(30))
|
||||
.expect("the only blocking worker must be occupied");
|
||||
|
||||
let mut operation =
|
||||
Box::pin(disk.write_all_internal(&path, InternalBuf::Ref(b"payload"), SyncMode::None, disk.io_root()));
|
||||
let queued = futures::poll!(operation.as_mut()).is_pending();
|
||||
let started_before_execution = !os::validate_native_creation_witnesses(std::slice::from_ref(&witness));
|
||||
let absent_before_execution = !path.exists();
|
||||
drop(operation);
|
||||
release_tx.send(()).expect("release the blocking worker");
|
||||
blocker.await.expect("blocking worker must drain");
|
||||
tokio::task::spawn_blocking(|| ())
|
||||
.await
|
||||
.expect("queued native open must drain before assertions");
|
||||
|
||||
assert!(queued, "the borrowed write must queue behind the occupied worker");
|
||||
assert!(
|
||||
absent_before_execution,
|
||||
"the real open must not run before the blocking worker is released"
|
||||
);
|
||||
assert!(
|
||||
started_before_execution,
|
||||
"the borrowed write must consume native history before queueing its open"
|
||||
);
|
||||
// The disk blocking queue skips an operation once its waiter is gone
|
||||
// (`run_cancellable` observes the closed result channel), so the open
|
||||
// never runs. The Started record taken before queueing must survive
|
||||
// that cancellation: the proof stays conservatively consumed.
|
||||
assert!(!path.exists(), "a cancelled borrowed write must not run its queued open");
|
||||
assert!(
|
||||
history.witness(&bucket, disk_id, Arc::clone(&disk.io_backend)).is_none(),
|
||||
"executor drain must not regrant native history to a cancelled publication"
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[tokio::test]
|
||||
async fn synchronous_direct_open_attempt_consumes_native_history() {
|
||||
let dir = tempfile::tempdir().expect("native disk directory");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 disk path")).expect("disk endpoint");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("native local disk");
|
||||
let bucket = format!("direct-io-open-attempt-{}", Uuid::new_v4());
|
||||
let witness = disk
|
||||
.publication_history
|
||||
.as_ref()
|
||||
.expect("native registration")
|
||||
.witness(&bucket, Uuid::new_v4(), Arc::clone(&disk.io_backend))
|
||||
.expect("fresh history");
|
||||
let path = disk.io_get_object_path(&bucket, "direct-child").expect("direct I/O path");
|
||||
|
||||
assert!(
|
||||
open_direct_writer(&path, &DirectIoWriteState::new()).is_err(),
|
||||
"the actual synchronous O_DIRECT entry must observe the missing bucket parent"
|
||||
);
|
||||
assert!(
|
||||
!os::validate_native_creation_witnesses(&[witness]),
|
||||
"a directly callable O_CREATE helper cannot retain retry permission after a failed publication attempt"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[tokio::test]
|
||||
async fn linux_queued_direct_open_cancellation_retains_started_history() {
|
||||
use crate::disk::os::prepared_publication_test_hooks as hooks;
|
||||
use std::sync::mpsc;
|
||||
|
||||
let dir = tempfile::tempdir().expect("native disk directory");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 disk path")).expect("disk endpoint");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("native local disk");
|
||||
let backend = Arc::new(StdBackend::new_without_fd_cache(disk.io_root().to_path_buf()));
|
||||
let actual_backend: Arc<dyn LocalIoBackend> = backend.clone();
|
||||
let history = disk.publication_history.as_ref().expect("native registration");
|
||||
let bucket = format!("native-queued-dio-{}", Uuid::new_v4());
|
||||
let disk_id = Uuid::new_v4();
|
||||
let witness = history
|
||||
.witness(&bucket, disk_id, Arc::clone(&actual_backend))
|
||||
.expect("fresh native history");
|
||||
let path = local_disk_object_path(disk.io_root(), "", &bucket).expect("first-level DIO path");
|
||||
let write_state = Arc::clone(&backend.direct_io_write);
|
||||
let idle_state_owners = Arc::strong_count(&write_state);
|
||||
let (entered_tx, entered_rx) = mpsc::channel();
|
||||
let (release_tx, release_rx) = mpsc::channel();
|
||||
let _hook = hooks::install_at(hooks::Stage::DirectWriteOpen, &path, move || {
|
||||
entered_tx.send(()).expect("signal real DIO executor entry");
|
||||
let _ = release_rx.recv();
|
||||
});
|
||||
|
||||
temp_env::async_with_vars([(ENV_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE, Some("true"))], async {
|
||||
let operation_backend = Arc::clone(&backend);
|
||||
let operation_bucket = bucket.clone();
|
||||
let operation = tokio::spawn(async move {
|
||||
operation_backend
|
||||
.open_write("", &operation_bucket, WriteMode::Truncate { size_hint: 0 })
|
||||
.await
|
||||
});
|
||||
tokio::task::spawn_blocking(move || entered_rx.recv_timeout(Duration::from_secs(30)))
|
||||
.await
|
||||
.expect("DIO entry waiter")
|
||||
.expect("the real blocking executor must enter");
|
||||
assert!(
|
||||
!os::validate_native_creation_witnesses(std::slice::from_ref(&witness)),
|
||||
"Started must be recorded before the queued DIO worker performs its own history lookup"
|
||||
);
|
||||
assert!(!path.exists(), "the keyed pause must precede the actual O_DIRECT open");
|
||||
operation.abort();
|
||||
match operation.await {
|
||||
Err(error) if error.is_cancelled() => {}
|
||||
Err(error) => panic!("DIO waiter failed instead of cancelling: {error}"),
|
||||
Ok(_) => panic!("the paused DIO waiter must be cancelled"),
|
||||
}
|
||||
release_tx.send(()).expect("release the real DIO open");
|
||||
timeout(Duration::from_secs(30), async {
|
||||
while Arc::strong_count(&write_state) > idle_state_owners {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("the detached DIO executor must drain");
|
||||
})
|
||||
.await;
|
||||
|
||||
assert!(path.is_file(), "cancelling the waiter must not discard the queued native open");
|
||||
std::fs::remove_file(&path).expect("remove the file published by the cancelled DIO waiter");
|
||||
assert!(
|
||||
history.witness(&bucket, disk_id, actual_backend).is_none(),
|
||||
"executor drain and physical removal must not regrant native history"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[tokio::test]
|
||||
async fn linux_mount_lease_alias_publication_consumes_same_history() {
|
||||
let dir = tempfile::tempdir().expect("native disk directory");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 disk path")).expect("disk endpoint");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("native local disk");
|
||||
let history = disk.publication_history.as_ref().expect("native registration");
|
||||
let disk_id = Uuid::new_v4();
|
||||
for (alias, root) in [("root", disk.root.as_path()), ("io", disk.io_root())] {
|
||||
let bucket = format!("native-alias-{alias}-{}", Uuid::new_v4());
|
||||
let witness = history
|
||||
.witness(&bucket, disk_id, Arc::clone(&disk.io_backend))
|
||||
.expect("fresh native history through both real root aliases");
|
||||
super::super::fs::mkdir(root.join(&bucket))
|
||||
.await
|
||||
.expect("publish through the actual directory entry");
|
||||
assert!(
|
||||
!os::validate_native_creation_witnesses(std::slice::from_ref(&witness)),
|
||||
"{alias}: publication through either physical-root alias must consume the same history"
|
||||
);
|
||||
std::fs::remove_dir(disk.root.join(&bucket)).expect("remove the physically published directory");
|
||||
assert!(
|
||||
history.witness(&bucket, disk_id, Arc::clone(&disk.io_backend)).is_none(),
|
||||
"{alias}: removal through the other alias must not regrant native history"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[tokio::test]
|
||||
async fn linux_native_history_is_registered_before_data_usage_layout() {
|
||||
use crate::disk::os::prepared_publication_test_hooks as hooks;
|
||||
|
||||
let dir = tempfile::tempdir().expect("native disk directory");
|
||||
let root = std::fs::canonicalize(dir.path()).expect("canonical disk root");
|
||||
let bucket = format!("native-before-layout-{}", Uuid::new_v4());
|
||||
let source = root.join(format!("layout-source-{}", Uuid::new_v4()));
|
||||
std::fs::create_dir(&source).expect("prepare an actual directory for publication");
|
||||
let destination = root.join(&bucket);
|
||||
let publication_path = destination.clone();
|
||||
let _hook = hooks::install_at(hooks::Stage::BeforeDataUsageLayout, &root, move || {
|
||||
super::super::fs::rename_std(&source, &publication_path)
|
||||
.expect("publish through the native entry before data usage layout");
|
||||
});
|
||||
let endpoint = Endpoint::try_from(root.to_str().expect("UTF-8 disk path")).expect("disk endpoint");
|
||||
let disk = LocalDisk::new(&endpoint, false)
|
||||
.await
|
||||
.expect("native local disk with real startup layout");
|
||||
let history = disk.publication_history.as_ref().expect("native registration");
|
||||
let disk_id = Uuid::new_v4();
|
||||
|
||||
let untouched = format!("native-layout-untouched-{}", Uuid::new_v4());
|
||||
assert!(
|
||||
history.witness(&untouched, disk_id, Arc::clone(&disk.io_backend)).is_some(),
|
||||
"startup layout must keep an untouched bucket's native history known"
|
||||
);
|
||||
assert!(
|
||||
history.witness(&bucket, disk_id, Arc::clone(&disk.io_backend)).is_none(),
|
||||
"registration must capture real publication before startup layout finishes"
|
||||
);
|
||||
std::fs::remove_dir(destination).expect("remove the physically published directory");
|
||||
assert!(
|
||||
history.witness(&bucket, disk_id, Arc::clone(&disk.io_backend)).is_none(),
|
||||
"startup completion and physical removal must retain Started"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn read_dir_entries_skip_symlinks_without_truncating_listing() {
|
||||
@@ -17026,6 +17337,109 @@ mod test {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rename_data_pending_restore_requires_parts_only_from_multipart_staging() {
|
||||
let dir = tempfile::tempdir().expect("disk directory should be created");
|
||||
let endpoint =
|
||||
Endpoint::try_from(dir.path().to_str().expect("disk path should be UTF-8")).expect("disk endpoint should parse");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
|
||||
let bucket = "restore-rename-controls";
|
||||
ensure_test_volume(&disk, bucket).await;
|
||||
for (case, source_volume, pending, requires_parts) in [
|
||||
("remote", RUSTFS_META_TMP_BUCKET, false, false),
|
||||
("pending-remote", RUSTFS_META_TMP_BUCKET, true, false),
|
||||
("pending-multipart", crate::disk::RUSTFS_META_MULTIPART_BUCKET, true, true),
|
||||
] {
|
||||
ensure_test_volume(&disk, source_volume).await;
|
||||
let staging = dir.path().join(source_volume).join(case);
|
||||
fs::create_dir_all(&staging).await.expect("staging parent should be created");
|
||||
let data_dir = Uuid::new_v4();
|
||||
let mut fi = test_file_info(case, Uuid::nil(), Some(data_dir), None);
|
||||
fi.transition_status = rustfs_filemeta::TRANSITION_COMPLETE.to_string();
|
||||
fi.transition_tier = "WARM".to_string();
|
||||
fi.transitioned_objname = "remote/object".to_string();
|
||||
if pending {
|
||||
fi.metadata.insert(
|
||||
rustfs_filemeta::metadata_keys::RESTORE.to_string(),
|
||||
"ongoing-request=\"true\"".to_string(),
|
||||
);
|
||||
}
|
||||
let result = disk.rename_data(source_volume, case, fi, bucket, case).await;
|
||||
let destination = dir.path().join(bucket).join(case);
|
||||
if requires_parts {
|
||||
assert!(result.is_err(), "missing multipart restore parts must reject the physical commit");
|
||||
assert!(
|
||||
!destination.join(STORAGE_FORMAT_FILE).exists(),
|
||||
"failed multipart restore must not publish metadata for absent local parts"
|
||||
);
|
||||
} else {
|
||||
result.expect("remote-only metadata commits must not require local staged parts");
|
||||
let stored = fs::read(destination.join(STORAGE_FORMAT_FILE))
|
||||
.await
|
||||
.expect("remote-only metadata should be committed");
|
||||
let restored = FileMeta::load(&stored)
|
||||
.expect("committed metadata should decode")
|
||||
.into_fileinfo(bucket, case, "", false, false, false)
|
||||
.expect("committed version should decode");
|
||||
assert!(restored.is_remote(), "remote-only commits must retain remote read routing");
|
||||
assert!(
|
||||
!destination.join(data_dir.to_string()).exists(),
|
||||
"remote-only metadata commit must not create a local payload directory"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rename_data_pending_multipart_restore_rejects_absent_data_directory() {
|
||||
assert_pending_multipart_restore_rejects_directory(None).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rename_data_pending_multipart_restore_rejects_nil_data_directory() {
|
||||
assert_pending_multipart_restore_rejects_directory(Some(Uuid::nil())).await;
|
||||
}
|
||||
|
||||
async fn assert_pending_multipart_restore_rejects_directory(data_dir: Option<Uuid>) {
|
||||
let dir = tempfile::tempdir().expect("disk directory should be created");
|
||||
let endpoint =
|
||||
Endpoint::try_from(dir.path().to_str().expect("disk path should be UTF-8")).expect("disk endpoint should parse");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
|
||||
let source_volume = crate::disk::RUSTFS_META_MULTIPART_BUCKET;
|
||||
let bucket = "restore-invalid-directory";
|
||||
let object = "object.bin";
|
||||
ensure_test_volume(&disk, source_volume).await;
|
||||
ensure_test_volume(&disk, bucket).await;
|
||||
let staging = dir.path().join(source_volume).join(object);
|
||||
fs::create_dir_all(&staging).await.expect("staging parent should be created");
|
||||
if let Some(data_dir) = data_dir {
|
||||
let data_path = staging.join(data_dir.to_string());
|
||||
fs::create_dir_all(&data_path)
|
||||
.await
|
||||
.expect("invalid staging directory should be created");
|
||||
fs::write(data_path.join("part.1"), b"x")
|
||||
.await
|
||||
.expect("nil directory fixture must contain a real part");
|
||||
}
|
||||
let mut fi = test_file_info(object, Uuid::nil(), data_dir, None);
|
||||
fi.transition_status = rustfs_filemeta::TRANSITION_COMPLETE.to_string();
|
||||
fi.transition_tier = "WARM".to_string();
|
||||
fi.transitioned_objname = "remote/object".to_string();
|
||||
fi.metadata.insert(
|
||||
rustfs_filemeta::metadata_keys::RESTORE.to_string(),
|
||||
"ongoing-request=\"true\"".to_string(),
|
||||
);
|
||||
let result = disk.rename_data(source_volume, object, fi, bucket, object).await;
|
||||
assert!(
|
||||
matches!(result, Err(DiskError::FileCorrupt)),
|
||||
"pending multipart restore must reject absent or nil staging identity as corrupt: {result:?}"
|
||||
);
|
||||
assert!(
|
||||
!dir.path().join(bucket).join(object).join(STORAGE_FORMAT_FILE).exists(),
|
||||
"invalid staging identity must fail before metadata publication"
|
||||
);
|
||||
}
|
||||
|
||||
// Seed a first PUT of `object` (no prior version) through the non-inline
|
||||
// rename_data path and return (disk, tempdir). The object dir and any prefix
|
||||
// dirs are created during the commit.
|
||||
@@ -31028,7 +31442,7 @@ mod test {
|
||||
assert_eq!(recorder.counter_value(counter, &[("outcome", "error")]), 2);
|
||||
assert_eq!(recorder.counter_value("rustfs_io_get_object_local_batch_read_bytes_total", &[]), 9);
|
||||
let stages = "rustfs_io_get_object_local_batch_stage_seconds";
|
||||
for (stage, success, errors) in [("metadata", 3, 1), ("open", 2, 0), ("read", 1, 0), ("execution", 0, 1)] {
|
||||
for (stage, success, errors) in [("open", 3, 1), ("metadata", 3, 0), ("read", 1, 0), ("execution", 0, 1)] {
|
||||
assert_eq!(
|
||||
recorder
|
||||
.histogram_values(stages, &[("stage", stage), ("outcome", "success")])
|
||||
@@ -31097,6 +31511,70 @@ mod test {
|
||||
assert_eq!(recorder.counter_value("rustfs_io_get_object_local_batch_read_bytes_total", &[]), 4);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn test_batch_shard_pread_rejects_nonregular_files_without_blocking() {
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
|
||||
let dir = tempfile::tempdir().expect("nonregular shard fixture");
|
||||
let fifo = dir.path().join("fifo");
|
||||
assert!(
|
||||
std::process::Command::new("mkfifo")
|
||||
.arg(&fifo)
|
||||
.status()
|
||||
.expect("run POSIX mkfifo for the shard fixture")
|
||||
.success()
|
||||
);
|
||||
let good = dir.path().join("good");
|
||||
std::fs::write(&good, b"data").expect("write regular shard");
|
||||
let mut batch = Box::pin(batch_shard_pread(vec![
|
||||
(dir.path().to_owned(), 0, 0),
|
||||
(fifo.clone(), 0, 0),
|
||||
(fifo.clone(), 0, 4),
|
||||
(good, 0, 4),
|
||||
]));
|
||||
let results = match tokio::time::timeout(Duration::from_secs(2), &mut batch).await {
|
||||
Ok(results) => results,
|
||||
Err(_) => {
|
||||
// Release a regressed blocking open before failing, so runtime
|
||||
// shutdown does not leave the test waiting on its worker.
|
||||
let _release = std::fs::OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.custom_flags(libc::O_NONBLOCK)
|
||||
.open(&fifo)
|
||||
.expect("release blocked FIFO reader");
|
||||
batch.await;
|
||||
panic!("a nonregular shard must not block its batch");
|
||||
}
|
||||
};
|
||||
for result in &results[..3] {
|
||||
assert!(matches!(result, Err(DiskError::FileCorrupt)), "nonregular shard: {result:?}");
|
||||
}
|
||||
assert_eq!(results[3].as_ref().expect("later regular shard remains readable").as_ref(), b"data");
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn test_batch_shard_pread_validates_descriptor_ranges() {
|
||||
let dir = tempfile::tempdir().expect("range validation fixture");
|
||||
let path = dir.path().join("shard");
|
||||
std::fs::write(&path, b"01234567").expect("write range fixture");
|
||||
let results = batch_shard_pread(vec![
|
||||
(path.clone(), 3, 5),
|
||||
(path.clone(), 8, 0),
|
||||
(path.clone(), 8, 1),
|
||||
(path.clone(), 9, 0),
|
||||
(path, usize::MAX, 1),
|
||||
])
|
||||
.await;
|
||||
assert_eq!(results[0].as_ref().expect("range ending at EOF").as_ref(), b"34567");
|
||||
assert!(results[1].as_ref().expect("empty range at EOF").is_empty());
|
||||
for result in &results[2..] {
|
||||
assert!(matches!(result, Err(DiskError::FileCorrupt)), "invalid range: {result:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn test_batch_shard_pread_basic() {
|
||||
|
||||
@@ -642,8 +642,20 @@ impl LocalDisk {
|
||||
};
|
||||
let dst_file_path = self.io_get_object_open_path(dst_volume, format!("{}/{}", dst_path, STORAGE_FORMAT_FILE).as_str())?;
|
||||
|
||||
// Multipart restore completion has validated staged local parts,
|
||||
// while its public status stays pending until finalization.
|
||||
let pending_multipart_restore = src_volume == crate::disk::RUSTFS_META_MULTIPART_BUCKET
|
||||
&& fi
|
||||
.metadata
|
||||
.get(rustfs_filemeta::metadata_keys::RESTORE)
|
||||
.and_then(|value| rustfs_filemeta::parse_restore_obj_status(value).ok())
|
||||
.is_some_and(|status| status.is_restore_in_progress == Some(true));
|
||||
if pending_multipart_restore && fi.data_dir.is_none_or(|dir| dir.is_nil()) {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
|
||||
// data_dir path (only used by the non-inline commit branch)
|
||||
let has_data_dir_path = if no_inline && !fi.is_remote() {
|
||||
let has_data_dir_path = if no_inline && (!fi.is_remote() || pending_multipart_restore) {
|
||||
if let Some(data_dir) = fi
|
||||
.data_dir
|
||||
.map(|dir| rustfs_utils::path::retain_slash(dir.to_string().as_str()))
|
||||
|
||||
@@ -613,6 +613,13 @@ impl DiskAPI for Disk {
|
||||
}
|
||||
}
|
||||
|
||||
async fn bucket_creation_witness(&self, volume: &str) -> Result<Option<os::NativeBucketCreationWitness>> {
|
||||
match self {
|
||||
Disk::Local(local_disk) => local_disk.bucket_creation_witness(volume).await,
|
||||
Disk::Remote(_) => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn make_volume(&self, volume: &str) -> Result<()> {
|
||||
match self {
|
||||
@@ -1470,6 +1477,11 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
|
||||
// NSScanner
|
||||
|
||||
// Volume operations.
|
||||
/// Unknown or remote backends cannot attest process-local publication history.
|
||||
async fn bucket_creation_witness(&self, _volume: &str) -> Result<Option<os::NativeBucketCreationWitness>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn make_volume(&self, volume: &str) -> Result<()>;
|
||||
async fn make_volumes(&self, volume: Vec<&str>) -> Result<()>;
|
||||
async fn list_volumes(&self) -> Result<Vec<VolumeInfo>>;
|
||||
@@ -2245,7 +2257,7 @@ impl DeleteRecoveryJournal {
|
||||
}
|
||||
let id = request.version_id.unwrap_or_default();
|
||||
let Some(original) = originals.get(&id) else {
|
||||
return false;
|
||||
return !id.is_nil() && !locals.contains_key(&id);
|
||||
};
|
||||
match locals.get(&id) {
|
||||
Some(local) => local == original,
|
||||
@@ -2295,7 +2307,10 @@ impl DeleteRecoveryJournal {
|
||||
};
|
||||
match (local, remote) {
|
||||
(Some(local), Some(remote)) => local == remote,
|
||||
(None, None) => self.original_digest.is_none() && authority.original_digest.is_none(),
|
||||
(None, None) => {
|
||||
(!(request.deleted && request.mark_deleted) && request.version_id.is_some_and(|id| !id.is_nil()))
|
||||
|| (self.original_digest.is_none() && authority.original_digest.is_none())
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
})
|
||||
@@ -3620,6 +3635,46 @@ mod delete_recovery_adoption_tests {
|
||||
};
|
||||
assert!(journal.matches_adoption_source(&[]), "certified non-null removal can already be absent");
|
||||
assert!(journal.matches_adoption_source(std::slice::from_ref(&source)));
|
||||
let missing = Uuid::new_v4();
|
||||
journal.requests.push(FileInfo {
|
||||
version_id: Some(missing),
|
||||
..Default::default()
|
||||
});
|
||||
journal.original_digest = Some([3; 32]);
|
||||
assert!(
|
||||
journal.matches_prepared_authority(&journal),
|
||||
"an explicit UUID absent from both prepared sources remains a no-op with original metadata"
|
||||
);
|
||||
assert!(journal.matches_adoption_source(&[]), "both non-null versions can remain absent");
|
||||
assert!(
|
||||
journal.matches_adoption_source(std::slice::from_ref(&source)),
|
||||
"a missing non-null version in the batch remains a no-op"
|
||||
);
|
||||
let mut unexpected = source.clone();
|
||||
unexpected.version_id = Some(missing);
|
||||
let mut foreign_preparation = journal.clone();
|
||||
foreign_preparation.original_versions.push(unexpected.clone());
|
||||
assert!(
|
||||
!journal.matches_prepared_authority(&foreign_preparation)
|
||||
&& !foreign_preparation.matches_prepared_authority(&journal),
|
||||
"absence from only one prepared source cannot establish a generation"
|
||||
);
|
||||
assert!(
|
||||
!journal.matches_adoption_source(&[source.clone(), unexpected]),
|
||||
"a version absent from the authority cannot be deleted locally"
|
||||
);
|
||||
for version_id in [None, Some(Uuid::nil())] {
|
||||
journal.requests.last_mut().expect("missing-version request").version_id = version_id;
|
||||
assert!(
|
||||
!journal.matches_prepared_authority(&journal),
|
||||
"a missing null version with original metadata cannot establish its generation"
|
||||
);
|
||||
assert!(
|
||||
!journal.matches_adoption_source(std::slice::from_ref(&source)),
|
||||
"a missing null version cannot establish its generation"
|
||||
);
|
||||
}
|
||||
journal.requests.pop();
|
||||
let mut foreign = source.clone();
|
||||
foreign.metadata_digest = [2; 32];
|
||||
assert!(!journal.matches_adoption_source(std::slice::from_ref(&foreign)));
|
||||
@@ -3635,6 +3690,25 @@ mod delete_recovery_adoption_tests {
|
||||
assert!(journal.matches_adoption_source(std::slice::from_ref(&journal.original_versions[0])));
|
||||
journal.requests.clear();
|
||||
assert!(!journal.matches_adoption_source(&journal.original_versions));
|
||||
|
||||
let mut marker = journal.clone();
|
||||
marker.requests = vec![FileInfo {
|
||||
version_id: Some(id),
|
||||
deleted: true,
|
||||
mark_deleted: true,
|
||||
..Default::default()
|
||||
}];
|
||||
marker.original_versions.clear();
|
||||
marker.original_digest = Some([3; 32]);
|
||||
assert!(
|
||||
!marker.matches_prepared_authority(&marker),
|
||||
"a marker with original metadata requires a source generation"
|
||||
);
|
||||
marker.original_digest = None;
|
||||
assert!(
|
||||
marker.matches_prepared_authority(&marker),
|
||||
"marker preparation without original metadata retains the existing allowance"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ use futures::TryStreamExt;
|
||||
use parking_lot::Mutex;
|
||||
use rustfs_utils::path::SLASH_SEPARATOR;
|
||||
use std::{
|
||||
collections::{HashMap, VecDeque},
|
||||
collections::{HashMap, HashSet, VecDeque},
|
||||
io,
|
||||
path::{Component, Path, PathBuf},
|
||||
sync::{Arc, LazyLock, Weak},
|
||||
@@ -333,10 +333,20 @@ pub(crate) mod prepared_publication_test_hooks {
|
||||
PreparedRename,
|
||||
Rename,
|
||||
Remove,
|
||||
MetadataReadAfterOpen,
|
||||
MetadataReadAfterRead,
|
||||
#[cfg(test)]
|
||||
Rollback,
|
||||
#[cfg(test)]
|
||||
DirFsync,
|
||||
#[cfg(test)]
|
||||
CreateDirectory,
|
||||
#[cfg(test)]
|
||||
NativeRootIdentity,
|
||||
#[cfg(all(test, target_os = "linux"))]
|
||||
DirectWriteOpen,
|
||||
#[cfg(all(test, target_os = "linux"))]
|
||||
BeforeDataUsageLayout,
|
||||
}
|
||||
|
||||
type Hook = Box<dyn FnOnce() + Send>;
|
||||
@@ -457,6 +467,8 @@ pub enum LocalPublicationStage {
|
||||
PreparedRename,
|
||||
Rename,
|
||||
Remove,
|
||||
MetadataReadAfterOpen,
|
||||
MetadataReadAfterRead,
|
||||
}
|
||||
|
||||
#[cfg(all(feature = "test-util", not(windows)))]
|
||||
@@ -469,6 +481,8 @@ impl LocalPublicationPause {
|
||||
LocalPublicationStage::PreparedRename => prepared_publication_test_hooks::Stage::PreparedRename,
|
||||
LocalPublicationStage::Rename => prepared_publication_test_hooks::Stage::Rename,
|
||||
LocalPublicationStage::Remove => prepared_publication_test_hooks::Stage::Remove,
|
||||
LocalPublicationStage::MetadataReadAfterOpen => prepared_publication_test_hooks::Stage::MetadataReadAfterOpen,
|
||||
LocalPublicationStage::MetadataReadAfterRead => prepared_publication_test_hooks::Stage::MetadataReadAfterRead,
|
||||
};
|
||||
let (entered_tx, entered) = oneshot::channel();
|
||||
let (release, release_rx) = std::sync::mpsc::channel::<()>();
|
||||
@@ -1713,6 +1727,226 @@ fn fsync_spawn_blocking<T: Send + 'static>(f: impl FnOnce() -> T + Send + 'stati
|
||||
}
|
||||
static DISK_VOLUME_MUTATION_LOCKS: LazyLock<Mutex<HashMap<PathBuf, Weak<RwLock<()>>>>> =
|
||||
LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
#[cfg(windows)]
|
||||
type NativeRootIdentity = (u64, [u8; 16]);
|
||||
#[cfg(not(windows))]
|
||||
type NativeRootIdentity = (u64, u64);
|
||||
|
||||
// Publication history outlives disk handles, cleanup, and metadata snapshots.
|
||||
// Reusing a pathname or descriptor for another physical root is not new proof.
|
||||
#[derive(Debug, Default)]
|
||||
struct NativePublicationHistory {
|
||||
roots: HashMap<NativeRootIdentity, NativeRootHistory>,
|
||||
aliases: HashMap<PathBuf, HashSet<NativeRootIdentity>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct NativeRootHistory {
|
||||
unknown: bool,
|
||||
started: HashSet<String>,
|
||||
}
|
||||
|
||||
static NATIVE_PUBLICATION_HISTORY: LazyLock<Mutex<NativePublicationHistory>> =
|
||||
LazyLock::new(|| Mutex::new(NativePublicationHistory::default()));
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct NativePublicationRegistration {
|
||||
identity: NativeRootIdentity,
|
||||
root: PathBuf,
|
||||
io_root: PathBuf,
|
||||
}
|
||||
|
||||
/// Opaque, process-local observation. Unsupported disks cannot construct it.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct NativeBucketCreationWitness {
|
||||
registration: NativePublicationRegistration,
|
||||
backend: Arc<dyn super::local::LocalIoBackend>,
|
||||
bucket: String,
|
||||
disk_id: uuid::Uuid,
|
||||
}
|
||||
|
||||
fn native_root_identity(path: &Path) -> Option<NativeRootIdentity> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
#[cfg(test)]
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::NativeRootIdentity, path);
|
||||
let metadata = std::fs::metadata(path).ok()?;
|
||||
metadata.is_dir().then(|| (metadata.dev(), metadata.ino()))
|
||||
}
|
||||
#[cfg(windows)]
|
||||
{
|
||||
// Resolve and validate the same directory anchor used for publication;
|
||||
// keep its volume and complete 128-bit file ID in the history key.
|
||||
let (_, directory) = open_windows_publication_root(path).ok()?;
|
||||
windows_file_identity(directory.handle.as_ref()).ok()
|
||||
}
|
||||
#[cfg(all(not(unix), not(windows)))]
|
||||
{
|
||||
let _ = path;
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn register_native_publication_root(root: &Path, io_root: &Path) -> Option<NativePublicationRegistration> {
|
||||
let identity = native_root_identity(root)?;
|
||||
if native_root_identity(io_root) != Some(identity) {
|
||||
return None;
|
||||
}
|
||||
let mut history = NATIVE_PUBLICATION_HISTORY.lock();
|
||||
history.roots.entry(identity).or_default();
|
||||
for alias in [root, io_root] {
|
||||
let alias: PathBuf = alias.components().collect();
|
||||
let previous = history.aliases.entry(alias).or_default();
|
||||
let collisions: Vec<_> = previous.iter().copied().filter(|old| *old != identity).collect();
|
||||
previous.insert(identity);
|
||||
if !collisions.is_empty() {
|
||||
history.roots.get_mut(&identity)?.unknown = true;
|
||||
for old in collisions {
|
||||
history.roots.get_mut(&old)?.unknown = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
Some(NativePublicationRegistration {
|
||||
identity,
|
||||
root: root.to_path_buf(),
|
||||
io_root: io_root.to_path_buf(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Record before queueing filesystem work. Cancellation never removes Started.
|
||||
pub(crate) fn record_native_publication(path: &Path) {
|
||||
let aliases: Vec<_> = {
|
||||
let history = NATIVE_PUBLICATION_HISTORY.lock();
|
||||
path.ancestors()
|
||||
.filter(|alias| history.aliases.contains_key(*alias))
|
||||
.map(Path::to_path_buf)
|
||||
.collect()
|
||||
};
|
||||
// Physical lookups must never hold the process-wide publication mutex.
|
||||
let observations: Vec<_> = aliases
|
||||
.into_iter()
|
||||
.filter_map(|alias| {
|
||||
let relative = path.strip_prefix(&alias).ok()?;
|
||||
let mut components = relative.components().filter(|part| !matches!(part, Component::CurDir));
|
||||
let first = components.next()?;
|
||||
let bucket = match first {
|
||||
Component::Normal(bucket) if components.all(|part| matches!(part, Component::Normal(_))) => {
|
||||
bucket.to_str().map(str::to_owned)
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
let physical = native_root_identity(&alias);
|
||||
Some((alias, physical, bucket))
|
||||
})
|
||||
.collect();
|
||||
let mut history = NATIVE_PUBLICATION_HISTORY.lock();
|
||||
for (alias, physical, bucket) in observations {
|
||||
// Aliases only accumulate identities. Include registration concurrent
|
||||
// with the stat, rather than using a stale captured identity list.
|
||||
let identities = history.aliases.get(&alias).cloned().unwrap_or_default();
|
||||
if physical.is_none() || identities.iter().any(|identity| Some(*identity) != physical) {
|
||||
for identity in identities {
|
||||
history.roots.entry(identity).or_default().unknown = true;
|
||||
}
|
||||
if let Some(identity) = physical {
|
||||
history.roots.entry(identity).or_default().unknown = true;
|
||||
}
|
||||
} else if let Some(identity) = physical {
|
||||
let root = history.roots.entry(identity).or_default();
|
||||
match bucket {
|
||||
Some(bucket) => {
|
||||
root.started.insert(bucket);
|
||||
}
|
||||
None => root.unknown = true,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl NativePublicationRegistration {
|
||||
pub(crate) fn invalidate(&self) {
|
||||
if let Some(root) = NATIVE_PUBLICATION_HISTORY.lock().roots.get_mut(&self.identity) {
|
||||
root.unknown = true;
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn witness(
|
||||
&self,
|
||||
bucket: &str,
|
||||
disk_id: uuid::Uuid,
|
||||
backend: Arc<dyn super::local::LocalIoBackend>,
|
||||
) -> Option<NativeBucketCreationWitness> {
|
||||
if disk_id.is_nil()
|
||||
|| !matches!(Path::new(bucket).components().next(), Some(Component::Normal(_)))
|
||||
|| Path::new(bucket).components().count() != 1
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let witness = NativeBucketCreationWitness {
|
||||
registration: self.clone(),
|
||||
backend,
|
||||
bucket: bucket.to_owned(),
|
||||
disk_id,
|
||||
};
|
||||
validate_native_creation_witnesses(std::slice::from_ref(&witness)).then_some(witness)
|
||||
}
|
||||
}
|
||||
|
||||
impl NativeBucketCreationWitness {
|
||||
pub(crate) fn matches(&self, other: &Self) -> bool {
|
||||
self.registration.identity == other.registration.identity
|
||||
&& Arc::ptr_eq(&self.backend, &other.backend)
|
||||
&& self.disk_id == other.disk_id
|
||||
&& self.bucket == other.bucket
|
||||
}
|
||||
|
||||
fn physical_root_is_current(&self) -> bool {
|
||||
let root = native_root_identity(&self.registration.root);
|
||||
let io_root = native_root_identity(&self.registration.io_root);
|
||||
if root == Some(self.registration.identity) && io_root == Some(self.registration.identity) {
|
||||
return true;
|
||||
}
|
||||
let mut history = NATIVE_PUBLICATION_HISTORY.lock();
|
||||
history.roots.entry(self.registration.identity).or_default().unknown = true;
|
||||
for identity in [root, io_root].into_iter().flatten() {
|
||||
history.roots.entry(identity).or_default().unknown = true;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn valid_in(&self, history: &NativePublicationHistory) -> bool {
|
||||
history
|
||||
.roots
|
||||
.get(&self.registration.identity)
|
||||
.is_some_and(|root| !root.unknown && !root.started.contains(&self.bucket))
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn validate_native_creation_witnesses(witnesses: &[NativeBucketCreationWitness]) -> bool {
|
||||
if witnesses.is_empty() || !witnesses.iter().all(NativeBucketCreationWitness::physical_root_is_current) {
|
||||
return false;
|
||||
}
|
||||
let history = NATIVE_PUBLICATION_HISTORY.lock();
|
||||
witnesses.iter().all(|witness| witness.valid_in(&history))
|
||||
}
|
||||
|
||||
pub(crate) fn consume_native_creation_witnesses(witnesses: &[NativeBucketCreationWitness]) -> bool {
|
||||
if witnesses.is_empty() || !witnesses.iter().all(NativeBucketCreationWitness::physical_root_is_current) {
|
||||
return false;
|
||||
}
|
||||
let mut history = NATIVE_PUBLICATION_HISTORY.lock();
|
||||
if !witnesses.iter().all(|witness| witness.valid_in(&history)) {
|
||||
return false;
|
||||
}
|
||||
for witness in witnesses {
|
||||
if let Some(root) = history.roots.get_mut(&witness.registration.identity) {
|
||||
root.started.insert(witness.bucket.clone());
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
type NamespaceMutationLock = AsyncMutex<()>;
|
||||
type NamespaceMutationLockRegistry = HashMap<PathBuf, Weak<NamespaceMutationLock>>;
|
||||
static DISK_NAMESPACE_MUTATION_LOCKS: LazyLock<Mutex<NamespaceMutationLockRegistry>> =
|
||||
@@ -2984,6 +3218,7 @@ pub(in crate::disk) async fn rename_with_namespace_owner(
|
||||
dst: &Path,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> io::Result<()> {
|
||||
record_native_publication(dst);
|
||||
if namespace_owner.is_none() {
|
||||
return tokio::fs::rename(src, dst).await;
|
||||
}
|
||||
@@ -3005,12 +3240,19 @@ pub(in crate::disk) async fn create_dir_all_with_namespace_owner(
|
||||
path: &Path,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> io::Result<()> {
|
||||
record_native_publication(path);
|
||||
if namespace_owner.is_none() {
|
||||
return tokio::fs::create_dir_all(path).await;
|
||||
}
|
||||
let path = path.to_path_buf();
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(&path, namespace_owner).await;
|
||||
run_blocking_namespace_operation(lease, move || std::fs::create_dir_all(path)).await
|
||||
run_blocking_namespace_operation(lease, move || {
|
||||
#[cfg(all(test, not(windows)))]
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::CreateDirectory, &path);
|
||||
record_native_publication(&path);
|
||||
std::fs::create_dir_all(path)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(name = "rename_all", level = "debug", skip_all)]
|
||||
@@ -3021,6 +3263,7 @@ pub(crate) async fn rename_all_with_owner(
|
||||
publication_root: &PublicationRoot,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
record_native_publication(dst_file_path.as_ref());
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(dst_file_path.as_ref(), namespace_owner).await;
|
||||
rename_all_with_lease(src_file_path, dst_file_path, base_dir, publication_root, lease).await
|
||||
}
|
||||
@@ -3032,6 +3275,7 @@ pub(crate) async fn rename_all_with_lease(
|
||||
publication_root: &PublicationRoot,
|
||||
lease: Arc<NamespaceMutationLease>,
|
||||
) -> Result<()> {
|
||||
record_native_publication(dst_file_path.as_ref());
|
||||
reliable_rename_inner_with_lease(
|
||||
src_file_path.as_ref().to_path_buf(),
|
||||
dst_file_path.as_ref().to_path_buf(),
|
||||
@@ -3132,6 +3376,7 @@ pub(crate) async fn rename_reconciled_metadata(
|
||||
owner: Option<Arc<dyn Send + Sync>>,
|
||||
authority: Arc<crate::bucket::lifecycle::legacy_transition_state_reconcile::TransitionStateReconcileAuthority>,
|
||||
) -> Result<()> {
|
||||
record_native_publication(&destination);
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(&destination, owner).await;
|
||||
run_blocking_namespace_operation(lease, move || {
|
||||
let preparation = prepare_rename_with_retry(&source, &destination, &base_dir, &publication_root)?;
|
||||
@@ -3782,6 +4027,7 @@ pub(crate) async fn rename_all_with_prepared_source(
|
||||
commit_guard: &RenameCommitGuard,
|
||||
lease: Arc<NamespaceMutationLease>,
|
||||
) -> Result<()> {
|
||||
record_native_publication(dst_file_path.as_ref());
|
||||
let src_file_path = src_file_path.as_ref().to_path_buf();
|
||||
let dst_file_path = dst_file_path.as_ref().to_path_buf();
|
||||
let base_dir = base_dir.as_ref().to_path_buf();
|
||||
@@ -3811,6 +4057,7 @@ pub(crate) async fn rename_all_with_commit_guard(
|
||||
commit_guard: &RenameCommitGuard,
|
||||
lease: Arc<NamespaceMutationLease>,
|
||||
) -> Result<()> {
|
||||
record_native_publication(dst_file_path.as_ref());
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
let source = src_file_path.as_ref().to_path_buf();
|
||||
@@ -3864,6 +4111,7 @@ pub(crate) async fn rename_all_ignore_missing_source_with_owner(
|
||||
publication_root: &PublicationRoot,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
record_native_publication(dst_file_path.as_ref());
|
||||
let src_file_path = src_file_path.as_ref();
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(dst_file_path.as_ref(), namespace_owner).await;
|
||||
match reliable_rename_inner_with_lease(
|
||||
@@ -3918,6 +4166,7 @@ async fn reliable_rename_inner(
|
||||
publication_root: &PublicationRoot,
|
||||
warn_on_missing_source: bool,
|
||||
) -> io::Result<()> {
|
||||
record_native_publication(dst_file_path.as_ref());
|
||||
let src_file_path = src_file_path.as_ref().to_path_buf();
|
||||
let dst_file_path = dst_file_path.as_ref().to_path_buf();
|
||||
let base_dir = base_dir.as_ref().to_path_buf();
|
||||
@@ -3941,6 +4190,7 @@ async fn reliable_rename_inner_with_lease(
|
||||
warn_on_missing_source: bool,
|
||||
lease: Arc<NamespaceMutationLease>,
|
||||
) -> io::Result<()> {
|
||||
record_native_publication(&dst_file_path);
|
||||
let operation = {
|
||||
let src_file_path = src_file_path.clone();
|
||||
let dst_file_path = dst_file_path.clone();
|
||||
@@ -4410,6 +4660,7 @@ fn rename_into_existing_parent(
|
||||
parent_guard: Option<&ExistingBaseDirectoryGuard>,
|
||||
#[cfg(target_os = "linux")] source_parent: &std::fs::File,
|
||||
) -> io::Result<()> {
|
||||
record_native_publication(dst_file_path);
|
||||
use rustix::fs::renameat;
|
||||
|
||||
let Some(parent_guard) = parent_guard else {
|
||||
@@ -4463,6 +4714,7 @@ fn rename_into_existing_parent(
|
||||
parent_guard: Option<&ExistingBaseDirectoryGuard>,
|
||||
source: &winapi_util::Handle,
|
||||
) -> io::Result<()> {
|
||||
record_native_publication(dst_file_path);
|
||||
use std::{
|
||||
mem::size_of,
|
||||
os::windows::{ffi::OsStrExt, io::AsRawHandle},
|
||||
@@ -4706,6 +4958,7 @@ impl PublicationRoot {
|
||||
}
|
||||
|
||||
pub(crate) fn create_truncate(&self, path: &Path) -> io::Result<std::fs::File> {
|
||||
record_native_publication(path);
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
use rustix::fs::{Mode, OFlags};
|
||||
@@ -4716,6 +4969,7 @@ impl PublicationRoot {
|
||||
}
|
||||
|
||||
pub(crate) fn create_truncate_with_parent(&self, path: &Path) -> io::Result<(std::fs::File, Option<std::fs::File>)> {
|
||||
record_native_publication(path);
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
use rustix::fs::{Mode, OFlags};
|
||||
@@ -4980,6 +5234,7 @@ impl LinuxCommitDirectories {
|
||||
}
|
||||
|
||||
fn rename(&self, source: &Path, destination: &Path, prepared: Option<&PreparedRenameSource>) -> io::Result<()> {
|
||||
record_native_publication(destination);
|
||||
let source_name = self.file_name(source, true)?;
|
||||
let destination_name = self.file_name(destination, false)?;
|
||||
let destination_parent = self.prepare_destination()?;
|
||||
@@ -7176,6 +7431,7 @@ pub(crate) fn mkdir_all_below_existing_base_std(
|
||||
base_dir: &Path,
|
||||
publication_root: &PublicationRoot,
|
||||
) -> io::Result<ExistingBaseDirectoryGuard> {
|
||||
record_native_publication(dir_path);
|
||||
let relative = existing_base_directory_suffix(dir_path, base_dir)?;
|
||||
|
||||
#[cfg(unix)]
|
||||
@@ -8865,6 +9121,77 @@ mod tests {
|
||||
assert_eq!(err.raw_os_error(), Some(disk_full));
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
fn windows_native_root_identity_matches_directory_handles() {
|
||||
let temp = tempdir().expect("create native identity fixture");
|
||||
let first = temp.path().join("first");
|
||||
let second = temp.path().join("second");
|
||||
std::fs::create_dir(&first).expect("create first root");
|
||||
std::fs::create_dir(&second).expect("create second root");
|
||||
let (_, first_anchor) = open_windows_publication_root(&first).expect("open first directory anchor");
|
||||
let (_, second_anchor) = open_windows_publication_root(&second).expect("open second directory anchor");
|
||||
let first_identity = windows_file_identity(first_anchor.handle.as_ref()).expect("query full first identity");
|
||||
let second_identity = windows_file_identity(second_anchor.handle.as_ref()).expect("query full second identity");
|
||||
|
||||
assert_eq!(native_root_identity(&first), Some(first_identity));
|
||||
assert_eq!(native_root_identity(&first.join(".")), Some(first_identity));
|
||||
assert_eq!(native_root_identity(&second), Some(second_identity));
|
||||
assert_ne!(first_identity, second_identity, "distinct physical roots must have distinct identities");
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[tokio::test]
|
||||
async fn windows_native_creation_witness_retains_started_history() {
|
||||
let temp = tempdir().expect("create native witness fixture");
|
||||
let root = std::fs::canonicalize(temp.path()).expect("canonical native root");
|
||||
let _publication_root = PublicationRoot::new(&root).expect("retain native directory anchor");
|
||||
let registration = register_native_publication_root(&root, &root.join(".")).expect("register same physical root");
|
||||
let backend: Arc<dyn super::super::local::LocalIoBackend> = Arc::new(super::super::local::StdBackend::new(root.clone()));
|
||||
let disk_id = uuid::Uuid::new_v4();
|
||||
let witness = registration
|
||||
.witness("published", disk_id, Arc::clone(&backend))
|
||||
.expect("fresh native witness");
|
||||
let bucket = root.join("published");
|
||||
|
||||
super::super::fs::mkdir(&bucket)
|
||||
.await
|
||||
.expect("publish through the actual native entry");
|
||||
assert!(bucket.is_dir(), "the production entry must create the directory");
|
||||
assert!(!validate_native_creation_witnesses(std::slice::from_ref(&witness)));
|
||||
std::fs::remove_dir(&bucket).expect("remove published directory");
|
||||
assert!(registration.witness("published", disk_id, Arc::clone(&backend)).is_none());
|
||||
let reconnected = register_native_publication_root(&root, &root).expect("reconnect same physical root");
|
||||
assert!(reconnected.witness("published", disk_id, Arc::clone(&backend)).is_none());
|
||||
|
||||
let unstarted = reconnected
|
||||
.witness("unstarted", disk_id, Arc::clone(&backend))
|
||||
.expect("untouched bucket keeps its witness");
|
||||
assert!(consume_native_creation_witnesses(std::slice::from_ref(&unstarted)));
|
||||
assert!(!validate_native_creation_witnesses(std::slice::from_ref(&unstarted)));
|
||||
assert!(reconnected.witness("unstarted", disk_id, backend).is_none());
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
fn windows_native_creation_witness_rejects_unknown_roots() {
|
||||
let temp = tempdir().expect("create unknown identity fixture");
|
||||
let root = temp.path().join("root");
|
||||
let other = temp.path().join("other");
|
||||
let file = temp.path().join("file");
|
||||
std::fs::create_dir(&root).expect("create native root");
|
||||
std::fs::create_dir(&other).expect("create distinct native root");
|
||||
std::fs::write(&file, b"not a directory").expect("create regular file");
|
||||
|
||||
assert!(native_root_identity(&temp.path().join("missing")).is_none());
|
||||
assert!(native_root_identity(&file).is_none());
|
||||
assert!(register_native_publication_root(&root, &other).is_none());
|
||||
let registration = register_native_publication_root(&root, &root).expect("register actual native root");
|
||||
let backend: Arc<dyn super::super::local::LocalIoBackend> = Arc::new(super::super::local::StdBackend::new(root));
|
||||
registration.invalidate();
|
||||
assert!(registration.witness("unstarted", uuid::Uuid::new_v4(), backend).is_none());
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
fn windows_rename_all_legacy_file_identity_distinguishes_sources() {
|
||||
@@ -11548,6 +11875,328 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, unix))]
|
||||
mod native_creation_history_tests {
|
||||
use super::*;
|
||||
|
||||
struct NativeFixture {
|
||||
_temp: tempfile::TempDir,
|
||||
root: PathBuf,
|
||||
registration: NativePublicationRegistration,
|
||||
backend: Arc<dyn super::super::local::LocalIoBackend>,
|
||||
disk_id: uuid::Uuid,
|
||||
}
|
||||
|
||||
impl NativeFixture {
|
||||
fn new() -> Self {
|
||||
let temp = tempfile::tempdir().expect("native root");
|
||||
let root = std::fs::canonicalize(temp.path()).expect("canonical root");
|
||||
let registration = register_native_publication_root(&root, &root).expect("native registration");
|
||||
let backend = Arc::new(super::super::local::StdBackend::new(root.clone()));
|
||||
Self {
|
||||
_temp: temp,
|
||||
root,
|
||||
registration,
|
||||
backend,
|
||||
disk_id: uuid::Uuid::new_v4(),
|
||||
}
|
||||
}
|
||||
|
||||
fn witness(&self, bucket: &str) -> Option<NativeBucketCreationWitness> {
|
||||
self.registration.witness(bucket, self.disk_id, Arc::clone(&self.backend))
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn every_native_directory_publication_entry_invalidates_retry() {
|
||||
let fixture = NativeFixture::new();
|
||||
for operation in [
|
||||
"recursive",
|
||||
"single",
|
||||
"open",
|
||||
"rename",
|
||||
"rename_std",
|
||||
"fallback",
|
||||
"owned_mkdir",
|
||||
"owned_rename",
|
||||
"handle_relative",
|
||||
"root_create",
|
||||
"root_create_with_parent",
|
||||
"guarded_rename",
|
||||
"guarded_prepared_rename",
|
||||
] {
|
||||
let bucket = format!("native-{operation}-{}", uuid::Uuid::new_v4());
|
||||
let path = fixture.root.join(&bucket);
|
||||
let witness = fixture.witness(&bucket).expect("fresh native witness");
|
||||
match operation {
|
||||
"recursive" => super::super::fs::make_dir_all(path.join("object/child"))
|
||||
.await
|
||||
.expect("recursive mkdir"),
|
||||
"single" => super::super::fs::mkdir(&path).await.expect("single mkdir"),
|
||||
"open" => {
|
||||
let file = super::super::fs::open_file(&path, super::super::fs::O_CREATE | super::super::fs::O_WRONLY)
|
||||
.await
|
||||
.expect("create file");
|
||||
drop(file);
|
||||
}
|
||||
"rename" | "rename_std" | "owned_rename" => {
|
||||
let source = fixture.root.join(format!("source-{}", uuid::Uuid::new_v4()));
|
||||
std::fs::create_dir(&source).expect("rename source");
|
||||
match operation {
|
||||
"rename" => super::super::fs::rename(&source, &path).await.expect("async rename"),
|
||||
"rename_std" => super::super::fs::rename_std(&source, &path).expect("sync rename"),
|
||||
_ => rename_with_namespace_owner(&source, &path, Some(Arc::new(())))
|
||||
.await
|
||||
.expect("owned rename"),
|
||||
}
|
||||
}
|
||||
"fallback" => make_dir_all(path.join("object/child"), &path)
|
||||
.await
|
||||
.expect("implicit volume fallback"),
|
||||
"owned_mkdir" => create_dir_all_with_namespace_owner(&path.join("child"), Some(Arc::new(())))
|
||||
.await
|
||||
.expect("owned recursive mkdir"),
|
||||
"handle_relative" => {
|
||||
let publication_root = PublicationRoot::new(&fixture.root).expect("publication root");
|
||||
let guard = mkdir_all_below_existing_base_std(&path.join("child"), &fixture.root, &publication_root)
|
||||
.expect("first-level handle-relative mkdir");
|
||||
drop(guard);
|
||||
}
|
||||
"root_create" | "root_create_with_parent" => {
|
||||
let publication_root = PublicationRoot::new(&fixture.root).expect("publication root");
|
||||
if operation == "root_create" {
|
||||
drop(publication_root.create_truncate(&path).expect("root-relative creation"));
|
||||
} else {
|
||||
drop(
|
||||
publication_root
|
||||
.create_truncate_with_parent(&path)
|
||||
.expect("root-relative creation with parent"),
|
||||
);
|
||||
}
|
||||
}
|
||||
"guarded_rename" | "guarded_prepared_rename" => {
|
||||
let publication_root = PublicationRoot::new(&fixture.root).expect("publication root");
|
||||
let source_parent = fixture.root.join(format!("source-{}", uuid::Uuid::new_v4()));
|
||||
std::fs::create_dir(&source_parent).expect("source directory");
|
||||
std::fs::create_dir(&path).expect("existing destination directory");
|
||||
let guard = prepare_rename_commit_guard(&source_parent, &path, &fixture.root, &publication_root)
|
||||
.expect("existing commit directories");
|
||||
let source = source_parent.join("xl.meta");
|
||||
let destination = path.join("xl.meta");
|
||||
assert!(
|
||||
validate_native_creation_witnesses(std::slice::from_ref(&witness)),
|
||||
"preparing existing directories must not mask the publication entry"
|
||||
);
|
||||
let lease = acquire_namespace_mutation_lease(&destination).await;
|
||||
if operation == "guarded_rename" {
|
||||
std::fs::write(&source, b"metadata").expect("source metadata");
|
||||
rename_all_with_commit_guard(&source, &destination, &fixture.root, &publication_root, &guard, lease)
|
||||
.await
|
||||
.expect("guarded publication");
|
||||
} else {
|
||||
let mut staged = create_prepared_rename_source_with_commit_guard(&source, &destination, &guard)
|
||||
.expect("prepared source");
|
||||
staged.write_all(b"metadata", false).expect("source metadata");
|
||||
rename_all_with_prepared_source(
|
||||
staged,
|
||||
&source,
|
||||
&destination,
|
||||
&fixture.root,
|
||||
&publication_root,
|
||||
&guard,
|
||||
lease,
|
||||
)
|
||||
.await
|
||||
.expect("prepared guarded publication");
|
||||
}
|
||||
assert_eq!(std::fs::read(&destination).expect("published metadata"), b"metadata");
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
assert!(
|
||||
!validate_native_creation_witnesses(std::slice::from_ref(&witness)),
|
||||
"{operation}: old proof remains live"
|
||||
);
|
||||
if path.is_dir() {
|
||||
std::fs::remove_dir_all(&path).expect("remove published directory");
|
||||
} else {
|
||||
std::fs::remove_file(&path).expect("remove published file");
|
||||
}
|
||||
assert!(fixture.witness(&bucket).is_none(), "{operation}: removal must not regrant proof");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelled_namespace_waiter_cannot_restore_native_proof() {
|
||||
let fixture = NativeFixture::new();
|
||||
let bucket = format!("native-cancel-{}", uuid::Uuid::new_v4());
|
||||
let path = fixture.root.join(&bucket);
|
||||
let witness = fixture.witness(&bucket).expect("fresh witness");
|
||||
let lease = acquire_namespace_mutation_lease(&path).await;
|
||||
let mut attempt = Box::pin(create_dir_all_with_namespace_owner(&path, Some(Arc::new(()))));
|
||||
assert!(matches!(futures::poll!(&mut attempt), std::task::Poll::Pending));
|
||||
assert!(
|
||||
!validate_native_creation_witnesses(std::slice::from_ref(&witness)),
|
||||
"Started must precede admission wait"
|
||||
);
|
||||
assert!(!path.exists(), "the namespace wait must still precede the syscall");
|
||||
drop(attempt);
|
||||
drop(lease);
|
||||
assert!(fixture.witness(&bucket).is_none(), "cancellation cannot undo an admitted attempt");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn queued_mkdir_cancellation_retains_lease_and_consumed_history() {
|
||||
use futures::FutureExt;
|
||||
use std::sync::mpsc;
|
||||
|
||||
let fixture = NativeFixture::new();
|
||||
let bucket = format!("native-queued-cancel-{}", uuid::Uuid::new_v4());
|
||||
let path = fixture.root.join(&bucket);
|
||||
let witness = fixture.witness(&bucket).expect("fresh witness");
|
||||
let (entered_tx, entered_rx) = mpsc::channel();
|
||||
let (release_tx, release_rx) = mpsc::channel();
|
||||
let _hook = prepared_publication_test_hooks::install_at(
|
||||
prepared_publication_test_hooks::Stage::CreateDirectory,
|
||||
&path,
|
||||
move || {
|
||||
entered_tx.send(()).expect("signal real executor entry");
|
||||
let _ = release_rx.recv();
|
||||
},
|
||||
);
|
||||
let operation_path = path.clone();
|
||||
let operation =
|
||||
tokio::spawn(async move { create_dir_all_with_namespace_owner(&operation_path, Some(Arc::new(()))).await });
|
||||
tokio::task::spawn_blocking(move || entered_rx.recv_timeout(std::time::Duration::from_secs(30)))
|
||||
.await
|
||||
.expect("entry waiter")
|
||||
.expect("the real blocking executor must enter");
|
||||
operation.abort();
|
||||
assert!(operation.await.expect_err("cancel the async waiter").is_cancelled());
|
||||
assert!(
|
||||
!validate_native_creation_witnesses(std::slice::from_ref(&witness)),
|
||||
"queued publication is permanently Started"
|
||||
);
|
||||
assert!(
|
||||
acquire_namespace_mutation_lease(&path).now_or_never().is_none(),
|
||||
"the actual blocked executor must still own the namespace lease"
|
||||
);
|
||||
release_tx.send(()).expect("release the syscall");
|
||||
let drained = tokio::time::timeout(std::time::Duration::from_secs(30), acquire_namespace_mutation_lease(&path))
|
||||
.await
|
||||
.expect("the lease must drain with the syscall");
|
||||
drop(drained);
|
||||
assert!(path.is_dir(), "the cancelled waiter cannot discard the already queued syscall");
|
||||
std::fs::remove_dir_all(&path).expect("remove the published directory");
|
||||
assert!(fixture.witness(&bucket).is_none(), "drain and removal cannot restore proof");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn publication_detects_retargeted_physical_alias_without_registration() {
|
||||
let source = NativeFixture::new();
|
||||
let destination = NativeFixture::new();
|
||||
let bucket = format!("native-retarget-{}", uuid::Uuid::new_v4());
|
||||
let witness = destination.witness(&bucket).expect("destination witness");
|
||||
let retired = source.root.with_extension("retired");
|
||||
std::fs::rename(&source.root, &retired).expect("move the original physical root");
|
||||
std::os::unix::fs::symlink(&destination.root, &source.root).expect("retarget the registered alias");
|
||||
super::super::fs::mkdir(source.root.join(&bucket))
|
||||
.await
|
||||
.expect("publish through the changed alias");
|
||||
assert!(
|
||||
!validate_native_creation_witnesses(std::slice::from_ref(&witness)),
|
||||
"publication-time alias drift must invalidate the actual destination root"
|
||||
);
|
||||
std::fs::remove_dir(destination.root.join(&bucket)).expect("remove the published bucket");
|
||||
assert!(destination.witness(&bucket).is_none(), "removal must not undo alias uncertainty");
|
||||
std::fs::remove_file(&source.root).expect("remove the symlink");
|
||||
std::fs::rename(retired, &source.root).expect("restore the test root");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blocked_physical_identity_lookup_does_not_block_other_publication() {
|
||||
use std::sync::mpsc;
|
||||
let stalled = NativeFixture::new();
|
||||
let healthy = NativeFixture::new();
|
||||
let witness = stalled.witness("unpublished").expect("stalled root witness");
|
||||
let (entered_tx, entered_rx) = mpsc::channel();
|
||||
let (release_tx, release_rx) = mpsc::channel();
|
||||
let _hook = prepared_publication_test_hooks::install_at(
|
||||
prepared_publication_test_hooks::Stage::NativeRootIdentity,
|
||||
&stalled.root,
|
||||
move || {
|
||||
entered_tx.send(()).expect("signal physical lookup");
|
||||
let _ = release_rx.recv();
|
||||
},
|
||||
);
|
||||
let validator = std::thread::spawn(move || validate_native_creation_witnesses(&[witness]));
|
||||
entered_rx
|
||||
.recv_timeout(std::time::Duration::from_secs(30))
|
||||
.expect("enter the actual identity lookup");
|
||||
let (published_tx, published_rx) = mpsc::channel();
|
||||
let path = healthy.root.join("published");
|
||||
let publisher = std::thread::spawn(move || {
|
||||
record_native_publication(&path);
|
||||
published_tx.send(()).expect("signal independent publication");
|
||||
});
|
||||
let published = published_rx.recv_timeout(std::time::Duration::from_secs(5));
|
||||
release_tx.send(()).expect("release the identity lookup");
|
||||
validator.join().expect("identity lookup thread");
|
||||
publisher.join().expect("publication thread");
|
||||
assert!(
|
||||
published.is_ok(),
|
||||
"a slow physical lookup must not retain the process-wide publication mutex"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_consume_is_atomic_and_history_survives_reconnect() {
|
||||
let fixture = NativeFixture::new();
|
||||
let first = format!("native-first-{}", uuid::Uuid::new_v4());
|
||||
let second = format!("native-second-{}", uuid::Uuid::new_v4());
|
||||
let first_witness = fixture.witness(&first).expect("first witness");
|
||||
let second_witness = fixture.witness(&second).expect("second witness");
|
||||
record_native_publication(&fixture.root.join(&second));
|
||||
assert!(!consume_native_creation_witnesses(&[first_witness.clone(), second_witness]));
|
||||
assert!(
|
||||
validate_native_creation_witnesses(std::slice::from_ref(&first_witness)),
|
||||
"failed cohort consume must be atomic"
|
||||
);
|
||||
assert!(consume_native_creation_witnesses(std::slice::from_ref(&first_witness)));
|
||||
assert!(!validate_native_creation_witnesses(std::slice::from_ref(&first_witness)));
|
||||
let reconnected = register_native_publication_root(&fixture.root, &fixture.root).expect("reconnected registration");
|
||||
let backend: Arc<dyn super::super::local::LocalIoBackend> =
|
||||
Arc::new(super::super::local::StdBackend::new(fixture.root.clone()));
|
||||
assert!(
|
||||
reconnected.witness(&first, fixture.disk_id, backend).is_none(),
|
||||
"reconnect must retain Started"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn physical_alias_reuse_is_unknown_and_backend_identity_is_real() {
|
||||
let temp = tempfile::tempdir().expect("alias parent");
|
||||
let root = temp.path().join("root");
|
||||
std::fs::create_dir(&root).expect("first physical root");
|
||||
let old = register_native_publication_root(&root, &root).expect("first registration");
|
||||
let backend: Arc<dyn super::super::local::LocalIoBackend> = Arc::new(super::super::local::StdBackend::new(root.clone()));
|
||||
let id = uuid::Uuid::new_v4();
|
||||
let witness = old.witness("same-name", id, Arc::clone(&backend)).expect("old witness");
|
||||
let other_backend: Arc<dyn super::super::local::LocalIoBackend> =
|
||||
Arc::new(super::super::local::StdBackend::new(root.clone()));
|
||||
let other = old.witness("same-name", id, other_backend).expect("new backend witness");
|
||||
assert!(!witness.matches(&other), "a different actual backend cannot reuse a pending observation");
|
||||
std::fs::rename(&root, temp.path().join("retired-root")).expect("retire physical root");
|
||||
std::fs::create_dir(&root).expect("replacement root at the same alias");
|
||||
let replacement = register_native_publication_root(&root, &root).expect("replacement registration");
|
||||
assert!(replacement.witness("same-name", id, backend).is_none(), "alias reuse must remain Unknown");
|
||||
assert!(
|
||||
!validate_native_creation_witnesses(&[witness]),
|
||||
"old physical identity must not validate at a new root"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, target_os = "linux"))]
|
||||
mod fd_relative_tests {
|
||||
use super::*;
|
||||
|
||||
@@ -17,8 +17,8 @@
|
||||
use crate::bucket::error::BucketMetadataError;
|
||||
use crate::disk::error::DiskError;
|
||||
use crate::storage_api_contracts::{error::StorageErrorCode, range::HTTPRangeError};
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
use rustfs_utils::path::decode_dir_object;
|
||||
use s3s::S3ErrorCode;
|
||||
|
||||
pub type Error = StorageError;
|
||||
pub type Result<T> = core::result::Result<T, Error>;
|
||||
@@ -1086,14 +1086,8 @@ impl From<rmp_serde::decode::Error> for Error {
|
||||
}
|
||||
}
|
||||
|
||||
impl From<s3s::xml::SerError> for Error {
|
||||
fn from(e: s3s::xml::SerError) -> Self {
|
||||
Error::other(e)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<s3s::xml::DeError> for Error {
|
||||
fn from(e: s3s::xml::DeError) -> Self {
|
||||
impl From<crate::bucket::utils::BucketConfigXmlError> for Error {
|
||||
fn from(e: crate::bucket::utils::BucketConfigXmlError) -> Self {
|
||||
Error::other(e)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,7 +20,6 @@
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use http::HeaderMap;
|
||||
use s3s::dto::ETag;
|
||||
use std::{collections::HashMap, io::Cursor, sync::Arc};
|
||||
use tokio::io::BufReader;
|
||||
|
||||
@@ -29,7 +28,7 @@ use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions};
|
||||
use crate::storage_api_contracts::range::HTTPRangeSpec;
|
||||
use rustfs_filemeta::ObjectPartInfo;
|
||||
use rustfs_rio::HashReader;
|
||||
use s3s::S3ErrorCode;
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
|
||||
//#[derive(Clone)]
|
||||
pub struct PutObjReader {
|
||||
@@ -150,24 +149,6 @@ pub fn new_getobjectreader<'a>(
|
||||
})
|
||||
}
|
||||
|
||||
/// Convert a raw stored ETag into the strongly-typed `s3s::dto::ETag`.
|
||||
///
|
||||
/// Supports already quoted (`"abc"`), weak (`W/"abc"`), or plain (`abc`) values.
|
||||
pub fn to_s3s_etag(etag: &str) -> ETag {
|
||||
if let Some(rest) = etag.strip_prefix("W/\"") {
|
||||
if let Some(body) = rest.strip_suffix('"') {
|
||||
return ETag::Weak(body.to_string());
|
||||
}
|
||||
return ETag::Weak(rest.to_string());
|
||||
}
|
||||
|
||||
if let Some(body) = etag.strip_prefix('"').and_then(|rest| rest.strip_suffix('"')) {
|
||||
return ETag::Strong(body.to_string());
|
||||
}
|
||||
|
||||
ETag::Strong(etag.to_string())
|
||||
}
|
||||
|
||||
pub fn get_raw_etag(metadata: &HashMap<String, String>) -> String {
|
||||
metadata
|
||||
.get("etag")
|
||||
@@ -232,28 +213,42 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_to_s3s_etag() {
|
||||
// Test unquoted ETag - should become strong etag
|
||||
assert_eq!(
|
||||
to_s3s_etag("6af8d12c0c74b78094884349f3c8a079"),
|
||||
ETag::Strong("6af8d12c0c74b78094884349f3c8a079".to_string())
|
||||
);
|
||||
fn test_new_getobjectreader_rejects_a_range_past_the_object_as_invalid_range() {
|
||||
let oi = multipart_object_info();
|
||||
let past_the_end = Some(HTTPRangeSpec {
|
||||
is_suffix_length: false,
|
||||
start: oi.size,
|
||||
end: oi.size + 10,
|
||||
});
|
||||
|
||||
assert_eq!(
|
||||
to_s3s_etag("\"6af8d12c0c74b78094884349f3c8a079\""),
|
||||
ETag::Strong("6af8d12c0c74b78094884349f3c8a079".to_string())
|
||||
);
|
||||
let err = new_getobjectreader(&past_the_end, &oi, &ObjectOptions::default(), &HeaderMap::new())
|
||||
.err()
|
||||
.expect("a range starting at the object size must be rejected");
|
||||
|
||||
assert_eq!(
|
||||
to_s3s_etag("W/\"6af8d12c0c74b78094884349f3c8a079\""),
|
||||
ETag::Weak("6af8d12c0c74b78094884349f3c8a079".to_string())
|
||||
);
|
||||
assert_eq!(err.code, S3ErrorCode::InvalidRange);
|
||||
assert_eq!(err.code.as_str(), "InvalidRange");
|
||||
assert_eq!(err.key, None);
|
||||
assert_eq!(err.bucket_name, None);
|
||||
}
|
||||
|
||||
assert_eq!(to_s3s_etag(""), ETag::Strong(String::new()));
|
||||
#[test]
|
||||
fn test_new_getobjectreader_rejects_an_unknown_size_without_a_range_as_invalid_range() {
|
||||
let oi = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
size: -1,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert_eq!(to_s3s_etag("\"incomplete"), ETag::Strong("\"incomplete".to_string()));
|
||||
let err = new_getobjectreader(&None, &oi, &ObjectOptions::default(), &HeaderMap::new())
|
||||
.err()
|
||||
.expect("an object of unknown size cannot be read without a range");
|
||||
|
||||
assert_eq!(to_s3s_etag("incomplete\""), ETag::Strong("incomplete\"".to_string()));
|
||||
assert_eq!(err.code, S3ErrorCode::InvalidRange);
|
||||
assert_eq!(err.code.as_str(), "InvalidRange");
|
||||
assert_eq!(err.message, "Invalid range");
|
||||
assert_eq!(err.key.as_deref(), Some("object"));
|
||||
assert_eq!(err.bucket_name.as_deref(), Some("bucket"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -959,6 +959,10 @@ pub struct ObjectOptions {
|
||||
/// Persisted bucket incarnation observed before authorization.
|
||||
pub expected_bucket_incarnation_id: Option<Uuid>,
|
||||
pub no_lock: bool,
|
||||
/// Require complete participant observations for internal multipart size decisions.
|
||||
/// This transient flag is never derived from client input or stored on disk.
|
||||
#[doc(hidden)]
|
||||
pub strict_multipart_part_sizes: bool,
|
||||
/// A genuine target write guard; loss fences alone do not authorize a borrowed write.
|
||||
#[doc(hidden)]
|
||||
pub write_lock_context: Option<WriteLockContext>,
|
||||
@@ -1013,6 +1017,10 @@ pub struct ObjectOptions {
|
||||
/// `ecstore` must leave it unset.
|
||||
#[doc(hidden)]
|
||||
pub data_movement_transferred_tier_versions: Option<crate::data_movement::TransferredTierVersions>,
|
||||
/// Marks the physical copy-back write of a RestoreObject operation. This
|
||||
/// is in-memory request context and must not be copied into object metadata.
|
||||
#[doc(hidden)]
|
||||
pub restore_copyback: bool,
|
||||
/// Internal, in-memory protection context. Never populated from S3 metadata.
|
||||
/// None selects the rollout default only for a new write, not for a rewrite.
|
||||
#[doc(hidden)]
|
||||
|
||||
@@ -275,16 +275,16 @@ pub(crate) type TypeLocalDiskSetDrives = Vec<Vec<Vec<Option<DiskStore>>>>;
|
||||
///
|
||||
/// # Returns
|
||||
/// * None
|
||||
pub fn set_global_region(region: s3s::region::Region) {
|
||||
pub fn set_global_region(region: rustfs_s3_types::Region) {
|
||||
current_ctx().set_region(region);
|
||||
}
|
||||
|
||||
/// Get the global region
|
||||
///
|
||||
/// # Returns
|
||||
/// * `Option<s3s::region::Region>` - The global region, if set
|
||||
/// * `Option<rustfs_s3_types::Region>` - The global region, if set
|
||||
///
|
||||
pub fn get_global_region() -> Option<s3s::region::Region> {
|
||||
pub fn get_global_region() -> Option<rustfs_s3_types::Region> {
|
||||
current_ctx().region()
|
||||
}
|
||||
|
||||
|
||||
@@ -52,7 +52,7 @@ use crate::services::event_notification::EventNotifier;
|
||||
use crate::services::tier::tier::TierConfigMgr;
|
||||
use crate::store::delete_recovery_runtime::DeleteRecoveryHints;
|
||||
use rustfs_lock::{GlobalLockManager, get_global_lock_manager};
|
||||
use s3s::region::Region;
|
||||
use rustfs_s3_types::Region;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::sync::{
|
||||
Arc, OnceLock,
|
||||
|
||||
@@ -44,10 +44,10 @@ use crate::{
|
||||
};
|
||||
use rustfs_concurrency::WorkloadAdmissionSnapshotProvider;
|
||||
use rustfs_config::server_config::{Config, get_global_server_config, set_global_server_config};
|
||||
use rustfs_gateway_types::persistence::PersistedLifecycleConfiguration;
|
||||
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
|
||||
use rustfs_lock::client::LockClient;
|
||||
use s3s::dto::BucketLifecycleConfiguration;
|
||||
use s3s::region::Region;
|
||||
use rustfs_s3_types::Region;
|
||||
use tokio::sync::{OwnedRwLockReadGuard, RwLock};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
#[cfg(test)]
|
||||
@@ -371,7 +371,7 @@ pub fn global_tier_config_mgr() -> Arc<RwLock<TierConfigMgr>> {
|
||||
get_global_tier_config_mgr()
|
||||
}
|
||||
|
||||
pub(crate) async fn bucket_lifecycle_config(bucket: &str) -> Option<BucketLifecycleConfiguration> {
|
||||
pub(crate) async fn bucket_lifecycle_config(bucket: &str) -> Option<PersistedLifecycleConfiguration> {
|
||||
GLOBAL_LIFECYCLE_SYS.get(bucket).await
|
||||
}
|
||||
|
||||
|
||||
@@ -703,6 +703,28 @@ pub async fn acquire_legacy_transition_state_reconcile_fleet_proof() -> Option<L
|
||||
.then_some(proof)
|
||||
}
|
||||
|
||||
pub async fn require_restore_copyback_fleet_capability() -> Result<()> {
|
||||
let Some(topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
return Ok(());
|
||||
#[cfg(not(any(test, feature = "test-util")))]
|
||||
return Err(Error::other("restore copy-back capability topology is unavailable"));
|
||||
};
|
||||
let Some(notification_sys) = get_global_notification_sys() else {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
return Ok(());
|
||||
#[cfg(not(any(test, feature = "test-util")))]
|
||||
return Err(Error::other("restore copy-back capability probe is unavailable"));
|
||||
};
|
||||
timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_restore_copyback_fleet(topology),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| Error::other("restore copy-back capability fleet probe timed out"))??;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn acquire_legacy_transition_state_reconcile_fleet_proof_from(
|
||||
state: &FleetCapabilityProofState,
|
||||
expected_topology: &str,
|
||||
@@ -1760,6 +1782,30 @@ impl NotificationSys {
|
||||
Ok(peer_epochs)
|
||||
}
|
||||
|
||||
async fn probe_restore_copyback_fleet(&self, topology_fingerprint: &str) -> Result<BTreeMap<String, Uuid>> {
|
||||
if self.peer_clients.len() != self.peer_topology_hosts.len() {
|
||||
return Err(Error::other("restore copy-back capability fleet membership is incomplete"));
|
||||
}
|
||||
let probes = self.peer_clients.iter().map(|client| async {
|
||||
let client = client
|
||||
.as_ref()
|
||||
.ok_or_else(|| Error::other("restore copy-back capability peer is unreachable"))?;
|
||||
client
|
||||
.probe_restore_copyback_capability(topology_fingerprint.to_string())
|
||||
.await
|
||||
});
|
||||
let mut peer_epochs = BTreeMap::new();
|
||||
for result in join_all(probes).await {
|
||||
let (peer, epoch) = result?;
|
||||
insert_remote_version_state_peer(&mut peer_epochs, peer, epoch)?;
|
||||
}
|
||||
let expected = self.peer_topology_hosts.iter().cloned().collect::<BTreeSet<_>>();
|
||||
if expected.len() != self.peer_topology_hosts.len() || peer_epochs.keys().ne(expected.iter()) {
|
||||
return Err(Error::other("restore copy-back capability fleet membership does not match topology"));
|
||||
}
|
||||
Ok(peer_epochs)
|
||||
}
|
||||
|
||||
async fn probe_transition_transaction_compaction_fleet(&self, topology_fingerprint: &str) -> Result<BTreeMap<String, Uuid>> {
|
||||
if self.peer_clients.len() != self.peer_topology_hosts.len() {
|
||||
return Err(Error::other(
|
||||
|
||||
@@ -1032,8 +1032,9 @@ mod tests {
|
||||
use crate::storage_api_contracts::object::ObjectIO as _;
|
||||
use http::HeaderMap;
|
||||
use rustfs_filemeta::{FileInfo, FileMeta, ObjectPartInfo, TransitionVersionState};
|
||||
use s3s::dto::{
|
||||
BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, NoncurrentVersionExpiration,
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedLifecycleConfiguration, PersistedLifecycleExpiration, PersistedLifecycleRule,
|
||||
PersistedNoncurrentVersionExpiration,
|
||||
};
|
||||
use std::time::Duration as StdDuration;
|
||||
use time::OffsetDateTime;
|
||||
@@ -1077,12 +1078,12 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn expired_delete_marker_lifecycle() -> BucketLifecycleConfiguration {
|
||||
BucketLifecycleConfiguration {
|
||||
fn expired_delete_marker_lifecycle() -> PersistedLifecycleConfiguration {
|
||||
PersistedLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
expiration: Some(LifecycleExpiration {
|
||||
rules: vec![PersistedLifecycleRule {
|
||||
status: "Enabled".to_owned(),
|
||||
expiration: Some(PersistedLifecycleExpiration {
|
||||
expired_object_delete_marker: Some(true),
|
||||
..Default::default()
|
||||
}),
|
||||
@@ -2110,11 +2111,11 @@ mod tests {
|
||||
))
|
||||
.await
|
||||
.expect("seed the expired source version");
|
||||
let lifecycle = BucketLifecycleConfiguration {
|
||||
let lifecycle = PersistedLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
expiration: Some(LifecycleExpiration {
|
||||
rules: vec![PersistedLifecycleRule {
|
||||
status: "Enabled".to_owned(),
|
||||
expiration: Some(PersistedLifecycleExpiration {
|
||||
days: Some(1),
|
||||
expired_object_all_versions: Some(true),
|
||||
..Default::default()
|
||||
@@ -2363,7 +2364,7 @@ mod tests {
|
||||
versioned,
|
||||
version_id,
|
||||
transition: crate::bucket::lifecycle::lifecycle::TransitionOptions {
|
||||
restore_request: s3s::dto::RestoreRequest {
|
||||
restore_request: rustfs_lifecycle::RestoreRequest {
|
||||
days: Some(1),
|
||||
..Default::default()
|
||||
},
|
||||
@@ -2443,7 +2444,7 @@ mod tests {
|
||||
async fn run_entry_with(
|
||||
&self,
|
||||
entry: MetaCacheEntry,
|
||||
lifecycle_config: Option<BucketLifecycleConfiguration>,
|
||||
lifecycle_config: Option<PersistedLifecycleConfiguration>,
|
||||
) -> RebalanceEntryOutcome {
|
||||
Box::pin(Arc::clone(&self.store).rebalance_entry(
|
||||
RebalanceEntryTarget {
|
||||
@@ -2779,17 +2780,17 @@ mod tests {
|
||||
}
|
||||
|
||||
/// A rule that expires every noncurrent version at once.
|
||||
fn noncurrent_expire_all_lifecycle() -> BucketLifecycleConfiguration {
|
||||
BucketLifecycleConfiguration {
|
||||
fn noncurrent_expire_all_lifecycle() -> PersistedLifecycleConfiguration {
|
||||
PersistedLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
rules: vec![PersistedLifecycleRule {
|
||||
status: "Enabled".to_owned(),
|
||||
expiration: None,
|
||||
abort_incomplete_multipart_upload: None,
|
||||
del_marker_expiration: None,
|
||||
filter: None,
|
||||
id: Some("noncurrent-expire-all".to_string()),
|
||||
noncurrent_version_expiration: Some(NoncurrentVersionExpiration {
|
||||
noncurrent_version_expiration: Some(PersistedNoncurrentVersionExpiration {
|
||||
noncurrent_days: Some(0),
|
||||
newer_noncurrent_versions: None,
|
||||
}),
|
||||
|
||||
@@ -67,8 +67,8 @@ use crate::error::{Error, Result};
|
||||
use crate::storage_api_contracts::range::HTTPRangeSpec;
|
||||
use rustfs_filemeta::TRANSITION_COMPLETE;
|
||||
use rustfs_filemeta::{FileInfo, MetaCacheEntry};
|
||||
use rustfs_gateway_types::persistence::PersistedReplicationConfiguration;
|
||||
use rustfs_rio::Index;
|
||||
use s3s::dto::ReplicationConfiguration;
|
||||
use serde::Serialize;
|
||||
use std::io::Cursor;
|
||||
use std::sync::Arc;
|
||||
@@ -2676,7 +2676,7 @@ fn test_resolve_rebalance_optional_bucket_config_result_passthrough() {
|
||||
let result = resolve_rebalance_optional_bucket_config_result(
|
||||
"bucket-a",
|
||||
"replication",
|
||||
Ok((ReplicationConfiguration::default(), OffsetDateTime::UNIX_EPOCH)),
|
||||
Ok((PersistedReplicationConfiguration::default(), OffsetDateTime::UNIX_EPOCH)),
|
||||
)
|
||||
.expect("bucket config should pass through");
|
||||
assert!(result.is_some());
|
||||
|
||||
@@ -34,9 +34,9 @@ pub struct RebalanceStats {
|
||||
#[derive(Debug, Default)]
|
||||
pub(super) struct RebalanceBucketConfigs {
|
||||
pub(super) bucket_incarnation_id: Option<uuid::Uuid>,
|
||||
pub(super) lifecycle_config: Option<s3s::dto::BucketLifecycleConfiguration>,
|
||||
pub(super) object_lock_config: Option<s3s::dto::ObjectLockConfiguration>,
|
||||
pub(super) replication_config: Option<(s3s::dto::ReplicationConfiguration, OffsetDateTime)>,
|
||||
pub(super) lifecycle_config: Option<rustfs_gateway_types::persistence::PersistedLifecycleConfiguration>,
|
||||
pub(super) object_lock_config: Option<rustfs_gateway_types::persistence::PersistedObjectLockConfiguration>,
|
||||
pub(super) replication_config: Option<(rustfs_gateway_types::persistence::PersistedReplicationConfiguration, OffsetDateTime)>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
|
||||
@@ -80,10 +80,11 @@ use crate::{
|
||||
store::ECStore,
|
||||
};
|
||||
use rustfs_filemeta::FileInfo;
|
||||
use rustfs_gateway_types::persistence::PersistedLifecycleConfiguration;
|
||||
use rustfs_rio::HashReader;
|
||||
use rustfs_s3_client::{admin_handler_utils::AdminError, provider_versions::ProviderVersionCapabilities};
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
use rustfs_utils::path::{SLASH_SEPARATOR, path_join};
|
||||
use s3s::{S3ErrorCode, dto::BucketLifecycleConfiguration};
|
||||
|
||||
use super::{
|
||||
tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_PERM_ERR},
|
||||
@@ -1036,7 +1037,7 @@ trait TierReferenceProofStore:
|
||||
.await
|
||||
}
|
||||
|
||||
async fn lifecycle_config_for_reference_proof(&self, bucket: &str) -> Result<Option<BucketLifecycleConfiguration>>;
|
||||
async fn lifecycle_config_for_reference_proof(&self, bucket: &str) -> Result<Option<PersistedLifecycleConfiguration>>;
|
||||
|
||||
async fn authoritative_bucket_tier_reference(
|
||||
self: Arc<Self>,
|
||||
@@ -1076,7 +1077,7 @@ impl TierReferenceProofStore for ECStore {
|
||||
Ok(buckets.into_values().collect())
|
||||
}
|
||||
|
||||
async fn lifecycle_config_for_reference_proof(&self, bucket: &str) -> Result<Option<BucketLifecycleConfiguration>> {
|
||||
async fn lifecycle_config_for_reference_proof(&self, bucket: &str) -> Result<Option<PersistedLifecycleConfiguration>> {
|
||||
match get_lifecycle_config(bucket).await {
|
||||
Ok((config, _updated_at)) => Ok(Some(config)),
|
||||
Err(Error::ConfigNotFound) => Ok(None),
|
||||
@@ -1338,7 +1339,7 @@ struct TierLifecycleReference<'a> {
|
||||
}
|
||||
|
||||
fn lifecycle_config_target_reference<'a>(
|
||||
config: &'a BucketLifecycleConfiguration,
|
||||
config: &'a PersistedLifecycleConfiguration,
|
||||
targets: &[TierMutationIntentTarget],
|
||||
) -> Option<TierLifecycleReference<'a>> {
|
||||
for rule in &config.rules {
|
||||
@@ -6927,9 +6928,8 @@ mod tests {
|
||||
endpoints::{Endpoints, PoolEndpoints, SetupType},
|
||||
};
|
||||
use crate::services::tier::tier_mutation_intent::TIER_MUTATION_INTENT_RECORD_PREFIX;
|
||||
use s3s::dto::{
|
||||
BucketLifecycleConfiguration, ExpirationStatus, LifecycleRule, NoncurrentVersionTransition, Transition,
|
||||
TransitionStorageClass,
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedLifecycleConfiguration, PersistedLifecycleRule, PersistedNoncurrentVersionTransition, PersistedTransition,
|
||||
};
|
||||
|
||||
#[test]
|
||||
@@ -7756,7 +7756,7 @@ mod tests {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl TierReferenceProofStore for LockingTierConfigStore {
|
||||
async fn lifecycle_config_for_reference_proof(&self, _bucket: &str) -> Result<Option<BucketLifecycleConfiguration>> {
|
||||
async fn lifecycle_config_for_reference_proof(&self, _bucket: &str) -> Result<Option<PersistedLifecycleConfiguration>> {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
@@ -10396,6 +10396,42 @@ mod tests {
|
||||
assert_eq!(observed.committed_config_etag.as_deref(), Some("etag-new"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn intent_advance_retries_cas_conflict_reload_lock_timeout() {
|
||||
let store = Arc::new(CasConfigStore::default());
|
||||
let mutation_id = uuid::Uuid::from_u128(39);
|
||||
let prepared = prepared_remove_intent("COLD-A", mutation_id);
|
||||
crate::services::tier::tier_mutation_intent::save_tier_mutation_intent_record(store.clone(), &prepared)
|
||||
.await
|
||||
.expect("prepared intent fixture should persist");
|
||||
|
||||
let committed = committed_remove_intent("COLD-A", mutation_id, "etag-new");
|
||||
let committed_bytes = committed.encode().expect("committed intent fixture should encode");
|
||||
let object = crate::services::tier::tier_mutation_intent::tier_mutation_intent_record_object_name(mutation_id)
|
||||
.expect("record object should build");
|
||||
let winning_etag = format!("etag-{}", store.next_etag.load(Ordering::SeqCst) + 1);
|
||||
store.rewrite_on_next_if_match(object.clone(), committed_bytes.clone()).await;
|
||||
*store.read_timeout_after_if_match_race.lock().await = Some(object.clone());
|
||||
|
||||
let (observed, applied) = crate::services::tier::tier_mutation_intent::advance_tier_mutation_intent_record_idempotent(
|
||||
store.clone(),
|
||||
mutation_id,
|
||||
TierMutationIntentState::Committed,
|
||||
Some("etag-new".to_string()),
|
||||
)
|
||||
.await
|
||||
.expect("CAS conflict reload should retry its typed lock timeout");
|
||||
|
||||
assert!(!applied);
|
||||
assert_eq!(observed, committed);
|
||||
assert!(store.read_timeout_after_if_match_race.lock().await.is_none());
|
||||
assert_eq!(
|
||||
store.objects.lock().await.get(&object).cloned(),
|
||||
Some((committed_bytes, winning_etag)),
|
||||
"retry must preserve the complete winning record and its object ETag"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn intent_advance_reconciles_matching_commit_after_final_cas_race() {
|
||||
let store = Arc::new(CasConfigStore::default());
|
||||
@@ -14867,6 +14903,7 @@ mod tests {
|
||||
objects: tokio::sync::Mutex<HashMap<String, (Vec<u8>, String)>>,
|
||||
legacy_state: tokio::sync::Mutex<Option<Vec<u8>>>,
|
||||
if_match_race_rewrite: tokio::sync::Mutex<std::collections::VecDeque<(String, Vec<u8>)>>,
|
||||
read_timeout_after_if_match_race: tokio::sync::Mutex<Option<String>>,
|
||||
next_etag: AtomicUsize,
|
||||
content_derived_etags: AtomicBool,
|
||||
fail_put: AtomicBool,
|
||||
@@ -14881,7 +14918,7 @@ mod tests {
|
||||
lock_manager: Arc<rustfs_lock::GlobalLockManager>,
|
||||
lock_requests: Mutex<Vec<(String, String)>>,
|
||||
listed_versions: Mutex<Vec<ObjectInfo>>,
|
||||
lifecycle_configs: Mutex<HashMap<String, BucketLifecycleConfiguration>>,
|
||||
lifecycle_configs: Mutex<HashMap<String, PersistedLifecycleConfiguration>>,
|
||||
}
|
||||
|
||||
impl Default for CasConfigStore {
|
||||
@@ -14892,6 +14929,7 @@ mod tests {
|
||||
objects: tokio::sync::Mutex::new(HashMap::new()),
|
||||
legacy_state: tokio::sync::Mutex::new(None),
|
||||
if_match_race_rewrite: tokio::sync::Mutex::new(std::collections::VecDeque::new()),
|
||||
read_timeout_after_if_match_race: tokio::sync::Mutex::new(None),
|
||||
next_etag: AtomicUsize::new(0),
|
||||
content_derived_etags: AtomicBool::new(false),
|
||||
fail_put: AtomicBool::new(false),
|
||||
@@ -14940,7 +14978,7 @@ mod tests {
|
||||
.retain(|version| version.bucket != bucket || version.name != object);
|
||||
}
|
||||
|
||||
fn add_lifecycle_config(&self, bucket: &str, config: BucketLifecycleConfiguration) {
|
||||
fn add_lifecycle_config(&self, bucket: &str, config: PersistedLifecycleConfiguration) {
|
||||
self.lifecycle_configs
|
||||
.lock()
|
||||
.expect("tier reference fixture should not poison")
|
||||
@@ -15049,6 +15087,19 @@ mod tests {
|
||||
crate::api::diagnostic::current_read_version_entry_source(),
|
||||
rustfs_io_metrics::s3_http_metrics::current_s3_http_operation_name(),
|
||||
));
|
||||
{
|
||||
// The queued rewrite keeps the initial read successful. Only
|
||||
// the first target read after that CAS race consumes the fault.
|
||||
let race_completed = self.if_match_race_rewrite.lock().await.is_empty();
|
||||
let mut timeout = self.read_timeout_after_if_match_race.lock().await;
|
||||
if race_completed && timeout.as_deref() == Some(object) {
|
||||
timeout.take();
|
||||
return Err(Error::Lock(rustfs_lock::LockError::Timeout {
|
||||
resource: format!("{bucket}/{object}@latest"),
|
||||
timeout: std::time::Duration::from_secs(5),
|
||||
}));
|
||||
}
|
||||
}
|
||||
let legacy_config_path = tier_config_path(TIER_CONFIG_LEGACY_FILE);
|
||||
let (data, etag) = if object == legacy_config_path {
|
||||
let state = self.legacy_state.lock().await;
|
||||
@@ -15630,7 +15681,7 @@ mod tests {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl TierReferenceProofStore for CasConfigStore {
|
||||
async fn lifecycle_config_for_reference_proof(&self, bucket: &str) -> Result<Option<BucketLifecycleConfiguration>> {
|
||||
async fn lifecycle_config_for_reference_proof(&self, bucket: &str) -> Result<Option<PersistedLifecycleConfiguration>> {
|
||||
Ok(self
|
||||
.lifecycle_configs
|
||||
.lock()
|
||||
@@ -15762,10 +15813,10 @@ mod tests {
|
||||
});
|
||||
store.add_lifecycle_config(
|
||||
"photos",
|
||||
BucketLifecycleConfiguration {
|
||||
PersistedLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
rules: vec![PersistedLifecycleRule {
|
||||
status: "Enabled".to_owned(),
|
||||
expiration: None,
|
||||
abort_incomplete_multipart_upload: None,
|
||||
del_marker_expiration: None,
|
||||
@@ -15774,10 +15825,10 @@ mod tests {
|
||||
noncurrent_version_expiration: None,
|
||||
noncurrent_version_transitions: None,
|
||||
prefix: None,
|
||||
transitions: Some(vec![Transition {
|
||||
transitions: Some(vec![PersistedTransition {
|
||||
days: Some(1),
|
||||
date: None,
|
||||
storage_class: Some(TransitionStorageClass::from_static("COLD-A")),
|
||||
storage_class: Some("COLD-A".to_owned()),
|
||||
}]),
|
||||
}],
|
||||
},
|
||||
@@ -16430,10 +16481,10 @@ mod tests {
|
||||
.save_tiering_config_if_current(store.clone(), None)
|
||||
.await
|
||||
.expect("reference proof fixture should persist");
|
||||
let config = BucketLifecycleConfiguration {
|
||||
let config = PersistedLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
rules: vec![PersistedLifecycleRule {
|
||||
status: "Enabled".to_owned(),
|
||||
expiration: None,
|
||||
abort_incomplete_multipart_upload: None,
|
||||
del_marker_expiration: None,
|
||||
@@ -16442,10 +16493,10 @@ mod tests {
|
||||
noncurrent_version_expiration: None,
|
||||
noncurrent_version_transitions: None,
|
||||
prefix: None,
|
||||
transitions: Some(vec![Transition {
|
||||
transitions: Some(vec![PersistedTransition {
|
||||
days: Some(1),
|
||||
date: None,
|
||||
storage_class: Some(TransitionStorageClass::from_static("COLD-A")),
|
||||
storage_class: Some("COLD-A".to_owned()),
|
||||
}]),
|
||||
}],
|
||||
};
|
||||
@@ -16602,10 +16653,10 @@ mod tests {
|
||||
old_backend_identity: Some(current_identity),
|
||||
new_backend_identity: None,
|
||||
};
|
||||
let config = BucketLifecycleConfiguration {
|
||||
let config = PersistedLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
rules: vec![PersistedLifecycleRule {
|
||||
status: "Enabled".to_owned(),
|
||||
expiration: None,
|
||||
abort_incomplete_multipart_upload: None,
|
||||
del_marker_expiration: None,
|
||||
@@ -16614,10 +16665,10 @@ mod tests {
|
||||
noncurrent_version_expiration: None,
|
||||
noncurrent_version_transitions: None,
|
||||
prefix: None,
|
||||
transitions: Some(vec![Transition {
|
||||
transitions: Some(vec![PersistedTransition {
|
||||
days: Some(1),
|
||||
date: None,
|
||||
storage_class: Some(TransitionStorageClass::from_static("COLD-A")),
|
||||
storage_class: Some("COLD-A".to_owned()),
|
||||
}]),
|
||||
}],
|
||||
};
|
||||
@@ -16647,20 +16698,20 @@ mod tests {
|
||||
old_backend_identity: Some(current_identity),
|
||||
new_backend_identity: None,
|
||||
};
|
||||
let config = BucketLifecycleConfiguration {
|
||||
let config = PersistedLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
rules: vec![PersistedLifecycleRule {
|
||||
status: "Enabled".to_owned(),
|
||||
expiration: None,
|
||||
abort_incomplete_multipart_upload: None,
|
||||
del_marker_expiration: None,
|
||||
filter: None,
|
||||
id: Some("move-noncurrent".to_string()),
|
||||
noncurrent_version_expiration: None,
|
||||
noncurrent_version_transitions: Some(vec![NoncurrentVersionTransition {
|
||||
noncurrent_version_transitions: Some(vec![PersistedNoncurrentVersionTransition {
|
||||
noncurrent_days: Some(1),
|
||||
newer_noncurrent_versions: None,
|
||||
storage_class: Some(TransitionStorageClass::from_static("COLD-A")),
|
||||
storage_class: Some("COLD-A".to_owned()),
|
||||
}]),
|
||||
prefix: None,
|
||||
transitions: None,
|
||||
@@ -16696,10 +16747,10 @@ mod tests {
|
||||
old_backend_identity: Some(current_identity),
|
||||
new_backend_identity: None,
|
||||
};
|
||||
let config = BucketLifecycleConfiguration {
|
||||
let config = PersistedLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
rules: vec![PersistedLifecycleRule {
|
||||
status: "Enabled".to_owned(),
|
||||
expiration: None,
|
||||
abort_incomplete_multipart_upload: None,
|
||||
del_marker_expiration: None,
|
||||
@@ -16708,10 +16759,10 @@ mod tests {
|
||||
noncurrent_version_expiration: None,
|
||||
noncurrent_version_transitions: None,
|
||||
prefix: None,
|
||||
transitions: Some(vec![Transition {
|
||||
transitions: Some(vec![PersistedTransition {
|
||||
days: Some(1),
|
||||
date: None,
|
||||
storage_class: Some(TransitionStorageClass::from_static("COLD-A")),
|
||||
storage_class: Some("COLD-A".to_owned()),
|
||||
}]),
|
||||
}],
|
||||
};
|
||||
|
||||
@@ -700,9 +700,25 @@ async fn advance_tier_mutation_intent_record_idempotent_at_prefix<S>(
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
let mut original_intent: Option<TierMutationIntent> = None;
|
||||
for attempt in 0..TIER_MUTATION_INTENT_ADVANCE_CAS_ATTEMPTS {
|
||||
let (mut intent, current_etag) =
|
||||
load_tier_mutation_intent_record_with_etag_at_prefix(api.clone(), prefix, mutation_id).await?;
|
||||
match load_tier_mutation_intent_record_with_etag_at_prefix(api.clone(), prefix, mutation_id).await {
|
||||
Ok(current) => current,
|
||||
Err(Error::Lock(rustfs_lock::LockError::Timeout { .. }))
|
||||
if attempt + 1 < TIER_MUTATION_INTENT_ADVANCE_CAS_ATTEMPTS =>
|
||||
{
|
||||
continue;
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
if let Some(original) = &original_intent {
|
||||
if !intent.same_identity_as(original) {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
} else {
|
||||
original_intent = Some(intent.clone());
|
||||
}
|
||||
let advanced = intent
|
||||
.advance_idempotent(next, committed_config_etag.clone())
|
||||
.map_err(tier_mutation_intent_store_error)?;
|
||||
@@ -713,7 +729,15 @@ where
|
||||
Ok(()) => return Ok((intent, true)),
|
||||
Err(Error::PreconditionFailed) => {
|
||||
let (mut current, _) =
|
||||
load_tier_mutation_intent_record_with_etag_at_prefix(api.clone(), prefix, mutation_id).await?;
|
||||
match load_tier_mutation_intent_record_with_etag_at_prefix(api.clone(), prefix, mutation_id).await {
|
||||
Ok(current) => current,
|
||||
Err(Error::Lock(rustfs_lock::LockError::Timeout { .. }))
|
||||
if attempt + 1 < TIER_MUTATION_INTENT_ADVANCE_CAS_ATTEMPTS =>
|
||||
{
|
||||
continue;
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
if !current.same_identity_as(&intent) {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
@@ -728,6 +752,11 @@ where
|
||||
}
|
||||
return Ok((current, false));
|
||||
}
|
||||
Err(Error::Lock(rustfs_lock::LockError::Timeout { .. }))
|
||||
if attempt + 1 < TIER_MUTATION_INTENT_ADVANCE_CAS_ATTEMPTS =>
|
||||
{
|
||||
continue;
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,7 +36,9 @@ use crate::services::tier::{
|
||||
use bytes::Bytes;
|
||||
use http::StatusCode;
|
||||
use rustfs_filemeta::metadata_keys;
|
||||
use rustfs_gateway_types::dto::{ObjectLockLegalHoldStatus, ObjectLockMode, ReplicationStatus};
|
||||
use rustfs_s3_client::credentials::{Credentials, SignatureType, Static, Value};
|
||||
use rustfs_s3_client::remote_error_code::RemoteErrorCode;
|
||||
use rustfs_s3_client::transition_api::{BucketLookupType, Options, TransitionClient, TransitionClientTimeouts, TransitionCore};
|
||||
use rustfs_s3_client::{
|
||||
admin_handler_utils::AdminError,
|
||||
@@ -49,10 +51,6 @@ use rustfs_utils::egress::validate_outbound_url;
|
||||
use rustfs_utils::http::headers::{
|
||||
CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE, EXPIRES, HeaderExt as _,
|
||||
};
|
||||
use s3s::{
|
||||
S3ErrorCode,
|
||||
dto::{ObjectLockLegalHoldStatus, ObjectLockRetentionMode, ReplicationStatus},
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
@@ -193,12 +191,15 @@ pub trait WarmBackend {
|
||||
.await
|
||||
{
|
||||
Ok(_) => Ok(TransitionCandidateProbe::VersionedPresent(remote_version_id.to_string())),
|
||||
Err(err) if matches!(to_error_response(&err).code, S3ErrorCode::InvalidRange) => {
|
||||
Err(err) if matches!(to_error_response(&err).code, RemoteErrorCode::INVALID_RANGE) => {
|
||||
Ok(TransitionCandidateProbe::VersionedPresent(remote_version_id.to_string()))
|
||||
}
|
||||
Err(err)
|
||||
if err.kind() == std::io::ErrorKind::NotFound
|
||||
|| matches!(to_error_response(&err).code, S3ErrorCode::NoSuchKey | S3ErrorCode::NoSuchVersion) =>
|
||||
|| matches!(
|
||||
to_error_response(&err).code,
|
||||
RemoteErrorCode::NO_SUCH_KEY | RemoteErrorCode::NO_SUCH_VERSION
|
||||
) =>
|
||||
{
|
||||
Ok(TransitionCandidateProbe::Missing)
|
||||
}
|
||||
@@ -218,9 +219,9 @@ pub fn build_transition_put_options(storage_class: String, mut metadata: HashMap
|
||||
let mut opts = PutObjectOptions {
|
||||
storage_class,
|
||||
send_content_md5: true,
|
||||
legalhold: ObjectLockLegalHoldStatus::from_static(""),
|
||||
legalhold: ObjectLockLegalHoldStatus::custom(""),
|
||||
internal: AdvancedPutOptions {
|
||||
replication_status: ReplicationStatus::from_static(""),
|
||||
replication_status: ReplicationStatus::custom(""),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
@@ -251,7 +252,7 @@ pub fn build_transition_put_options(storage_class: String, mut metadata: HashMap
|
||||
}
|
||||
|
||||
if let Some(mode) = metadata.lookup(metadata_keys::OBJECT_LOCK_MODE) {
|
||||
opts.mode = ObjectLockRetentionMode::from(mode.to_ascii_uppercase());
|
||||
opts.mode = ObjectLockMode::custom(mode.to_ascii_uppercase());
|
||||
}
|
||||
|
||||
if let Some(retain_until_date) = metadata
|
||||
@@ -262,7 +263,7 @@ pub fn build_transition_put_options(storage_class: String, mut metadata: HashMap
|
||||
}
|
||||
|
||||
if let Some(legalhold) = metadata.lookup(metadata_keys::OBJECT_LOCK_LEGAL_HOLD) {
|
||||
opts.legalhold = ObjectLockLegalHoldStatus::from(legalhold.to_ascii_uppercase());
|
||||
opts.legalhold = ObjectLockLegalHoldStatus::custom(legalhold.to_ascii_uppercase());
|
||||
}
|
||||
|
||||
// Promoted keys are read above through `lookup`, which accepts more than
|
||||
@@ -679,7 +680,7 @@ fn is_remote_tier_auth_rejection(err: &std::io::Error) -> bool {
|
||||
matches!(response.status_code, StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN)
|
||||
|| matches!(
|
||||
response.code,
|
||||
S3ErrorCode::AccessDenied | S3ErrorCode::InvalidAccessKeyId | S3ErrorCode::SignatureDoesNotMatch
|
||||
RemoteErrorCode::ACCESS_DENIED | RemoteErrorCode::INVALID_ACCESS_KEY_ID | RemoteErrorCode::SIGNATURE_DOES_NOT_MATCH
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1304,7 +1305,7 @@ mod tests {
|
||||
|
||||
struct AuthRejectingProbePutBackend {
|
||||
status_code: StatusCode,
|
||||
code: S3ErrorCode,
|
||||
code: RemoteErrorCode,
|
||||
probes: Arc<AtomicUsize>,
|
||||
removes: Arc<AtomicUsize>,
|
||||
}
|
||||
@@ -1767,10 +1768,10 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn check_warm_backend_reports_auth_rejected_probe_put_as_invalid_credentials() {
|
||||
for (status_code, code) in [
|
||||
(StatusCode::FORBIDDEN, S3ErrorCode::AccessDenied),
|
||||
(StatusCode::UNAUTHORIZED, S3ErrorCode::Custom("".into())),
|
||||
(StatusCode::OK, S3ErrorCode::InvalidAccessKeyId),
|
||||
(StatusCode::OK, S3ErrorCode::SignatureDoesNotMatch),
|
||||
(StatusCode::FORBIDDEN, RemoteErrorCode::ACCESS_DENIED),
|
||||
(StatusCode::UNAUTHORIZED, RemoteErrorCode::custom("")),
|
||||
(StatusCode::OK, RemoteErrorCode::INVALID_ACCESS_KEY_ID),
|
||||
(StatusCode::OK, RemoteErrorCode::SIGNATURE_DOES_NOT_MATCH),
|
||||
] {
|
||||
let probes = Arc::new(AtomicUsize::new(0));
|
||||
let removes = Arc::new(AtomicUsize::new(0));
|
||||
@@ -2165,18 +2166,31 @@ mod tests {
|
||||
metadata_keys::OBJECT_LOCK_LEGAL_HOLD.to_string(),
|
||||
ObjectLockLegalHoldStatus::ON.to_string(),
|
||||
);
|
||||
metadata.insert(
|
||||
metadata_keys::OBJECT_LOCK_MODE.to_string(),
|
||||
ObjectLockRetentionMode::GOVERNANCE.to_string(),
|
||||
);
|
||||
metadata.insert(metadata_keys::OBJECT_LOCK_MODE.to_string(), ObjectLockMode::GOVERNANCE.to_string());
|
||||
|
||||
let opts = build_transition_put_options("COLD".to_string(), metadata);
|
||||
|
||||
assert_eq!(opts.mode.as_str(), ObjectLockRetentionMode::GOVERNANCE);
|
||||
assert_eq!(opts.legalhold.as_str(), ObjectLockLegalHoldStatus::ON);
|
||||
assert_eq!(opts.mode, ObjectLockMode::GOVERNANCE);
|
||||
assert_eq!(opts.legalhold, ObjectLockLegalHoldStatus::ON);
|
||||
assert_ne!(opts.retain_until_date, OffsetDateTime::UNIX_EPOCH);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_transition_put_options_sends_no_object_lock_or_replication_headers_by_default() {
|
||||
let opts = build_transition_put_options("COLD".to_string(), HashMap::new());
|
||||
assert_eq!(opts.mode.as_str(), "");
|
||||
assert_eq!(opts.legalhold.as_str(), "");
|
||||
assert_eq!(opts.internal.replication_status.as_str(), "");
|
||||
let header = opts.header();
|
||||
for name in [
|
||||
"x-amz-object-lock-mode",
|
||||
"x-amz-object-lock-legal-hold",
|
||||
"x-amz-replication-status",
|
||||
] {
|
||||
assert!(header.get(name).is_none(), "{name} must not be sent: {header:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_transition_put_options_filters_promoted_headers_from_user_metadata() {
|
||||
let mut metadata = HashMap::new();
|
||||
|
||||
@@ -28,6 +28,8 @@ use crate::services::tier::{
|
||||
},
|
||||
};
|
||||
use http::HeaderMap;
|
||||
use rustfs_gateway_types::dto::Status as BucketVersioningStatus;
|
||||
use rustfs_s3_client::remote_error_code::RemoteErrorCode;
|
||||
use rustfs_s3_client::{
|
||||
api_error_response::to_error_response,
|
||||
api_get_options::GetObjectOptions,
|
||||
@@ -42,7 +44,6 @@ use rustfs_s3_client::{
|
||||
};
|
||||
use rustfs_utils::egress::validate_outbound_url_allow_private;
|
||||
use rustfs_utils::path::SLASH_SEPARATOR;
|
||||
use s3s::{S3ErrorCode, dto::BucketVersioningStatus};
|
||||
|
||||
pub struct WarmBackendS3 {
|
||||
pub client: Arc<TransitionClient>,
|
||||
@@ -61,8 +62,8 @@ enum RemoteBucketVersioning {
|
||||
|
||||
fn remote_bucket_versioning_from_status(status: Option<&str>) -> Result<RemoteBucketVersioning, std::io::Error> {
|
||||
Ok(match status {
|
||||
Some(BucketVersioningStatus::ENABLED) => RemoteBucketVersioning::Enabled,
|
||||
Some(BucketVersioningStatus::SUSPENDED) => RemoteBucketVersioning::Suspended,
|
||||
Some(status) if status == BucketVersioningStatus::ENABLED.as_str() => RemoteBucketVersioning::Enabled,
|
||||
Some(status) if status == BucketVersioningStatus::SUSPENDED.as_str() => RemoteBucketVersioning::Suspended,
|
||||
Some(status) => {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
@@ -237,7 +238,7 @@ impl WarmBackendS3 {
|
||||
}
|
||||
Err(err) => {
|
||||
let response = to_error_response(&err);
|
||||
if response.code == S3ErrorCode::NoSuchKey {
|
||||
if response.code == RemoteErrorCode::NO_SUCH_KEY {
|
||||
Ok(TransitionCandidateProbe::Missing)
|
||||
} else {
|
||||
Err(err)
|
||||
@@ -324,7 +325,7 @@ impl WarmBackendS3 {
|
||||
}
|
||||
Err(err) => {
|
||||
let response = to_error_response(&err);
|
||||
if response.code == S3ErrorCode::NoSuchKey {
|
||||
if response.code == RemoteErrorCode::NO_SUCH_KEY {
|
||||
Ok(TransitionCandidateProbe::Missing)
|
||||
} else {
|
||||
Err(err)
|
||||
@@ -688,7 +689,7 @@ mod tests {
|
||||
.probe_transition_candidate("forbidden-probe")
|
||||
.await
|
||||
.expect_err("an authorization failure must not be mistaken for a missing key");
|
||||
assert_eq!(to_error_response(&err).code, S3ErrorCode::AccessDenied);
|
||||
assert_eq!(to_error_response(&err).code, RemoteErrorCode::ACCESS_DENIED);
|
||||
assert_eq!(
|
||||
backend
|
||||
.probe_transition_candidate("delete-marker-hidden")
|
||||
@@ -1066,11 +1067,13 @@ mod tests {
|
||||
RemoteBucketVersioning::Disabled
|
||||
);
|
||||
assert_eq!(
|
||||
remote_bucket_versioning_from_status(Some(BucketVersioningStatus::ENABLED)).expect("enabled status should parse"),
|
||||
remote_bucket_versioning_from_status(Some(BucketVersioningStatus::ENABLED.as_str()))
|
||||
.expect("enabled status should parse"),
|
||||
RemoteBucketVersioning::Enabled
|
||||
);
|
||||
assert_eq!(
|
||||
remote_bucket_versioning_from_status(Some(BucketVersioningStatus::SUSPENDED)).expect("suspended status should parse"),
|
||||
remote_bucket_versioning_from_status(Some(BucketVersioningStatus::SUSPENDED.as_str()))
|
||||
.expect("suspended status should parse"),
|
||||
RemoteBucketVersioning::Suspended
|
||||
);
|
||||
let err = remote_bucket_versioning_from_status(Some("UnexpectedStatus"))
|
||||
|
||||
@@ -18,7 +18,6 @@ use std::{
|
||||
sync::atomic::{AtomicBool, Ordering},
|
||||
};
|
||||
|
||||
use s3s::header::{X_AMZ_DELETE_MARKER, X_AMZ_VERSION_ID};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::services::tier::{
|
||||
@@ -128,7 +127,7 @@ impl WarmBackend for WarmBackendWasabi {
|
||||
self.ensure_unversioned().await?;
|
||||
}
|
||||
let (headers, reader) = self.s3.get_with_headers(object, rv, opts).await?;
|
||||
if rv.is_empty() && (headers.contains_key(X_AMZ_VERSION_ID) || headers.contains_key(X_AMZ_DELETE_MARKER)) {
|
||||
if rv.is_empty() && (headers.contains_key("x-amz-version-id") || headers.contains_key("x-amz-delete-marker")) {
|
||||
self.versioning_drifted.store(true, Ordering::Release);
|
||||
return Err(io::Error::other(
|
||||
"Wasabi bucket versioning changed while reading an unversioned tier object",
|
||||
|
||||
@@ -280,7 +280,7 @@ pub(in crate::set_disk) fn resolve_read_part_from_responses(
|
||||
continue;
|
||||
}
|
||||
|
||||
if !parts[part_idx].etag.is_empty() {
|
||||
if !parts[part_idx].etag.is_empty() && parts[part_idx].error.is_none() {
|
||||
present_count += 1;
|
||||
let part = &parts[part_idx];
|
||||
let key = (part.etag.as_str(), part.number, part.size, part.actual_size, part.integrity.as_ref());
|
||||
@@ -1118,9 +1118,9 @@ pub(in crate::set_disk) fn fill_deferred_bitrot_readers(
|
||||
// offset can be advanced to the failing stripe. A parity reader that was
|
||||
// opened eagerly during setup is pinned at the stripe-0 stream position
|
||||
// and could never be engaged mid-object, so swap it for an unopened
|
||||
// deferred reader carrying a stripe handle. The eager open already proved
|
||||
// the shard is reachable; no shard bytes were read from it, and the
|
||||
// ready/error bookkeeping that quorum decisions rely on is left untouched.
|
||||
// deferred reader carrying a stripe handle. The eager setup already proved
|
||||
// the shard is reachable; the ready/error bookkeeping that quorum decisions
|
||||
// rely on is left untouched.
|
||||
// Gate off (default): keep the eagerly opened parity readers exactly as
|
||||
// before — the lockstep path reads them on every stripe.
|
||||
if !demand_bound_lockstep {
|
||||
@@ -1217,8 +1217,8 @@ pub(in crate::set_disk) fn record_bitrot_reader_setup_strategy(
|
||||
}
|
||||
|
||||
/// When all online shards are local and mmap-read is enabled, materialize
|
||||
/// shard bytes with one `batch_shard_pread` instead of per-shard
|
||||
/// `spawn_blocking` via `open_disk_reader`.
|
||||
/// shard bytes in batches instead of per-shard `spawn_blocking` via
|
||||
/// `open_disk_reader`. Demand-bound reads add parity batches only as needed.
|
||||
#[cfg(unix)]
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn try_create_bitrot_readers_via_batch_pread(
|
||||
@@ -1232,6 +1232,10 @@ async fn try_create_bitrot_readers_via_batch_pread(
|
||||
shard_size: usize,
|
||||
checksum_algo: HashAlgorithm,
|
||||
skip_verify_bitrot: bool,
|
||||
data_shards: usize,
|
||||
parity_shards: usize,
|
||||
mode: BitrotReaderSetupMode,
|
||||
strategy: BitrotReaderSetupStrategy,
|
||||
) -> Option<BitrotReaderSetup> {
|
||||
use crate::disk::local::batch_shard_pread;
|
||||
use rustfs_io_metrics::{LocalBatchReadDecision, record_local_batch_read_decision};
|
||||
@@ -1275,42 +1279,62 @@ async fn try_create_bitrot_readers_via_batch_pread(
|
||||
return None;
|
||||
}
|
||||
|
||||
let requests: Vec<_> = batch_items.iter().map(|(_, p, off, len)| (p.clone(), *off, *len)).collect();
|
||||
#[cfg(test)]
|
||||
for (disk_index, _, _, _) in &batch_items {
|
||||
disk_call_counters::record(object, disk_call_counters::KIND_DISK_SHARD_OPEN, *disk_index);
|
||||
}
|
||||
record_decision(LocalBatchReadDecision::Selected);
|
||||
let batch_results = batch_shard_pread(requests).await;
|
||||
|
||||
// Legacy lockstep consumes every shard, so keep its coalesced batch even
|
||||
// when setup prefers data blocks. Only demand-bound decode can leave the
|
||||
// parity readers unopened; preading them here would discard their bytes
|
||||
// when fill_deferred_bitrot_readers installs stripe-aligned reserves.
|
||||
let data_first = strategy != BitrotReaderSetupStrategy::AllShards
|
||||
&& crate::erasure::coding::decode::get_lockstep_data_shards_only_enabled();
|
||||
let mut setup = BitrotReaderSetup::new(disks.len());
|
||||
for (i, (idx, _, _, _)) in batch_items.iter().enumerate() {
|
||||
setup.mark_scheduled(*idx);
|
||||
match &batch_results[i] {
|
||||
Ok(bytes) => {
|
||||
let reader = BitrotReader::new(
|
||||
ShardReader::InMemory(Cursor::new(bytes.clone())),
|
||||
shard_size,
|
||||
checksum_algo.clone(),
|
||||
skip_verify_bitrot,
|
||||
);
|
||||
setup.apply_reader_result(*idx, Ok(Some(reader)));
|
||||
}
|
||||
Err(e) => {
|
||||
setup.apply_reader_result(*idx, Err(e.clone()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (idx, disk_op) in disks.iter().enumerate() {
|
||||
if setup.scheduled[idx] {
|
||||
continue;
|
||||
}
|
||||
setup.mark_scheduled(idx);
|
||||
if disk_op.is_none() {
|
||||
for (idx, disk) in disks.iter().enumerate() {
|
||||
if disk.is_none() {
|
||||
setup.mark_scheduled(idx);
|
||||
setup.apply_reader_result(idx, Ok(None));
|
||||
}
|
||||
}
|
||||
let mut batch_size = if data_first {
|
||||
batch_items.iter().take_while(|(idx, _, _, _)| *idx < data_shards).count()
|
||||
} else {
|
||||
batch_items.len()
|
||||
};
|
||||
let mut pending = batch_items.into_iter();
|
||||
record_decision(LocalBatchReadDecision::Selected);
|
||||
loop {
|
||||
let mut indices = Vec::with_capacity(batch_size.min(pending.len()));
|
||||
let requests: Vec<_> = pending
|
||||
.by_ref()
|
||||
.take(batch_size)
|
||||
.map(|(idx, path, offset, length)| {
|
||||
setup.mark_scheduled(idx);
|
||||
indices.push(idx);
|
||||
#[cfg(test)]
|
||||
disk_call_counters::record(object, disk_call_counters::KIND_DISK_SHARD_OPEN, idx);
|
||||
(path, offset, length)
|
||||
})
|
||||
.collect();
|
||||
if !requests.is_empty() {
|
||||
for (idx, result) in indices.into_iter().zip(batch_shard_pread(requests).await) {
|
||||
let result = result.map(|bytes| {
|
||||
Some(BitrotReader::new(
|
||||
ShardReader::InMemory(Cursor::new(bytes)),
|
||||
shard_size,
|
||||
checksum_algo.clone(),
|
||||
skip_verify_bitrot,
|
||||
))
|
||||
});
|
||||
setup.apply_reader_result(idx, result);
|
||||
}
|
||||
}
|
||||
if !data_first || setup.has_setup_quorum(data_shards, parity_shards, mode) || pending.len() == 0 {
|
||||
break;
|
||||
}
|
||||
// A missing/corrupt data file needs replacement parity. Reuse the
|
||||
// canonical target, including the extra verification source required
|
||||
// for reconstruction, and retry only the outstanding deficit.
|
||||
batch_size = setup
|
||||
.scheduling_target(data_shards, parity_shards, mode)
|
||||
.saturating_sub(setup.available_shards());
|
||||
}
|
||||
|
||||
Some(setup)
|
||||
}
|
||||
@@ -1328,6 +1352,10 @@ async fn try_create_bitrot_readers_via_batch_pread(
|
||||
_shard_size: usize,
|
||||
_checksum_algo: HashAlgorithm,
|
||||
_skip_verify_bitrot: bool,
|
||||
_data_shards: usize,
|
||||
_parity_shards: usize,
|
||||
_mode: BitrotReaderSetupMode,
|
||||
_strategy: BitrotReaderSetupStrategy,
|
||||
) -> Option<BitrotReaderSetup> {
|
||||
rustfs_io_metrics::record_local_batch_read_decision(rustfs_io_metrics::LocalBatchReadDecision::UnsupportedPlatform, None);
|
||||
None
|
||||
@@ -1517,6 +1545,10 @@ pub(in crate::set_disk) async fn create_bitrot_readers_until_quorum_with_prefere
|
||||
shard_size,
|
||||
checksum_algo.clone(),
|
||||
skip_verify_bitrot,
|
||||
data_shards,
|
||||
parity_shards,
|
||||
mode,
|
||||
strategy,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -7889,6 +7921,329 @@ mod tests {
|
||||
(dirs, disks)
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
async fn local_batch_quorum_fixture(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
) -> (Vec<TempDir>, Vec<Option<DiskStore>>, Vec<FileInfo>, Vec<u8>) {
|
||||
let payload: Vec<_> = (0..1024 * 1024)
|
||||
.map(|index| u8::try_from(index % 251).expect("bounded byte"))
|
||||
.collect();
|
||||
let (dirs, disks) = call_counter_local_disks(bucket, 8).await;
|
||||
let mut files = inline_metadata_fanout_fileinfos_with_geometry(bucket, object, &payload, false, 4, 4).await;
|
||||
for (file, disk) in files.iter_mut().zip(&disks) {
|
||||
let path = format!("{object}/{}/part.1", file.data_dir.expect("fixture data directory"));
|
||||
disk.as_ref()
|
||||
.expect("fixture disk")
|
||||
.write_all(bucket, &path, file.data.take().expect("encoded fixture shard"))
|
||||
.await
|
||||
.expect("external shard fixture should write");
|
||||
}
|
||||
(dirs, disks, files, payload)
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
async fn local_batch_quorum_setup(
|
||||
files: &[FileInfo],
|
||||
disks: &[Option<DiskStore>],
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
mode: BitrotReaderSetupMode,
|
||||
) -> BitrotReaderSetup {
|
||||
let erasure = coding::Erasure::new_with_options(4, 4, files[0].erasure.block_size, false);
|
||||
create_bitrot_readers_until_quorum_with_preference(
|
||||
files,
|
||||
disks,
|
||||
bucket,
|
||||
object,
|
||||
1,
|
||||
0,
|
||||
erasure.shard_file_offset(0, 1024 * 1024, 1024 * 1024),
|
||||
erasure.shard_size(),
|
||||
HashAlgorithm::HighwayHash256S,
|
||||
false,
|
||||
true,
|
||||
4,
|
||||
4,
|
||||
mode,
|
||||
false,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
async fn local_batch_quorum_decode(setup: BitrotReaderSetup, block_size: usize) -> (Vec<u8>, Option<std::io::Error>) {
|
||||
let erasure = coding::Erasure::new_with_options(4, 4, block_size, false);
|
||||
let mut output = Vec::new();
|
||||
let (_, error) = erasure
|
||||
.decode_with_stripe_handles_and_reopeners(
|
||||
&mut output,
|
||||
setup.readers,
|
||||
0,
|
||||
1024 * 1024,
|
||||
1024 * 1024,
|
||||
None,
|
||||
setup.deferred_stripe_handles,
|
||||
setup.deferred_reopeners,
|
||||
)
|
||||
.await;
|
||||
(output, error)
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn local_batch_quorum_healthy_reads_only_data_when_demand_bound() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(ENV_RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP, Some("true")),
|
||||
("RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE", Some("false")),
|
||||
],
|
||||
crate::erasure::coding::decode::with_decode_read_policy(
|
||||
crate::erasure::coding::decode::DecodeReadPolicy::DemandBound,
|
||||
async {
|
||||
let bucket = "batch-data-only";
|
||||
let object = "healthy-one-mib";
|
||||
let (_dirs, disks, files, payload) = local_batch_quorum_fixture(bucket, object).await;
|
||||
let calls = disk_call_counters::observe(object);
|
||||
let setup = local_batch_quorum_setup(&files, &disks, bucket, object, BitrotReaderSetupMode::ReadQuorum).await;
|
||||
assert_eq!(calls.total(disk_call_counters::KIND_DISK_SHARD_OPEN), 4);
|
||||
assert_eq!(setup.available_shards(), 4);
|
||||
assert_eq!(setup.deferred_shards(), 4);
|
||||
for index in 4..8 {
|
||||
assert_eq!(calls.for_disk(disk_call_counters::KIND_DISK_SHARD_OPEN, index), 0);
|
||||
assert!(!setup.attempted[index], "parity must stay unopened until needed");
|
||||
}
|
||||
let (output, error) = local_batch_quorum_decode(setup, files[0].erasure.block_size).await;
|
||||
assert!(error.is_none(), "healthy decode failed: {error:?}");
|
||||
assert_eq!(output, payload);
|
||||
assert_eq!(calls.total(disk_call_counters::KIND_DISK_SHARD_OPEN), 4);
|
||||
},
|
||||
),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn local_batch_quorum_preserves_legacy_and_all_shards_override() {
|
||||
for (data_first, demand_bound) in [("true", "false"), ("false", "true")] {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(ENV_RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP, Some(data_first)),
|
||||
("RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE", Some(demand_bound)),
|
||||
],
|
||||
async {
|
||||
let bucket = "batch-legacy";
|
||||
let object = "all-shards";
|
||||
let (_dirs, disks, files, payload) = local_batch_quorum_fixture(bucket, object).await;
|
||||
let calls = disk_call_counters::observe(object);
|
||||
let setup = local_batch_quorum_setup(&files, &disks, bucket, object, BitrotReaderSetupMode::ReadQuorum).await;
|
||||
assert_eq!(calls.total(disk_call_counters::KIND_DISK_SHARD_OPEN), 8);
|
||||
assert_eq!(setup.available_shards(), 8);
|
||||
let (output, error) = local_batch_quorum_decode(setup, files[0].erasure.block_size).await;
|
||||
assert!(error.is_none(), "legacy decode failed: {error:?}");
|
||||
assert_eq!(output, payload);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn local_batch_quorum_recovers_missing_truncated_and_bitrot_data() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(ENV_RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP, Some("true")),
|
||||
("RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE", Some("true")),
|
||||
],
|
||||
async {
|
||||
for fault in ["missing", "truncated", "bitrot"] {
|
||||
let bucket = "batch-degraded";
|
||||
let object = format!("degraded-{fault}");
|
||||
let (dirs, disks, files, payload) = local_batch_quorum_fixture(bucket, &object).await;
|
||||
let path = dirs[0]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join(&object)
|
||||
.join(files[0].data_dir.expect("fixture directory").to_string())
|
||||
.join("part.1");
|
||||
match fault {
|
||||
"missing" => tokio::fs::remove_file(&path).await.expect("remove data shard"),
|
||||
"truncated" => tokio::fs::write(&path, b"short").await.expect("truncate data shard"),
|
||||
_ => {
|
||||
let mut bytes = tokio::fs::read(&path).await.expect("read bitrot fixture");
|
||||
bytes[32] ^= 0xff;
|
||||
tokio::fs::write(&path, bytes)
|
||||
.await
|
||||
.expect("corrupt data without changing length");
|
||||
}
|
||||
}
|
||||
let setup =
|
||||
local_batch_quorum_setup(&files, &disks, bucket, &object, BitrotReaderSetupMode::ReadQuorum).await;
|
||||
assert_eq!(setup.available_shards(), 4);
|
||||
if fault == "bitrot" {
|
||||
assert_eq!(setup.attempted_shards(), 4, "bitrot is detected on decode, not pread");
|
||||
} else {
|
||||
assert_eq!(setup.attempted_shards(), 5, "setup reads only one replacement parity shard");
|
||||
}
|
||||
let (output, error) = local_batch_quorum_decode(setup, files[0].erasure.block_size).await;
|
||||
assert!(error.is_none(), "{fault} should recover: {error:?}");
|
||||
assert_eq!(output, payload, "{fault} must reconstruct the exact body");
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn local_batch_quorum_keeps_extra_reconstruction_verification_source() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_READER_SETUP, Some("true")),
|
||||
("RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE", Some("true")),
|
||||
],
|
||||
async {
|
||||
let bucket = "batch-verification";
|
||||
let object = "missing-data";
|
||||
let (_dirs, mut disks, files, payload) = local_batch_quorum_fixture(bucket, object).await;
|
||||
disks[0] = None;
|
||||
let calls = disk_call_counters::observe(object);
|
||||
let setup =
|
||||
local_batch_quorum_setup(&files, &disks, bucket, object, BitrotReaderSetupMode::VerifyReconstruction).await;
|
||||
assert_eq!(setup.available_shards(), 5, "degraded reconstruction needs quorum plus verification");
|
||||
assert_eq!(calls.total(disk_call_counters::KIND_DISK_SHARD_OPEN), 5);
|
||||
assert!(setup.ready[4] && setup.ready[5], "two parity sources must be available");
|
||||
assert!(!setup.attempted[6] && !setup.attempted[7], "unneeded parity stays deferred");
|
||||
let (output, error) = local_batch_quorum_decode(setup, files[0].erasure.block_size).await;
|
||||
assert!(error.is_none(), "verified reconstruction failed: {error:?}");
|
||||
assert_eq!(output, payload);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn local_batch_quorum_fails_without_read_quorum() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(ENV_RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP, Some("true")),
|
||||
("RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE", Some("true")),
|
||||
],
|
||||
async {
|
||||
let bucket = "batch-no-quorum";
|
||||
let object = "all-data-offline";
|
||||
let (_dirs, mut disks, files, _payload) = local_batch_quorum_fixture(bucket, object).await;
|
||||
disks[..5].fill(None);
|
||||
let setup = local_batch_quorum_setup(&files, &disks, bucket, object, BitrotReaderSetupMode::ReadQuorum).await;
|
||||
assert_eq!(setup.available_shards(), 3);
|
||||
assert!(!setup.has_setup_quorum(4, 4, BitrotReaderSetupMode::ReadQuorum));
|
||||
assert_eq!(setup.deferred_shards(), 0);
|
||||
let (output, error) = local_batch_quorum_decode(setup, files[0].erasure.block_size).await;
|
||||
assert!(error.is_some(), "insufficient shards must fail closed");
|
||||
assert!(output.is_empty(), "no unverified bytes may be emitted");
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn local_batch_quorum_recovers_late_bitrot_after_range_offset() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(ENV_RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP, Some("true")),
|
||||
("RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE", Some("true")),
|
||||
],
|
||||
async {
|
||||
let bucket = "batch-ranged";
|
||||
let object = "late-bitrot";
|
||||
let (_dirs, disks) = call_counter_local_disks(bucket, 8).await;
|
||||
let erasure = coding::Erasure::new_with_options(4, 4, 64, false);
|
||||
let payload: Vec<u8> = (0..192).collect();
|
||||
let mut shards = vec![Vec::new(); 8];
|
||||
for block in payload.chunks(64) {
|
||||
for (index, shard) in erasure
|
||||
.encode_data(block)
|
||||
.expect("encode range stripe")
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
{
|
||||
let mut writer = coding::BitrotWriterWrapper::new(
|
||||
coding::CustomWriter::new_inline_buffer(),
|
||||
erasure.shard_size(),
|
||||
HashAlgorithm::HighwayHash256S,
|
||||
);
|
||||
writer.write(&shard).await.expect("write range stripe");
|
||||
writer.shutdown().await.expect("finish range stripe");
|
||||
shards[index].extend(writer.into_inline_data().expect("range stripe bytes"));
|
||||
}
|
||||
}
|
||||
// Corrupt the third stripe; the first requested stripe is healthy.
|
||||
shards[0][2 * (32 + erasure.shard_size()) + 32] ^= 0xff;
|
||||
let mut files = vec![FileInfo::new(object, 4, 4); 8];
|
||||
let data_dir = Uuid::new_v4();
|
||||
for ((file, disk), bytes) in files.iter_mut().zip(&disks).zip(shards) {
|
||||
file.data_dir = Some(data_dir);
|
||||
disk.as_ref()
|
||||
.expect("fixture disk")
|
||||
.write_all(bucket, &format!("{object}/{data_dir}/part.1"), Bytes::from(bytes))
|
||||
.await
|
||||
.expect("write multi-stripe shard");
|
||||
}
|
||||
let calls = disk_call_counters::observe(object);
|
||||
let setup = create_bitrot_readers_until_quorum_with_preference(
|
||||
&files,
|
||||
&disks,
|
||||
bucket,
|
||||
object,
|
||||
1,
|
||||
erasure.shard_size(),
|
||||
2 * erasure.shard_size(),
|
||||
erasure.shard_size(),
|
||||
HashAlgorithm::HighwayHash256S,
|
||||
false,
|
||||
true,
|
||||
4,
|
||||
4,
|
||||
BitrotReaderSetupMode::ReadQuorum,
|
||||
false,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(calls.total(disk_call_counters::KIND_DISK_SHARD_OPEN), 4);
|
||||
let mut output = Vec::new();
|
||||
let (written, error) = erasure
|
||||
.decode_with_stripe_handles_and_reopeners(
|
||||
&mut output,
|
||||
setup.readers,
|
||||
64,
|
||||
128,
|
||||
payload.len(),
|
||||
None,
|
||||
setup.deferred_stripe_handles,
|
||||
setup.deferred_reopeners,
|
||||
)
|
||||
.await;
|
||||
assert!(error.is_none(), "late bitrot must recover from stripe-aligned parity: {error:?}");
|
||||
assert_eq!(written, 128);
|
||||
assert_eq!(output, payload[64..]);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn local_batch_read_decisions_follow_the_actual_reader_setup_entry() {
|
||||
@@ -7957,6 +8312,10 @@ mod tests {
|
||||
4,
|
||||
HashAlgorithm::None,
|
||||
false,
|
||||
1,
|
||||
1,
|
||||
BitrotReaderSetupMode::ReadQuorum,
|
||||
BitrotReaderSetupStrategy::AllShards,
|
||||
)
|
||||
.await
|
||||
.is_none(),
|
||||
@@ -7974,6 +8333,10 @@ mod tests {
|
||||
4,
|
||||
HashAlgorithm::HighwayHash256S,
|
||||
false,
|
||||
1,
|
||||
1,
|
||||
BitrotReaderSetupMode::ReadQuorum,
|
||||
BitrotReaderSetupStrategy::AllShards,
|
||||
)
|
||||
.await
|
||||
.is_none(),
|
||||
@@ -7993,6 +8356,10 @@ mod tests {
|
||||
4,
|
||||
HashAlgorithm::None,
|
||||
false,
|
||||
1,
|
||||
1,
|
||||
BitrotReaderSetupMode::ReadQuorum,
|
||||
BitrotReaderSetupStrategy::AllShards,
|
||||
)
|
||||
.await
|
||||
.is_none()
|
||||
@@ -8020,6 +8387,10 @@ mod tests {
|
||||
4,
|
||||
HashAlgorithm::None,
|
||||
false,
|
||||
1,
|
||||
1,
|
||||
BitrotReaderSetupMode::ReadQuorum,
|
||||
BitrotReaderSetupStrategy::AllShards,
|
||||
)
|
||||
.await
|
||||
.is_none(),
|
||||
@@ -8037,6 +8408,10 @@ mod tests {
|
||||
4,
|
||||
HashAlgorithm::None,
|
||||
false,
|
||||
1,
|
||||
1,
|
||||
BitrotReaderSetupMode::ReadQuorum,
|
||||
BitrotReaderSetupStrategy::AllShards,
|
||||
)
|
||||
.await
|
||||
.is_none()
|
||||
@@ -8054,6 +8429,10 @@ mod tests {
|
||||
4,
|
||||
HashAlgorithm::None,
|
||||
false,
|
||||
1,
|
||||
1,
|
||||
BitrotReaderSetupMode::ReadQuorum,
|
||||
BitrotReaderSetupStrategy::AllShards,
|
||||
)
|
||||
.await
|
||||
.is_none()
|
||||
@@ -12823,6 +13202,20 @@ mod tests {
|
||||
assert_eq!(err, DiskError::ErasureReadQuorum);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_read_part_does_not_count_error_response_with_etag_as_a_vote() {
|
||||
let healthy = read_part_test_part(1, "same-etag");
|
||||
let mut failed = healthy.clone();
|
||||
failed.error = Some("metadata read failed".to_owned());
|
||||
let responses = vec![Some(vec![healthy.clone()]), Some(vec![healthy]), Some(vec![failed])];
|
||||
let err = resolve_read_part_from_responses("bucket", "upload/part.1.meta", 1, 0, 1, &responses, 3)
|
||||
.expect_err("an error-bearing response must not supply the third vote");
|
||||
assert_eq!(err, DiskError::ErasureReadQuorum);
|
||||
let part = resolve_read_part_from_responses("bucket", "upload/part.1.meta", 1, 0, 1, &responses, 2)
|
||||
.expect("the two healthy matching responses remain a valid quorum");
|
||||
assert!(part.error.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_read_part_accepts_alternate_missing_error_markers() {
|
||||
let responses = vec![
|
||||
|
||||
@@ -338,17 +338,26 @@ fn abort_slot(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn abort_certificate_cohort(
|
||||
pub(in crate::set_disk) fn abort_certificate_cohort(
|
||||
certificate: &MultipartAbortCertificate,
|
||||
ids: &[Option<Uuid>],
|
||||
authority: &ProofVerificationContext,
|
||||
) -> Result<()> {
|
||||
certificate.verify_with_authority(authority)?;
|
||||
if ids.len() != certificate.disk_ids.len()
|
||||
|| ids.iter().copied().collect::<Option<Vec<_>>>().as_ref() != Some(&certificate.disk_ids)
|
||||
{
|
||||
if ids.len() != certificate.disk_ids.len() {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
let mut missing = false;
|
||||
for (actual, expected) in ids.iter().zip(&certificate.disk_ids) {
|
||||
match actual {
|
||||
Some(actual) if actual != expected => return Err(DiskError::FileCorrupt),
|
||||
Some(_) => {}
|
||||
None => missing = true,
|
||||
}
|
||||
}
|
||||
if missing {
|
||||
return Err(DiskError::ErasureWriteQuorum);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -154,8 +154,8 @@ use rustfs_utils::http::headers::{
|
||||
};
|
||||
use rustfs_utils::http::{
|
||||
SUFFIX_ACTUAL_OBJECT_SIZE_CAP, SUFFIX_ACTUAL_SIZE, SUFFIX_BUCKET_INCARNATION_ID, SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE,
|
||||
SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_RESTORE_OPERATION_ID, SUFFIX_RESTORE_WORKER_LOCK, contains_key_str, get_header_map,
|
||||
get_str, insert_str, is_object_encryption_marker, remove_header_map,
|
||||
SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_RESTORE_COPYBACK, SUFFIX_RESTORE_OPERATION_ID, SUFFIX_RESTORE_WORKER_LOCK,
|
||||
contains_key_str, get_header_map, get_str, insert_str, is_object_encryption_marker, remove_header_map,
|
||||
};
|
||||
use rustfs_utils::{
|
||||
HashAlgorithm,
|
||||
@@ -283,6 +283,23 @@ fn ensure_decommission_tier_free_version_commit_fence(bucket: &str, object: &str
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Names a `bucket/object` key prefix whose best-effort rename-tail repairs
|
||||
/// are not submitted. A root-heal fixture arms it so the explicit admin heal
|
||||
/// remains the only owner of objects written while their target was offline.
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
const RENAME_TAIL_HEAL_SUPPRESS_PREFIX_ENV: &str = "RUSTFS_E2E_SUPPRESS_RENAME_TAIL_HEAL_PREFIX";
|
||||
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
fn rename_tail_heal_suppressed_for_e2e(bucket: &str, object: Option<&str>) -> bool {
|
||||
let Some(prefix) = std::env::var(RENAME_TAIL_HEAL_SUPPRESS_PREFIX_ENV)
|
||||
.ok()
|
||||
.filter(|prefix| !prefix.is_empty())
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
object.is_some_and(|object| format!("{bucket}/{object}").starts_with(&prefix))
|
||||
}
|
||||
|
||||
impl SetDisks {
|
||||
pub(super) async fn require_current_restore_operation_id(
|
||||
&self,
|
||||
@@ -486,10 +503,16 @@ pub(crate) fn strip_internal_multipart_metadata(metadata: &mut HashMap<String, S
|
||||
rustfs_utils::http::metadata_compat::remove_str(metadata, SUFFIX_BUCKET_INCARNATION_ID);
|
||||
}
|
||||
|
||||
fn should_persist_encryption_original_size(metadata: &HashMap<String, String>) -> bool {
|
||||
pub(crate) fn should_persist_encryption_original_size(metadata: &HashMap<String, String>) -> bool {
|
||||
metadata.keys().any(|key| is_object_encryption_marker(key))
|
||||
}
|
||||
|
||||
/// Reject ambiguous part sizes unless the upload records a transformation.
|
||||
pub(crate) fn multipart_part_logical_size(stored_size: usize, actual_size: i64, transformed: bool) -> Option<usize> {
|
||||
let actual_size = usize::try_from(actual_size).ok()?;
|
||||
(transformed || actual_size == stored_size).then_some(actual_size)
|
||||
}
|
||||
|
||||
/// Per-set memoized capacity dirty scope.
|
||||
///
|
||||
/// The disk endpoint/path identity of each slot in a set is immutable for the
|
||||
@@ -4245,14 +4268,23 @@ pub(super) struct MultipartWriteQuorumContext<'a> {
|
||||
part_number: Option<usize>,
|
||||
}
|
||||
|
||||
fn record_multipart_write_quorum_failure(
|
||||
stage: &'static str,
|
||||
errs: &[Option<DiskError>],
|
||||
write_quorum: usize,
|
||||
) -> crate::disk::error_reduce::WriteQuorumFailureSummary {
|
||||
let summary = build_write_quorum_failure_summary(errs, OBJECT_OP_IGNORED_ERRS, write_quorum);
|
||||
runtime_sources::record_erasure_write_quorum_failure(stage, summary.dominant_error_label);
|
||||
summary
|
||||
}
|
||||
|
||||
fn log_multipart_write_quorum_failure(
|
||||
context: MultipartWriteQuorumContext<'_>,
|
||||
errs: &[Option<DiskError>],
|
||||
write_quorum: usize,
|
||||
returned_error: &DiskError,
|
||||
) {
|
||||
let summary = build_write_quorum_failure_summary(errs, OBJECT_OP_IGNORED_ERRS, write_quorum);
|
||||
runtime_sources::record_erasure_write_quorum_failure(context.stage, summary.dominant_error_label);
|
||||
let summary = record_multipart_write_quorum_failure(context.stage, errs, write_quorum);
|
||||
warn!(
|
||||
target: "rustfs_ecstore::set_disk",
|
||||
event = EVENT_SET_DISK_MULTIPART,
|
||||
@@ -5134,6 +5166,11 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
if rename_tail_heal_suppressed_for_e2e(&request.bucket, request.object_prefix.as_deref()) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let _ = rustfs_heal_contracts::heal_channel::send_heal_request(request).await;
|
||||
Ok(())
|
||||
}
|
||||
@@ -13914,10 +13951,7 @@ mod tests {
|
||||
let requested_until = now + Duration::from_secs(60 * 60 * 24);
|
||||
|
||||
let mut user_defined = HashMap::new();
|
||||
user_defined.insert(
|
||||
metadata_keys::OBJECT_LOCK_MODE.to_string(),
|
||||
s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(),
|
||||
);
|
||||
user_defined.insert(metadata_keys::OBJECT_LOCK_MODE.to_string(), "COMPLIANCE".to_string());
|
||||
user_defined.insert(
|
||||
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
|
||||
existing_until.format(&time::format_description::well_known::Rfc3339).unwrap(),
|
||||
@@ -13929,7 +13963,7 @@ mod tests {
|
||||
};
|
||||
let opts = ObjectOptions {
|
||||
object_lock_retention: Some(crate::storage_api_contracts::object::ObjectLockRetentionOptions {
|
||||
mode: Some(s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string()),
|
||||
mode: Some("COMPLIANCE".to_string()),
|
||||
retain_until: Some(requested_until),
|
||||
bypass_governance: true,
|
||||
}),
|
||||
@@ -13949,10 +13983,7 @@ mod tests {
|
||||
let requested_until = now + Duration::from_secs(60 * 60 * 24);
|
||||
|
||||
let mut user_defined = HashMap::new();
|
||||
user_defined.insert(
|
||||
metadata_keys::OBJECT_LOCK_MODE.to_string(),
|
||||
s3s::dto::ObjectLockRetentionMode::GOVERNANCE.to_string(),
|
||||
);
|
||||
user_defined.insert(metadata_keys::OBJECT_LOCK_MODE.to_string(), "GOVERNANCE".to_string());
|
||||
user_defined.insert(
|
||||
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
|
||||
existing_until.format(&time::format_description::well_known::Rfc3339).unwrap(),
|
||||
@@ -13964,7 +13995,7 @@ mod tests {
|
||||
};
|
||||
let opts = ObjectOptions {
|
||||
object_lock_retention: Some(crate::storage_api_contracts::object::ObjectLockRetentionOptions {
|
||||
mode: Some(s3s::dto::ObjectLockRetentionMode::GOVERNANCE.to_string()),
|
||||
mode: Some("GOVERNANCE".to_string()),
|
||||
retain_until: Some(requested_until),
|
||||
bypass_governance: true,
|
||||
}),
|
||||
@@ -13979,10 +14010,7 @@ mod tests {
|
||||
async fn test_check_object_lock_delete_blocks_compliance_version_delete() {
|
||||
let retain_until = OffsetDateTime::now_utc() + Duration::from_secs(60 * 60 * 24 * 60);
|
||||
let mut user_defined = HashMap::new();
|
||||
user_defined.insert(
|
||||
metadata_keys::OBJECT_LOCK_MODE.to_string(),
|
||||
s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(),
|
||||
);
|
||||
user_defined.insert(metadata_keys::OBJECT_LOCK_MODE.to_string(), "COMPLIANCE".to_string());
|
||||
user_defined.insert(
|
||||
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
|
||||
retain_until.format(&time::format_description::well_known::Rfc3339).unwrap(),
|
||||
@@ -14024,10 +14052,7 @@ mod tests {
|
||||
},
|
||||
restore_expires: Some(restore_expiry),
|
||||
user_defined: Arc::new(HashMap::from([
|
||||
(
|
||||
metadata_keys::OBJECT_LOCK_MODE.to_string(),
|
||||
s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(),
|
||||
),
|
||||
(metadata_keys::OBJECT_LOCK_MODE.to_string(), "COMPLIANCE".to_string()),
|
||||
(
|
||||
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
|
||||
retain_until.format(&time::format_description::well_known::Rfc3339).unwrap(),
|
||||
@@ -14098,10 +14123,7 @@ mod tests {
|
||||
async fn test_check_object_lock_delete_allows_versioned_delete_marker_creation() {
|
||||
let retain_until = OffsetDateTime::now_utc() + Duration::from_secs(60 * 60 * 24 * 60);
|
||||
let mut user_defined = HashMap::new();
|
||||
user_defined.insert(
|
||||
metadata_keys::OBJECT_LOCK_MODE.to_string(),
|
||||
s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(),
|
||||
);
|
||||
user_defined.insert(metadata_keys::OBJECT_LOCK_MODE.to_string(), "COMPLIANCE".to_string());
|
||||
user_defined.insert(
|
||||
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
|
||||
retain_until.format(&time::format_description::well_known::Rfc3339).unwrap(),
|
||||
@@ -14126,10 +14148,7 @@ mod tests {
|
||||
fn governance_retained_obj_info() -> ObjectInfo {
|
||||
let retain_until = OffsetDateTime::now_utc() + Duration::from_secs(60 * 60 * 24 * 60);
|
||||
let mut user_defined = HashMap::new();
|
||||
user_defined.insert(
|
||||
metadata_keys::OBJECT_LOCK_MODE.to_string(),
|
||||
s3s::dto::ObjectLockRetentionMode::GOVERNANCE.to_string(),
|
||||
);
|
||||
user_defined.insert(metadata_keys::OBJECT_LOCK_MODE.to_string(), "GOVERNANCE".to_string());
|
||||
user_defined.insert(
|
||||
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
|
||||
retain_until.format(&time::format_description::well_known::Rfc3339).unwrap(),
|
||||
@@ -14179,10 +14198,7 @@ mod tests {
|
||||
async fn test_check_object_lock_delete_blocks_replicated_compliance_version_purge() {
|
||||
let retain_until = OffsetDateTime::now_utc() + Duration::from_secs(60 * 60 * 24 * 60);
|
||||
let mut user_defined = HashMap::new();
|
||||
user_defined.insert(
|
||||
metadata_keys::OBJECT_LOCK_MODE.to_string(),
|
||||
s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(),
|
||||
);
|
||||
user_defined.insert(metadata_keys::OBJECT_LOCK_MODE.to_string(), "COMPLIANCE".to_string());
|
||||
user_defined.insert(
|
||||
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
|
||||
retain_until.format(&time::format_description::well_known::Rfc3339).unwrap(),
|
||||
@@ -14237,11 +14253,11 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_object_lock_delete_check_required_keeps_object_lock_config_buckets() {
|
||||
use s3s::dto::{ObjectLockConfiguration, ObjectLockEnabled};
|
||||
use rustfs_gateway_types::persistence::PersistedObjectLockConfiguration as ObjectLockConfiguration;
|
||||
|
||||
let mut bm = crate::bucket::metadata::BucketMetadata::new("lock-config-bucket");
|
||||
bm.object_lock_config = Some(ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
object_lock_enabled: Some("Enabled".to_owned()),
|
||||
..Default::default()
|
||||
});
|
||||
assert!(object_lock_delete_check_required(Some(&bm)));
|
||||
@@ -17138,8 +17154,8 @@ mod tests {
|
||||
// otherwise `from_file_info` never synthesizes the null version id and
|
||||
// the branch under test is not reached.
|
||||
let suspended = crate::bucket::replication::DeleteReplicationConfigSnapshot::from_configs_for_test(
|
||||
s3s::dto::VersioningConfiguration {
|
||||
status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::SUSPENDED)),
|
||||
rustfs_gateway_types::persistence::PersistedVersioningConfiguration {
|
||||
status: Some("Suspended".to_owned()),
|
||||
..Default::default()
|
||||
},
|
||||
None,
|
||||
|
||||
@@ -40,9 +40,35 @@ pub(crate) mod test_hooks {
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
type Hook = Arc<dyn Fn() -> BoxFuture<'static, ()> + Send + Sync>;
|
||||
type TransactionLockHook = Arc<dyn Fn(&rustfs_lock::NamespaceLockGuard) + Send + Sync>;
|
||||
|
||||
static BEFORE_GC_FINALIZE: Mutex<Vec<(String, String, Hook)>> = Mutex::new(Vec::new());
|
||||
static CLEANUP_ATTEMPTS: Mutex<Vec<(String, String)>> = Mutex::new(Vec::new());
|
||||
static AFTER_TRANSACTION_LOCK: Mutex<Vec<(String, String, TransactionLockHook)>> = Mutex::new(Vec::new());
|
||||
|
||||
pub(crate) fn after_transaction_lock_acquired(bucket: &str, object: &str, hook: TransactionLockHook) {
|
||||
AFTER_TRANSACTION_LOCK
|
||||
.lock()
|
||||
.expect("transaction lock hooks")
|
||||
.push((bucket.to_owned(), object.to_owned(), hook));
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn run_after_transaction_lock_acquired(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
guard: &rustfs_lock::NamespaceLockGuard,
|
||||
) {
|
||||
let hook = {
|
||||
let mut hooks = AFTER_TRANSACTION_LOCK.lock().expect("transaction lock hooks");
|
||||
hooks
|
||||
.iter()
|
||||
.position(|(hook_bucket, hook_object, _)| hook_bucket == bucket && hook_object == object)
|
||||
.map(|index| hooks.remove(index).2)
|
||||
};
|
||||
if let Some(hook) = hook {
|
||||
hook(guard);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn record_cleanup_attempt(bucket: &str, object: &str) {
|
||||
CLEANUP_ATTEMPTS
|
||||
@@ -112,6 +138,7 @@ pub(crate) struct DeleteRecoveryContext<'a> {
|
||||
/// Only the locked heal path may repair a live marker-creation source.
|
||||
pub repair_marker_source: bool,
|
||||
pub namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
pub transaction_guard: Option<Arc<rustfs_lock::NamespaceLockGuard>>,
|
||||
pub recovery_ticket: Option<crate::store::delete_recovery_runtime::DeleteRecoveryTicket>,
|
||||
pub published_hint: Option<(Uuid, &'a [Result<DeleteRecoveryInspection>])>,
|
||||
pub can_commit: &'a (dyn Fn() -> bool + Sync),
|
||||
@@ -125,6 +152,7 @@ impl<'a> DeleteRecoveryContext<'a> {
|
||||
explicit_delete: false,
|
||||
repair_marker_source: false,
|
||||
namespace_owner: None,
|
||||
transaction_guard: None,
|
||||
recovery_ticket: None,
|
||||
published_hint: None,
|
||||
can_commit,
|
||||
@@ -373,9 +401,22 @@ impl SetDisks {
|
||||
write_quorum: usize,
|
||||
mut context: DeleteRecoveryContext<'_>,
|
||||
) -> Result<()> {
|
||||
if context.namespace_owner.is_none() {
|
||||
context.namespace_owner = Some(self.ctx.begin_namespace_commit());
|
||||
}
|
||||
let transaction_guard = match context.transaction_guard.take() {
|
||||
Some(guard) => guard,
|
||||
None => self
|
||||
.acquire_delete_transaction_lock(bucket, object)
|
||||
.await
|
||||
.map_err(DiskError::other)?,
|
||||
};
|
||||
let outer_can_commit = context.can_commit;
|
||||
let can_commit = || outer_can_commit() && !transaction_guard.is_lock_lost();
|
||||
context.can_commit = &can_commit;
|
||||
context.transaction_guard = Some(Arc::clone(&transaction_guard));
|
||||
let owner = context
|
||||
.namespace_owner
|
||||
.take()
|
||||
.unwrap_or_else(|| self.ctx.begin_namespace_commit());
|
||||
context.namespace_owner = Some(Arc::new((owner, Arc::clone(&transaction_guard))));
|
||||
let can_commit = context.can_commit;
|
||||
let states = inspect(disks, bucket, object, transaction).await;
|
||||
let untouched_absence = |state: &Result<DeleteRecoveryInspection>| matches!(state, Ok(state) if state.state == DeleteRecoveryState::Absent && state.current_digest.is_none());
|
||||
@@ -547,8 +588,25 @@ impl SetDisks {
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
context: DeleteRecoveryContext<'_>,
|
||||
mut context: DeleteRecoveryContext<'_>,
|
||||
) -> Result<DeleteRecoverySummary> {
|
||||
let transaction_guard = match context.transaction_guard.take() {
|
||||
Some(guard) => guard,
|
||||
None => self
|
||||
.acquire_delete_transaction_lock(bucket, object)
|
||||
.await
|
||||
.map_err(DiskError::other)?,
|
||||
};
|
||||
let outer_can_commit = context.can_commit;
|
||||
let can_commit = || outer_can_commit() && !transaction_guard.is_lock_lost();
|
||||
context.can_commit = &can_commit;
|
||||
if !context.dry_run {
|
||||
let owner = context
|
||||
.namespace_owner
|
||||
.take()
|
||||
.unwrap_or_else(|| self.ctx.begin_namespace_commit());
|
||||
context.namespace_owner = Some(Arc::new((owner, Arc::clone(&transaction_guard))));
|
||||
}
|
||||
let DeleteRecoveryContext {
|
||||
incarnation,
|
||||
dry_run,
|
||||
@@ -558,6 +616,7 @@ impl SetDisks {
|
||||
published_hint,
|
||||
can_commit,
|
||||
recovery_ticket: _,
|
||||
transaction_guard: _,
|
||||
} = context;
|
||||
let disks = self.disk_inventory().await;
|
||||
let complete_inventory = disks.len() == self.set_drive_count && disks.iter().all(Option::is_some);
|
||||
@@ -1257,6 +1316,135 @@ mod tests {
|
||||
assert_eq!(body(&store, bucket, None).await, original);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_deferred_batch_discards_released_alias_fence_and_preserves_caller_fence() {
|
||||
use crate::bucket::metadata_sys::ObjectLockConfigState;
|
||||
use crate::object_api::{NamespaceLockFence, NamespaceLockSignalTestFence, ObjectLockConfigSnapshot};
|
||||
use crate::storage_api_contracts::object::ObjectToDelete;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
for lose_caller in [false, true] {
|
||||
let bucket = if lose_caller {
|
||||
"deferred-batch-lost-caller"
|
||||
} else {
|
||||
"deferred-batch-released-alias"
|
||||
};
|
||||
let (_dirs, store, disks) = fixture(bucket).await;
|
||||
put(&store, bucket, &[27; 4096], false).await;
|
||||
let transaction = applied(&store, &disks, bucket, None, disks.len()).await;
|
||||
let before = join_all(
|
||||
disks
|
||||
.iter()
|
||||
.map(|disk| disk.inspect_delete_transaction(bucket, "object", transaction)),
|
||||
)
|
||||
.await
|
||||
.into_iter()
|
||||
.collect::<Result<Vec<_>>>()
|
||||
.expect("capture pending recovery evidence");
|
||||
let mut set = (*store.pools[0].disk_set[0]).clone();
|
||||
set.lockers = vec![Arc::new(rustfs_lock::LocalClient::with_manager(Arc::new(
|
||||
rustfs_lock::GlobalLockManager::Enabled(Arc::new(rustfs_lock::FastObjectLockManager::new())),
|
||||
)))];
|
||||
set.ctx
|
||||
.update_erasure_type(crate::layout::endpoints::SetupType::DistErasure)
|
||||
.await;
|
||||
|
||||
let (caller_fence, caller_lost) = NamespaceLockFence::loss_handle_for_test();
|
||||
let released_alias_lost = Arc::new(AtomicBool::new(false));
|
||||
let signal_fence = Arc::new(Mutex::new(None));
|
||||
let captured_signal_fence = Arc::clone(&signal_fence);
|
||||
let captured_alias_lost = Arc::clone(&released_alias_lost);
|
||||
test_hooks::after_transaction_lock_acquired(
|
||||
bucket,
|
||||
"object",
|
||||
Arc::new(move |guard| {
|
||||
let signal = guard.lock_lost_signal().expect("distributed alias signal");
|
||||
let fence = NamespaceLockSignalTestFence::install_with_loss_handle(&signal, Arc::clone(&captured_alias_lost));
|
||||
// Retain the signal so a later allocation cannot reuse its
|
||||
// registry identity after the original guard is dropped.
|
||||
*captured_signal_fence.lock().expect("captured alias fence") = Some((fence, signal));
|
||||
}),
|
||||
);
|
||||
let replay_started = Arc::new(AtomicBool::new(false));
|
||||
let captured_replay_started = Arc::clone(&replay_started);
|
||||
let expired_alias = Arc::clone(&released_alias_lost);
|
||||
test_hooks::after_transaction_lock_acquired(
|
||||
bucket,
|
||||
"object",
|
||||
Arc::new(move |_| {
|
||||
// Acquiring the same transaction key again proves the first
|
||||
// lease was released. Only its copied fence becomes stale.
|
||||
expired_alias.store(true, Ordering::Release);
|
||||
caller_lost.store(lose_caller, Ordering::Release);
|
||||
captured_replay_started.store(true, Ordering::Release);
|
||||
}),
|
||||
);
|
||||
|
||||
let (deleted, errors) = set
|
||||
.delete_objects(
|
||||
bucket,
|
||||
vec![ObjectToDelete {
|
||||
object_name: "object".into(),
|
||||
..Default::default()
|
||||
}],
|
||||
ObjectOptions {
|
||||
namespace_lock_fence: Some(caller_fence),
|
||||
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
|
||||
ObjectLockConfigState::ConfirmedAbsent,
|
||||
))),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
replay_started.load(Ordering::Acquire),
|
||||
"missing metadata must take the deferred replay path"
|
||||
);
|
||||
assert!(
|
||||
released_alias_lost.load(Ordering::Acquire),
|
||||
"the completed batch alias must be marked lost"
|
||||
);
|
||||
if lose_caller {
|
||||
assert!(errors[0].is_some(), "losing the original caller fence must reject deferred replay");
|
||||
for (disk, original) in disks.iter().zip(before) {
|
||||
let after = disk
|
||||
.inspect_delete_transaction(bucket, "object", transaction)
|
||||
.await
|
||||
.expect("retained proof");
|
||||
assert_eq!(
|
||||
after.fingerprint, original.fingerprint,
|
||||
"lost caller authority must preserve every prepared journal"
|
||||
);
|
||||
}
|
||||
} else {
|
||||
assert!(
|
||||
errors.iter().all(Option::is_none),
|
||||
"released alias loss must not reject fresh recovery: {errors:?}"
|
||||
);
|
||||
assert!(
|
||||
!deleted[0].found,
|
||||
"fresh unversioned recovery commits absence before the idempotent batch retry"
|
||||
);
|
||||
for disk in &disks {
|
||||
assert!(
|
||||
discover(disk, bucket, Some("object"))
|
||||
.await
|
||||
.expect("final recovery evidence")
|
||||
.is_empty()
|
||||
);
|
||||
assert_eq!(
|
||||
disk.read_version("", bucket, "object", "", &ReadOptions::default())
|
||||
.await
|
||||
.expect_err("replayed delete must remove the source"),
|
||||
DiskError::FileNotFound
|
||||
);
|
||||
}
|
||||
}
|
||||
drop(signal_fence);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_corrupt_bucket_does_not_prevent_other_bucket_rollback() {
|
||||
let bad_bucket = "recover-corrupt-bucket";
|
||||
@@ -2125,6 +2313,86 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_batch_with_missing_version_recovers_after_preparation_failure() {
|
||||
use crate::disk::RUSTFS_META_TMP_BUCKET;
|
||||
use crate::storage_api_contracts::object::ObjectToDelete;
|
||||
|
||||
let bucket = "recover-batch-missing-version";
|
||||
let (dirs, store, disks) = fixture(bucket).await;
|
||||
let deleted = put(&store, bucket, &vec![48; 4096], true).await.expect("version to delete");
|
||||
let remaining = vec![49; 4096];
|
||||
let retained = put(&store, bucket, &remaining, true).await.expect("version to retain");
|
||||
let missing = Uuid::new_v4();
|
||||
let temporary = dirs[3].path().join(RUSTFS_META_TMP_BUCKET);
|
||||
let saved = dirs[3].path().join("saved-delete-tmp");
|
||||
tokio::fs::rename(&temporary, &saved)
|
||||
.await
|
||||
.expect("save returning disk's staging directory");
|
||||
tokio::fs::write(&temporary, b"block preparation")
|
||||
.await
|
||||
.expect("fail staging after the recovery preflight");
|
||||
|
||||
let (objects, errors) = store
|
||||
.delete_objects(
|
||||
bucket,
|
||||
[deleted, missing]
|
||||
.into_iter()
|
||||
.map(|version| ObjectToDelete {
|
||||
object_name: "object".into(),
|
||||
version_id: Some(version),
|
||||
..Default::default()
|
||||
})
|
||||
.collect(),
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert_eq!(objects.len(), 2);
|
||||
assert!(errors.iter().all(Option::is_none), "batch deletion succeeds at quorum: {errors:?}");
|
||||
assert!(
|
||||
disks[3]
|
||||
.discover_delete_transactions(bucket, Some("object"), None, 100)
|
||||
.await
|
||||
.expect("inspect failed preparation")
|
||||
.transactions
|
||||
.is_empty(),
|
||||
"returning disk must adopt the quorum decision without a local preparation"
|
||||
);
|
||||
tokio::fs::remove_file(&temporary).await.expect("remove preparation failure");
|
||||
tokio::fs::rename(&saved, &temporary)
|
||||
.await
|
||||
.expect("restore returning disk staging");
|
||||
|
||||
let result = store
|
||||
.recover_delete_transactions(bucket, &HealOpts::default())
|
||||
.await
|
||||
.expect("recover batch containing a missing non-null version");
|
||||
assert!(result.detail.contains("blocked=0"), "{}", result.detail);
|
||||
assert_eq!(body(&store, bucket, Some(retained)).await, remaining);
|
||||
for version in [deleted, missing] {
|
||||
assert_eq!(
|
||||
disks[3]
|
||||
.read_version(bucket, bucket, "object", &version.to_string(), &ReadOptions::default())
|
||||
.await
|
||||
.expect_err("returning disk must not retain or create a deleted version"),
|
||||
DiskError::FileVersionNotFound
|
||||
);
|
||||
}
|
||||
for disk in &disks {
|
||||
assert!(
|
||||
disk.discover_delete_transactions(bucket, Some("object"), None, 100)
|
||||
.await
|
||||
.expect("inspect converged batch")
|
||||
.transactions
|
||||
.is_empty(),
|
||||
"every participant must converge and release the decision"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_dry_run_matches_adoption_of_already_absent_non_null_version() {
|
||||
let bucket = "recover-preview-non-null";
|
||||
@@ -2491,6 +2759,140 @@ mod tests {
|
||||
.expect("no retained version decision may keep the empty bucket nonempty");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_partial_batch_receipt_handles_missing_versions() {
|
||||
use crate::disk::RUSTFS_META_TMP_BUCKET;
|
||||
use crate::set_disk::ops::object::DeleteCleanupBarrier;
|
||||
use crate::storage_api_contracts::object::ObjectToDelete;
|
||||
|
||||
for include_missing in [false, true] {
|
||||
let bucket = "recover-prepared-batch-missing-versions";
|
||||
let (dirs, store, disks) = fixture(bucket).await;
|
||||
metadata_sys::update_in(
|
||||
&store.ctx,
|
||||
bucket,
|
||||
crate::bucket::metadata::BUCKET_VERSIONING_CONFIG,
|
||||
b"<VersioningConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
|
||||
)
|
||||
.await
|
||||
.expect("enable supported versioned bucket");
|
||||
let first = put(&store, bucket, &vec![81; 1024 * 1024], true)
|
||||
.await
|
||||
.expect("first version");
|
||||
let second = put(&store, bucket, &vec![82; 4096], true).await.expect("retained version");
|
||||
let missing = Uuid::new_v4();
|
||||
let barrier = DeleteCleanupBarrier::install(bucket);
|
||||
let deleting = store.clone();
|
||||
let task = tokio::spawn(async move {
|
||||
let mut versions = vec![first];
|
||||
if include_missing {
|
||||
versions.push(missing);
|
||||
}
|
||||
deleting
|
||||
.delete_objects(
|
||||
bucket,
|
||||
versions
|
||||
.into_iter()
|
||||
.map(|version| ObjectToDelete {
|
||||
object_name: "object".into(),
|
||||
version_id: Some(version),
|
||||
..Default::default()
|
||||
})
|
||||
.collect(),
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
let transactions = disks[0]
|
||||
.discover_delete_transactions(bucket, Some("object"), None, 100)
|
||||
.await
|
||||
.expect("discover batch preparation")
|
||||
.transactions;
|
||||
assert_eq!(transactions.len(), 1);
|
||||
let transaction = transactions[0].transaction;
|
||||
for disk in &disks {
|
||||
let state = disk
|
||||
.inspect_delete_transaction(bucket, "object", transaction)
|
||||
.await
|
||||
.expect("inspect prepared batch");
|
||||
assert_eq!(state.state, DeleteRecoveryState::NewPrepared);
|
||||
let journal = state.journal.expect("prepared journal");
|
||||
assert!(journal.original_digest.is_some());
|
||||
assert!(journal.original_versions.iter().all(|v| v.version_id != Some(missing)));
|
||||
}
|
||||
// Fail durable receipt staging on two participants after metadata quorum.
|
||||
for dir in dirs.iter().skip(2) {
|
||||
tokio::fs::rename(dir.path().join(RUSTFS_META_TMP_BUCKET), dir.path().join("saved-receipt-staging"))
|
||||
.await
|
||||
.expect("save receipt staging directory");
|
||||
tokio::fs::write(dir.path().join(RUSTFS_META_TMP_BUCKET), b"receipt staging I/O failure")
|
||||
.await
|
||||
.expect("inject receipt staging failure");
|
||||
}
|
||||
barrier.release();
|
||||
let (_, errors) = task.await.expect("batch deletion must join");
|
||||
drop(barrier);
|
||||
assert_eq!(errors.len(), if include_missing { 2 } else { 1 });
|
||||
assert!(errors.iter().all(Option::is_some), "partial receipts are indeterminate: {errors:?}");
|
||||
let mut receipts = 0;
|
||||
let mut prepared = 0;
|
||||
for disk in &disks {
|
||||
match disk
|
||||
.inspect_delete_transaction(bucket, "object", transaction)
|
||||
.await
|
||||
.expect("partial decision")
|
||||
.state
|
||||
{
|
||||
DeleteRecoveryState::QuorumCommitted => receipts += 1,
|
||||
DeleteRecoveryState::NewPrepared => prepared += 1,
|
||||
state => panic!("unexpected post-publication state: {state:?}"),
|
||||
}
|
||||
}
|
||||
assert_eq!((receipts, prepared), (2, 2));
|
||||
for dir in dirs.iter().skip(2) {
|
||||
tokio::fs::remove_file(dir.path().join(RUSTFS_META_TMP_BUCKET))
|
||||
.await
|
||||
.expect("remove injected staging failure");
|
||||
tokio::fs::rename(dir.path().join("saved-receipt-staging"), dir.path().join(RUSTFS_META_TMP_BUCKET))
|
||||
.await
|
||||
.expect("restore receipt staging");
|
||||
}
|
||||
store
|
||||
.delete_object(
|
||||
bucket,
|
||||
"object",
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(second.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("later version deletion succeeds");
|
||||
store
|
||||
.recover_delete_transactions(bucket, &HealOpts::default())
|
||||
.await
|
||||
.expect("certified batch must recover after later version deletion");
|
||||
for disk in &disks {
|
||||
assert!(
|
||||
disk.discover_delete_transactions(bucket, Some("object"), None, 100)
|
||||
.await
|
||||
.expect("settled transaction inventory")
|
||||
.transactions
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
store
|
||||
.delete_bucket(bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("settled receipts must not block removal of the empty bucket");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_partial_receipt_converges_after_later_version_absence() {
|
||||
let bucket = "recover-partial-chained-versions";
|
||||
|
||||
@@ -3634,6 +3634,18 @@ impl SetDisks {
|
||||
return Ok((result, err.map(|e| e.into())));
|
||||
}
|
||||
|
||||
// Keep the transaction key through both replay and the metadata heal
|
||||
// based on that replay. The raw target key always precedes this key.
|
||||
let transaction_guard =
|
||||
if _write_lock_guard.is_some() && !opts.read_repair && !crate::bucket::utils::is_meta_bucketname(bucket) {
|
||||
Some(self.acquire_delete_transaction_lock(bucket, object).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let locks_lost = || {
|
||||
_write_lock_guard.as_deref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
};
|
||||
let mut recovery_removed_version = false;
|
||||
if !opts.read_repair
|
||||
&& _write_lock_guard.is_some()
|
||||
@@ -3655,12 +3667,17 @@ impl SetDisks {
|
||||
dry_run: opts.dry_run,
|
||||
explicit_delete,
|
||||
repair_marker_source: true,
|
||||
transaction_guard: transaction_guard.clone(),
|
||||
namespace_owner: (!opts.dry_run).then(|| {
|
||||
Arc::new((scope.clone(), self.ctx.begin_namespace_commit(), _write_lock_guard.clone()))
|
||||
as Arc<dyn Send + Sync>
|
||||
Arc::new((
|
||||
scope.clone(),
|
||||
self.ctx.begin_namespace_commit(),
|
||||
_write_lock_guard.clone(),
|
||||
transaction_guard.clone(),
|
||||
)) as Arc<dyn Send + Sync>
|
||||
}),
|
||||
..crate::set_disk::DeleteRecoveryContext::new(Some(scope.incarnation), &|| {
|
||||
scope.check().is_ok() && !_write_lock_guard.as_deref().is_some_and(|guard| guard.is_lock_lost())
|
||||
scope.check().is_ok() && !locks_lost()
|
||||
})
|
||||
},
|
||||
)
|
||||
@@ -3752,7 +3769,7 @@ impl SetDisks {
|
||||
&& errs
|
||||
.iter()
|
||||
.all(|err| matches!(err, Some(DiskError::FileNotFound | DiskError::FileVersionNotFound)))
|
||||
&& !_write_lock_guard.as_deref().is_some_and(|guard| guard.is_lock_lost())
|
||||
&& !locks_lost()
|
||||
{
|
||||
*absence = Some(HealedObjectAbsence {
|
||||
pool_index: self.pool_index,
|
||||
@@ -3767,8 +3784,10 @@ impl SetDisks {
|
||||
// Pass no_lock=true since we already obtained write lock (or are already called with no_lock=true)
|
||||
let mut inner_opts = *opts;
|
||||
inner_opts.no_lock = true;
|
||||
let commit_fence = || !_write_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost());
|
||||
let namespace_owner = _write_lock_guard.as_ref().map(|guard| guard.clone() as Arc<dyn Send + Sync>);
|
||||
let commit_fence = || !locks_lost();
|
||||
let namespace_owner = _write_lock_guard
|
||||
.as_ref()
|
||||
.map(|guard| Arc::new((Arc::clone(guard), transaction_guard.clone())) as Arc<dyn Send + Sync>);
|
||||
let (result, err) = pin_object_heal_future(|| {
|
||||
self.heal_object_with_explicit_version_regen(
|
||||
bucket,
|
||||
@@ -3815,7 +3834,7 @@ impl SetDisks {
|
||||
result,
|
||||
err.map(Into::into),
|
||||
absence,
|
||||
_write_lock_guard.as_deref().is_some_and(|guard| guard.is_lock_lost()),
|
||||
locks_lost(),
|
||||
bucket,
|
||||
object,
|
||||
));
|
||||
@@ -3827,7 +3846,7 @@ impl SetDisks {
|
||||
result,
|
||||
err.map(Into::into),
|
||||
absence,
|
||||
_write_lock_guard.as_deref().is_some_and(|guard| guard.is_lock_lost()),
|
||||
locks_lost(),
|
||||
bucket,
|
||||
object,
|
||||
))
|
||||
|
||||
@@ -44,6 +44,21 @@ impl crate::storage_api_contracts::namespace::NamespaceLocking for SetDisks {
|
||||
|
||||
#[tracing::instrument(level = "trace", skip(self))]
|
||||
async fn new_ns_lock(&self, bucket: &str, object: &str) -> Result<NamespaceLockWrapper> {
|
||||
// Lock the file the disk resolves, not the spelling: on Windows `a\b`
|
||||
// and `a/b` share one xl.meta, so they must share one lock. The key is
|
||||
// folded by the requesting node only; lock servers compare it verbatim.
|
||||
let resource = ObjectKey {
|
||||
bucket: Arc::from(bucket),
|
||||
object: Arc::from(object_name_disk_identity(object).as_ref()),
|
||||
version: None,
|
||||
};
|
||||
|
||||
Ok(self.namespace_lock_for_key(resource).await)
|
||||
}
|
||||
}
|
||||
|
||||
impl SetDisks {
|
||||
async fn namespace_lock_for_key(&self, resource: ObjectKey) -> NamespaceLockWrapper {
|
||||
// Resolved from this set's own instance context (backlog#1052), not the
|
||||
// ambient facade: the facade tracks whichever context is currently
|
||||
// published process-wide, so a second instance (or, in tests, another
|
||||
@@ -69,20 +84,33 @@ impl crate::storage_api_contracts::namespace::NamespaceLocking for SetDisks {
|
||||
NamespaceLock::with_local_manager_shared(self.set_lock_namespace.clone(), self.local_lock_manager.clone())
|
||||
};
|
||||
|
||||
// Lock the file the disk resolves, not the spelling: on Windows `a\b`
|
||||
// and `a/b` share one xl.meta, so they must share one lock. The key is
|
||||
// folded by the requesting node only; lock servers compare it verbatim.
|
||||
let resource = ObjectKey {
|
||||
bucket: Arc::from(bucket),
|
||||
object: Arc::from(object_name_disk_identity(object).as_ref()),
|
||||
version: None,
|
||||
};
|
||||
|
||||
Ok(NamespaceLockWrapper::new(set_lock, resource, self.locker_owner.clone()))
|
||||
NamespaceLockWrapper::new(set_lock, resource, self.locker_owner.clone())
|
||||
}
|
||||
|
||||
/// Raw object locks precede this set-local transaction lock. Its versioned
|
||||
/// resource cannot collide with normal namespace keys, which use None.
|
||||
/// Keeping the target set's clients avoids nesting overlapping raw quorums.
|
||||
pub(in crate::set_disk) async fn acquire_delete_transaction_lock(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
) -> Result<Arc<rustfs_lock::NamespaceLockGuard>> {
|
||||
let resource = ObjectKey::with_version(
|
||||
bucket,
|
||||
object_name_disk_identity(object).as_ref(),
|
||||
format!("rustfs:delete-transaction:{}:{}", self.pool_index, self.set_index),
|
||||
);
|
||||
let guard = self
|
||||
.namespace_lock_for_key(resource)
|
||||
.await
|
||||
.get_write_lock(super::super::get_lock_acquire_timeout())
|
||||
.await
|
||||
.map_err(|error| self.map_namespace_lock_error(bucket, object, "delete_transaction", error))?;
|
||||
#[cfg(test)]
|
||||
super::delete_recovery::test_hooks::run_after_transaction_lock_acquired(bucket, object, &guard);
|
||||
Ok(Arc::new(guard))
|
||||
}
|
||||
}
|
||||
|
||||
impl SetDisks {
|
||||
#[allow(dead_code, reason = "lock diagnostics formatter with no caller in this port (backlog#1823)")]
|
||||
pub(in crate::set_disk) fn format_lock_error(&self, bucket: &str, object: &str, mode: &str, err: &LockResult) -> String {
|
||||
match err {
|
||||
|
||||
@@ -38,13 +38,14 @@ use super::super::{
|
||||
ObjectLockDiagGuard, ObjectOptions, ObjectPartInfo, OffsetDateTime, PartInfo, PutObjReader, RUSTFS_META_MULTIPART_BUCKET,
|
||||
RUSTFS_META_TMP_BUCKET, RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY, Result, SLASH_SEPARATOR,
|
||||
SUFFIX_ACTUAL_OBJECT_SIZE_CAP, SUFFIX_ACTUAL_SIZE, SUFFIX_BUCKET_INCARNATION_ID, SUFFIX_COMPRESSION_SIZE,
|
||||
SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_RESTORE_OPERATION_ID, SUFFIX_RESTORE_WORKER_LOCK, SetDisks, SmallWritePath, StorageError,
|
||||
Uuid, WriteLayout, check_object_lock_for_deletion_with_state, classify_multipart_part_write_path, coding,
|
||||
complete_multipart_part_error, complete_multipart_part_error_result, complete_part_checksum, contains_key_str,
|
||||
create_bitrot_writer, debug, disk, error, file_info_is_valid_for_metadata, get_complete_multipart_md5, get_header_map,
|
||||
get_str, insert_str, is_err_object_not_found, is_err_version_not_found, is_min_allowed_part_size,
|
||||
log_multipart_write_quorum_failure, parts_after_marker, path_join_buf, prepare_complete_multipart_parts,
|
||||
record_compression_total_memory, reduce_read_quorum_errs, reduce_write_quorum_errs, remove_header_map, resolve_write_layout,
|
||||
SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_RESTORE_COPYBACK, SUFFIX_RESTORE_OPERATION_ID, SUFFIX_RESTORE_WORKER_LOCK, SetDisks,
|
||||
SmallWritePath, StorageError, Uuid, WriteLayout, check_object_lock_for_deletion_with_state,
|
||||
classify_multipart_part_write_path, coding, complete_multipart_part_error, complete_multipart_part_error_result,
|
||||
complete_part_checksum, contains_key_str, create_bitrot_writer, debug, disk, error, file_info_is_valid_for_metadata,
|
||||
get_complete_multipart_md5, get_header_map, get_str, insert_str, is_err_object_not_found, is_err_version_not_found,
|
||||
is_min_allowed_part_size, log_multipart_write_quorum_failure, multipart_part_logical_size, parts_after_marker, path_join_buf,
|
||||
prepare_complete_multipart_parts, record_compression_total_memory, record_multipart_write_quorum_failure,
|
||||
reduce_read_quorum_errs, reduce_write_quorum_errs, remove_header_map, resolve_write_layout,
|
||||
restore_commit_operation_id_from_metadata, should_persist_encryption_original_size, strip_internal_multipart_metadata,
|
||||
to_object_err, warn,
|
||||
};
|
||||
@@ -233,6 +234,19 @@ fn admitted_multipart_size(current: u64, candidate: u64, limit: u64) -> Result<u
|
||||
Ok(total)
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
enum MultipartMetadataQuorum {
|
||||
Read,
|
||||
Write,
|
||||
Cleanup,
|
||||
}
|
||||
|
||||
impl From<bool> for MultipartMetadataQuorum {
|
||||
fn from(write: bool) -> Self {
|
||||
if write { Self::Write } else { Self::Read }
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct StaleMultipartCleanupGuard {
|
||||
file_info: FileInfo,
|
||||
upload_path: String,
|
||||
@@ -410,6 +424,8 @@ impl StaleMultipartCleanupGuard {
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum MultipartCommitPause {
|
||||
NewUploadBeforeLockLost,
|
||||
#[cfg(test)]
|
||||
ListPartsAfterMetadata,
|
||||
PutPartBeforeLockAcquire,
|
||||
PutPartBeforeLockLost,
|
||||
PutPartAfterCapacityAdmission,
|
||||
@@ -824,7 +840,8 @@ where
|
||||
}
|
||||
|
||||
if successful_responses < read_quorum {
|
||||
if empty_upload_fallback_possible(successful_responses, &errs) {
|
||||
// A single missing directory cannot speak for unavailable disks.
|
||||
if !errs.is_empty() && errs.iter().all(|err| matches!(err, Some(DiskError::FileNotFound))) {
|
||||
return Err(DiskError::FileNotFound);
|
||||
}
|
||||
|
||||
@@ -834,6 +851,51 @@ where
|
||||
Ok((errs, object_parts))
|
||||
}
|
||||
|
||||
// Size-dependent decisions must not discard a part merely because only a
|
||||
// minority of its files was observed. All participants must answer, and each
|
||||
// observed part must have a complete file/metadata pair on a read quorum.
|
||||
fn validate_part_size_observations(
|
||||
errs: &[Option<DiskError>],
|
||||
object_parts: &[Vec<String>],
|
||||
read_quorum: usize,
|
||||
) -> disk::error::Result<Vec<usize>> {
|
||||
if errs.iter().any(Option::is_some) {
|
||||
return Err(DiskError::ErasureReadQuorum);
|
||||
}
|
||||
let mut counts: HashMap<usize, usize> = HashMap::new();
|
||||
for entries in object_parts {
|
||||
let mut observed: HashMap<usize, u8> = HashMap::new();
|
||||
for entry in entries {
|
||||
let Some(name) = entry.strip_prefix("part.") else {
|
||||
continue;
|
||||
};
|
||||
let (number, flag) = match name.strip_suffix(".meta") {
|
||||
Some(number) => (number, 2),
|
||||
None => (name, 1),
|
||||
};
|
||||
let parsed = number.parse::<usize>().map_err(|_| DiskError::FileCorrupt)?;
|
||||
if !(1..=MAX_PARTS_COUNT).contains(&parsed)
|
||||
|| !number.as_bytes().first().is_some_and(|digit| (b'1'..=b'9').contains(digit))
|
||||
{
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
*observed.entry(parsed).or_default() |= flag;
|
||||
}
|
||||
for (number, flags) in observed {
|
||||
if flags != 3 {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
*counts.entry(number).or_default() += 1;
|
||||
}
|
||||
}
|
||||
if counts.values().any(|&count| count < read_quorum) {
|
||||
return Err(DiskError::ErasureReadQuorum);
|
||||
}
|
||||
let mut part_numbers: Vec<_> = counts.into_keys().collect();
|
||||
part_numbers.sort_unstable();
|
||||
Ok(part_numbers)
|
||||
}
|
||||
|
||||
fn reduce_quorum_part_numbers(object_parts: Vec<Vec<String>>, read_quorum: usize) -> Vec<usize> {
|
||||
let mut part_quorum_map: HashMap<usize, usize> = HashMap::new();
|
||||
|
||||
@@ -1004,7 +1066,7 @@ impl SetDisks {
|
||||
]),
|
||||
SLASH_SEPARATOR
|
||||
);
|
||||
let part_numbers = match Self::list_parts(&online_disks, &part_path, read_quorum).await {
|
||||
let part_numbers = match Self::list_parts(&online_disks, &part_path, read_quorum, true).await {
|
||||
Ok(parts) => parts,
|
||||
Err(DiskError::FileNotFound) => return Ok(0),
|
||||
Err(err) => return Err(to_object_err(err.into(), vec![bucket, object, upload_id])),
|
||||
@@ -1016,28 +1078,45 @@ impl SetDisks {
|
||||
.iter()
|
||||
.map(|number| format!("{part_path}part.{number}.meta"))
|
||||
.collect::<Vec<_>>();
|
||||
let existing_parts =
|
||||
Self::read_parts(&online_disks, RUSTFS_META_MULTIPART_BUCKET, &part_meta_paths, &part_numbers, read_quorum)
|
||||
.await
|
||||
.map_err(|err| to_object_err(err.into(), vec![bucket, object, upload_id]))?;
|
||||
let existing_parts = Self::read_parts(
|
||||
&online_disks,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&part_meta_paths,
|
||||
&part_numbers,
|
||||
fi.write_quorum(self.default_write_quorum()),
|
||||
)
|
||||
.await
|
||||
.map_err(|err| to_object_err(err.into(), vec![bucket, object, upload_id]))?;
|
||||
|
||||
existing_parts.into_iter().try_fold(0_u64, |total, part| {
|
||||
if part.error.is_some() || part.number == replacing_part {
|
||||
return if part.error.is_some() {
|
||||
Err(Error::PartMissingOrCorrupt)
|
||||
} else {
|
||||
Ok(total)
|
||||
};
|
||||
}
|
||||
let part_size = u64::try_from(part.actual_size).map_err(|_| Error::PartMissingOrCorrupt)?;
|
||||
total.checked_add(part_size).ok_or_else(|| {
|
||||
Error::InvalidArgument(
|
||||
"multipart upload".to_string(),
|
||||
SUFFIX_MAX_TOTAL_OBJECT_SIZE.to_string(),
|
||||
"logical size overflow".to_string(),
|
||||
)
|
||||
if existing_parts.len() != part_numbers.len() {
|
||||
return Err(Error::PartMissingOrCorrupt);
|
||||
}
|
||||
let transformed = fi.is_compressed() || should_persist_encryption_original_size(&fi.metadata);
|
||||
existing_parts
|
||||
.into_iter()
|
||||
.zip(part_numbers)
|
||||
.try_fold(0_u64, |total, (part, number)| {
|
||||
if part.number != number {
|
||||
return Err(Error::PartMissingOrCorrupt);
|
||||
}
|
||||
if part.error.is_some() || part.number == replacing_part {
|
||||
return if part.error.is_some() {
|
||||
Err(Error::PartMissingOrCorrupt)
|
||||
} else {
|
||||
Ok(total)
|
||||
};
|
||||
}
|
||||
let part_size = multipart_part_logical_size(part.size, part.actual_size, transformed)
|
||||
.and_then(|size| u64::try_from(size).ok())
|
||||
.ok_or(Error::PartMissingOrCorrupt)?;
|
||||
total.checked_add(part_size).ok_or_else(|| {
|
||||
Error::InvalidArgument(
|
||||
"multipart upload".to_string(),
|
||||
SUFFIX_MAX_TOTAL_OBJECT_SIZE.to_string(),
|
||||
"logical size overflow".to_string(),
|
||||
)
|
||||
})
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
async fn discover_multipart_upload_paths(
|
||||
@@ -1139,7 +1218,7 @@ impl SetDisks {
|
||||
}
|
||||
let upload_id = runtime_sources::deployment_upload_id(raw_upload_id);
|
||||
let file_info = match self
|
||||
.check_multipart_upload_path_exists(bucket, object, &upload_id, &upload_path, false, true)
|
||||
.check_multipart_upload_path_exists(bucket, object, &upload_id, &upload_path, MultipartMetadataQuorum::Read, true)
|
||||
.await
|
||||
{
|
||||
Ok((file_info, _)) => file_info,
|
||||
@@ -1226,7 +1305,7 @@ impl SetDisks {
|
||||
.acquire_write_lock_diag("abort_data_movement_multipart", RUSTFS_META_MULTIPART_BUCKET, &upload_path)
|
||||
.await?;
|
||||
let file_info = match self
|
||||
.check_multipart_upload_path_exists(bucket, object, upload_id, &upload_path, true, true)
|
||||
.check_multipart_upload_path_exists(bucket, object, upload_id, &upload_path, MultipartMetadataQuorum::Write, true)
|
||||
.await
|
||||
{
|
||||
Ok((file_info, _)) => file_info,
|
||||
@@ -1269,6 +1348,7 @@ impl SetDisks {
|
||||
disks: &[Option<DiskStore>],
|
||||
part_path: &str,
|
||||
read_quorum: usize,
|
||||
strict_size: bool,
|
||||
) -> disk::error::Result<Vec<usize>> {
|
||||
let mut futures = Vec::with_capacity(disks.len());
|
||||
let part_path = part_path.to_string();
|
||||
@@ -1277,21 +1357,24 @@ impl SetDisks {
|
||||
let part_path = part_path.clone();
|
||||
futures.push(async move {
|
||||
if let Some(disk) = disk {
|
||||
disk.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, part_path.as_str(), -1)
|
||||
match disk
|
||||
.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, part_path.as_str(), -1)
|
||||
.await
|
||||
{
|
||||
Err(DiskError::FileNotFound) if strict_size => Ok(Vec::new()),
|
||||
result => result,
|
||||
}
|
||||
} else {
|
||||
Err(DiskError::DiskNotFound)
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
let mut errs = Vec::with_capacity(disks.len());
|
||||
let mut object_parts = Vec::with_capacity(disks.len());
|
||||
|
||||
let (collected_errs, collected_parts) = collect_list_parts_results(futures, read_quorum).await?;
|
||||
errs.extend(collected_errs);
|
||||
object_parts.extend(collected_parts);
|
||||
let (errs, object_parts) = collect_list_parts_results(futures, read_quorum).await?;
|
||||
|
||||
if strict_size {
|
||||
return validate_part_size_observations(&errs, &object_parts, read_quorum);
|
||||
}
|
||||
if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) {
|
||||
return Err(err);
|
||||
}
|
||||
@@ -1333,7 +1416,7 @@ impl SetDisks {
|
||||
data_movement: bool,
|
||||
) -> Result<(FileInfo, Vec<FileInfo>)> {
|
||||
let upload_id_path = Self::validated_multipart_upload_path(bucket, object, upload_id, data_movement)?;
|
||||
self.check_multipart_upload_path_exists(bucket, object, upload_id, &upload_id_path, write, false)
|
||||
self.check_multipart_upload_path_exists(bucket, object, upload_id, &upload_id_path, write.into(), false)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -1373,7 +1456,7 @@ impl SetDisks {
|
||||
object: &str,
|
||||
upload_id: &str,
|
||||
upload_id_path: &str,
|
||||
write: bool,
|
||||
quorum_mode: MultipartMetadataQuorum,
|
||||
include_recovery: bool,
|
||||
) -> Result<(FileInfo, Vec<FileInfo>)> {
|
||||
// Slow metadata I/O must not hold the topology registry's read lock.
|
||||
@@ -1489,22 +1572,26 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
let mut quorum = read_quorum as usize;
|
||||
if write {
|
||||
if quorum_mode != MultipartMetadataQuorum::Read {
|
||||
quorum = write_quorum as usize;
|
||||
|
||||
if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, quorum) {
|
||||
log_multipart_write_quorum_failure(
|
||||
MultipartWriteQuorumContext {
|
||||
stage: MULTIPART_WRITE_QUORUM_UPLOAD_METADATA,
|
||||
bucket,
|
||||
object,
|
||||
upload_id,
|
||||
part_number: None,
|
||||
},
|
||||
&errs,
|
||||
quorum,
|
||||
&err,
|
||||
);
|
||||
if quorum_mode == MultipartMetadataQuorum::Write {
|
||||
log_multipart_write_quorum_failure(
|
||||
MultipartWriteQuorumContext {
|
||||
stage: MULTIPART_WRITE_QUORUM_UPLOAD_METADATA,
|
||||
bucket,
|
||||
object,
|
||||
upload_id,
|
||||
part_number: None,
|
||||
},
|
||||
&errs,
|
||||
quorum,
|
||||
&err,
|
||||
);
|
||||
} else {
|
||||
record_multipart_write_quorum_failure(MULTIPART_WRITE_QUORUM_UPLOAD_METADATA, &errs, quorum);
|
||||
}
|
||||
return Err(map_error(err));
|
||||
}
|
||||
} else if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, quorum) {
|
||||
@@ -1523,7 +1610,7 @@ impl SetDisks {
|
||||
.acquire_write_lock_diag("stale_multipart_cleanup", RUSTFS_META_MULTIPART_BUCKET, upload_path)
|
||||
.await?;
|
||||
let (file_info, _) = self
|
||||
.check_multipart_upload_path_exists("", "", "", upload_path, true, true)
|
||||
.check_multipart_upload_path_exists("", "", "", upload_path, MultipartMetadataQuorum::Cleanup, true)
|
||||
.await?;
|
||||
let write_quorum = file_info.write_quorum(self.default_write_quorum());
|
||||
Ok(StaleMultipartCleanupGuard {
|
||||
@@ -2634,6 +2721,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
ensure_data_movement_upload_access(&fi, bucket, object, upload_id, opts)?;
|
||||
ensure_multipart_bucket_incarnation(&self.ctx, &fi, bucket, object, upload_id, opts.expected_bucket_incarnation_id)
|
||||
.await?;
|
||||
#[cfg(test)]
|
||||
pause_multipart_commit(bucket, object, MultipartCommitPause::ListPartsAfterMetadata).await;
|
||||
|
||||
let upload_id_path = Self::validated_multipart_upload_path(bucket, object, upload_id, opts.data_movement)?;
|
||||
|
||||
@@ -2682,16 +2771,17 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
SLASH_SEPARATOR
|
||||
);
|
||||
|
||||
let mut part_numbers = match Self::list_parts(&online_disks, &part_path, read_quorum).await {
|
||||
Ok(parts) => parts,
|
||||
Err(err) => {
|
||||
if err == DiskError::FileNotFound {
|
||||
return Ok(ret);
|
||||
}
|
||||
let mut part_numbers =
|
||||
match Self::list_parts(&online_disks, &part_path, read_quorum, opts.strict_multipart_part_sizes).await {
|
||||
Ok(parts) => parts,
|
||||
Err(err) => {
|
||||
if err == DiskError::FileNotFound {
|
||||
return Ok(ret);
|
||||
}
|
||||
|
||||
return Err(to_object_err(err.into(), vec![bucket, object]));
|
||||
}
|
||||
};
|
||||
return Err(to_object_err(err.into(), vec![bucket, object]));
|
||||
}
|
||||
};
|
||||
|
||||
if part_numbers.is_empty() {
|
||||
return Ok(ret);
|
||||
@@ -2708,11 +2798,29 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
.map(|v| format!("{part_path}part.{v}.meta"))
|
||||
.collect::<Vec<String>>();
|
||||
|
||||
let object_parts =
|
||||
Self::read_parts(&online_disks, RUSTFS_META_MULTIPART_BUCKET, &part_meta_paths, &part_numbers, read_quorum)
|
||||
.await
|
||||
.map_err(|e| to_object_err(e.into(), vec![bucket, object, upload_id]))?;
|
||||
let object_parts = Self::read_parts(
|
||||
&online_disks,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&part_meta_paths,
|
||||
&part_numbers,
|
||||
if opts.strict_multipart_part_sizes {
|
||||
fi.write_quorum(self.default_write_quorum())
|
||||
} else {
|
||||
read_quorum
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(|e| to_object_err(e.into(), vec![bucket, object, upload_id]))?;
|
||||
|
||||
if opts.strict_multipart_part_sizes
|
||||
&& (object_parts.len() != part_numbers.len()
|
||||
|| object_parts
|
||||
.iter()
|
||||
.zip(&part_numbers)
|
||||
.any(|(part, number)| part.number != *number || part.error.is_some() || part.etag.is_empty()))
|
||||
{
|
||||
return Err(Error::PartMissingOrCorrupt);
|
||||
}
|
||||
let mut count = max_parts;
|
||||
|
||||
for part in object_parts.iter() {
|
||||
@@ -3032,7 +3140,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
// its full write quorum and the upload's incarnation/lifecycle fences.
|
||||
let upload_id_path = Self::validated_multipart_upload_path(bucket, object, upload_id, opts.data_movement)?;
|
||||
let (fi, _) = self
|
||||
.check_multipart_upload_path_exists(bucket, object, upload_id, &upload_id_path, false, true)
|
||||
.check_multipart_upload_path_exists(bucket, object, upload_id, &upload_id_path, MultipartMetadataQuorum::Read, true)
|
||||
.await?;
|
||||
ensure_data_movement_upload_access(&fi, bucket, object, upload_id, opts)?;
|
||||
ensure_multipart_bucket_incarnation(&self.ctx, &fi, bucket, object, upload_id, opts.expected_bucket_incarnation_id)
|
||||
@@ -3192,7 +3300,14 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
}
|
||||
SourceRecovery::Missing => {
|
||||
return match self
|
||||
.check_multipart_upload_path_exists(bucket, object, upload_id, &upload_id_path, false, true)
|
||||
.check_multipart_upload_path_exists(
|
||||
bucket,
|
||||
object,
|
||||
upload_id,
|
||||
&upload_id_path,
|
||||
MultipartMetadataQuorum::Read,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => Err(Error::ErasureWriteQuorum),
|
||||
@@ -3318,7 +3433,14 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
crate::object_api::ENCRYPTED_PART_LAYOUT_CANDIDATE_SUFFIX,
|
||||
);
|
||||
rustfs_utils::http::metadata_compat::remove_str(&mut fi.metadata, crate::object_api::ENCRYPTED_PART_LAYOUT_QUORUM_SUFFIX);
|
||||
if expected_restore_operation_id.is_some() {
|
||||
// Pending copy-back keeps its recovery identity until finalization.
|
||||
if expected_restore_operation_id.is_some()
|
||||
&& !fi
|
||||
.metadata
|
||||
.get(metadata_keys::RESTORE)
|
||||
.and_then(|value| rustfs_filemeta::parse_restore_obj_status(value).ok())
|
||||
.is_some_and(|status| status.is_restore_in_progress == Some(true))
|
||||
{
|
||||
rustfs_utils::http::metadata_compat::remove_str(&mut fi.metadata, SUFFIX_RESTORE_OPERATION_ID);
|
||||
rustfs_utils::http::metadata_compat::remove_str(&mut fi.metadata, SUFFIX_RESTORE_WORKER_LOCK);
|
||||
}
|
||||
@@ -3703,6 +3825,13 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
fi.metadata.remove(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM);
|
||||
fi.metadata.remove(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM_TYPE);
|
||||
strip_internal_multipart_metadata(&mut fi.metadata);
|
||||
if opts.restore_copyback {
|
||||
insert_str(
|
||||
&mut fi.metadata,
|
||||
SUFFIX_RESTORE_COPYBACK,
|
||||
rustfs_utils::http::metadata_compat::RESTORE_COPYBACK_PROTOCOL_V1.to_string(),
|
||||
);
|
||||
}
|
||||
|
||||
// A previous coordinator can stop after a minority publication without
|
||||
// undo. A fresh final directory keeps this attempt separate from old
|
||||
@@ -7492,7 +7621,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn collect_list_parts_results_returns_file_not_found_for_empty_upload_dirs() {
|
||||
async fn collect_list_parts_results_rejects_unavailable_empty_upload_dirs() {
|
||||
let tasks: Vec<_> = vec![
|
||||
(5_u64, Err(DiskError::FileNotFound)),
|
||||
(10, Err(DiskError::DiskNotFound)),
|
||||
@@ -7507,11 +7636,372 @@ mod tests {
|
||||
|
||||
let err = collect_list_parts_results(tasks, 2)
|
||||
.await
|
||||
.expect_err("missing multipart directories should be treated as empty uploads");
|
||||
.expect_err("an unavailable disk cannot prove an empty upload");
|
||||
|
||||
assert_eq!(err, DiskError::ErasureReadQuorum);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn list_parts_rejects_disk_loss_after_upload_metadata_was_read() {
|
||||
let (dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-list-loss-after-metadata";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disks, bucket).await;
|
||||
let opts = ObjectOptions {
|
||||
max_parity: true,
|
||||
..Default::default()
|
||||
};
|
||||
let (upload_id, _) = stage_upload_with_create_opts(&set, bucket, object, &[1; 32], &opts).await;
|
||||
let (fi, _) = set
|
||||
.check_upload_id_exists(bucket, object, &upload_id, false)
|
||||
.await
|
||||
.expect("read initial metadata");
|
||||
let upload_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
let part_dir = dirs[0]
|
||||
.path()
|
||||
.join(RUSTFS_META_MULTIPART_BUCKET)
|
||||
.join(upload_path)
|
||||
.join(fi.data_dir.expect("upload data directory").to_string());
|
||||
let held_dir = part_dir.with_extension("held-for-test");
|
||||
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::ListPartsAfterMetadata);
|
||||
let query_set = Arc::clone(&set);
|
||||
let query_id = upload_id.clone();
|
||||
let query = tokio::spawn(async move {
|
||||
query_set
|
||||
.list_object_parts(bucket, object, &query_id, None, 100, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
tokio::fs::rename(&part_dir, &held_dir)
|
||||
.await
|
||||
.expect("simulate the surviving disk missing the part directory");
|
||||
{
|
||||
let mut live = set.disks.write().await;
|
||||
for slot in 1..4 {
|
||||
live[slot] = None;
|
||||
}
|
||||
}
|
||||
barrier.release();
|
||||
let result = query.await.expect("list task must finish");
|
||||
drop(barrier);
|
||||
tokio::fs::rename(&held_dir, &part_dir)
|
||||
.await
|
||||
.expect("restore the original directory");
|
||||
{
|
||||
let mut live = set.disks.write().await;
|
||||
for slot in 1..4 {
|
||||
live[slot] = Some(disks[slot].clone());
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
matches!(result, Err(StorageError::InsufficientReadQuorum(..))),
|
||||
"disk loss must fail rather than return an empty part list: {result:?}"
|
||||
);
|
||||
let recovered = set
|
||||
.list_object_parts(bucket, object, &upload_id, None, 100, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("listing resumes after the original disks return");
|
||||
assert_eq!(recovered.parts.len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn collect_list_parts_results_proves_all_directories_missing() {
|
||||
let tasks = (0..4)
|
||||
.map(|_| async { Err::<Vec<String>, _>(DiskError::FileNotFound) })
|
||||
.collect();
|
||||
let err = collect_list_parts_results(tasks, 2)
|
||||
.await
|
||||
.expect_err("all missing directories are empty");
|
||||
assert_eq!(err, DiskError::FileNotFound);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strict_multipart_part_sizes_reject_incomplete_observations() {
|
||||
let pair = || vec!["part.1".to_string(), "part.1.meta".to_string()];
|
||||
let all_online = vec![None; 4];
|
||||
assert!(validate_part_size_observations(&all_online, &[vec![], vec![], vec![], vec![]], 2).is_ok());
|
||||
assert_eq!(
|
||||
validate_part_size_observations(&all_online, &[pair(), pair(), vec![], vec![]], 2),
|
||||
Ok(vec![1])
|
||||
);
|
||||
let unordered = vec!["part.9.meta", "part.1", "part.9", "part.1.meta", "part.1"]
|
||||
.into_iter()
|
||||
.map(str::to_owned)
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(
|
||||
validate_part_size_observations(&all_online, &[unordered.clone(), unordered, vec![], vec![]], 2),
|
||||
Ok(vec![1, 9])
|
||||
);
|
||||
assert_eq!(
|
||||
validate_part_size_observations(&all_online, &[pair(), vec![], vec![], vec![]], 2),
|
||||
Err(DiskError::ErasureReadQuorum)
|
||||
);
|
||||
assert_eq!(
|
||||
validate_part_size_observations(&all_online, &[vec!["part.1.meta".to_string()], pair(), pair(), vec![]], 2),
|
||||
Err(DiskError::FileCorrupt)
|
||||
);
|
||||
for invalid in ["part.0", "part.10001", "part.01", "part.+1", "part.1x"] {
|
||||
let entries = vec![invalid.to_string(), format!("{invalid}.meta")];
|
||||
assert_eq!(
|
||||
validate_part_size_observations(&all_online, &[entries.clone(), entries.clone(), entries.clone(), entries], 2),
|
||||
Err(DiskError::FileCorrupt),
|
||||
"noncanonical part names are not valid size evidence even on every disk"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
validate_part_size_observations(
|
||||
&[None, None, None, Some(DiskError::DiskNotFound)],
|
||||
&[pair(), pair(), pair(), vec![]],
|
||||
2
|
||||
),
|
||||
Err(DiskError::ErasureReadQuorum)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn strict_multipart_capped_upload_rejects_unavailable_participant_and_recovers() {
|
||||
let (_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "strict-multipart-size-bucket";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disks, bucket).await;
|
||||
let mut metadata = HashMap::new();
|
||||
insert_str(&mut metadata, SUFFIX_MAX_TOTAL_OBJECT_SIZE, "10".to_string());
|
||||
let opts = ObjectOptions {
|
||||
max_parity: true,
|
||||
user_defined: metadata,
|
||||
..Default::default()
|
||||
};
|
||||
let (upload_id, _) = stage_upload_with_create_opts(&set, bucket, object, &[1; 6], &opts).await;
|
||||
|
||||
set.disks.write().await[3] = None;
|
||||
let mut part = PutObjReader::from_vec(vec![2; 4]);
|
||||
let err = set
|
||||
.put_object_part(bucket, object, &upload_id, 2, &mut part, &ObjectOptions::default())
|
||||
.await;
|
||||
assert!(err.is_err(), "an unavailable disk must not produce a size admission: {err:?}");
|
||||
set.disks.write().await[3] = Some(disks[3].clone());
|
||||
|
||||
// Two distinct part identities at read quorum cannot prove which size
|
||||
// belongs to the acknowledged write; require the write quorum instead.
|
||||
let (other_id, _) = stage_upload_with_create_opts(&set, bucket, "other", &[9; 2], &ObjectOptions::default()).await;
|
||||
let (fi, _) = set
|
||||
.check_upload_id_exists(bucket, object, &upload_id, false)
|
||||
.await
|
||||
.expect("read capped upload metadata");
|
||||
let (other_fi, _) = set
|
||||
.check_upload_id_exists(bucket, "other", &other_id, false)
|
||||
.await
|
||||
.expect("read other upload metadata");
|
||||
let path = format!(
|
||||
"{}/{}/part.1.meta",
|
||||
SetDisks::get_upload_id_dir(bucket, object, &upload_id),
|
||||
fi.data_dir.expect("data dir")
|
||||
);
|
||||
let other_path = format!(
|
||||
"{}/{}/part.1.meta",
|
||||
SetDisks::get_upload_id_dir(bucket, "other", &other_id),
|
||||
other_fi.data_dir.expect("data dir")
|
||||
);
|
||||
let mut originals = Vec::new();
|
||||
for disk in disks.iter().take(2) {
|
||||
originals.push(
|
||||
disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &path)
|
||||
.await
|
||||
.expect("save original part metadata"),
|
||||
);
|
||||
let conflicting = disk
|
||||
.read_all(RUSTFS_META_MULTIPART_BUCKET, &other_path)
|
||||
.await
|
||||
.expect("read conflicting part metadata");
|
||||
disk.write_all(RUSTFS_META_MULTIPART_BUCKET, &path, conflicting)
|
||||
.await
|
||||
.expect("install split part metadata");
|
||||
}
|
||||
let mut split = PutObjReader::from_vec(vec![5; 4]);
|
||||
let split_result = set
|
||||
.put_object_part(bucket, object, &upload_id, 2, &mut split, &ObjectOptions::default())
|
||||
.await;
|
||||
for (disk, original) in disks.iter().zip(originals) {
|
||||
disk.write_all(RUSTFS_META_MULTIPART_BUCKET, &path, original)
|
||||
.await
|
||||
.expect("restore original part metadata");
|
||||
}
|
||||
assert!(
|
||||
split_result.is_err(),
|
||||
"split part identities must not authorize a size decision: {split_result:?}"
|
||||
);
|
||||
|
||||
let mut originals = Vec::new();
|
||||
for disk in disks.iter().take(3) {
|
||||
let original = disk
|
||||
.read_all(RUSTFS_META_MULTIPART_BUCKET, &path)
|
||||
.await
|
||||
.expect("save original part metadata");
|
||||
let mut stale = ObjectPartInfo::unmarshal(&original).expect("decode original part metadata");
|
||||
stale.actual_size = 1;
|
||||
disk.write_all(
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&path,
|
||||
Bytes::from(stale.marshal_msg().expect("encode stale part")),
|
||||
)
|
||||
.await
|
||||
.expect("install stale size on write quorum");
|
||||
originals.push(original);
|
||||
}
|
||||
let mut undercounted = PutObjReader::from_vec(vec![6; 4]);
|
||||
let result = set
|
||||
.put_object_part(bucket, object, &upload_id, 2, &mut undercounted, &ObjectOptions::default())
|
||||
.await;
|
||||
for (disk, original) in disks.iter().zip(originals) {
|
||||
disk.write_all(RUSTFS_META_MULTIPART_BUCKET, &path, original)
|
||||
.await
|
||||
.expect("restore original part metadata");
|
||||
}
|
||||
assert!(
|
||||
matches!(result, Err(StorageError::PartMissingOrCorrupt)),
|
||||
"smaller logical size must fail closed: {result:?}"
|
||||
);
|
||||
|
||||
let mut originals = Vec::new();
|
||||
for disk in disks.iter().take(3) {
|
||||
let original = disk
|
||||
.read_all(RUSTFS_META_MULTIPART_BUCKET, &path)
|
||||
.await
|
||||
.expect("save original part metadata");
|
||||
let mut wrong_identity = ObjectPartInfo::unmarshal(&original).expect("decode original part metadata");
|
||||
wrong_identity.number = 2;
|
||||
disk.write_all(
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&path,
|
||||
Bytes::from(wrong_identity.marshal_msg().expect("encode wrong identity")),
|
||||
)
|
||||
.await
|
||||
.expect("install mismatched part identity on write quorum");
|
||||
originals.push(original);
|
||||
}
|
||||
let list_result = set
|
||||
.list_object_parts(
|
||||
bucket,
|
||||
object,
|
||||
&upload_id,
|
||||
None,
|
||||
100,
|
||||
&ObjectOptions {
|
||||
strict_multipart_part_sizes: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let mut replacement = PutObjReader::from_vec(vec![7; 4]);
|
||||
let result = set
|
||||
.put_object_part(bucket, object, &upload_id, 2, &mut replacement, &ObjectOptions::default())
|
||||
.await;
|
||||
for (disk, original) in disks.iter().zip(originals) {
|
||||
disk.write_all(RUSTFS_META_MULTIPART_BUCKET, &path, original)
|
||||
.await
|
||||
.expect("restore original part metadata");
|
||||
}
|
||||
assert!(
|
||||
matches!(list_result, Err(StorageError::PartMissingOrCorrupt)),
|
||||
"strict listing must reject mismatched part identity: {list_result:?}"
|
||||
);
|
||||
assert!(
|
||||
matches!(result, Err(StorageError::PartMissingOrCorrupt)),
|
||||
"mismatched part identity must not bypass the cap: {result:?}"
|
||||
);
|
||||
|
||||
let mut too_large = PutObjReader::from_vec(vec![3; 5]);
|
||||
let err = set
|
||||
.put_object_part(bucket, object, &upload_id, 2, &mut too_large, &ObjectOptions::default())
|
||||
.await;
|
||||
assert!(
|
||||
matches!(err, Err(StorageError::EntityTooLarge(11, 10))),
|
||||
"existing part must count: {err:?}"
|
||||
);
|
||||
let mut fits = PutObjReader::from_vec(vec![4; 4]);
|
||||
set.put_object_part(bucket, object, &upload_id, 2, &mut fits, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("the upload should resume at the exact limit after all disks recover");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn capped_multipart_concurrent_parts_cannot_exceed_logical_limit() {
|
||||
let (_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "concurrent-capped-multipart";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disks, bucket).await;
|
||||
let mut metadata = HashMap::new();
|
||||
insert_str(&mut metadata, SUFFIX_MAX_TOTAL_OBJECT_SIZE, "10".to_owned());
|
||||
let opts = ObjectOptions {
|
||||
max_parity: true,
|
||||
user_defined: metadata,
|
||||
..Default::default()
|
||||
};
|
||||
let upload = set
|
||||
.new_multipart_upload(bucket, object, &opts)
|
||||
.await
|
||||
.expect("create capped upload");
|
||||
let barrier = Arc::new(tokio::sync::Barrier::new(3));
|
||||
let mut tasks = Vec::new();
|
||||
for (number, byte) in [(1_usize, 0x41_u8), (2, 0x42)] {
|
||||
let set = Arc::clone(&set);
|
||||
let barrier = Arc::clone(&barrier);
|
||||
let upload_id = upload.upload_id.clone();
|
||||
tasks.push(tokio::spawn(async move {
|
||||
let mut body = PutObjReader::from_vec(vec![byte; 6]);
|
||||
barrier.wait().await;
|
||||
set.put_object_part(bucket, object, &upload_id, number, &mut body, &ObjectOptions::default())
|
||||
.await
|
||||
}));
|
||||
}
|
||||
barrier.wait().await;
|
||||
let mut accepted = Vec::new();
|
||||
let mut rejected = 0;
|
||||
for task in tasks {
|
||||
match tokio::time::timeout(Duration::from_secs(30), task)
|
||||
.await
|
||||
.expect("concurrent part deadline")
|
||||
.expect("part task")
|
||||
{
|
||||
Ok(part) => accepted.push(part),
|
||||
Err(StorageError::EntityTooLarge(12, 10)) => rejected += 1,
|
||||
Err(other) => panic!("unexpected concurrent admission result: {other:?}"),
|
||||
}
|
||||
}
|
||||
assert_eq!(accepted.len(), 1, "only one six-byte part fits the ten-byte limit");
|
||||
assert_eq!(rejected, 1);
|
||||
let listed = set
|
||||
.list_object_parts(bucket, object, &upload.upload_id, None, 100, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read final part listing");
|
||||
assert_eq!(listed.parts.len(), 1);
|
||||
assert_eq!(listed.parts[0].part_num, accepted[0].part_num);
|
||||
let winner = accepted.remove(0);
|
||||
let part_number = winner.part_num;
|
||||
set.clone()
|
||||
.complete_multipart_upload(
|
||||
bucket,
|
||||
object,
|
||||
&upload.upload_id,
|
||||
vec![CompletePart {
|
||||
part_num: part_number,
|
||||
etag: winner.etag,
|
||||
..Default::default()
|
||||
}],
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("complete the admitted part");
|
||||
let mut reader = set
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read completed object");
|
||||
let mut actual = Vec::new();
|
||||
reader.stream.read_to_end(&mut actual).await.expect("read whole object");
|
||||
assert_eq!(actual, vec![if part_number == 1 { 0x41 } else { 0x42 }; 6]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn collect_list_parts_results_fails_early_when_file_not_found_fallback_is_impossible() {
|
||||
let started = Instant::now();
|
||||
@@ -8131,9 +8621,7 @@ mod tests {
|
||||
assert!(listed.parts.is_empty(), "part lock loss before rename must not publish the part");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn put_object_part_fences_upload_lock_loss_before_rename() {
|
||||
async fn put_object_part_fences_upload_lock_loss_before_rename_case(capped: bool) {
|
||||
let target = Arc::new(std::sync::RwLock::new(None));
|
||||
let refresh_calls = Arc::new(AtomicUsize::new(0));
|
||||
let lockers: Vec<Arc<dyn LockClient>> = (0..4)
|
||||
@@ -8145,10 +8633,12 @@ mod tests {
|
||||
let bucket = "multipart-put-part-lock-loss-bucket";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
let mut create_opts = ObjectOptions::default();
|
||||
if capped {
|
||||
insert_str(&mut create_opts.user_defined, SUFFIX_MAX_TOTAL_OBJECT_SIZE, "8192".to_owned());
|
||||
}
|
||||
let upload = temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("false"))], async {
|
||||
set_disks
|
||||
.new_multipart_upload(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
set_disks.new_multipart_upload(bucket, object, &create_opts).await
|
||||
})
|
||||
.await
|
||||
.expect("multipart upload should be created");
|
||||
@@ -8199,6 +8689,18 @@ mod tests {
|
||||
assert!(listed.parts.is_empty(), "lock loss before rename must not publish the part");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn put_object_part_fences_upload_lock_loss_before_rename() {
|
||||
put_object_part_fences_upload_lock_loss_before_rename_case(false).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn capped_multipart_part_fences_upload_lock_loss_before_rename() {
|
||||
put_object_part_fences_upload_lock_loss_before_rename_case(true).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn complete_encrypted_multipart_marks_quorum_validated_layout() {
|
||||
temp_env::async_with_vars([(crate::object_api::ENV_RUSTFS_ENCRYPTED_RANGE_SEEK, Some("true"))], async {
|
||||
@@ -11133,7 +11635,7 @@ mod tests {
|
||||
*disk = None;
|
||||
}
|
||||
let (info, _) = set
|
||||
.check_multipart_upload_path_exists(bucket, object, &upload, &path, false, true)
|
||||
.check_multipart_upload_path_exists(bucket, object, &upload, &path, MultipartMetadataQuorum::Read, true)
|
||||
.await
|
||||
.expect("four primary plus four recovery copies reach metadata read quorum");
|
||||
assert_eq!(info.name, path);
|
||||
@@ -11154,7 +11656,7 @@ mod tests {
|
||||
*disk = None;
|
||||
}
|
||||
assert!(
|
||||
set.check_multipart_upload_path_exists(bucket, object, &upload, &path, false, true)
|
||||
set.check_multipart_upload_path_exists(bucket, object, &upload, &path, MultipartMetadataQuorum::Read, true)
|
||||
.await
|
||||
.is_err(),
|
||||
"two files on each of four disks must not count as eight voters"
|
||||
@@ -14471,7 +14973,7 @@ mod tests {
|
||||
object,
|
||||
&u_new,
|
||||
&SetDisks::get_upload_id_dir(bucket, object, &u_new),
|
||||
true,
|
||||
MultipartMetadataQuorum::Write,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
@@ -14516,7 +15018,7 @@ mod tests {
|
||||
object,
|
||||
&upload,
|
||||
&SetDisks::get_upload_id_dir(bucket, object, &upload),
|
||||
true,
|
||||
MultipartMetadataQuorum::Write,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
@@ -14599,7 +15101,7 @@ mod tests {
|
||||
assert_completion_files_preserved(&disk_stores, RUSTFS_META_MULTIPART_BUCKET, &source_files).await;
|
||||
set_disks
|
||||
.check_multipart_upload_path_exists(
|
||||
bucket, object, &u_new, &SetDisks::get_upload_id_dir(bucket, object, &u_new), true, true,
|
||||
bucket, object, &u_new, &SetDisks::get_upload_id_dir(bucket, object, &u_new), MultipartMetadataQuorum::Write, true,
|
||||
)
|
||||
.await
|
||||
.expect("upload authority must survive the post-commit crash");
|
||||
|
||||
@@ -363,6 +363,202 @@ mod native_abort_closure_tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stale_cleanup_reclaims_certified_one_and_two_disk_residuals_after_reopen() {
|
||||
for survivors in [1, 2] {
|
||||
let (dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "native-stale-certified";
|
||||
let object = format!("survivors-{survivors}");
|
||||
make_bucket_on_all(&disks, bucket).await;
|
||||
let opts = ObjectOptions {
|
||||
max_parity: true,
|
||||
..Default::default()
|
||||
};
|
||||
let (upload, _) = stage_upload_with_create_opts(&set, bucket, &object, &[0x61; 4096], &opts).await;
|
||||
let path = SetDisks::get_upload_id_dir(bucket, &object, &upload);
|
||||
let before = originals(&disks, &path).await;
|
||||
{
|
||||
let physical = disks.iter().cloned().map(Some).collect::<Vec<_>>();
|
||||
let SourceCleanupAuthority::Native { certificate, snapshots } =
|
||||
prepare_source_cleanup(&physical, bucket, &object, &path, true, &set.multipart_proof_verification_context())
|
||||
.await
|
||||
.expect("close every original and persist the certificate")
|
||||
else {
|
||||
panic!("native upload must retain a native certificate")
|
||||
};
|
||||
for slot in 0..(4 - survivors) {
|
||||
let response = operation(
|
||||
&disks[slot],
|
||||
&path,
|
||||
MultipartSourceOperation::DeleteAborted {
|
||||
expected: snapshots[slot].clone().expect("actual certified source"),
|
||||
certificate: certificate.as_ref().clone(),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(response, MultipartSourceResponse::AbortDeleted(_)));
|
||||
}
|
||||
}
|
||||
let (reopened, cold) = cold_reopen(disks, set).await;
|
||||
for slot in (4 - survivors)..4 {
|
||||
assert_eq!(
|
||||
retained_native_multipart_original(&reopened[slot], slot, bucket, &object, &path).await,
|
||||
before[slot]
|
||||
);
|
||||
}
|
||||
cold.lock_stale_multipart_cleanup(&path)
|
||||
.await
|
||||
.expect("complete certificate authorizes the background retry")
|
||||
.delete(&cold)
|
||||
.await
|
||||
.expect("background retry removes certified survivors");
|
||||
for dir in &dirs {
|
||||
assert!(!dir.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&path).exists());
|
||||
}
|
||||
assert!(
|
||||
cold.lock_stale_multipart_cleanup(&path).await.is_err(),
|
||||
"a repeated pass must not recreate the retired upload"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn stale_certified_residual_survives_lock_loss_then_converges() {
|
||||
let target = Arc::new(std::sync::RwLock::new(None));
|
||||
let refresh_calls = Arc::new(AtomicUsize::new(0));
|
||||
let lockers: Vec<Arc<dyn LockClient>> = (0..4)
|
||||
.map(|_| {
|
||||
Arc::new(SelectiveLockLossClient::new(Arc::clone(&target), Arc::clone(&refresh_calls))) as Arc<dyn LockClient>
|
||||
})
|
||||
.collect();
|
||||
let (dirs, disks, set) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
|
||||
let bucket = "stale-certified-lock-loss";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disks, bucket).await;
|
||||
let opts = ObjectOptions {
|
||||
max_parity: true,
|
||||
..Default::default()
|
||||
};
|
||||
let (upload, _) = stage_upload_with_create_opts(&set, bucket, object, &[0x63; 4096], &opts).await;
|
||||
let path = SetDisks::get_upload_id_dir(bucket, object, &upload);
|
||||
let (fi, _) = set
|
||||
.check_upload_id_exists(bucket, object, &upload, false)
|
||||
.await
|
||||
.expect("source metadata before closure");
|
||||
let part_path = format!("{path}/{}/part.1", fi.data_dir.expect("source data directory"));
|
||||
let original = originals(&disks, &path).await;
|
||||
let surviving_part = disks[3]
|
||||
.read_all(RUSTFS_META_MULTIPART_BUCKET, &part_path)
|
||||
.await
|
||||
.expect("snapshot surviving shard");
|
||||
{
|
||||
let physical = disks.iter().cloned().map(Some).collect::<Vec<_>>();
|
||||
let SourceCleanupAuthority::Native { certificate, snapshots } =
|
||||
prepare_source_cleanup(&physical, bucket, object, &path, true, &set.multipart_proof_verification_context())
|
||||
.await
|
||||
.expect("all original participants close and persist the certificate")
|
||||
else {
|
||||
panic!("native certificate required")
|
||||
};
|
||||
let authority = set.multipart_proof_verification_context();
|
||||
let mut ids = certificate.disk_ids.iter().copied().map(Some).collect::<Vec<_>>();
|
||||
ids[0] = None;
|
||||
assert_eq!(
|
||||
crate::set_disk::core::multipart_source::abort_certificate_cohort(&certificate, &ids, &authority),
|
||||
Err(DiskError::ErasureWriteQuorum),
|
||||
"an unavailable participant is a retryable proof gap"
|
||||
);
|
||||
ids[1] = Some(Uuid::new_v4());
|
||||
assert_eq!(
|
||||
crate::set_disk::core::multipart_source::abort_certificate_cohort(&certificate, &ids, &authority),
|
||||
Err(DiskError::FileCorrupt),
|
||||
"a missing participant cannot hide a known identity conflict"
|
||||
);
|
||||
ids[0] = Some(Uuid::new_v4());
|
||||
ids[1] = Some(certificate.disk_ids[1]);
|
||||
assert_eq!(
|
||||
crate::set_disk::core::multipart_source::abort_certificate_cohort(&certificate, &ids, &authority),
|
||||
Err(DiskError::FileCorrupt),
|
||||
"a known different identity remains a certificate conflict"
|
||||
);
|
||||
for slot in 0..3 {
|
||||
let response = operation(
|
||||
&disks[slot],
|
||||
&path,
|
||||
MultipartSourceOperation::DeleteAborted {
|
||||
expected: snapshots[slot].clone().expect("certified source"),
|
||||
certificate: certificate.as_ref().clone(),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(response, MultipartSourceResponse::AbortDeleted(_)));
|
||||
}
|
||||
}
|
||||
assert!(dirs[3].path().join(RUSTFS_META_MULTIPART_BUCKET).join(&path).exists());
|
||||
let offline = set.disks.write().await[0]
|
||||
.take()
|
||||
.expect("take an original participant offline");
|
||||
let rejected = set.lock_stale_multipart_cleanup(&path).await;
|
||||
set.disks.write().await[0] = Some(offline);
|
||||
let err = rejected
|
||||
.err()
|
||||
.expect("a certificate must not substitute for an unavailable participant");
|
||||
assert!(
|
||||
matches!(err, StorageError::ErasureWriteQuorum | StorageError::InsufficientWriteQuorum(..)),
|
||||
"the missing participant must be reported as a quorum failure: {err:?}"
|
||||
);
|
||||
assert_eq!(retained_native_multipart_original(&disks[3], 3, bucket, object, &path).await, original[3]);
|
||||
assert_eq!(
|
||||
disks[3]
|
||||
.read_all(RUSTFS_META_MULTIPART_BUCKET, &part_path)
|
||||
.await
|
||||
.expect("surviving shard while a participant was offline"),
|
||||
surviving_part
|
||||
);
|
||||
tokio::time::pause();
|
||||
*target.write().expect("lock-loss target") =
|
||||
Some(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, path.clone()));
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let guard = set
|
||||
.lock_stale_multipart_cleanup(&path)
|
||||
.await
|
||||
.expect("certificate admits the one-disk residual");
|
||||
tokio::time::advance(Duration::from_secs(11)).await;
|
||||
for _ in 0..20 {
|
||||
if guard.is_lock_lost() {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
assert!(refresh_calls.load(Ordering::Acquire) > 0, "the real lock refresh must fail");
|
||||
assert!(guard.is_lock_lost(), "the cleanup guard must observe lock loss");
|
||||
// Real disk I/O must not run under Tokio's auto-advancing paused clock:
|
||||
// a pending blocking operation could otherwise exhaust its timeout.
|
||||
tokio::time::resume();
|
||||
let err = guard.delete(&set).await.expect_err("lock loss must fence physical deletion");
|
||||
assert!(matches!(err, StorageError::NamespaceLockQuorumUnavailable { .. }), "{err:?}");
|
||||
assert_eq!(retained_native_multipart_original(&disks[3], 3, bucket, object, &path).await, original[3]);
|
||||
assert_eq!(
|
||||
disks[3]
|
||||
.read_all(RUSTFS_META_MULTIPART_BUCKET, &part_path)
|
||||
.await
|
||||
.expect("surviving shard after loss"),
|
||||
surviving_part
|
||||
);
|
||||
|
||||
*target.write().expect("restore lock service") = None;
|
||||
set.lock_stale_multipart_cleanup(&path)
|
||||
.await
|
||||
.expect("reacquire the cleanup lock")
|
||||
.delete(&set)
|
||||
.await
|
||||
.expect("the certified residual converges after lock recovery");
|
||||
for dir in &dirs {
|
||||
assert!(!dir.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&path).exists());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn native_abort_partial_delete_surviving_producer_confirms_current_and_historical_exact_target() {
|
||||
use crate::set_disk::core::io_primitives::{rename_fault_injection, rollback_fault_injection};
|
||||
@@ -507,7 +703,7 @@ mod native_abort_closure_tests {
|
||||
let path = SetDisks::get_upload_id_dir(bucket, object, &upload);
|
||||
pin_historical_unmarked_upload(&disks, bucket, &path).await;
|
||||
let before = originals(&disks, &path).await;
|
||||
set.check_multipart_upload_path_exists(bucket, object, &upload, &path, false, true)
|
||||
set.check_multipart_upload_path_exists(bucket, object, &upload, &path, MultipartMetadataQuorum::Read, true)
|
||||
.await
|
||||
.expect("readonly legacy cleanup identity lookup");
|
||||
assert_eq!(
|
||||
@@ -786,7 +982,7 @@ mod native_abort_closure_tests {
|
||||
"known orphan native evidence remains unknown: {inspected:?}"
|
||||
);
|
||||
let readonly = set
|
||||
.check_multipart_upload_path_exists(bucket, object, &upload, &path, false, true)
|
||||
.check_multipart_upload_path_exists(bucket, object, &upload, &path, MultipartMetadataQuorum::Read, true)
|
||||
.await;
|
||||
assert!(
|
||||
matches!(
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -27,6 +27,7 @@ use std::sync::Arc;
|
||||
struct RestoreFinalizeBarrierState {
|
||||
bucket: String,
|
||||
object: String,
|
||||
before_lock: bool,
|
||||
arrived: tokio::sync::Notify,
|
||||
release: tokio::sync::Notify,
|
||||
}
|
||||
@@ -43,9 +44,18 @@ pub(in crate::set_disk) struct RestoreFinalizeBarrier {
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
impl RestoreFinalizeBarrier {
|
||||
pub(in crate::set_disk) fn install(bucket: &str, object: &str) -> Self {
|
||||
Self::install_at(bucket, object, false)
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn install_before_lock(bucket: &str, object: &str) -> Self {
|
||||
Self::install_at(bucket, object, true)
|
||||
}
|
||||
|
||||
fn install_at(bucket: &str, object: &str, before_lock: bool) -> Self {
|
||||
let state = Arc::new(RestoreFinalizeBarrierState {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
before_lock,
|
||||
arrived: tokio::sync::Notify::new(),
|
||||
release: tokio::sync::Notify::new(),
|
||||
});
|
||||
@@ -81,13 +91,13 @@ impl Drop for RestoreFinalizeBarrier {
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
async fn maybe_pause_restore_finalize(bucket: &str, object: &str) {
|
||||
async fn maybe_pause_restore_finalize(bucket: &str, object: &str, before_lock: bool) {
|
||||
let barrier = RESTORE_FINALIZE_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("restore finalize barrier mutex should not poison")
|
||||
.as_ref()
|
||||
.filter(|barrier| barrier.bucket == bucket && barrier.object == object)
|
||||
.filter(|barrier| barrier.bucket == bucket && barrier.object == object && barrier.before_lock == before_lock)
|
||||
.cloned();
|
||||
if let Some(barrier) = barrier {
|
||||
barrier.arrived.notify_one();
|
||||
@@ -150,6 +160,8 @@ impl SetDisks {
|
||||
obj_info: &ObjectInfo,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<ObjectInfo> {
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
maybe_pause_restore_finalize(bucket, object, true).await;
|
||||
let expected = RestoreCleanupIdentity::from_object_info(obj_info);
|
||||
let expected_operation_id = restore_operation_id_from_metadata(&opts.user_defined)?;
|
||||
let expected_etag = obj_info
|
||||
@@ -208,7 +220,7 @@ impl SetDisks {
|
||||
return Err(Error::other("restored object changed before restore metadata finalization"));
|
||||
}
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
maybe_pause_restore_finalize(bucket, object).await;
|
||||
maybe_pause_restore_finalize(bucket, object, false).await;
|
||||
let restore_expiry =
|
||||
lifecycle::expected_expiry_time(OffsetDateTime::now_utc(), opts.transition.restore_request.days.unwrap_or(1));
|
||||
fi.metadata.insert(
|
||||
@@ -219,7 +231,14 @@ impl SetDisks {
|
||||
}
|
||||
.to_string(),
|
||||
);
|
||||
fi.transition_status = lifecycle::TRANSITION_COMPLETE.to_string();
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut fi.metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_STATUS,
|
||||
lifecycle::TRANSITION_COMPLETE.to_string(),
|
||||
);
|
||||
for suffix in [
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_COPYBACK,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_OPERATION_ID,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_WORKER_LOCK,
|
||||
] {
|
||||
@@ -315,10 +334,20 @@ impl SetDisks {
|
||||
if !expected.matches_file_info(&fi, &expected_etag) {
|
||||
return Ok(());
|
||||
}
|
||||
let copyback_data_dir = rustfs_utils::http::metadata_compat::get_consistent_str(
|
||||
&fi.metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_COPYBACK,
|
||||
)
|
||||
.filter(|value| *value == rustfs_utils::http::metadata_compat::RESTORE_COPYBACK_PROTOCOL_V1)
|
||||
.and_then(|_| fi.data_dir.filter(|data_dir| !data_dir.is_nil()));
|
||||
ensure_restore_metadata_lock_held(bucket, object, opts, "restore_cleanup_metadata")?;
|
||||
fi.metadata.remove(metadata_keys::RESTORE);
|
||||
fi.metadata.remove(metadata_keys::RESTORE_EXPIRY_DAYS);
|
||||
fi.metadata.remove(metadata_keys::RESTORE_REQUEST_DATE);
|
||||
rustfs_utils::http::metadata_compat::remove_str(
|
||||
&mut fi.metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_COPYBACK,
|
||||
);
|
||||
rustfs_utils::http::metadata_compat::remove_str(
|
||||
&mut fi.metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_OPERATION_ID,
|
||||
@@ -348,6 +377,20 @@ impl SetDisks {
|
||||
)
|
||||
.await?;
|
||||
self.invalidate_get_object_metadata_cache(bucket, object).await;
|
||||
if let Some(data_dir) = copyback_data_dir {
|
||||
let cleanup = self
|
||||
.commit_rename_data_dir_and_mark_capacity(
|
||||
&disks,
|
||||
bucket,
|
||||
object,
|
||||
&data_dir.to_string(),
|
||||
"",
|
||||
self.default_write_quorum(),
|
||||
)
|
||||
.await;
|
||||
self.report_old_data_dir_cleanup(bucket, object, &data_dir.to_string(), &cleanup)
|
||||
.await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -274,3 +274,452 @@ async fn prepared_snapshot_transition_duplicate_and_late_get_use_committed_remot
|
||||
"late GET must route to the remote generation instead of reopening the released local source"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn single_part_restore_copyback_stays_pending_before_finalization() {
|
||||
restore_copyback_stays_pending_before_finalization(false).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn multipart_restore_copyback_stays_pending_before_finalization() {
|
||||
restore_copyback_stays_pending_before_finalization(true).await;
|
||||
}
|
||||
|
||||
async fn restore_copyback_stays_pending_before_finalization(multipart: bool) {
|
||||
use crate::bucket::lifecycle::bucket_lifecycle_ops::LifecycleOps as _;
|
||||
use crate::set_disk::replication::RestoreFinalizeBarrier;
|
||||
use crate::storage_api_contracts::multipart::{CompletePart, MultipartOperations as _};
|
||||
use rustfs_gateway_types::persistence::PersistedLifecycleConfiguration;
|
||||
use rustfs_lifecycle::Lifecycle as _;
|
||||
|
||||
let (dirs, set_disks) = make_local_set_disks(4, 2).await;
|
||||
let bucket = "restore-before-finalization-bucket";
|
||||
let object = "object.bin";
|
||||
let payload = vec![0x5au8; 6 * 1024 * 1024];
|
||||
set_disks
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let write_opts = ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
};
|
||||
let original = if multipart {
|
||||
let upload = set_disks
|
||||
.new_multipart_upload(bucket, object, &write_opts)
|
||||
.await
|
||||
.expect("source multipart upload should be created");
|
||||
let mut parts = Vec::new();
|
||||
for (index, bytes) in payload.chunks(5 * 1024 * 1024).enumerate() {
|
||||
let mut reader = PutObjReader::from_vec(bytes.to_vec());
|
||||
let part = set_disks
|
||||
.put_object_part(bucket, object, &upload.upload_id, index + 1, &mut reader, &write_opts)
|
||||
.await
|
||||
.expect("source part should be written");
|
||||
parts.push(CompletePart {
|
||||
part_num: part.part_num,
|
||||
etag: part.etag,
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
set_disks
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, &upload.upload_id, parts, &write_opts)
|
||||
.await
|
||||
.expect("source multipart upload should complete")
|
||||
} else {
|
||||
let mut reader = PutObjReader::from_vec(payload.clone());
|
||||
set_disks
|
||||
.put_object(bucket, object, &mut reader, &write_opts)
|
||||
.await
|
||||
.expect("source object should be written")
|
||||
};
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&set_disks.instance_ctx().tier_config_mgr(), &tier_name).await;
|
||||
set_disks
|
||||
.transition_object(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
transition: TransitionOptions {
|
||||
status: TRANSITION_PENDING.to_string(),
|
||||
tier: tier_name.clone(),
|
||||
etag: original.etag.clone().expect("source ETag should be present"),
|
||||
..Default::default()
|
||||
},
|
||||
version_id: original.version_id.map(|version| version.to_string()),
|
||||
mod_time: original.mod_time,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("source should transition before restore");
|
||||
|
||||
let operation_id = Uuid::new_v4().to_string();
|
||||
let mut operation_metadata = std::collections::HashMap::new();
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut operation_metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_OPERATION_ID,
|
||||
operation_id.clone(),
|
||||
);
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut operation_metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_WORKER_LOCK,
|
||||
rustfs_utils::http::metadata_compat::RESTORE_WORKER_LOCK_PROTOCOL_V1.to_string(),
|
||||
);
|
||||
operation_metadata.insert(
|
||||
rustfs_filemeta::metadata_keys::RESTORE.to_string(),
|
||||
"ongoing-request=\"true\"".to_string(),
|
||||
);
|
||||
set_disks
|
||||
.put_object_metadata(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
eval_metadata: Some(operation_metadata.clone()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("the restore operation generation should be installed");
|
||||
let barrier = RestoreFinalizeBarrier::install_before_lock(bucket, object);
|
||||
let restore_set = Arc::clone(&set_disks);
|
||||
let restore = tokio::spawn(async move {
|
||||
let mut opts = ObjectOptions {
|
||||
user_defined: operation_metadata,
|
||||
..Default::default()
|
||||
};
|
||||
opts.transition.restore_request.days = Some(1);
|
||||
restore_set.restore_transitioned_object(bucket, object, &opts).await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("copy-back must reach the gap before finalization acquires its lock");
|
||||
let pending = set_disks
|
||||
.get_object_info(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("HEAD must remain available before finalization");
|
||||
let pending_data_dir = pending
|
||||
.data_dir
|
||||
.expect("pending copy-back must retain its local data directory");
|
||||
for dir in &dirs {
|
||||
assert!(
|
||||
dir.path()
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(pending_data_dir.to_string())
|
||||
.exists(),
|
||||
"pending copy-back must retain its committed local data directory"
|
||||
);
|
||||
}
|
||||
let status = parse_restore_obj_status(
|
||||
pending
|
||||
.user_defined
|
||||
.get(rustfs_filemeta::metadata_keys::RESTORE)
|
||||
.expect("copy-back must retain restore metadata"),
|
||||
)
|
||||
.expect("pending restore status should parse");
|
||||
assert_eq!(
|
||||
pending.transitioned_object.status, TRANSITION_COMPLETE,
|
||||
"copy-back must retain the remote identity needed by recovery and cleanup"
|
||||
);
|
||||
backend.set_unreachable(true).await;
|
||||
let mut restored_before_finalization = Vec::new();
|
||||
set_disks
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("pending copy-back must be readable locally before metadata finalization")
|
||||
.stream
|
||||
.read_to_end(&mut restored_before_finalization)
|
||||
.await
|
||||
.expect("pending local body should drain");
|
||||
assert_eq!(restored_before_finalization, payload, "pending local copy must preserve every byte");
|
||||
set_disks
|
||||
.transition_object(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
transition: TransitionOptions {
|
||||
status: TRANSITION_PENDING.to_string(),
|
||||
tier: pending.transitioned_object.tier.clone(),
|
||||
etag: pending.etag.clone().expect("pending ETag should be present"),
|
||||
..Default::default()
|
||||
},
|
||||
version_id: pending.version_id.map(|version| version.to_string()),
|
||||
mod_time: pending.mod_time,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("a queued transition must skip the pending restore copy-back");
|
||||
let after_queued_transition = set_disks
|
||||
.get_object_info(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("restore metadata should remain readable after a queued transition");
|
||||
assert_eq!(
|
||||
after_queued_transition.transitioned_object.status, TRANSITION_COMPLETE,
|
||||
"queued transition must not remove local copy-back data before finalization"
|
||||
);
|
||||
// Evaluate beyond any pre-copy deadline without sleeping or advancing the
|
||||
// process clock. Pending copy-back must never become a restore-expiry candidate.
|
||||
let lifecycle = PersistedLifecycleConfiguration::default();
|
||||
let expiry_evaluation_time = OffsetDateTime::now_utc() + time::Duration::days(2);
|
||||
let action = lifecycle
|
||||
.eval_inner(&pending.to_lifecycle_opts(), expiry_evaluation_time, 0)
|
||||
.await
|
||||
.action;
|
||||
|
||||
barrier.release();
|
||||
tokio::time::timeout(Duration::from_secs(30), restore)
|
||||
.await
|
||||
.expect("released restore should complete promptly")
|
||||
.expect("restore task should join")
|
||||
.expect("copy-back should finalize successfully");
|
||||
|
||||
assert!(status.on_going(), "copy-back must not advertise completion before metadata finalization");
|
||||
assert!(status.expiry().is_none(), "pending copy-back must not expose a completed expiry");
|
||||
assert!(
|
||||
!action.delete_restored(),
|
||||
"scanner must not reclaim copy-back before metadata finalization"
|
||||
);
|
||||
|
||||
for (suffix, expected) in [
|
||||
(rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_OPERATION_ID, operation_id.as_str()),
|
||||
(
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_WORKER_LOCK,
|
||||
rustfs_utils::http::metadata_compat::RESTORE_WORKER_LOCK_PROTOCOL_V1,
|
||||
),
|
||||
] {
|
||||
for prefix in [
|
||||
rustfs_utils::http::RUSTFS_INTERNAL_PREFIX,
|
||||
rustfs_utils::http::MINIO_INTERNAL_PREFIX,
|
||||
] {
|
||||
assert_eq!(
|
||||
pending.user_defined.get(&format!("{prefix}{suffix}")).map(String::as_str),
|
||||
Some(expected),
|
||||
"pending copy-back must retain each recovery marker alias"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let completed = set_disks
|
||||
.get_object_info(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("finalized metadata should be readable");
|
||||
assert_eq!(
|
||||
completed.data_dir,
|
||||
Some(pending_data_dir),
|
||||
"finalization must retain the local copy-back data directory"
|
||||
);
|
||||
assert_eq!(
|
||||
completed.transitioned_object.status, TRANSITION_COMPLETE,
|
||||
"finalization must restore the completed transition state"
|
||||
);
|
||||
let completed_status = parse_restore_obj_status(
|
||||
completed
|
||||
.user_defined
|
||||
.get(rustfs_filemeta::metadata_keys::RESTORE)
|
||||
.expect("finalized restore status should be present"),
|
||||
)
|
||||
.expect("finalized restore status should parse");
|
||||
assert!(!completed_status.on_going(), "finalization must publish completion");
|
||||
assert!(completed_status.expiry().is_some(), "finalization must publish the completed expiry");
|
||||
for suffix in [
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_OPERATION_ID,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_WORKER_LOCK,
|
||||
] {
|
||||
for prefix in [
|
||||
rustfs_utils::http::RUSTFS_INTERNAL_PREFIX,
|
||||
rustfs_utils::http::MINIO_INTERNAL_PREFIX,
|
||||
] {
|
||||
assert!(
|
||||
!completed.user_defined.contains_key(&format!("{prefix}{suffix}")),
|
||||
"finalization must remove every recovery marker alias"
|
||||
);
|
||||
}
|
||||
}
|
||||
let completed_action = lifecycle
|
||||
.eval_inner(&completed.to_lifecycle_opts(), expiry_evaluation_time, 0)
|
||||
.await
|
||||
.action;
|
||||
assert!(
|
||||
completed_action.delete_restored(),
|
||||
"the same lifecycle evaluator must expire a completed restore beyond its deadline"
|
||||
);
|
||||
backend.set_unreachable(true).await;
|
||||
let mut restored = Vec::new();
|
||||
set_disks
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("completed restore must be readable with the remote tier unavailable")
|
||||
.stream
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("local restored body should drain");
|
||||
assert_eq!(restored, payload, "finalized local copy must preserve every byte");
|
||||
for dir in &dirs {
|
||||
assert!(
|
||||
dir.path()
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(pending_data_dir.to_string())
|
||||
.exists(),
|
||||
"completed restore must retain its local data directory"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn restore_copyback_overwrite_preserves_remote_cleanup_owner() {
|
||||
assert_transition_overwrite_preserves_remote_cleanup_owner(true).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn completed_transition_overwrite_preserves_remote_cleanup_owner() {
|
||||
assert_transition_overwrite_preserves_remote_cleanup_owner(false).await;
|
||||
}
|
||||
|
||||
async fn assert_transition_overwrite_preserves_remote_cleanup_owner(during_copyback: bool) {
|
||||
use crate::set_disk::replication::RestoreFinalizeBarrier;
|
||||
|
||||
let (dirs, set_disks) = make_local_set_disks(4, 2).await;
|
||||
let bucket = "restore-copyback-cleanup-owner";
|
||||
let object = "object.bin";
|
||||
let payload = vec![0x37u8; 6 * 1024 * 1024];
|
||||
let write_opts = ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
};
|
||||
set_disks
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("cleanup probe bucket should be created");
|
||||
let mut reader = PutObjReader::from_vec(payload);
|
||||
let original = set_disks
|
||||
.put_object(bucket, object, &mut reader, &write_opts)
|
||||
.await
|
||||
.expect("cleanup probe source should be written");
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&set_disks.instance_ctx().tier_config_mgr(), &tier_name).await;
|
||||
set_disks
|
||||
.transition_object(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
transition: TransitionOptions {
|
||||
status: TRANSITION_PENDING.to_string(),
|
||||
tier: tier_name.clone(),
|
||||
etag: original.etag.clone().expect("source ETag should be present"),
|
||||
..Default::default()
|
||||
},
|
||||
mod_time: original.mod_time,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("cleanup probe source should transition");
|
||||
let source = set_disks
|
||||
.get_object_info(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("transitioned cleanup source should resolve");
|
||||
assert_eq!(backend.object_count().await, 1, "transition must retain the real remote cleanup source");
|
||||
let mut barrier = None;
|
||||
let mut restore = None;
|
||||
if during_copyback {
|
||||
let operation_id = Uuid::new_v4();
|
||||
let mut metadata = std::collections::HashMap::new();
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_OPERATION_ID,
|
||||
operation_id.to_string(),
|
||||
);
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_WORKER_LOCK,
|
||||
rustfs_utils::http::metadata_compat::RESTORE_WORKER_LOCK_PROTOCOL_V1.to_string(),
|
||||
);
|
||||
metadata.insert(
|
||||
rustfs_filemeta::metadata_keys::RESTORE.to_string(),
|
||||
"ongoing-request=\"true\"".to_string(),
|
||||
);
|
||||
set_disks
|
||||
.put_object_metadata(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
eval_metadata: Some(metadata.clone()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("cleanup probe restore generation should be installed");
|
||||
let installed = RestoreFinalizeBarrier::install_before_lock(bucket, object);
|
||||
let restore_set = Arc::clone(&set_disks);
|
||||
restore = Some(tokio::spawn(async move {
|
||||
let mut opts = ObjectOptions {
|
||||
user_defined: metadata,
|
||||
..Default::default()
|
||||
};
|
||||
opts.transition.restore_request.days = Some(1);
|
||||
restore_set.restore_transitioned_object(bucket, object, &opts).await
|
||||
}));
|
||||
tokio::time::timeout(Duration::from_secs(30), installed.wait_until_paused())
|
||||
.await
|
||||
.expect("real restore should publish local bytes before finalization");
|
||||
let pending = set_disks
|
||||
.get_object_info(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("copy-back metadata should resolve");
|
||||
assert!(pending.restore_ongoing, "probe must hit an unfinished real restore");
|
||||
barrier = Some(installed);
|
||||
}
|
||||
let mut replacement = PutObjReader::from_vec(vec![0x69u8; 6 * 1024 * 1024]);
|
||||
set_disks
|
||||
.put_object(bucket, object, &mut replacement, &write_opts)
|
||||
.await
|
||||
.expect("ordinary overwrite must succeed while restore finalization is paused");
|
||||
if let Some(barrier) = barrier {
|
||||
barrier.release();
|
||||
}
|
||||
if let Some(restore) = restore {
|
||||
tokio::time::timeout(Duration::from_secs(30), restore)
|
||||
.await
|
||||
.expect("superseded restore must finish")
|
||||
.expect("superseded restore must not panic")
|
||||
.expect_err("restore finalization must not revive the overwritten generation");
|
||||
}
|
||||
let mut cleanup_owners = Vec::new();
|
||||
for dir in &dirs {
|
||||
let encoded = match tokio::fs::read(dir.path().join(bucket).join(object).join(crate::disk::STORAGE_FORMAT_FILE)).await {
|
||||
Ok(encoded) => encoded,
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
|
||||
cleanup_owners.push(0);
|
||||
continue;
|
||||
}
|
||||
Err(err) => panic!("cleanup source metadata should be readable: {err}"),
|
||||
};
|
||||
let versions = rustfs_filemeta::FileMeta::load(&encoded)
|
||||
.expect("cleanup source metadata should decode")
|
||||
.get_file_info_versions(bucket, object, false)
|
||||
.expect("cleanup source versions should decode");
|
||||
let count = versions
|
||||
.free_versions
|
||||
.iter()
|
||||
.filter(|version| {
|
||||
version.transition_tier == source.transitioned_object.tier
|
||||
&& version.transitioned_objname == source.transitioned_object.name
|
||||
})
|
||||
.count();
|
||||
cleanup_owners.push(count);
|
||||
}
|
||||
let remote_objects = backend.object_count().await;
|
||||
println!("restore_cleanup_probe during_copyback={during_copyback} owners={cleanup_owners:?} remote_objects={remote_objects}");
|
||||
assert!(
|
||||
remote_objects == 0 || cleanup_owners.iter().filter(|count| **count > 0).count() >= 3,
|
||||
"ordinary overwrite must remove the remote object or retain its cleanup owner at write quorum: during_copyback={during_copyback}, owners={cleanup_owners:?}, remote_objects={remote_objects}"
|
||||
);
|
||||
}
|
||||
|
||||
+1062
-58
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user