mirror of
https://github.com/rustfs/rustfs.git
synced 2026-10-27 22:37:13 +00:00
Compare commits
17 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 4eba959910 | |||
| e4c2a373d5 | |||
| b33cf840b5 | |||
| 2f8d82ac50 | |||
| 59bc6d7c94 | |||
| a56104206e | |||
| be485e6cab | |||
| fbd94e129d | |||
| cf5a9e0ba3 | |||
| 254592717a | |||
| aba9e2e9e6 | |||
| 924700a24f | |||
| b8eef694c7 | |||
| 26d55bb04e | |||
| c9f7e48a44 | |||
| 11e1662c90 | |||
| 7ea3aa2a93 |
@@ -6,7 +6,17 @@
|
||||
- Route every new sync/flush through the durability-mode and bucket override
|
||||
gates; mode `none` must not pay the new fsync.
|
||||
- Keep blocking filesystem/CPU work off async runtime threads, but do not split
|
||||
one small operation into many `spawn_blocking` round trips.
|
||||
one small operation into many `spawn_blocking` round trips; coalesce `open`,
|
||||
`read`/`write`, `fsync`, and `close` into one blocking closure per disk.
|
||||
- Check for per-disk syscall amplification (`disks × parts/files`): pin a
|
||||
directory once (`openat2` on Linux) and reuse the handle across sibling reads
|
||||
instead of re-walking ancestor components per file or re-opening identical
|
||||
`source_parent == destination_parent` directories.
|
||||
- After `fstat`/`file.metadata()`, require `Vec::with_capacity(len + 1)` before
|
||||
`read_to_end` so small metadata reads do not trigger 6–7 exponential-growth
|
||||
`read()` syscalls.
|
||||
- Avoid redundant `store.load()` or `root_missing()` round-trips under the same
|
||||
exclusive lock when the state or original bytes were just loaded or persisted.
|
||||
- Measure lock hold time across I/O and compare acquisition order for ABBA.
|
||||
- Keep cleanup, extra stat/rename, and diagnostics out of the PUT commit critical
|
||||
section when they need not be there.
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-linux=14becc458b9eb24add510d94ee6cd04ba1fb19907d598effe1524e5242400d10
|
||||
sha256-darwin=14becc458b9eb24add510d94ee6cd04ba1fb19907d598effe1524e5242400d10
|
||||
sha256-linux=6190506b9e4bdb60fd392544a7bc47be708077bb4fe7a2ef689d047455e1af86
|
||||
sha256-darwin=6190506b9e4bdb60fd392544a7bc47be708077bb4fe7a2ef689d047455e1af86
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=596ca4ce6661a1ccd5c8278a7781fd46922ebe5e23377a254471ec5f4fabfeb0
|
||||
sha256-linux=1f3d1d167fbd58a72b46a255edb111c7ee0abdb1ae37ac00400d1239cacdd5e1
|
||||
sha256-darwin=a67a347f26be186c598b5beaf68e6bcbe1855350a35e1423dd5b8a81bcba06a5
|
||||
sha256-linux=ad4fabe811c6cca1aba89fad34ccd404b5832a6a955875c2826f6e0fa1ea4f1f
|
||||
|
||||
@@ -1 +1 @@
|
||||
sha256=4eac269f9a59b2f9be4e7a87ea6aa38454f5ebe10ede35c9b8d4218137d2cd35
|
||||
sha256=e4ee3898b840a8374cb02c64342cc15f8d7ecd1becaf4b4a2a4657290f9e7ff1
|
||||
|
||||
@@ -1 +1 @@
|
||||
sha256=57ffdd620dec9cba08a411b8ad681f4eb906cf5764ffed3c7c992084f5fc6092
|
||||
sha256=f065e7071aac9d48a7c1b301f328fa7cc14f11c4e7dc46da3e55057c4ea29bd5
|
||||
|
||||
@@ -70,11 +70,6 @@ s3s-footprint-check: ## Check the s3s dependency footprint ratchet stays frozen
|
||||
@echo "📦 Checking s3s footprint ratchet..."
|
||||
./scripts/check_s3s_footprint.sh
|
||||
|
||||
.PHONY: test-count-ratchet-check
|
||||
test-count-ratchet-check: ## Check no workspace crate lost tests or gained #[ignore] against the committed baseline
|
||||
@echo "🧪 Checking per-crate test-count ratchet..."
|
||||
./scripts/check_test_count_ratchet.sh
|
||||
|
||||
.PHONY: fips-wording-check
|
||||
fips-wording-check: ## Check docs and crates/kms do not over-claim crypto capabilities
|
||||
@echo "📣 Checking cryptographic capability wording guard..."
|
||||
|
||||
@@ -20,11 +20,11 @@ planning-docs-check: ## Check that no planning-type documents are committed
|
||||
./scripts/check_no_planning_docs.sh
|
||||
|
||||
.PHONY: pre-commit
|
||||
pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check test-count-ratchet-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
|
||||
pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
|
||||
@echo "✅ All pre-commit checks passed!"
|
||||
|
||||
.PHONY: pre-pr
|
||||
pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check test-count-ratchet-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check log-analyzer-rules-check offline-enrollment-e2e-check clippy-check test ## Run full pre-PR checks with clippy and tests
|
||||
pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check log-analyzer-rules-check offline-enrollment-e2e-check clippy-check test ## Run full pre-PR checks with clippy and tests
|
||||
@echo "✅ All pre-PR checks passed!"
|
||||
|
||||
.PHONY: dev-check
|
||||
|
||||
@@ -50,10 +50,9 @@ script-tests: ## Run shell script tests
|
||||
./scripts/test_fuzz_runner.sh
|
||||
./scripts/test_python_bin.sh
|
||||
./scripts/check_embedded_secrets.sh --self-test
|
||||
./scripts/check_s3s_footprint.sh --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/codemods/s3_error_to_s3_types.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_e2e_binary.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_inspect_btf_prototypes.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_release_build_workflow.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_release_glibc.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_upgrade_workflow.py
|
||||
|
||||
@@ -345,12 +345,6 @@ threads-required = "num-test-threads"
|
||||
filter = 'package(rustfs-ecstore) & test(=store::init::tests::dispatch_manifest_rollback_bounded_concurrency_reaches_tail_behind_slow_member)'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# These short-deadline real-disk progress probes need the test run's capacity
|
||||
# for scanner and object-lock handoffs; keep their assertions and deadlines.
|
||||
[[profile.default.overrides]]
|
||||
filter = '(package(rustfs-scanner) & (test(=scanner::tests::cycle_persist_failure::later_cycle_post_commit_failure_reloads_durable_state) | test(=scanner::tests::running_main_loop_catches_up_pause_cleared_after_startup_observe))) | (package(rustfs-ecstore) & test(=store::init::tests::restored_transitioned_delete_causally_enqueues_free_version))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ci profile — the strict CI gate (ci.yml `cargo nextest run --profile ci`)
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -419,12 +413,6 @@ threads-required = "num-test-threads"
|
||||
filter = 'package(rustfs-scanner) & (test(/^scanner::backlog::tests::native_/) | test(=scanner_folder::tests::mrf_ownership::mrf_ownership_manager_completion_preserves_scanner_pending) | test(/^scanner_io::tests::service_cohort::/))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# Keep short-deadline real-disk progress probes from competing with unrelated
|
||||
# tests for scanner and object-lock capacity. Assertions and deadlines stay on.
|
||||
[[profile.ci.overrides]]
|
||||
filter = '(package(rustfs-scanner) & (test(=scanner::tests::cycle_persist_failure::later_cycle_post_commit_failure_reloads_durable_state) | test(=scanner::tests::running_main_loop_catches_up_pause_cleared_after_startup_observe))) | (package(rustfs-ecstore) & test(=store::init::tests::restored_transitioned_delete_causally_enqueues_free_version))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
[[profile.ci.scripts]]
|
||||
filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(batch_transitioned_delete_uses_free_version_per_item|decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|dispatched_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|force_tier_remove_blocks_on_physical_free_version_hidden_by_other_pool|legacy_unknown_transition_delete_falls_back_for_single_batch_and_blocks_prefix|multi_pool_(recursive_prefix_rejects_legacy_or_hidden_merge_loser_before_delete|same_remote_tuple_(batch|single)_delete_waits_for_all_sources|same_tuple_recursive_prefix_uses_one_journal_owner|transitioned_delete_persists_one_free_version_per_remote_tuple)|recursive_prefix_partial_(pool|set)_failure_keeps_prepared_cleanup_owners|restored_transitioned_delete_uses_free_version_as_cleanup_owner|stable_transitioned_recursive_prefix_delete_uses_journal_owners|suspended_null_transition_delete_uses_free_version_as_sole_owner|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)|transitioned_delete_(free_version_replays_after_store_restart|local_quorum_failure_rolls_back_without_cleanup_owner|uses_free_version_as_cleanup_owner)|versioned_delete_marker_keeps_transitioned_source_and_remote_object|versioned_explicit_transition_delete_preserves_other_version_then_allows_bucket_delete))$/)'
|
||||
setup = 'ecstore-large-stack'
|
||||
@@ -684,14 +672,6 @@ test-group = 'ecstore-serial-flaky'
|
||||
# the PR-lane e2e coverage for scanner usage snapshots consumed by quota and
|
||||
# admin surfaces. It uses the same single-node, random-port, isolated-temp-dir
|
||||
# fixture as the existing smoke modules.
|
||||
#
|
||||
# Dual-stack differential (rustfs/backlog#2740): `dual_stack_differential::`
|
||||
# starts two single-node servers from the same binary, one per
|
||||
# RUSTFS_S3_STACK value, on random ports with isolated temp dirs, and runs one
|
||||
# SDK script covering every operation of `impl S3 for FS` against both. Its
|
||||
# comparator unit tests need no server. It meets the admission criteria
|
||||
# unchanged; the module clause admits a new differential test without another
|
||||
# filter edit.
|
||||
[profile.e2e-smoke]
|
||||
default-filter = """
|
||||
package(e2e_test) & (
|
||||
@@ -700,7 +680,6 @@ default-filter = """
|
||||
| test(/^reliant::lifecycle::/)
|
||||
| test(/^conditional_delete_objects_test::/)
|
||||
| test(/^reliant::tiering::/)
|
||||
| test(/^dual_stack_differential::/)
|
||||
| test(/^upgrade_compatibility_test::upgrade_write_readiness_tests::/)
|
||||
| test(/^on_demand_migration::(get_basic_test::(get_miss_pulls_inline_and_serves_locally_afterwards|head_miss_answers_from_the_source_without_persisting)|interaction_test::test_odm_admin_config_is_redacted_and_status_counts_match_the_source)$/)
|
||||
)
|
||||
|
||||
@@ -1,37 +0,0 @@
|
||||
# Files that may still reference s3s (`s3s::` or `s3s_sigv4`) when Phase 1 of
|
||||
# the gateway migration ends (rustfs/backlog#2734 task T0.6, rustfs/backlog#2741).
|
||||
# Consumed only by `scripts/check_s3s_footprint.sh --mode allowlist`.
|
||||
#
|
||||
# One entry per line: a repository-relative file path or a git `:(glob)`
|
||||
# pathspec. `**` matches across directories, `*` stays within one path
|
||||
# component, and a directory entry matches everything under it. Blank lines
|
||||
# and lines starting with `#` are ignored; an entry may not contain whitespace,
|
||||
# so a note belongs on its own comment line.
|
||||
#
|
||||
# Every Rust file listed by `git grep -l '\bs3s\(::\|_sigv4\)' -- '*.rs'` that
|
||||
# no entry matches fails the allowlist mode. Task T1.9 switches CI to that
|
||||
# mode; until then `--dry-run` prints the files still outside this list, which
|
||||
# is the Phase 1 tracking list.
|
||||
#
|
||||
# Never add crates/** other than crates/e2e_test: the engine crates leave s3s
|
||||
# entirely, and the script rejects such an entry. Only the s3s-facing edge of
|
||||
# the rustfs binary and the e2e oracle keep it.
|
||||
|
||||
# rustfs binary: the s3s-facing edge that survives Phase 1
|
||||
rustfs/src/storage/ecfs.rs
|
||||
rustfs/src/storage/s3_compat/**
|
||||
rustfs/src/storage/access.rs
|
||||
rustfs/src/auth.rs
|
||||
rustfs/src/error.rs
|
||||
rustfs/src/storage_api.rs
|
||||
rustfs/src/app/storage_api.rs
|
||||
rustfs/src/admin/router.rs
|
||||
rustfs/src/app/metadata_route.rs
|
||||
rustfs/src/server/http.rs
|
||||
rustfs/src/server/layer.rs
|
||||
rustfs/src/server/hybrid.rs
|
||||
rustfs/src/protocols/client.rs
|
||||
|
||||
# e2e oracle: the test target that verifies S3 behaviour against s3s
|
||||
crates/e2e_test/src/fake_s3_target/**
|
||||
crates/e2e_test/src/**
|
||||
@@ -1,17 +0,0 @@
|
||||
# Exact baseline for the direct s3s footprint, consumed only by
|
||||
# scripts/check_s3s_footprint.sh (rustfs/backlog#1677; rustfs/backlog#2734 T0.4).
|
||||
#
|
||||
# Each value is the exact measurement of the tree, not a ceiling. CI fails when a
|
||||
# measured counter differs from its line here in either direction:
|
||||
# - a larger measurement means new code widened the s3s surface: route it
|
||||
# through the gateway abstractions instead; never raise a value here;
|
||||
# - a smaller measurement means the PR shrank the surface: lower the matching
|
||||
# value to the measured number in the same PR.
|
||||
#
|
||||
# files Rust files referencing s3s paths, repo-wide minus crates/e2e_test
|
||||
# s3_error_lines lines holding s3_error!, same scope, minus those the script
|
||||
# header's rule clears (rustfs_s3_types's macro); fails closed
|
||||
# ecstore_files Rust files referencing s3s paths under crates/ecstore/src
|
||||
files=65
|
||||
s3_error_lines=289
|
||||
ecstore_files=0
|
||||
@@ -1,64 +0,0 @@
|
||||
# Per-crate test-count baseline consumed by scripts/check_test_count_ratchet.sh
|
||||
# (rustfs/backlog#2734 section 1 rule 3; T0.3 in rustfs/backlog#2738).
|
||||
#
|
||||
# One row per workspace member: <package>, tests, ignored, tab-separated.
|
||||
# tests counts #[test], #[tokio::test], #[tokio::test(...)] and #[test_case(...)]
|
||||
# attribute lines under the package directory; ignored counts #[ignore] and
|
||||
# #[ignore = "..."]. Both columns are exact: CI fails when a crate's tests fall
|
||||
# below or rise above its row, or when its ignored count changes. Raise a row in
|
||||
# the PR that adds tests; lowering one is a reviewed statement that tests were
|
||||
# intentionally removed. Regenerate with:
|
||||
# scripts/check_test_count_ratchet.sh --update
|
||||
e2e_test 988 20
|
||||
rustfs 5979 64
|
||||
rustfs-audit 70 0
|
||||
rustfs-checksums 26 0
|
||||
rustfs-common 36 0
|
||||
rustfs-concurrency 54 0
|
||||
rustfs-config 43 0
|
||||
rustfs-connect-symbol-catalog 6 0
|
||||
rustfs-credentials 24 0
|
||||
rustfs-crypto 67 0
|
||||
rustfs-data-usage 55 0
|
||||
rustfs-ecstore 7312 2
|
||||
rustfs-extension-schema 23 0
|
||||
rustfs-filemeta 352 0
|
||||
rustfs-heal 1001 3
|
||||
rustfs-heal-contracts 7 0
|
||||
rustfs-iam 394 0
|
||||
rustfs-io-core 49 0
|
||||
rustfs-io-metrics 151 0
|
||||
rustfs-keystone 29 0
|
||||
rustfs-kms 715 25
|
||||
rustfs-license 2 0
|
||||
rustfs-lifecycle 155 0
|
||||
rustfs-lock 138 0
|
||||
rustfs-log-analyzer 90 0
|
||||
rustfs-madmin 210 0
|
||||
rustfs-notify 137 0
|
||||
rustfs-object-capacity 92 0
|
||||
rustfs-object-data-cache 140 0
|
||||
rustfs-obs 381 0
|
||||
rustfs-policy 733 0
|
||||
rustfs-protocols 739 1
|
||||
rustfs-protos 73 0
|
||||
rustfs-replication 209 0
|
||||
rustfs-rio 206 0
|
||||
rustfs-rio-v2 34 6
|
||||
rustfs-s3-client 69 0
|
||||
rustfs-s3-ops 8 0
|
||||
rustfs-s3-types 103 0
|
||||
rustfs-s3select-api 207 0
|
||||
rustfs-s3select-query 189 2
|
||||
rustfs-scanner 1213 20
|
||||
rustfs-scanner-metrics 90 0
|
||||
rustfs-security-governance 20 0
|
||||
rustfs-signer 31 0
|
||||
rustfs-storage-api 34 0
|
||||
rustfs-targets 603 26
|
||||
rustfs-test-utils 1 0
|
||||
rustfs-tls-runtime 43 0
|
||||
rustfs-trusted-proxies 68 0
|
||||
rustfs-utils 247 0
|
||||
rustfs-zip 30 0
|
||||
zstd 11 0
|
||||
@@ -86,10 +86,6 @@ runs:
|
||||
shell: bash
|
||||
run: ./scripts/check_s3s_footprint.sh
|
||||
|
||||
- name: Check per-crate test-count ratchet
|
||||
shell: bash
|
||||
run: ./scripts/check_test_count_ratchet.sh
|
||||
|
||||
- name: Check cryptographic capability wording
|
||||
shell: bash
|
||||
run: ./scripts/check_fips_wording.sh
|
||||
|
||||
@@ -74,7 +74,6 @@ Notes:
|
||||
- `.github/workflows/e2e-s3tests.yml` (weekly full s3-tests sweep)
|
||||
- `.github/workflows/mint.yml` (weekly multi-SDK mint run)
|
||||
- `.github/workflows/fuzz.yml` (nightly fuzz corpus jobs)
|
||||
- `.github/workflows/performance-ab.yml` (nightly warp A/B gate)
|
||||
- **Pending:** the ci-7 e2e nightly-full workflow does not exist yet
|
||||
(backlog#1149 ci-7). When it lands, wire it up with the snippet above.
|
||||
|
||||
|
||||
@@ -18,7 +18,6 @@
|
||||
"never_ran_grace_until": "2026-09-08T00:00:00Z"
|
||||
},
|
||||
{ "workflow": ".github/workflows/nightly-gnu.yml", "max_age_hours": 36 },
|
||||
{ "workflow": ".github/workflows/performance-ab.yml", "max_age_hours": 36 },
|
||||
{
|
||||
"workflow": ".github/workflows/runner-hygiene.yml",
|
||||
"max_age_hours": 792,
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
|
||||
This repository is public. For a public repository, GitHub's standard hosted runners (`ubuntu-latest`, `windows-latest`, `macos-latest`) cost nothing and have no minute cap, whatever plan the organisation is on. GitHub bills only two things: larger hosted runners, which have custom labels and are not used here, and standard runners in private repositories.
|
||||
|
||||
Every other label in this repository is capacity RustFS pays for by the hour whether or not a job is running well:
|
||||
Every other label in this repository is capacity RustFS pays for. ARC pods and lab machines accrue cost while the machine exists. A Ubicloud label is metered for the minutes a job holds its VM:
|
||||
|
||||
| Label | Paid by | What it is | Use it for |
|
||||
|---|---|---|---|
|
||||
@@ -15,7 +15,7 @@ Every other label in this repository is capacity RustFS pays for by the hour whe
|
||||
| `windows-latest`, `macos-latest` | nobody | GitHub-hosted standard runners | Windows and macOS targets |
|
||||
| `sm-standard-2` | RustFS | ARC pod, Ubuntu 24.04, minimal image: no `gh`, no C toolchain; Docker only on the `dind-` label | only a job that a hosted runner cannot run (see the placement rule) |
|
||||
| `sm-standard-4` | RustFS | ARC pod, Ubuntu 24.04, 14 CPUs and 28 GB (the label is not the core count), large disk | workspace-scale Cargo builds |
|
||||
| `sm-standard-4-arm` | RustFS | ARC pod, aarch64; reached through the `build.yml` matrix, so `actionlint.yaml` does not list it | native aarch64 release builds |
|
||||
| `ubicloud-standard-16-arm` | RustFS | Ubicloud VM, alias of `ubicloud-standard-16-arm-ubuntu-2404`: Ubuntu 24.04 arm64, 16 vCPU, 48 GB, 300 GB disk. Reached through the `build.yml` matrix, so `actionlint.yaml` does not list it | native aarch64 release builds (`linux-aarch64-musl`, `linux-aarch64-gnu`) |
|
||||
| `dind-sm-standard-2` | RustFS | ARC pod with Docker-in-Docker | image builds that need `buildx` and a registry push |
|
||||
| `smoke-testing`, `pf-testing` | RustFS | long-lived lab machines | the packaged functional chain and performance suites |
|
||||
|
||||
@@ -40,7 +40,7 @@ A hosted runner is also the only choice for a few things the pods cannot do: `mo
|
||||
|
||||
- Every `shared-key` has exactly one writer. `cache-warm.yml` owns `ci-dev`, `ci-feat-rio`, `ci-feat-proto` and `ci-uring`; every other `./.github/actions/setup` call states `cache-save-if` explicitly, and `scripts/security/check_cache_save_if.sh` fails the build when one does not.
|
||||
- All readers of a key run on the same image as its writer. Keys that live on hosted runners say so in their name (`ci-uring` is warmed on `ubuntu-latest`; `ci-e2e-distributed-hosted`, `e2e-upgrade-server-hosted`, `e2e-upgrade-tests-hosted`).
|
||||
- Moving a job between pools therefore means renaming its key. The new key is empty until a run from `main` (a scheduled run, or a manual dispatch of that workflow from `main`) saves it; until then that lane does a cold build.
|
||||
- Moving a job between pools therefore means renaming its key. The new key is empty until a run from `main` (a scheduled run, or a manual dispatch of that workflow from `main`) saves it; until then that lane does a cold build. The `build.yml` aarch64 release legs append `-ubicloud-arm` for this reason. `cache-warm.yml` does not write those keys.
|
||||
- `CARGO_BUILD_JOBS` and every other `CARGO*`, `CC*`, `CFLAGS*`, `CXX*`, `CMAKE*` and `RUST*` variable is hashed into the key. `ci.yml` sets `CARGO_BUILD_JOBS` at step level for that reason; promoting it to job level rotates the key for every lane that shares it.
|
||||
|
||||
## Where self-hosted minutes leak
|
||||
@@ -62,7 +62,7 @@ gh api "repos/rustfs/rustfs/actions/runs/<run_id>/jobs?per_page=100" \
|
||||
--jq '.jobs[] | "\(.conclusion)\t\(.labels | join(","))\t\(.started_at)\t\(.completed_at)\t\(.name)"'
|
||||
```
|
||||
|
||||
Only rows whose label starts with `sm-`, `dind-`, `smoke-` or `pf-` cost anything. At the October 2026 review, a full pull-request run of `ci.yml` spent about 150 self-hosted minutes, a third of it in jobs that never compiled; treat that as the scale of what a placement mistake costs, not as a current figure.
|
||||
Only rows whose label starts with `sm-`, `dind-`, `ubicloud-`, `smoke-` or `pf-` cost anything. At the October 2026 review, a full pull-request run of `ci.yml` spent about 150 self-hosted minutes, a third of it in jobs that never compiled; treat that as the scale of what a placement mistake costs, not as a current figure.
|
||||
|
||||
## Checklist for a new or moved job
|
||||
|
||||
|
||||
@@ -29,7 +29,6 @@ on:
|
||||
- 'scripts/test_package_service_scripts.sh'
|
||||
- 'scripts/test_package_versions.sh'
|
||||
- 'scripts/test_docker_workflow.py'
|
||||
- 'scripts/security/check_performance_ab_workflow.sh'
|
||||
- 'scripts/security/check_preview_release_workflow.sh'
|
||||
- 'scripts/security/check_tier_artifact_workflow.sh'
|
||||
- 'scripts/security/check_workflow_pins.sh'
|
||||
@@ -48,7 +47,6 @@ on:
|
||||
- 'scripts/test_package_service_scripts.sh'
|
||||
- 'scripts/test_package_versions.sh'
|
||||
- 'scripts/test_docker_workflow.py'
|
||||
- 'scripts/security/check_performance_ab_workflow.sh'
|
||||
- 'scripts/security/check_preview_release_workflow.sh'
|
||||
- 'scripts/security/check_tier_artifact_workflow.sh'
|
||||
- 'scripts/security/check_workflow_pins.sh'
|
||||
@@ -155,9 +153,6 @@ jobs:
|
||||
- name: Check preview release workflow policy
|
||||
run: ./scripts/security/check_preview_release_workflow.sh
|
||||
|
||||
- name: Check performance A/B workflow trust boundary
|
||||
run: ./scripts/security/check_performance_ab_workflow.sh
|
||||
|
||||
- name: Check tier evidence workflow isolation
|
||||
run: ./scripts/security/check_tier_artifact_workflow.sh
|
||||
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
name: BTF prototype reference
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- "scripts/inspect_btf_prototypes.py"
|
||||
- "scripts/test_inspect_btf_prototypes.py"
|
||||
- "scripts/fixtures/btf_prototypes.c"
|
||||
- ".github/workflows/btf-prototype-reference.yml"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: btf-prototype-reference-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
reference:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- name: Check prototype and CLI contracts
|
||||
run: python3 -B scripts/test_inspect_btf_prototypes.py
|
||||
- name: Install only missing native reference tools
|
||||
run: |
|
||||
select_bpftool() {
|
||||
local candidate
|
||||
local candidates=()
|
||||
if command -v bpftool >/dev/null; then
|
||||
candidates+=("$(command -v bpftool)")
|
||||
fi
|
||||
for candidate in /usr/lib/linux-tools-*/bpftool /usr/lib/linux-tools/*/bpftool; do
|
||||
if [[ -x "$candidate" ]]; then
|
||||
candidates+=("$candidate")
|
||||
fi
|
||||
done
|
||||
while IFS= read -r candidate; do
|
||||
if [[ -x "$candidate" ]] && file -Lb "$candidate" | grep -q 'ELF' \
|
||||
&& "$candidate" version >/dev/null 2>&1; then
|
||||
printf '%s\n' "$candidate"
|
||||
return 0
|
||||
fi
|
||||
done < <(printf '%s\n' "${candidates[@]}" | sort -Vr)
|
||||
return 1
|
||||
}
|
||||
packages=()
|
||||
command -v clang >/dev/null || packages+=(clang)
|
||||
command -v llvm-objcopy >/dev/null || packages+=(llvm)
|
||||
select_bpftool >/dev/null || packages+=(linux-tools-generic)
|
||||
if (( ${#packages[@]} )); then
|
||||
sudo apt-get update
|
||||
sudo apt-get install --no-install-recommends -y "${packages[@]}"
|
||||
fi
|
||||
# File parsing needs a working binary, not a host-kernel wrapper.
|
||||
selected_bpftool="$(select_bpftool)"
|
||||
"$selected_bpftool" version
|
||||
printf 'BPFTOOL=%s\n' "$selected_bpftool" >> "$GITHUB_ENV"
|
||||
- name: Compare compiled declarations with bpftool
|
||||
run: |
|
||||
clang -target bpf -O2 -g -c scripts/fixtures/btf_prototypes.c -o "$RUNNER_TEMP/prototypes.o"
|
||||
llvm-objcopy --dump-section ".BTF=$RUNNER_TEMP/prototypes.btf" "$RUNNER_TEMP/prototypes.o"
|
||||
python3 -B scripts/test_inspect_btf_prototypes.py --native-reference "$RUNNER_TEMP/prototypes.btf"
|
||||
@@ -181,12 +181,18 @@ jobs:
|
||||
selected="all"
|
||||
fi
|
||||
|
||||
|
||||
# linux-aarch64-* need a real aarch64 host (placement rule: hardware)
|
||||
# so the packaged binary and console smoke execute the artifact.
|
||||
# ubicloud-standard-16-arm is that host: Ubuntu 24.04 arm64, alias of
|
||||
# ubicloud-standard-16-arm-ubuntu-2404 (16 vCPU, 48 GB). linux-x86_64-* stay on the
|
||||
# ARC sm-standard-4 pool. GNU rows keep cross=true so cargo-zigbuild
|
||||
# pins the glibc 2.31 floor on the same architecture; musl stays a
|
||||
# native cargo build.
|
||||
all='{"include":[
|
||||
{"target_id":"linux-x86_64-musl","os":"sm-standard-4","target":"x86_64-unknown-linux-musl","cross":false,"platform":"linux","rustflags":"","features":"","glibc":""},
|
||||
{"target_id":"linux-aarch64-musl","os":"sm-standard-4-arm","target":"aarch64-unknown-linux-musl","cross":false,"platform":"linux","rustflags":"","features":"","glibc":""},
|
||||
{"target_id":"linux-aarch64-musl","os":"ubicloud-standard-16-arm","target":"aarch64-unknown-linux-musl","cross":false,"platform":"linux","rustflags":"","features":"","glibc":""},
|
||||
{"target_id":"linux-x86_64-gnu","os":"sm-standard-4","target":"x86_64-unknown-linux-gnu","cross":true,"platform":"linux","rustflags":"","features":"pyroscope","glibc":"2.31"},
|
||||
{"target_id":"linux-aarch64-gnu","os":"sm-standard-4-arm","target":"aarch64-unknown-linux-gnu","cross":true,"platform":"linux","rustflags":"","features":"pyroscope","glibc":"2.31"},
|
||||
{"target_id":"linux-aarch64-gnu","os":"ubicloud-standard-16-arm","target":"aarch64-unknown-linux-gnu","cross":true,"platform":"linux","rustflags":"","features":"pyroscope","glibc":"2.31"},
|
||||
{"target_id":"macos-aarch64","os":"macos-latest","target":"aarch64-apple-darwin","cross":false,"platform":"macos","rustflags":"","features":"pyroscope","glibc":""},
|
||||
{"target_id":"windows-x86_64","os":"windows-latest","target":"x86_64-pc-windows-msvc","cross":false,"platform":"windows","rustflags":"","features":"","glibc":""}
|
||||
]}'
|
||||
@@ -244,7 +250,11 @@ jobs:
|
||||
with:
|
||||
rust-version: stable
|
||||
target: ${{ matrix.target }}
|
||||
cache-shared-key: build-${{ matrix.target }}${{ matrix.glibc && format('-glibc-{0}', matrix.glibc) || '' }}
|
||||
# rust-cache keys include OS and architecture but not the runner image.
|
||||
# The Ubicloud aarch64 legs therefore use their own suffix so they
|
||||
# cannot restore a cache written on a different arm64 image.
|
||||
# x86_64, macOS, and Windows keys stay unsuffixed.
|
||||
cache-shared-key: build-${{ matrix.target }}${{ matrix.glibc && format('-glibc-{0}', matrix.glibc) || '' }}${{ matrix.os == 'ubicloud-standard-16-arm' && '-ubicloud-arm' || '' }}
|
||||
# main only. A cache saved on refs/tags/X is scoped to that tag: no
|
||||
# other tag, no main run and no PR can restore it, so every release
|
||||
# cycle wrote up to 12 entries of 1-2GB (preview tag plus final tag,
|
||||
@@ -764,9 +774,9 @@ jobs:
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# The self-hosted Linux runners do not ship the aws CLI (GitHub-hosted
|
||||
# images did). Install the official CLI bundle on demand so R2 uploads
|
||||
# survive a fresh runner without relying on a distro package.
|
||||
# ARC pods (the x86_64 legs) do not ship the aws CLI. Ubicloud arm64
|
||||
# images follow the GitHub runner image and often do. Install the
|
||||
# official CLI bundle when it is missing so R2 uploads work on either.
|
||||
if ! command -v aws >/dev/null 2>&1; then
|
||||
echo "aws CLI not found on runner; installing..."
|
||||
if command -v apt-get >/dev/null 2>&1; then
|
||||
|
||||
@@ -16,9 +16,7 @@ name: Continuous Integration
|
||||
|
||||
on:
|
||||
push:
|
||||
# gateway/integration receives the daily merge of main as a direct push
|
||||
# (rustfs/backlog#2734), so its pushes must prove the result like main's.
|
||||
branches: [ main, gateway/integration ]
|
||||
branches: [ main ]
|
||||
paths-ignore:
|
||||
- "**.md"
|
||||
- "docs/**"
|
||||
@@ -739,8 +737,6 @@ jobs:
|
||||
needs: [ build-rustfs-debug-binary ]
|
||||
runs-on: sm-standard-2
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
RUSTFS_S3_STACK: legacy
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -859,105 +855,18 @@ jobs:
|
||||
path: ${{ runner.temp }}/rustfs-e2e-*/rustfs.log
|
||||
retention-days: 3
|
||||
|
||||
# The same e2e-smoke suite against the same debug binary with the gateway
|
||||
# S3 stack selected (rustfs/backlog#2734). Report-only: it is outside the
|
||||
# Test and Lint aggregate and becomes required at T2.10. The archive and run
|
||||
# steps must stay identical to e2e-tests; scripts/ci_gate.py enforces that.
|
||||
e2e-tests-gateway:
|
||||
name: e2e-smoke (gateway, report-only until T2.10)
|
||||
needs: [ build-rustfs-debug-binary ]
|
||||
runs-on: sm-standard-2
|
||||
timeout-minutes: 30
|
||||
continue-on-error: true
|
||||
env:
|
||||
RUSTFS_S3_STACK: gateway
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: ci-dev
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install awscurl
|
||||
run: |
|
||||
python3 -m pip install --user --upgrade pip "awscurl==0.44"
|
||||
echo "AWSCURL_PATH=$HOME/.local/bin/awscurl" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Verify awscurl
|
||||
run: test -x "$AWSCURL_PATH"
|
||||
|
||||
- name: Download debug binary
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: rustfs-debug-binary
|
||||
path: target/debug
|
||||
|
||||
- name: Make binary executable
|
||||
run: chmod +x ./target/debug/rustfs
|
||||
|
||||
- name: Archive e2e smoke test binaries
|
||||
env:
|
||||
NEXTEST_ARCHIVE: ${{ runner.temp }}/rustfs-e2e-smoke.tar.zst
|
||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-smoke-list.json
|
||||
run: |
|
||||
cargo nextest archive --profile e2e-smoke -p e2e_test --archive-file "${NEXTEST_ARCHIVE}"
|
||||
cargo nextest list --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" --message-format json > "${NEXTEST_LISTING}"
|
||||
python3 ./scripts/check_test_wiring.py --check-profile e2e-smoke "${NEXTEST_LISTING}"
|
||||
./scripts/check_security_smoke_count.sh check "${NEXTEST_LISTING}"
|
||||
|
||||
- name: Run e2e smoke suite
|
||||
env:
|
||||
NEXTEST_ARCHIVE: ${{ runner.temp }}/rustfs-e2e-smoke.tar.zst
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-smoke-logs
|
||||
SAMPLER_INTERVAL_SECS: "15"
|
||||
run: |
|
||||
./scripts/ci/resource_sampler.sh start e2e-smoke
|
||||
trap './scripts/ci/resource_sampler.sh stop' EXIT
|
||||
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" \
|
||||
--status-level all --final-status-level all --failure-output final
|
||||
|
||||
- name: Upload e2e smoke diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-smoke-gateway-diagnostics-${{ github.run_number }}
|
||||
path: |
|
||||
${{ runner.temp }}/rustfs-e2e-smoke-logs/
|
||||
${{ runner.temp }}/rustfs-e2e-smoke-list.json
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Upload e2e smoke JUnit report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-smoke-gateway-junit-${{ github.run_number }}
|
||||
path: target/nextest/e2e-smoke/junit.xml
|
||||
if-no-files-found: warn
|
||||
|
||||
e2e-full:
|
||||
name: End-to-End Tests (full merge gate)
|
||||
# Merge gate only (backlog#1149 ci-5): the never-automated user-visible
|
||||
# suites — KMS, object_lock, multipart_auth, quota, checksum, encryption,
|
||||
# security-boundary, ... — via the e2e-full nextest profile. Too heavy for
|
||||
# every PR, so it is gated to main and gateway/integration pushes, the merge
|
||||
# queue, and manual dispatch. protocols / the 7 cluster suites / replication
|
||||
# / #[ignore] are owned by other lanes (see .config/nextest.toml profile.e2e-full).
|
||||
# every PR, so it is gated to main pushes, the merge queue, and manual
|
||||
# dispatch. protocols / the 7 cluster suites / replication / #[ignore] are
|
||||
# owned by other lanes (see .config/nextest.toml profile.e2e-full).
|
||||
if: >-
|
||||
github.event_name == 'workflow_dispatch' ||
|
||||
github.event_name == 'merge_group' ||
|
||||
(github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/gateway/integration'))
|
||||
(github.event_name == 'push' && github.ref == 'refs/heads/main')
|
||||
needs: [ build-rustfs-debug-binary ]
|
||||
runs-on: sm-standard-2
|
||||
timeout-minutes: 55
|
||||
|
||||
@@ -45,7 +45,7 @@ on:
|
||||
# 07:00 UTC Sunday — staggered clear of the other Sunday crons: ci (00:00),
|
||||
# build (01:00), e2e-s3tests (02:00), audit (03:00), nix-flake-update
|
||||
# (05:00), mint (06:00), and the daily fuzz (02:00), minio-interop (03:17),
|
||||
# e2e-replication-nightly (04:00) and performance-ab (06:00) lanes.
|
||||
# and e2e-replication-nightly (04:00) lanes.
|
||||
- cron: "43 7 * * 0"
|
||||
|
||||
concurrency:
|
||||
|
||||
@@ -38,8 +38,8 @@ name: e2e-nightly
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
# 04:00 UTC nightly — staggered clear of fuzz/e2e-s3tests (02:00),
|
||||
# stale (01:30) and performance-ab (06:00).
|
||||
# 04:00 UTC nightly — staggered clear of fuzz/e2e-s3tests (02:00)
|
||||
# and stale (01:30).
|
||||
- cron: "29 4 * * *"
|
||||
|
||||
# Only alert-on-failure needs more than read access; it declares its own
|
||||
|
||||
@@ -1,384 +0,0 @@
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Warp A/B budget gate for the hotpath series (rustfs/backlog#935 HP-14, item 4).
|
||||
#
|
||||
# Two entry points, honestly scoped:
|
||||
# * schedule (nightly, on main): post-merge detection — catches a regression
|
||||
# within 24h of landing, not before merge.
|
||||
# * workflow_dispatch: an explicitly selected trusted ref.
|
||||
# The dispatch input can run the gate with --allow-regression so a deliberate
|
||||
# correctness cost (e.g. the #4221 fsync durability fix) is recorded, not
|
||||
# blocked (rustfs/backlog#935 correction 1).
|
||||
|
||||
name: Performance A/B
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "31 6 * * *" # 06:31 UTC nightly, against main
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
duration:
|
||||
description: "warp duration per round"
|
||||
required: false
|
||||
default: "12s"
|
||||
type: string
|
||||
allow_regression:
|
||||
description: "Pass the gate despite a FAIL (deliberate tradeoff)"
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
RUST_BACKTRACE: 1
|
||||
|
||||
jobs:
|
||||
warp-ab:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Warp A/B budget gate
|
||||
runs-on: sm-standard-2
|
||||
# A normal nightly restores the last successful binary and builds only the
|
||||
# candidate; daily access keeps that cache warm. A cache miss may build both
|
||||
# and needs room for the A/B run plus artifact and cache publication.
|
||||
timeout-minutes: 180
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
fetch-depth: 0 # baseline may be an earlier successful scheduled head
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: warp-ab-${{ hashFiles('**/Cargo.lock') }}
|
||||
cache-save-if: ${{ github.ref == 'refs/heads/main' }}
|
||||
|
||||
- name: Install warp
|
||||
run: |
|
||||
set -euo pipefail
|
||||
WARP_VERSION="1.0.0"
|
||||
curl -fsSL "https://github.com/minio/warp/releases/download/v${WARP_VERSION}/warp_Linux_x86_64.tar.gz" \
|
||||
| sudo tar -xz -C /usr/local/bin warp
|
||||
warp --version
|
||||
|
||||
- name: Decide exemption
|
||||
id: exempt
|
||||
env:
|
||||
INPUT_ALLOW_REGRESSION: ${{ github.event.inputs.allow_regression }}
|
||||
run: |
|
||||
allow="false"
|
||||
if [[ "$INPUT_ALLOW_REGRESSION" == "true" ]]; then
|
||||
allow="true"
|
||||
fi
|
||||
echo "allow_regression=$allow" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# A failed regression run must keep comparing against the last known-good
|
||||
# scheduled head. Otherwise the next nightly would absorb the regression
|
||||
# into its baseline and turn green without a fix.
|
||||
- name: Find last successful scheduled baseline
|
||||
id: scheduled_baseline
|
||||
if: github.event_name == 'schedule'
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
result-encoding: string
|
||||
script: |
|
||||
const { data } = await github.rest.actions.listWorkflowRuns({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
workflow_id: "performance-ab.yml",
|
||||
event: "schedule",
|
||||
status: "success",
|
||||
per_page: 1,
|
||||
});
|
||||
return data.workflow_runs[0]?.head_sha ?? "";
|
||||
|
||||
# Manual runs compare a selected ref with current main. Scheduled runs
|
||||
# compare current main with the last successful scheduled head. With no
|
||||
# history, the first run measures the candidate against itself and seeds
|
||||
# that head only if the complete rig succeeds.
|
||||
- name: Resolve baseline / candidate commits
|
||||
id: commits
|
||||
env:
|
||||
SCHEDULED_BASELINE_SHA: ${{ steps.scheduled_baseline.outputs.result }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
candidate_sha="$(git rev-parse HEAD)"
|
||||
if [[ "${{ github.event_name }}" == "schedule" ]]; then
|
||||
baseline_sha="${SCHEDULED_BASELINE_SHA:-$candidate_sha}"
|
||||
else
|
||||
baseline_sha="$(git rev-parse origin/main)"
|
||||
fi
|
||||
git cat-file -e "${baseline_sha}^{commit}"
|
||||
if ! git merge-base --is-ancestor "$baseline_sha" "$candidate_sha"; then
|
||||
echo "::error::baseline $baseline_sha is not an ancestor of candidate $candidate_sha; update the selected ref before comparing" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "baseline_sha=$baseline_sha" >> "$GITHUB_OUTPUT"
|
||||
echo "candidate_sha=$candidate_sha" >> "$GITHUB_OUTPUT"
|
||||
echo "baseline commit: $baseline_sha"
|
||||
echo "candidate commit: $candidate_sha"
|
||||
|
||||
# Exact-key restore of the candidate binary saved by its successful
|
||||
# scheduled run. A miss leaves cache-hit unset and falls back to a source
|
||||
# build of that known-good head.
|
||||
- name: Restore cached baseline binary
|
||||
id: baseline_cache
|
||||
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
|
||||
with:
|
||||
path: baseline-bin/rustfs
|
||||
key: rustfs-baseline-${{ steps.commits.outputs.baseline_sha }}
|
||||
|
||||
# Self-heal: on a nightly/dispatch run where the candidate commit IS the
|
||||
# baseline commit, a cache miss would make the rig build the same commit
|
||||
# twice (~65min per side with the post-#4806 LTO profile — no job budget
|
||||
# fits that). Build it once here, reuse it for both phases, and save it
|
||||
# back to the cache so the next run hits.
|
||||
- name: Build baseline on cache miss (same-commit self-heal)
|
||||
id: selfheal
|
||||
if: >-
|
||||
steps.baseline_cache.outputs.cache-hit != 'true' &&
|
||||
steps.commits.outputs.baseline_sha == steps.commits.outputs.candidate_sha
|
||||
run: |
|
||||
set -euo pipefail
|
||||
cargo build --release --bin rustfs
|
||||
mkdir -p baseline-bin
|
||||
cp target/release/rustfs baseline-bin/rustfs
|
||||
echo "built=true" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Build baseline on cache miss (different candidate)
|
||||
id: baseline_build
|
||||
if: >-
|
||||
steps.baseline_cache.outputs.cache-hit != 'true' &&
|
||||
steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha
|
||||
run: |
|
||||
set -euo pipefail
|
||||
baseline_root="$RUNNER_TEMP/rustfs-baseline-${{ github.run_id }}"
|
||||
baseline_target="$RUNNER_TEMP/rustfs-baseline-target-${{ github.run_id }}"
|
||||
git worktree add --detach "$baseline_root" "${{ steps.commits.outputs.baseline_sha }}"
|
||||
cargo build --release --manifest-path "$baseline_root/Cargo.toml" --bin rustfs --target-dir "$baseline_target"
|
||||
mkdir -p baseline-bin
|
||||
cp "$baseline_target/release/rustfs" baseline-bin/rustfs
|
||||
git worktree remove --force "$baseline_root"
|
||||
echo "built=true" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Build candidate binary
|
||||
id: candidate_build
|
||||
if: steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha
|
||||
run: |
|
||||
set -euo pipefail
|
||||
cargo build --release --bin rustfs
|
||||
mkdir -p candidate-bin
|
||||
cp target/release/rustfs candidate-bin/rustfs
|
||||
echo "built=true" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Save self-healed baseline to cache
|
||||
if: steps.selfheal.outputs.built == 'true' || steps.baseline_build.outputs.built == 'true'
|
||||
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
|
||||
with:
|
||||
path: baseline-bin/rustfs
|
||||
key: rustfs-baseline-${{ steps.commits.outputs.baseline_sha }}
|
||||
|
||||
- name: Run warp A/B and gate
|
||||
id: ab
|
||||
env:
|
||||
INPUT_DURATION: ${{ github.event.inputs.duration }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# The formal runner executes A1 baseline -> B1 candidate -> B2 candidate
|
||||
# -> A2 baseline for each workload and drive-sync cell. It requires three
|
||||
# rounds per leg to emit tail latency and error-rate evidence.
|
||||
# --health-timeout 180 outlasts the server's own 120s startup-readiness
|
||||
# budget, which the rig's previous 60s health poll undershot (the first
|
||||
# two nightly failures). perf-6 recalibrates these once the noise study
|
||||
# lands.
|
||||
duration="${INPUT_DURATION:-12s}"
|
||||
baseline_sha="${{ steps.commits.outputs.baseline_sha }}"
|
||||
candidate_sha="${{ steps.commits.outputs.candidate_sha }}"
|
||||
baseline_hit="${{ steps.baseline_cache.outputs.cache-hit }}"
|
||||
selfheal_built="${{ steps.selfheal.outputs.built }}"
|
||||
baseline_built="${{ steps.baseline_build.outputs.built }}"
|
||||
candidate_built="${{ steps.candidate_build.outputs.built }}"
|
||||
|
||||
args=(--duration "$duration" --rounds 3 --cooldown 5 --health-timeout 180 --baseline-revision "$baseline_sha" --candidate-revision "$candidate_sha")
|
||||
|
||||
if [[ "$baseline_hit" == "true" || "$selfheal_built" == "true" || "$baseline_built" == "true" ]]; then
|
||||
chmod +x baseline-bin/rustfs
|
||||
base_bin="$PWD/baseline-bin/rustfs"
|
||||
args+=(--baseline-bin "$base_bin")
|
||||
if [[ "$baseline_hit" == "true" ]]; then
|
||||
base_src="actions-cache (rustfs-baseline-$baseline_sha)"
|
||||
elif [[ "$selfheal_built" == "true" ]]; then
|
||||
base_src="source build (cache self-heal, saved as rustfs-baseline-$baseline_sha)"
|
||||
else
|
||||
base_src="isolated baseline source build (saved as rustfs-baseline-$baseline_sha)"
|
||||
fi
|
||||
if [[ "$candidate_sha" == "$baseline_sha" ]]; then
|
||||
# No commits landed since the last successful baseline, so reuse
|
||||
# the one binary for both phases and measure only rig drift.
|
||||
args+=(--candidate-bin "$base_bin")
|
||||
cand_src="same binary as baseline (same commit)"
|
||||
elif [[ "$candidate_built" == "true" ]]; then
|
||||
chmod +x candidate-bin/rustfs
|
||||
args+=(--candidate-bin "$PWD/candidate-bin/rustfs")
|
||||
cand_src="source build of the checked-out ref"
|
||||
else
|
||||
echo "::error::candidate binary was not built" >&2
|
||||
exit 2
|
||||
fi
|
||||
else
|
||||
echo "::error::baseline binary was not restored or built" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
echo "baseline binary: $base_src"
|
||||
echo "candidate binary: $cand_src"
|
||||
|
||||
if [[ "${{ steps.exempt.outputs.allow_regression }}" == "true" ]]; then
|
||||
args+=(--allow-regression --exemption-reason "workflow dispatch override")
|
||||
fi
|
||||
# Do not let a gate FAIL abort the job here; capture status and surface
|
||||
# it after the step summary is written.
|
||||
set +e
|
||||
bash scripts/run_hotpath_warp_abba.sh "${args[@]}"
|
||||
echo "status=$?" >> "$GITHUB_OUTPUT"
|
||||
set -e
|
||||
# Locate the newest run dir + gate.md for the summary/comment/artifact
|
||||
# steps. On a startup failure there is no gate.md, but the run dir still
|
||||
# holds server-logs/ for diagnosis.
|
||||
# Run dirs are UTC-timestamp names (no special chars); ls is safe here.
|
||||
# shellcheck disable=SC2012
|
||||
run_dir="$(ls -td target/hotpath-abba/*/ 2>/dev/null | head -n1 || true)"
|
||||
echo "run_dir=${run_dir%/}" >> "$GITHUB_OUTPUT"
|
||||
# shellcheck disable=SC2012
|
||||
gate_md="$(ls -t target/hotpath-abba/*/candidate_gate.md 2>/dev/null | head -n1 || true)"
|
||||
echo "gate_md=$gate_md" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Upload A/B results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: hotpath-warp-ab-${{ github.run_number }}
|
||||
# Includes per-cell median_summary.csv / baseline_compare.csv, both gates,
|
||||
# and server-logs/ (rustfs.log + startup env per phase) so a failed run
|
||||
# is diagnosable. Short retention: this is churny nightly debug data.
|
||||
path: target/hotpath-abba/
|
||||
if-no-files-found: warn
|
||||
retention-days: 14
|
||||
|
||||
- name: Write gate summary
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
status="${{ steps.ab.outputs.status }}"
|
||||
gate_md="${{ steps.ab.outputs.gate_md }}"
|
||||
run_dir="${{ steps.ab.outputs.run_dir }}"
|
||||
{
|
||||
echo "## Hotpath warp A/B — run ${{ github.run_number }}"
|
||||
echo
|
||||
if [[ "$status" == "0" ]]; then
|
||||
echo "Rig/gate exit: \`0\` (pass or warn)."
|
||||
else
|
||||
echo "Rig/gate exit: \`${status:-unknown}\` — **FAILED**."
|
||||
fi
|
||||
echo
|
||||
if [[ -n "$gate_md" && -f "$gate_md" ]]; then
|
||||
cat "$gate_md"
|
||||
else
|
||||
echo "No \`gate.md\` produced — the rig failed **before** the gate"
|
||||
echo "(most likely server startup / health). Failing phase(s) below;"
|
||||
echo "full logs in the \`hotpath-warp-ab-${{ github.run_number }}\` artifact."
|
||||
if [[ -n "$run_dir" && -d "$run_dir/server-logs" ]]; then
|
||||
for f in "$run_dir"/server-logs/*.log; do
|
||||
[[ -f "$f" ]] || continue
|
||||
echo
|
||||
echo "<details><summary>$(basename "$f")</summary>"
|
||||
echo
|
||||
echo '```'
|
||||
tail -n 30 "$f"
|
||||
echo '```'
|
||||
echo "</details>"
|
||||
done
|
||||
fi
|
||||
fi
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Stage successful candidate baseline
|
||||
if: >-
|
||||
steps.ab.outputs.status == '0' &&
|
||||
steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha
|
||||
run: |
|
||||
set -euo pipefail
|
||||
cp candidate-bin/rustfs baseline-bin/rustfs
|
||||
|
||||
- name: Cache successful candidate baseline
|
||||
if: >-
|
||||
steps.ab.outputs.status == '0' &&
|
||||
steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha
|
||||
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
|
||||
with:
|
||||
path: baseline-bin/rustfs
|
||||
key: rustfs-baseline-${{ steps.commits.outputs.candidate_sha }}
|
||||
|
||||
# Scheduled failure alerting is handled by the alert-on-failure job below
|
||||
# (perf-2 consuming ci-8's schedule-failure-issue composite action).
|
||||
|
||||
- name: Enforce gate
|
||||
if: always()
|
||||
run: |
|
||||
status="${{ steps.ab.outputs.status }}"
|
||||
if [[ -z "$status" ]]; then
|
||||
echo "::error::warp A/B setup failed before the rig ran. Check the first failed workflow step." >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ "$status" != "0" ]]; then
|
||||
echo "::error::warp A/B budget gate failed (exit $status). See the step summary / gate.md artifact." >&2
|
||||
exit "$status"
|
||||
fi
|
||||
echo "warp A/B budget gate passed."
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [warp-ab]
|
||||
# `always()` is required: without it this job is skipped when a needed
|
||||
# job fails. Alerts only for scheduled (nightly) runs (backlog#1149
|
||||
# ci-8); manual dispatch failures are already watched by a human.
|
||||
# `cancelled` is included alongside `failure` on purpose: a job that hits
|
||||
# timeout-minutes ends as `cancelled`, and the 2026-07-11..07-14 nightly
|
||||
# timeouts went silent precisely because the guard was failure-only. The
|
||||
# composite action already reports cancelled/timed-out jobs in the issue
|
||||
# body.
|
||||
if: >-
|
||||
github.repository == 'rustfs/rustfs' && (
|
||||
always() && github.event_name == 'schedule' &&
|
||||
(contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
|
||||
)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Open or update failure-tracking issue
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}
|
||||
@@ -16,8 +16,8 @@
|
||||
#
|
||||
# Forces a job failure and then runs .github/actions/schedule-failure-issue
|
||||
# through the exact `needs` + `always() && contains(needs.*.result,
|
||||
# 'failure')` wiring used by the real consumers (e2e-s3tests, mint, fuzz,
|
||||
# performance-ab). Dispatch it twice to verify both the issue-creation and
|
||||
# 'failure')` wiring used by the real consumers (e2e-s3tests, mint, fuzz).
|
||||
# Dispatch it twice to verify both the issue-creation and
|
||||
# the dedupe-comment paths, then close the resulting
|
||||
# "[scheduled-failure] Schedule Failure Alert Drill" issue in rustfs/backlog
|
||||
# (the default target repository). Requires the BACKLOG_ISSUE_TOKEN secret
|
||||
|
||||
@@ -28,7 +28,6 @@ on:
|
||||
- "mint"
|
||||
- "minio-interop"
|
||||
- "Nightly GNU Build"
|
||||
- "Performance A/B"
|
||||
- "Runner Hygiene"
|
||||
types: [completed]
|
||||
|
||||
|
||||
+1
-1
@@ -155,7 +155,7 @@ default build (lifecycle:
|
||||
- ⚠️ PARTIALLY VIOLATED: serving-side `s3s` references remain in ecstore
|
||||
(bucket metadata/replication/lifecycle DTOs and error mapping). The
|
||||
count is ratcheted shrink-only by `scripts/check_s3s_footprint.sh`
|
||||
(the `ecstore_files` line of `.config/s3s-footprint-baseline.txt`; the `object_lock` module was converted to
|
||||
(`S3S_ECSTORE_FILES_BASELINE`; the `object_lock` module was converted to
|
||||
storage-level types as the first ratchet step). Target state: the
|
||||
baseline reaches zero and ecstore's `Cargo.toml` drops `s3s`.
|
||||
|
||||
|
||||
Generated
+12
-80
@@ -8545,15 +8545,6 @@ version = "1.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
|
||||
|
||||
[[package]]
|
||||
name = "quick-xml"
|
||||
version = "0.41.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quick-xml"
|
||||
version = "0.42.0"
|
||||
@@ -9418,7 +9409,7 @@ dependencies = [
|
||||
"proptest",
|
||||
"prost 0.14.4",
|
||||
"pyroscope",
|
||||
"quick-xml 0.42.0",
|
||||
"quick-xml",
|
||||
"quinn",
|
||||
"rand 0.10.3",
|
||||
"rcgen",
|
||||
@@ -9437,7 +9428,6 @@ dependencies = [
|
||||
"rustfs-ecstore",
|
||||
"rustfs-extension-schema",
|
||||
"rustfs-filemeta",
|
||||
"rustfs-gateway-types",
|
||||
"rustfs-heal",
|
||||
"rustfs-heal-contracts",
|
||||
"rustfs-iam",
|
||||
@@ -9716,7 +9706,6 @@ dependencies = [
|
||||
"rustfs-data-usage",
|
||||
"rustfs-erasure-codec",
|
||||
"rustfs-filemeta",
|
||||
"rustfs-gateway-types",
|
||||
"rustfs-heal-contracts",
|
||||
"rustfs-io-metrics",
|
||||
"rustfs-lifecycle",
|
||||
@@ -9822,48 +9811,6 @@ dependencies = [
|
||||
"xxhash-rust",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-gateway-stream"
|
||||
version = "0.2.0"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=21283e874c2381bab611e9347acbc5b92e9e2b3c#21283e874c2381bab611e9347acbc5b92e9e2b3c"
|
||||
dependencies = [
|
||||
"bitflags 2.13.2",
|
||||
"bytes",
|
||||
"http 1.5.0",
|
||||
"http-body",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-gateway-types"
|
||||
version = "0.32.0+aws.2026-10-02"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=21283e874c2381bab611e9347acbc5b92e9e2b3c#21283e874c2381bab611e9347acbc5b92e9e2b3c"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"crc-fast",
|
||||
"futures-core",
|
||||
"hex",
|
||||
"http 1.5.0",
|
||||
"md-5 0.11.0",
|
||||
"percent-encoding",
|
||||
"rustfs-gateway-stream",
|
||||
"rustfs-gateway-xml",
|
||||
"s3s",
|
||||
"sha1 0.11.0",
|
||||
"sha2 0.11.1",
|
||||
"time",
|
||||
"twox-hash",
|
||||
"unicode-normalization",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-gateway-xml"
|
||||
version = "0.4.0"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=21283e874c2381bab611e9347acbc5b92e9e2b3c#21283e874c2381bab611e9347acbc5b92e9e2b3c"
|
||||
dependencies = [
|
||||
"quick-xml 0.41.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-heal"
|
||||
version = "1.0.2"
|
||||
@@ -10134,10 +10081,10 @@ dependencies = [
|
||||
"metrics-util",
|
||||
"proptest",
|
||||
"rustfs-config",
|
||||
"rustfs-gateway-types",
|
||||
"rustfs-replication",
|
||||
"rustfs-scanner-metrics",
|
||||
"rustfs-storage-api",
|
||||
"s3s",
|
||||
"serial_test",
|
||||
"temp-env",
|
||||
"time",
|
||||
@@ -10200,6 +10147,7 @@ dependencies = [
|
||||
"reqwest",
|
||||
"rmp-serde",
|
||||
"rustfs-signer",
|
||||
"s3s",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sysinfo",
|
||||
@@ -10239,7 +10187,7 @@ dependencies = [
|
||||
"jiff",
|
||||
"metrics",
|
||||
"percent-encoding",
|
||||
"quick-xml 0.42.0",
|
||||
"quick-xml",
|
||||
"rayon",
|
||||
"rustc-hash",
|
||||
"rustfs-config",
|
||||
@@ -10420,13 +10368,10 @@ dependencies = [
|
||||
"rustfs-config",
|
||||
"rustfs-credentials",
|
||||
"rustfs-ecstore",
|
||||
"rustfs-gateway-stream",
|
||||
"rustfs-gateway-types",
|
||||
"rustfs-iam",
|
||||
"rustfs-keystone",
|
||||
"rustfs-policy",
|
||||
"rustfs-rio",
|
||||
"rustfs-s3-types",
|
||||
"rustfs-storage-api",
|
||||
"rustfs-test-utils",
|
||||
"rustfs-tls-runtime",
|
||||
@@ -10434,13 +10379,13 @@ dependencies = [
|
||||
"rustfs-trusted-proxies",
|
||||
"rustfs-utils",
|
||||
"rustls",
|
||||
"s3s",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha1 0.11.0",
|
||||
"sha2 0.11.1",
|
||||
"socket2",
|
||||
"subtle",
|
||||
"sync_wrapper",
|
||||
"tempfile",
|
||||
"thiserror 2.0.21",
|
||||
"time",
|
||||
@@ -10490,7 +10435,7 @@ dependencies = [
|
||||
"regex",
|
||||
"rmp",
|
||||
"rmp-serde",
|
||||
"rustfs-gateway-types",
|
||||
"s3s",
|
||||
"serde",
|
||||
"time",
|
||||
"tokio",
|
||||
@@ -10585,13 +10530,11 @@ dependencies = [
|
||||
"hyper-util",
|
||||
"lazy_static",
|
||||
"md-5 0.11.0",
|
||||
"quick-xml 0.42.0",
|
||||
"quick-xml",
|
||||
"rand 0.10.3",
|
||||
"rustfs-checksums",
|
||||
"rustfs-config",
|
||||
"rustfs-gateway-types",
|
||||
"rustfs-rio",
|
||||
"rustfs-s3-types",
|
||||
"rustfs-signer",
|
||||
"rustfs-storage-api",
|
||||
"rustfs-tls-runtime",
|
||||
@@ -10625,19 +10568,9 @@ dependencies = [
|
||||
name = "rustfs-s3-types"
|
||||
version = "1.0.2"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"hotpath",
|
||||
"http 1.5.0",
|
||||
"http-body",
|
||||
"http-body-util",
|
||||
"hyper",
|
||||
"hyper-util",
|
||||
"pin-project-lite",
|
||||
"s3s",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -10661,10 +10594,9 @@ dependencies = [
|
||||
"parking_lot 0.13.0",
|
||||
"rustfs-common",
|
||||
"rustfs-ecstore",
|
||||
"rustfs-gateway-types",
|
||||
"rustfs-s3-types",
|
||||
"rustfs-storage-api",
|
||||
"rustfs-test-utils",
|
||||
"s3s",
|
||||
"serde_json",
|
||||
"serial_test",
|
||||
"thiserror 2.0.21",
|
||||
@@ -10690,6 +10622,7 @@ dependencies = [
|
||||
"parking_lot 0.13.0",
|
||||
"rustfs-s3select-api",
|
||||
"rustfs-test-utils",
|
||||
"s3s",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tokio",
|
||||
@@ -10718,7 +10651,6 @@ dependencies = [
|
||||
"rustfs-data-usage",
|
||||
"rustfs-ecstore",
|
||||
"rustfs-filemeta",
|
||||
"rustfs-gateway-types",
|
||||
"rustfs-heal",
|
||||
"rustfs-heal-contracts",
|
||||
"rustfs-lifecycle",
|
||||
@@ -10775,10 +10707,9 @@ dependencies = [
|
||||
"bytes",
|
||||
"hotpath",
|
||||
"http 1.5.0",
|
||||
"http-body",
|
||||
"hyper",
|
||||
"rustfs-s3-types",
|
||||
"rustfs-utils",
|
||||
"s3s",
|
||||
"serde_urlencoded",
|
||||
"thiserror 2.0.21",
|
||||
"time",
|
||||
@@ -10836,6 +10767,7 @@ dependencies = [
|
||||
"rustfs-utils",
|
||||
"rustls",
|
||||
"rustls-native-certs",
|
||||
"s3s",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.11.1",
|
||||
@@ -11227,7 +11159,7 @@ dependencies = [
|
||||
"numeric_cast",
|
||||
"phf 0.14.0",
|
||||
"pin-project-lite",
|
||||
"quick-xml 0.42.0",
|
||||
"quick-xml",
|
||||
"regex",
|
||||
"s3s-multipart",
|
||||
"s3s-rfc2047",
|
||||
|
||||
+1
-13
@@ -48,7 +48,7 @@ members = [
|
||||
"crates/replication", # Replication contracts and wire formats
|
||||
"crates/concurrency", # Concurrency management for RustFS - timeout, locking, backpressure, and I/O scheduling
|
||||
"crates/s3-client", # S3 client for engine-side consumption of remote S3 endpoints (tiering, transition targets)
|
||||
"crates/s3-types", # S3 event and error type definitions
|
||||
"crates/s3-types", # S3 event type definitions
|
||||
"crates/s3-ops", # S3 operation definitions and mapping
|
||||
"crates/s3select-api", # S3 Select API interface
|
||||
"crates/s3select-query", # S3 Select query engine
|
||||
@@ -143,13 +143,6 @@ rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.2" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.2" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.2" }
|
||||
|
||||
# The gateway's S3 types: the DTOs and persisted bucket-configuration codecs the
|
||||
# engine crates move onto from s3s (rustfs/backlog#2745). The one pin of the
|
||||
# gateway in this workspace; no crate may enable a `compat-s3s*` feature except
|
||||
# the s3s-facing edge of the rustfs binary.
|
||||
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "21283e874c2381bab611e9347acbc5b92e9e2b3c", version = "0.32.0", default-features = false }
|
||||
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "21283e874c2381bab611e9347acbc5b92e9e2b3c", version = "0.2.0", default-features = false }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-channel = "2.5.0"
|
||||
async_zip = { default-features = false, version = "0.0.19" }
|
||||
@@ -333,10 +326,6 @@ rust-embed = { version = "8.13.0" }
|
||||
rustc-hash = { version = "2.1.3" }
|
||||
s3s = { git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fcac87e3b99fd5bd9e176b442c0", features = ["minio"] }
|
||||
s3s-sigv4 = { git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fcac87e3b99fd5bd9e176b442c0" }
|
||||
# The same s3s pin under the name the legacy edge of the gateway migration uses
|
||||
# (crates/s3-types, feature `compat-s3s`). Keep `rev` identical to `s3s` above.
|
||||
# DELETE BY T4.2 (rustfs/backlog#2784).
|
||||
legacy-s3s = { package = "s3s", git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fcac87e3b99fd5bd9e176b442c0", features = ["minio"] }
|
||||
serial_test = "4.0.1"
|
||||
shadow-rs = { default-features = false, version = "2.0.0" }
|
||||
siphasher = "1.0.4"
|
||||
@@ -345,7 +334,6 @@ compact_str = "0.10.0"
|
||||
snap = "1.1.2"
|
||||
starshard = { version = "2.4.0" }
|
||||
strum = { version = "0.28.0" }
|
||||
sync_wrapper = "1.0.2"
|
||||
sysinfo = "0.39.6"
|
||||
temp-env = "0.3.6"
|
||||
tempfile = "3.27.0"
|
||||
|
||||
@@ -100,6 +100,7 @@ when restoring `/rustfs/console` for a custom OEM build.
|
||||
## S3 API environment variables
|
||||
|
||||
- `RUSTFS_API_OBJECT_MAX_VERSIONS` caps the number of retained versions for a single object. It defaults to `9223372036854775807`, matching MinIO's practical-unlimited default. Set a positive integer to enforce a lower per-object metadata bound.
|
||||
- `RUSTFS_VEEAM_SOSAPI_ENABLE` serves Veeam SOSAPI `system.xml` and `capacity.xml` for an authenticated GET or HEAD of those exact bucket-root keys. It defaults to `true`. Set it to `false` to restore the normal object path, including deletion of a real object previously stored at that key.
|
||||
- `MINIO_API_OBJECT_MAX_VERSIONS` is accepted as a compatibility alias when the canonical RustFS variable is not set.
|
||||
|
||||
## Distributed endpoint locality
|
||||
|
||||
@@ -222,6 +222,16 @@ pub const ENV_RUSTFS_TLS_PATH: &str = "RUSTFS_TLS_PATH";
|
||||
/// Environment variable for server KMS enable.
|
||||
pub const ENV_RUSTFS_KMS_ENABLE: &str = "RUSTFS_KMS_ENABLE";
|
||||
|
||||
/// Environment variable for Veeam SOSAPI virtual system objects.
|
||||
///
|
||||
/// When true (the default), authenticated GET and HEAD of the two Veeam
|
||||
/// SOSAPI keys return generated XML. When false, those keys use the normal
|
||||
/// object path so an operator can remove a real object stored there earlier.
|
||||
pub const ENV_VEEAM_SOSAPI_ENABLE: &str = "RUSTFS_VEEAM_SOSAPI_ENABLE";
|
||||
|
||||
/// Default Veeam SOSAPI enablement.
|
||||
pub const DEFAULT_VEEAM_SOSAPI_ENABLE: bool = true;
|
||||
|
||||
/// Default KMS enable for server-side encryption
|
||||
/// This is the default value for enabling KMS encryption for server-side encryption.
|
||||
/// Default value: false
|
||||
|
||||
@@ -17,8 +17,7 @@
|
||||
//! when PublicAccessBlock configuration is missing or explicitly set.
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use crate::sts_query_compat_test::{OPA_AUTH_TOKEN, OpaMock};
|
||||
use aws_sdk_s3::types::{PublicAccessBlockConfiguration, Tag, Tagging};
|
||||
use aws_sdk_s3::types::PublicAccessBlockConfiguration;
|
||||
use tracing::info;
|
||||
|
||||
async fn setup_public_bucket(
|
||||
@@ -70,98 +69,6 @@ async fn anonymous_get_object(
|
||||
local_http_client().get(&url).send().await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_plain_http_matches_secure_transport_false_bucket_policy() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
assert!(env.url.starts_with("http://"), "fixture must exercise plaintext HTTP: {}", env.url);
|
||||
|
||||
let bucket = "secure-transport-policy";
|
||||
let key = "plain-http.txt";
|
||||
let admin = env.create_s3_client();
|
||||
admin.create_bucket().bucket(bucket).send().await?;
|
||||
admin
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(aws_sdk_s3::primitives::ByteStream::from_static(b"plain http"))
|
||||
.send()
|
||||
.await?;
|
||||
admin
|
||||
.put_bucket_policy()
|
||||
.bucket(bucket)
|
||||
.policy(
|
||||
serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Principal": "*",
|
||||
"Action": "s3:GetObject",
|
||||
"Resource": format!("arn:aws:s3:::{bucket}/*"),
|
||||
"Condition": {"Bool": {"aws:SecureTransport": "false"}}
|
||||
}]
|
||||
})
|
||||
.to_string(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let response = anonymous_get_object(&env, bucket, key).await?;
|
||||
assert_eq!(response.status().as_u16(), 200);
|
||||
assert_eq!(response.bytes().await?.as_ref(), b"plain http");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Root credentials have no attached IAM policy, so these requests depend on
|
||||
/// the owner flag reaching policy evaluation unchanged.
|
||||
#[tokio::test]
|
||||
async fn test_root_requests_keep_policy_owner_flag() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let opa = OpaMock::start().await?;
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_with_env(
|
||||
vec![],
|
||||
&[
|
||||
("RUSTFS_POLICY_PLUGIN_URL", opa.url.as_str()),
|
||||
("RUSTFS_POLICY_PLUGIN_AUTH_TOKEN", OPA_AUTH_TOKEN),
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
|
||||
let bucket = "root-owner-flag";
|
||||
let key = "owner.txt";
|
||||
let client = env.create_s3_client();
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(aws_sdk_s3::primitives::ByteStream::from_static(b"owner access"))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
// Duplicate stored tags are ambiguous. An owner request must not enter the
|
||||
// OPA tag-loading path, which rejects this metadata before evaluation.
|
||||
client
|
||||
.put_bucket_tagging()
|
||||
.bucket(bucket)
|
||||
.tagging(
|
||||
Tagging::builder()
|
||||
.tag_set(Tag::builder().key("department").value("finance").build()?)
|
||||
.tag_set(Tag::builder().key("department").value("engineering").build()?)
|
||||
.build()?,
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
let object = client.get_object().bucket(bucket).key(key).send().await?;
|
||||
|
||||
assert_eq!(object.body.collect().await?.into_bytes().as_ref(), b"owner access");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Issue #2036: Anonymous GetObject should succeed when bucket policy allows it
|
||||
/// and no PublicAccessBlock configuration exists (ConfigNotFound).
|
||||
#[tokio::test]
|
||||
|
||||
@@ -13,18 +13,20 @@
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{
|
||||
DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, retrying_get_equals, unique_bucket,
|
||||
wait_for_ready, wait_until,
|
||||
DistCluster, DistLayout, TestResult, assert_object_bytes, cluster_admin_ok, get_object_bytes, payload_for, put_object,
|
||||
retrying_get_equals, retrying_put, s3_probe_client, s3_probe_error_code, unique_bucket, wait_for_ready, wait_until,
|
||||
};
|
||||
use crate::chaos::{census_object_version_on_disk, signed_admin_post};
|
||||
use crate::common::{build_test_s3_config, init_logging};
|
||||
use crate::fault_proxy::FaultMode;
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::SdkError;
|
||||
use aws_sdk_s3::operation::{get_object::GetObjectError, put_object::PutObjectError};
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::{Barrier, mpsc};
|
||||
use tokio::time::timeout;
|
||||
use tokio::time::{Instant, timeout};
|
||||
|
||||
#[tokio::test]
|
||||
async fn kill_and_restart_node_preserves_objects() -> TestResult {
|
||||
@@ -44,6 +46,133 @@ async fn kill_and_restart_node_preserves_objects() -> TestResult {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ec4_partial_node_return_restores_read_write_quorum() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start_with_env(
|
||||
DistLayout::FourByFour,
|
||||
&[
|
||||
("RUSTFS_ERASURE_SET_DRIVE_COUNT", "16"),
|
||||
("RUSTFS_STORAGE_CLASS_STANDARD", "EC:4"),
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
let storage = cluster_admin_ok(&dist.cluster, http::Method::GET, "/rustfs/admin/v3/storageinfo", None).await?;
|
||||
let storage: serde_json::Value = serde_json::from_str(&storage)?;
|
||||
let storage: rustfs_madmin::StorageInfo = serde_json::from_value(storage["info"].clone())?;
|
||||
assert_eq!(storage.disks.len(), 16);
|
||||
assert_eq!(storage.backend.total_sets, vec![1]);
|
||||
assert_eq!(storage.backend.drives_per_set, vec![16]);
|
||||
assert_eq!(storage.backend.standard_sc_parity, Some(4));
|
||||
|
||||
let bucket = unique_bucket("partial-ec4");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let client = s3_probe_client(&dist.client(0)?);
|
||||
let baseline = payload_for("partial-recovery/baseline", 8 * 1024 * 1024);
|
||||
put_object(&client, &bucket, "baseline.bin", baseline.clone()).await?;
|
||||
wait_until(
|
||||
Duration::from_secs(30),
|
||||
|| async {
|
||||
for node in &dist.cluster.nodes {
|
||||
for directory in &node.data_dirs {
|
||||
let census = census_object_version_on_disk(std::path::Path::new(directory), &bucket, "baseline.bin", None)?;
|
||||
if !census.is_complete() {
|
||||
return Ok(false);
|
||||
}
|
||||
assert_eq!(census.data_blocks, Some(12), "the object must require twelve data shards");
|
||||
assert_eq!(census.parity_blocks, Some(4), "the object must have four parity shards");
|
||||
assert!(!census.expected_part_numbers.is_empty(), "the baseline must use physical shards");
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
},
|
||||
"the baseline is complete on all sixteen EC:4 drives",
|
||||
)
|
||||
.await?;
|
||||
|
||||
dist.cluster.stop_node(2)?;
|
||||
dist.cluster.stop_node(3)?;
|
||||
let read_error = timeout(Duration::from_secs(60), get_object_bytes(&client, &bucket, "baseline.bin"))
|
||||
.await?
|
||||
.expect_err("eight unavailable drives must prevent the baseline GET");
|
||||
assert!(
|
||||
matches!(
|
||||
s3_probe_error_code(read_error.as_ref()),
|
||||
Some("SlowDown" | "SlowDownRead" | "ServiceUnavailable")
|
||||
),
|
||||
"quorum loss must produce an availability error: {read_error:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
read_error
|
||||
.downcast_ref::<SdkError<GetObjectError>>()
|
||||
.and_then(SdkError::raw_response)
|
||||
.map(|response| response.status().as_u16()),
|
||||
Some(503),
|
||||
"read quorum loss must remain a service availability failure"
|
||||
);
|
||||
let write_error = timeout(Duration::from_secs(60), put_object(&client, &bucket, "unavailable.bin", baseline.clone()))
|
||||
.await?
|
||||
.expect_err("eight unavailable drives must prevent a successful PUT");
|
||||
assert!(
|
||||
matches!(s3_probe_error_code(write_error.as_ref()), Some("SlowDown" | "ServiceUnavailable")),
|
||||
"quorum loss must produce an availability error: {write_error:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
write_error
|
||||
.downcast_ref::<SdkError<PutObjectError>>()
|
||||
.and_then(SdkError::raw_response)
|
||||
.map(|response| response.status().as_u16()),
|
||||
Some(503),
|
||||
"write quorum loss must remain a service availability failure"
|
||||
);
|
||||
|
||||
// Recover only four drives. The remaining node must stay stopped while
|
||||
// every live coordinator serves old data and acknowledges new writes.
|
||||
dist.cluster.start_node_process(2).await?;
|
||||
let recovery_deadline = Instant::now() + Duration::from_secs(90);
|
||||
let mut recovered_objects = Vec::new();
|
||||
for node in 0..3 {
|
||||
assert!(dist.cluster.nodes[3].process.is_none(), "the fourth node must remain unavailable");
|
||||
let client = s3_probe_client(&dist.client(node)?);
|
||||
retrying_get_equals(
|
||||
&client,
|
||||
&bucket,
|
||||
"baseline.bin",
|
||||
&baseline,
|
||||
recovery_deadline.saturating_duration_since(Instant::now()),
|
||||
)
|
||||
.await?;
|
||||
let key = format!("partial-{node}.bin");
|
||||
let body = payload_for(&key, 1024 * 1024);
|
||||
retrying_put(
|
||||
&client,
|
||||
&bucket,
|
||||
&key,
|
||||
body.clone(),
|
||||
recovery_deadline.saturating_duration_since(Instant::now()),
|
||||
)
|
||||
.await?;
|
||||
recovered_objects.push((key, body));
|
||||
}
|
||||
for node in 0..3 {
|
||||
assert!(dist.cluster.nodes[3].process.is_none(), "reads must complete before all drives return");
|
||||
let client = s3_probe_client(&dist.client(node)?);
|
||||
assert_object_bytes(&client, &bucket, "baseline.bin", &baseline).await?;
|
||||
for (key, body) in &recovered_objects {
|
||||
assert_object_bytes(&client, &bucket, key, body).await?;
|
||||
}
|
||||
}
|
||||
|
||||
dist.cluster.start_node(3).await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
let client = s3_probe_client(&dist.client(3)?);
|
||||
assert_object_bytes(&client, &bucket, "baseline.bin", &baseline).await?;
|
||||
for (key, body) in &recovered_objects {
|
||||
assert_object_bytes(&client, &bucket, key, body).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn full_cluster_restart_preserves_objects() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -1181,11 +1181,16 @@ pub(crate) async fn retrying_get_equals(
|
||||
)
|
||||
.into()),
|
||||
Err(error) => {
|
||||
if matches!(s3_probe_error_code(error.as_ref()), Some("SlowDown" | "ServiceUnavailable")) {
|
||||
Ok(false)
|
||||
} else {
|
||||
Err(error)
|
||||
}
|
||||
let retryable = match s3_probe_error_code(error.as_ref()) {
|
||||
Some("SlowDown" | "ServiceUnavailable") => true,
|
||||
// GET preserves this compatibility code for a lost data quorum.
|
||||
Some("SlowDownRead") => error
|
||||
.downcast_ref::<SdkError<GetObjectError>>()
|
||||
.and_then(SdkError::raw_response)
|
||||
.is_some_and(|response| response.status().as_u16() == 503),
|
||||
_ => false,
|
||||
};
|
||||
if retryable { Ok(false) } else { Err(error) }
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -1324,6 +1329,62 @@ mod retry_tests {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn retrying_get_allows_read_quorum_recovery() -> TestResult {
|
||||
use aws_sdk_s3::config::Region;
|
||||
use aws_smithy_http_client::test_util::{ReplayEvent, StaticReplayClient};
|
||||
use aws_smithy_types::body::SdkBody;
|
||||
|
||||
let endpoint = "http://read-quorum.example.invalid";
|
||||
let event = |status: u16, body: &'static str| {
|
||||
ReplayEvent::new(
|
||||
http::Request::builder()
|
||||
.method(Method::GET)
|
||||
.uri(format!("{endpoint}/retry-probe/key?x-id=GetObject"))
|
||||
.body(SdkBody::empty())
|
||||
.expect("build replay request"),
|
||||
http::Response::builder()
|
||||
.status(status)
|
||||
.header("content-type", "application/xml")
|
||||
.header("content-length", body.len())
|
||||
.body(SdkBody::from(body))
|
||||
.expect("build replay response"),
|
||||
)
|
||||
};
|
||||
for status in [503, 403, 500] {
|
||||
let transport = StaticReplayClient::new(vec![
|
||||
event(status, "<Error><Code>SlowDownRead</Code></Error>"),
|
||||
event(200, "expected bytes"),
|
||||
]);
|
||||
let client = Client::from_conf(
|
||||
aws_sdk_s3::Config::builder()
|
||||
.behavior_version_latest()
|
||||
.credentials_provider(Credentials::new(FAKE_ACCESS_KEY, FAKE_SECRET_KEY, None, None, "read-quorum-test"))
|
||||
.region(Region::new("us-east-1"))
|
||||
.endpoint_url(endpoint)
|
||||
.force_path_style(true)
|
||||
.retry_config(RetryConfig::disabled())
|
||||
.http_client(transport.clone())
|
||||
.build(),
|
||||
);
|
||||
let result = retrying_get_equals(&client, "retry-probe", "key", b"expected bytes", Duration::from_secs(5)).await;
|
||||
if status == 503 {
|
||||
result?;
|
||||
assert_eq!(
|
||||
transport.actual_requests().count(),
|
||||
2,
|
||||
"the probe must retry the read-quorum error exactly once"
|
||||
);
|
||||
transport.assert_requests_match(&[]);
|
||||
} else {
|
||||
let error = result.expect_err("the compatibility code must not hide a non-503 failure");
|
||||
assert_eq!(s3_probe_error_code(error.as_ref()), Some("SlowDownRead"));
|
||||
assert_eq!(transport.actual_requests().count(), 1, "non-availability errors must fail immediately");
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_polls_distinguish_committed_and_replicated_keys() -> TestResult {
|
||||
let target = FakeS3Target::start().await?;
|
||||
|
||||
@@ -1638,6 +1638,191 @@ async fn late_complete_packets_cannot_publish_after_abort_and_bucket_recreation(
|
||||
"refused late packets must not recreate staging"
|
||||
);
|
||||
}
|
||||
late_part_packets_preserve_retired_and_fresh_uploads().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn late_part_packets_preserve_retired_and_fresh_uploads() -> TestResult {
|
||||
for (method, settles) in [("PreparePartTransaction", false), ("SettlePartTransaction", true)] {
|
||||
for recreate_bucket in [false, true] {
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
|
||||
let key = "late-part-transaction";
|
||||
let previous = payload_for("committed-before-late-part", 4096);
|
||||
clients[0]
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from(previous.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
let upload = clients[0].create_multipart_upload().bucket(&bucket).key(key).send().await?;
|
||||
let upload_id = upload.upload_id().ok_or("missing original upload ID")?;
|
||||
clients[0]
|
||||
.upload_part()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.upload_id(upload_id)
|
||||
.part_number(1)
|
||||
.body(ByteStream::from(vec![0x41; 9 * 1024 * 1024]))
|
||||
.send()
|
||||
.await?;
|
||||
let path = upload_path_scope(&bucket, key, upload_id)?;
|
||||
let hash = sha256_hex(format!("{bucket}/{key}").as_bytes());
|
||||
let roots = cluster
|
||||
.nodes
|
||||
.iter()
|
||||
.flat_map(|node| &node.data_dirs)
|
||||
.map(|root| Path::new(root).join(RUSTFS_META_MULTIPART_BUCKET).join(&path))
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(roots.len(), 4);
|
||||
let original = multipart_sources(&cluster, Some(&hash))?;
|
||||
assert_eq!(original.len(), 4);
|
||||
assert!(
|
||||
original.values().all(|source| source
|
||||
.files
|
||||
.keys()
|
||||
.any(|path| path.file_name().and_then(|name| name.to_str()) == Some("part.1"))),
|
||||
"all selected peers originally own the acknowledged part"
|
||||
);
|
||||
let scope = upload_scope(upload_id)?;
|
||||
// Retain only the original attempt; recovery of the same upload
|
||||
// must be able to issue fresh settlement requests through the gate.
|
||||
let pauses = proxies[1..3]
|
||||
.iter()
|
||||
.map(|proxy| proxy.pause_late(&[method], 1, &scope))
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
let replacement = clients[0]
|
||||
.upload_part()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.upload_id(upload_id)
|
||||
.part_number(1)
|
||||
.body(ByteStream::from(vec![0x62; 8 * 1024 * 1024]));
|
||||
let caller = tokio::spawn(async move { replacement.send().await });
|
||||
wait_for_faults(&pauses, &[method]).await?;
|
||||
let failure = tokio::time::timeout(DEADLINE, caller)
|
||||
.await??
|
||||
.expect_err("two retained peers cannot acknowledge part replacement");
|
||||
assert_unavailable(&failure, "storage_write_quorum", None);
|
||||
assert!(pauses.iter().all(|pause| pause.hits() == 1 && pause.responses() == 0));
|
||||
let expected_actions = if settles { (1, 0) } else { (0, 0) };
|
||||
assert!(
|
||||
pauses.iter().all(|pause| pause.retained_settle_actions() == expected_actions),
|
||||
"settlement capture must be a real Commit, not Rollback"
|
||||
);
|
||||
abort_after_fault(&clients[0], &bucket, key, upload_id, false).await?;
|
||||
assert!(
|
||||
roots.iter().all(|root| !root.exists()),
|
||||
"ordinary Abort physically retires all original roots"
|
||||
);
|
||||
assert!(multipart_sources(&cluster, Some(&hash))?.is_empty());
|
||||
for client in &clients {
|
||||
assert_object_bytes_after_fault(client, &bucket, key, &previous).await?;
|
||||
}
|
||||
let protected = if recreate_bucket {
|
||||
clients[0].delete_object().bucket(&bucket).key(key).send().await?;
|
||||
clients[0].delete_bucket().bucket(&bucket).send().await?;
|
||||
clients[1].create_bucket().bucket(&bucket).send().await?;
|
||||
let fresh = payload_for("recreated-before-late-part", 4096);
|
||||
clients[1]
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from(fresh.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
fresh
|
||||
} else {
|
||||
previous
|
||||
};
|
||||
let fresh = clients[2].create_multipart_upload().bucket(&bucket).key(key).send().await?;
|
||||
let fresh_id = fresh.upload_id().ok_or("missing fresh upload ID")?;
|
||||
let fresh_body = payload_for("fresh-active-part", 4096);
|
||||
let fresh_part = clients[2]
|
||||
.upload_part()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.upload_id(fresh_id)
|
||||
.part_number(1)
|
||||
.body(ByteStream::from(fresh_body.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
let before = multipart_sources(&cluster, Some(&hash))?;
|
||||
assert_eq!(before.len(), 4, "the new upload exists on every original participant");
|
||||
for pause in &pauses {
|
||||
pause.release();
|
||||
}
|
||||
wait_until(
|
||||
DEADLINE,
|
||||
|| async {
|
||||
Ok(pauses.iter().all(|pause| {
|
||||
pause.responses() == pause.hits()
|
||||
&& if settles {
|
||||
pause.successful_mutation_replies() == pause.hits() && pause.refused_mutations() == 0
|
||||
} else {
|
||||
pause.refused_mutations() == pause.hits() && pause.successful_mutation_replies() == 0
|
||||
}
|
||||
}))
|
||||
},
|
||||
"every original part packet returns its typed authenticated terminal outcome",
|
||||
)
|
||||
.await?;
|
||||
assert!(roots.iter().all(|root| !root.exists()), "late part packets cannot recreate retired roots");
|
||||
assert!(
|
||||
multipart_sources(&cluster, Some(&hash))? == before,
|
||||
"late packets cannot change a fresh upload's shard or control bytes"
|
||||
);
|
||||
for client in &clients {
|
||||
assert_object_bytes_after_fault(client, &bucket, key, &protected).await?;
|
||||
let listed = client
|
||||
.list_parts()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.upload_id(fresh_id)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(listed.parts().len(), 1);
|
||||
assert_eq!(listed.parts()[0].e_tag(), fresh_part.e_tag());
|
||||
let missing = client
|
||||
.list_parts()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.upload_id(upload_id)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("old upload remains retired");
|
||||
assert_eq!(missing.raw_response().expect("old upload service response").status().as_u16(), 404);
|
||||
assert_eq!(missing.as_service_error().and_then(|error| error.meta().code()), Some("NoSuchUpload"));
|
||||
}
|
||||
clients[2]
|
||||
.complete_multipart_upload()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.upload_id(fresh_id)
|
||||
.multipart_upload(
|
||||
CompletedMultipartUpload::builder()
|
||||
.parts(
|
||||
CompletedPart::builder()
|
||||
.part_number(1)
|
||||
.e_tag(fresh_part.e_tag().ok_or("missing fresh part ETag")?)
|
||||
.build(),
|
||||
)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
for client in &clients {
|
||||
assert_object_bytes_after_fault(client, &bucket, key, &fresh_body).await?;
|
||||
}
|
||||
assert!(multipart_sources(&cluster, Some(&hash))?.is_empty());
|
||||
assert_no_uploads_after_fault(&clients, &bucket).await?;
|
||||
eprintln!(
|
||||
"MPU late part: method={method}, recreate_bucket={recreate_bucket}, retired_roots=4, delivered=2, refused={}, successful_replies={}, fresh_sources_unchanged=4, verified_readers=4",
|
||||
pauses.iter().map(RpcPause::refused_mutations).sum::<usize>(),
|
||||
pauses.iter().map(RpcPause::successful_mutation_replies).sum::<usize>()
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -1,512 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! What one stack answered, and how two answers are compared.
|
||||
//!
|
||||
//! An [`Observation`] keeps only what a client acts on: the HTTP status, the S3 error code, the
|
||||
//! allowlisted response headers and a few typed fields decoded from the response. The error
|
||||
//! message is kept for diagnostics and never compared: message wording is judged by the wire
|
||||
//! corpus, not here. The comparator is pure so it can be tested without a server.
|
||||
|
||||
use serde::Serialize;
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
/// Response headers compared exactly (after [`normalized_header_value`]).
|
||||
const HEADER_ALLOWLIST: [&str; 7] = [
|
||||
"content-type",
|
||||
"x-amz-version-id",
|
||||
"etag",
|
||||
"x-amz-delete-marker",
|
||||
"content-range",
|
||||
"accept-ranges",
|
||||
"x-amz-request-charged",
|
||||
];
|
||||
|
||||
/// Every response header with this prefix is compared as well.
|
||||
const HEADER_ALLOWLIST_PREFIX: &str = "x-amz-server-side-encryption";
|
||||
|
||||
/// Placeholder for a field one side did not send.
|
||||
const ABSENT: &str = "<absent>";
|
||||
|
||||
/// The lowercase header name if the comparator looks at this header.
|
||||
pub(crate) fn compared_header(name: &str) -> Option<String> {
|
||||
let name = name.to_ascii_lowercase();
|
||||
(HEADER_ALLOWLIST.contains(&name.as_str()) || name.starts_with(HEADER_ALLOWLIST_PREFIX)).then_some(name)
|
||||
}
|
||||
|
||||
/// A version ID is minted per process, so its value never matches across two servers; only
|
||||
/// whether it is the literal `null` (an unversioned write) carries meaning.
|
||||
fn normalized_header_value(name: &str, value: &str) -> String {
|
||||
if name == "x-amz-version-id" && value != "null" {
|
||||
"<version-id>".to_owned()
|
||||
} else {
|
||||
value.to_owned()
|
||||
}
|
||||
}
|
||||
|
||||
/// One request's outcome as a client sees it.
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub(crate) struct Observation {
|
||||
/// The S3 operation, named as in the `s3s::S3` trait (`GetObject`).
|
||||
pub op: &'static str,
|
||||
/// Which request of the script this was (`normal`, `negative`, or a setup label).
|
||||
pub case: &'static str,
|
||||
/// HTTP status; 0 when no response arrived.
|
||||
pub status: u16,
|
||||
/// The S3 error code, or the SDK's error kind when there was no service error.
|
||||
pub error_code: Option<String>,
|
||||
/// Allowlisted headers only, lowercase names, normalized values.
|
||||
pub headers: BTreeMap<String, String>,
|
||||
/// Typed values decoded from the response body, never free text.
|
||||
pub fields: BTreeMap<String, String>,
|
||||
/// Diagnostic only; never compared.
|
||||
#[serde(skip)]
|
||||
pub message: Option<String>,
|
||||
}
|
||||
|
||||
impl Observation {
|
||||
/// Builds an observation, keeping only the allowlisted headers.
|
||||
pub(crate) fn new<'h>(
|
||||
op: &'static str,
|
||||
case: &'static str,
|
||||
status: u16,
|
||||
error_code: Option<String>,
|
||||
headers: impl IntoIterator<Item = (&'h str, &'h str)>,
|
||||
) -> Self {
|
||||
let headers = headers
|
||||
.into_iter()
|
||||
.filter_map(|(name, value)| compared_header(name).map(|name| (name, value)))
|
||||
.map(|(name, value)| {
|
||||
let value = normalized_header_value(&name, value);
|
||||
(name, value)
|
||||
})
|
||||
.collect();
|
||||
Self {
|
||||
op,
|
||||
case,
|
||||
status,
|
||||
error_code,
|
||||
headers,
|
||||
fields: BTreeMap::new(),
|
||||
message: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn with_field(mut self, name: &str, value: impl Into<String>) -> Self {
|
||||
self.fields.insert(name.to_owned(), value.into());
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn is_success(&self) -> bool {
|
||||
(200..300).contains(&self.status)
|
||||
}
|
||||
}
|
||||
|
||||
/// One field on which the two stacks answered differently.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
|
||||
pub(crate) struct Difference {
|
||||
pub op: String,
|
||||
pub case: String,
|
||||
pub field: String,
|
||||
pub a: String,
|
||||
pub b: String,
|
||||
}
|
||||
|
||||
/// Every difference between two transcripts of the same script, matched by `(op, case)`.
|
||||
pub(crate) fn compare(a: &[Observation], b: &[Observation]) -> Vec<Difference> {
|
||||
let mut differences = Vec::new();
|
||||
let (index_a, duplicates_a) = index(a);
|
||||
let (index_b, duplicates_b) = index(b);
|
||||
for (op, case) in duplicates_a.union(&duplicates_b) {
|
||||
let count = |side: &[Observation]| {
|
||||
side.iter()
|
||||
.filter(|observation| (observation.op, observation.case) == (*op, *case))
|
||||
.count()
|
||||
.to_string()
|
||||
};
|
||||
differences.push(difference(op, case, "duplicate_case", count(a), count(b)));
|
||||
}
|
||||
let keys: BTreeSet<_> = index_a.keys().chain(index_b.keys()).copied().collect();
|
||||
for key @ (op, case) in keys {
|
||||
match (index_a.get(&key), index_b.get(&key)) {
|
||||
(Some(a), Some(b)) => compare_one(a, b, &mut differences),
|
||||
(a, b) => {
|
||||
let presence = |side: Option<&&Observation>| if side.is_some() { "present" } else { ABSENT }.to_owned();
|
||||
differences.push(difference(op, case, "observation", presence(a), presence(b)));
|
||||
}
|
||||
}
|
||||
}
|
||||
differences
|
||||
}
|
||||
|
||||
type Key = (&'static str, &'static str);
|
||||
|
||||
fn index(side: &[Observation]) -> (BTreeMap<Key, &Observation>, BTreeSet<Key>) {
|
||||
let mut index = BTreeMap::new();
|
||||
let mut duplicates = BTreeSet::new();
|
||||
for observation in side {
|
||||
let key = (observation.op, observation.case);
|
||||
if index.insert(key, observation).is_some() {
|
||||
duplicates.insert(key);
|
||||
}
|
||||
}
|
||||
(index, duplicates)
|
||||
}
|
||||
|
||||
fn difference(op: &str, case: &str, field: &str, a: String, b: String) -> Difference {
|
||||
Difference {
|
||||
op: op.to_owned(),
|
||||
case: case.to_owned(),
|
||||
field: field.to_owned(),
|
||||
a,
|
||||
b,
|
||||
}
|
||||
}
|
||||
|
||||
fn compare_one(a: &Observation, b: &Observation, differences: &mut Vec<Difference>) {
|
||||
let mut field = |name: String, value_a: Option<&str>, value_b: Option<&str>| {
|
||||
if value_a != value_b {
|
||||
let show = |value: Option<&str>| value.unwrap_or(ABSENT).to_owned();
|
||||
differences.push(difference(a.op, a.case, &name, show(value_a), show(value_b)));
|
||||
}
|
||||
};
|
||||
field(
|
||||
"status".to_owned(),
|
||||
Some(a.status.to_string().as_str()),
|
||||
Some(b.status.to_string().as_str()),
|
||||
);
|
||||
field("error_code".to_owned(), a.error_code.as_deref(), b.error_code.as_deref());
|
||||
let headers: BTreeSet<_> = a.headers.keys().chain(b.headers.keys()).collect();
|
||||
for name in headers {
|
||||
field(
|
||||
format!("header.{name}"),
|
||||
a.headers.get(name).map(String::as_str),
|
||||
b.headers.get(name).map(String::as_str),
|
||||
);
|
||||
}
|
||||
let fields: BTreeSet<_> = a.fields.keys().chain(b.fields.keys()).collect();
|
||||
for name in fields {
|
||||
field(
|
||||
format!("body.{name}"),
|
||||
a.fields.get(name).map(String::as_str),
|
||||
b.fields.get(name).map(String::as_str),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Operations observed on both sides.
|
||||
pub(crate) fn compared_ops(a: &[Observation], b: &[Observation]) -> BTreeSet<&'static str> {
|
||||
let ops_b: BTreeSet<_> = b.iter().map(|observation| observation.op).collect();
|
||||
a.iter()
|
||||
.map(|observation| observation.op)
|
||||
.filter(|op| ops_b.contains(op))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// `(op, case)` pairs observed on both sides.
|
||||
pub(crate) fn compared_cases(a: &[Observation], b: &[Observation]) -> usize {
|
||||
let cases_b: BTreeSet<_> = b.iter().map(|observation| (observation.op, observation.case)).collect();
|
||||
a.iter()
|
||||
.map(|observation| (observation.op, observation.case))
|
||||
.collect::<BTreeSet<_>>()
|
||||
.intersection(&cases_b)
|
||||
.count()
|
||||
}
|
||||
|
||||
/// The JSON report written for every run.
|
||||
#[derive(Debug, Serialize)]
|
||||
pub(crate) struct Report {
|
||||
pub schema: u32,
|
||||
pub run: String,
|
||||
pub stack_a: String,
|
||||
pub stack_b: String,
|
||||
pub region_a: Option<String>,
|
||||
pub region_b: Option<String>,
|
||||
pub compared_ops: usize,
|
||||
pub compared_cases: usize,
|
||||
pub differences: Vec<Difference>,
|
||||
}
|
||||
|
||||
impl Report {
|
||||
/// Writes the report as `<directory>/<run>.json` and returns its path.
|
||||
pub(crate) fn write_to(&self, directory: &Path) -> std::io::Result<PathBuf> {
|
||||
std::fs::create_dir_all(directory)?;
|
||||
let path = directory.join(format!("{}.json", self.run));
|
||||
let json = serde_json::to_vec_pretty(self).map_err(std::io::Error::other)?;
|
||||
std::fs::write(&path, json)?;
|
||||
Ok(path)
|
||||
}
|
||||
}
|
||||
|
||||
/// One line per difference, for a failing assertion.
|
||||
pub(crate) fn render(differences: &[Difference]) -> String {
|
||||
differences
|
||||
.iter()
|
||||
.map(|d| format!("({}, {}, {}, a={:?}, b={:?})", d.op, d.case, d.field, d.a, d.b))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn ok(op: &'static str) -> Observation {
|
||||
Observation::new(op, "normal", 200, None, [("content-type", "application/xml")])
|
||||
}
|
||||
|
||||
fn fields(differences: &[Difference]) -> Vec<&str> {
|
||||
differences.iter().map(|d| d.field.as_str()).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identical_transcripts_have_no_difference() {
|
||||
let a = vec![
|
||||
ok("HeadBucket"),
|
||||
ok("GetBucketLocation").with_field("location_constraint", "us-east-1"),
|
||||
];
|
||||
let b = a.clone();
|
||||
assert_eq!(compare(&a, &b), Vec::new());
|
||||
assert_eq!(compared_ops(&a, &b).len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_status_difference_is_reported() {
|
||||
let a = vec![ok("PutBucketAcl")];
|
||||
let b = vec![Observation::new(
|
||||
"PutBucketAcl",
|
||||
"normal",
|
||||
403,
|
||||
None,
|
||||
[("content-type", "application/xml")],
|
||||
)];
|
||||
assert_eq!(
|
||||
compare(&a, &b),
|
||||
vec![Difference {
|
||||
op: "PutBucketAcl".into(),
|
||||
case: "normal".into(),
|
||||
field: "status".into(),
|
||||
a: "200".into(),
|
||||
b: "403".into(),
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_error_code_difference_is_reported() {
|
||||
let a = vec![Observation::new("GetObject", "negative", 404, Some("NoSuchKey".into()), [])];
|
||||
let b = vec![Observation::new(
|
||||
"GetObject",
|
||||
"negative",
|
||||
404,
|
||||
Some("NoSuchBucket".into()),
|
||||
[],
|
||||
)];
|
||||
let differences = compare(&a, &b);
|
||||
assert_eq!(fields(&differences), ["error_code"]);
|
||||
assert_eq!((differences[0].a.as_str(), differences[0].b.as_str()), ("NoSuchKey", "NoSuchBucket"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_error_code_on_one_side_only_is_reported() {
|
||||
let a = vec![Observation::new("HeadBucket", "negative", 404, Some("NotFound".into()), [])];
|
||||
let b = vec![Observation::new("HeadBucket", "negative", 404, None, [])];
|
||||
let differences = compare(&a, &b);
|
||||
assert_eq!(fields(&differences), ["error_code"]);
|
||||
assert_eq!(differences[0].b, ABSENT);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_allowlisted_header_difference_is_reported() {
|
||||
let a = vec![Observation::new("HeadObject", "normal", 200, None, [("ETag", "\"aa\"")])];
|
||||
let b = vec![Observation::new("HeadObject", "normal", 200, None, [("etag", "\"bb\"")])];
|
||||
assert_eq!(fields(&compare(&a, &b)), ["header.etag"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_header_sent_by_one_side_only_is_reported() {
|
||||
let a = vec![Observation::new(
|
||||
"DeleteObject",
|
||||
"normal",
|
||||
204,
|
||||
None,
|
||||
[("x-amz-delete-marker", "true")],
|
||||
)];
|
||||
let b = vec![Observation::new("DeleteObject", "normal", 204, None, [])];
|
||||
let differences = compare(&a, &b);
|
||||
assert_eq!(fields(&differences), ["header.x-amz-delete-marker"]);
|
||||
assert_eq!((differences[0].a.as_str(), differences[0].b.as_str()), ("true", ABSENT));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_server_side_encryption_header_is_compared() {
|
||||
let header = "x-amz-server-side-encryption-customer-key-md5";
|
||||
let a = vec![Observation::new("PutObject", "sse-c", 200, None, [(header, "one")])];
|
||||
let b = vec![Observation::new("PutObject", "sse-c", 200, None, [(header, "two")])];
|
||||
assert_eq!(fields(&compare(&a, &b)), [format!("header.{header}")]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn headers_outside_the_allowlist_are_not_compared() {
|
||||
let a = vec![Observation::new(
|
||||
"ListBuckets",
|
||||
"normal",
|
||||
200,
|
||||
None,
|
||||
[
|
||||
("date", "Mon"),
|
||||
("x-amz-request-id", "1"),
|
||||
("server", "a"),
|
||||
("content-length", "10"),
|
||||
],
|
||||
)];
|
||||
let b = vec![Observation::new(
|
||||
"ListBuckets",
|
||||
"normal",
|
||||
200,
|
||||
None,
|
||||
[
|
||||
("date", "Tue"),
|
||||
("x-amz-request-id", "2"),
|
||||
("server", "b"),
|
||||
("content-length", "11"),
|
||||
],
|
||||
)];
|
||||
assert_eq!(compare(&a, &b), Vec::new());
|
||||
assert!(a[0].headers.is_empty(), "non-allowlisted headers were kept: {:?}", a[0].headers);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn distinct_version_ids_are_not_a_difference() {
|
||||
let a = vec![Observation::new(
|
||||
"PutObject",
|
||||
"normal",
|
||||
200,
|
||||
None,
|
||||
[("x-amz-version-id", "0b1c")],
|
||||
)];
|
||||
let b = vec![Observation::new(
|
||||
"PutObject",
|
||||
"normal",
|
||||
200,
|
||||
None,
|
||||
[("x-amz-version-id", "9f8e")],
|
||||
)];
|
||||
assert_eq!(compare(&a, &b), Vec::new());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_null_version_id_against_a_real_one_is_reported() {
|
||||
let a = vec![Observation::new(
|
||||
"PutObject",
|
||||
"normal",
|
||||
200,
|
||||
None,
|
||||
[("x-amz-version-id", "null")],
|
||||
)];
|
||||
let b = vec![Observation::new(
|
||||
"PutObject",
|
||||
"normal",
|
||||
200,
|
||||
None,
|
||||
[("x-amz-version-id", "9f8e")],
|
||||
)];
|
||||
assert_eq!(fields(&compare(&a, &b)), ["header.x-amz-version-id"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_body_field_difference_is_reported() {
|
||||
let a = vec![ok("GetBucketLocation").with_field("location_constraint", "us-east-1")];
|
||||
let b = vec![ok("GetBucketLocation").with_field("location_constraint", "eu-west-1")];
|
||||
let differences = compare(&a, &b);
|
||||
assert_eq!(fields(&differences), ["body.location_constraint"]);
|
||||
assert_eq!((differences[0].a.as_str(), differences[0].b.as_str()), ("us-east-1", "eu-west-1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_body_field_on_one_side_only_is_reported() {
|
||||
let a = vec![ok("ListObjectsV2").with_field("keys", "[]")];
|
||||
let b = vec![ok("ListObjectsV2")];
|
||||
assert_eq!(fields(&compare(&a, &b)), ["body.keys"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn error_messages_are_never_compared() {
|
||||
let mut a = Observation::new("GetObject", "negative", 404, Some("NoSuchKey".into()), []);
|
||||
let mut b = a.clone();
|
||||
a.message = Some("The specified key does not exist.".into());
|
||||
b.message = Some("Object not found".into());
|
||||
assert_eq!(compare(&[a], &[b]), Vec::new());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_case_observed_on_one_side_only_is_reported() {
|
||||
let a = vec![ok("HeadBucket"), ok("ListBuckets")];
|
||||
let b = vec![ok("HeadBucket")];
|
||||
let differences = compare(&a, &b);
|
||||
assert_eq!(fields(&differences), ["observation"]);
|
||||
assert_eq!(differences[0].op, "ListBuckets");
|
||||
assert_eq!(compared_ops(&a, &b).into_iter().collect::<Vec<_>>(), ["HeadBucket"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_case_recorded_twice_is_reported() {
|
||||
let a = vec![ok("HeadBucket"), ok("HeadBucket")];
|
||||
let b = vec![ok("HeadBucket")];
|
||||
assert_eq!(fields(&compare(&a, &b)), ["duplicate_case"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compared_ops_counts_operations_not_cases() {
|
||||
let negative = Observation::new("HeadBucket", "negative", 404, Some("NotFound".into()), []);
|
||||
let a = vec![ok("HeadBucket"), negative, ok("ListBuckets")];
|
||||
let b = a.clone();
|
||||
assert_eq!(compared_ops(&a, &b).len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_report_lists_each_difference_as_op_field_a_b() {
|
||||
let directory = std::env::temp_dir().join(format!("dual-stack-report-{}", uuid::Uuid::new_v4()));
|
||||
let report = Report {
|
||||
schema: 1,
|
||||
run: "unit".into(),
|
||||
stack_a: "legacy".into(),
|
||||
stack_b: "legacy".into(),
|
||||
region_a: None,
|
||||
region_b: Some("eu-west-1".into()),
|
||||
compared_ops: 1,
|
||||
compared_cases: 1,
|
||||
differences: compare(
|
||||
&[ok("GetBucketLocation").with_field("location_constraint", "us-east-1")],
|
||||
&[ok("GetBucketLocation").with_field("location_constraint", "eu-west-1")],
|
||||
),
|
||||
};
|
||||
let path = report.write_to(&directory).expect("write the report");
|
||||
let json: serde_json::Value = serde_json::from_slice(&std::fs::read(&path).expect("read the report")).expect("parse");
|
||||
let _ = std::fs::remove_dir_all(&directory);
|
||||
assert_eq!(path.file_name().and_then(|name| name.to_str()), Some("unit.json"));
|
||||
assert_eq!(json["compared_ops"], 1);
|
||||
assert_eq!(
|
||||
json["differences"],
|
||||
serde_json::json!([{
|
||||
"op": "GetBucketLocation",
|
||||
"case": "normal",
|
||||
"field": "body.location_constraint",
|
||||
"a": "us-east-1",
|
||||
"b": "eu-west-1",
|
||||
}])
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,199 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Dual-stack differential: one `rustfs` binary, two processes, one SDK script (rustfs/backlog#2740).
|
||||
//!
|
||||
//! Stack A always runs `RUSTFS_S3_STACK=legacy`. Stack B runs the value of `RUSTFS_E2E_STACK_B`
|
||||
//! (default `legacy` until the gateway stack is assembled), and `RUSTFS_E2E_STACK_B_REGION`, when
|
||||
//! set, starts B in another region so that a difference can be injected by hand. Both servers
|
||||
//! answer the script in `scenarios`, which covers every operation of `impl S3 for FS`; `compare`
|
||||
//! reports each `(op, field, a, b)` on which the answers differ, and every run writes its report
|
||||
//! to `<target>/dual-stack/<run>.json`. Two processes, not two embedded servers: bucket metadata
|
||||
//! is process-global, so two stacks in one process would read each other's buckets.
|
||||
|
||||
mod compare;
|
||||
mod scenarios;
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, workspace_root};
|
||||
use compare::{Difference, Observation, Report, compare, compared_cases, compared_ops, render};
|
||||
use std::path::PathBuf;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
type BoxError = Box<dyn std::error::Error + Send + Sync>;
|
||||
|
||||
const STACK_A: &str = "legacy";
|
||||
const ENV_STACK_B: &str = "RUSTFS_E2E_STACK_B";
|
||||
const ENV_STACK_B_REGION: &str = "RUSTFS_E2E_STACK_B_REGION";
|
||||
/// Becomes `gateway` once rustfs/backlog#2734 T2.5 assembles the gateway stack.
|
||||
const DEFAULT_STACK_B: &str = "legacy";
|
||||
/// The region the negative control gives stack A explicitly, so an inherited `RUSTFS_REGION`
|
||||
/// cannot erase the injected difference. Not `us-east-1`: that region answers GetBucketLocation
|
||||
/// with an empty LocationConstraint and re-creating an owned bucket with 200, so it would not
|
||||
/// report its own name and would add CreateBucket differences to the control.
|
||||
const CONTROL_REGION_A: &str = "us-west-2";
|
||||
const CONTROL_REGION_B: &str = "eu-west-1";
|
||||
/// `normal` cases the legacy stack must answer with a 2xx. The comparison alone cannot tell a
|
||||
/// script that exercises real behavior from one in which both stacks fail the same way. Five of
|
||||
/// the 73 fail on legacy by design of this script: PutBucketReplication (no registered remote
|
||||
/// target), GetBucketReplication (so no configuration), PutBucketNotificationConfiguration (no
|
||||
/// notification target configured), RestoreObject (the object is not archived) and
|
||||
/// GetObjectTorrent (legacy answers `NoSuchKey`).
|
||||
const MIN_NORMAL_SUCCESSES: usize = 68;
|
||||
|
||||
struct PairConfig {
|
||||
stack_b: String,
|
||||
region_a: Option<String>,
|
||||
region_b: Option<String>,
|
||||
}
|
||||
|
||||
impl PairConfig {
|
||||
fn from_env() -> Self {
|
||||
Self {
|
||||
stack_b: std::env::var(ENV_STACK_B).unwrap_or_else(|_| DEFAULT_STACK_B.to_owned()),
|
||||
region_a: None,
|
||||
region_b: std::env::var(ENV_STACK_B_REGION).ok(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Starts one server on `stack`, overriding its region only when asked.
|
||||
async fn start_stack(stack: &str, region: Option<&str>) -> Result<RustFSTestEnvironment, BoxError> {
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
let mut vars = vec![("RUSTFS_S3_STACK", stack)];
|
||||
if let Some(region) = region {
|
||||
vars.push(("RUSTFS_REGION", region));
|
||||
}
|
||||
env.start_rustfs_server_without_cleanup_with_env(&vars).await?;
|
||||
Ok(env)
|
||||
}
|
||||
|
||||
/// Starts stack A and stack B side by side, each with its own port and data directory.
|
||||
async fn spawn_pair(config: &PairConfig) -> Result<(RustFSTestEnvironment, RustFSTestEnvironment), BoxError> {
|
||||
tokio::try_join!(
|
||||
start_stack(STACK_A, config.region_a.as_deref()),
|
||||
start_stack(&config.stack_b, config.region_b.as_deref())
|
||||
)
|
||||
}
|
||||
|
||||
fn report_directory() -> PathBuf {
|
||||
let target = std::env::var_os("CARGO_TARGET_DIR").map_or_else(|| PathBuf::from("target"), PathBuf::from);
|
||||
let target = if target.is_absolute() {
|
||||
target
|
||||
} else {
|
||||
workspace_root().join(target)
|
||||
};
|
||||
target.join("dual-stack")
|
||||
}
|
||||
|
||||
struct Run {
|
||||
report: Report,
|
||||
path: PathBuf,
|
||||
a: Vec<Observation>,
|
||||
}
|
||||
|
||||
/// Runs the script against both stacks concurrently and writes the report.
|
||||
async fn run_differential(label: &str, config: &PairConfig) -> Run {
|
||||
let (server_a, server_b) = spawn_pair(config).await.expect("start stack A and stack B");
|
||||
let (a, b) = tokio::join!(Box::pin(scenarios::run(&server_a)), Box::pin(scenarios::run(&server_b)));
|
||||
let millis = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map_or(0, |elapsed| elapsed.as_millis());
|
||||
let report = Report {
|
||||
schema: 1,
|
||||
run: format!("{label}-{millis}-{}", std::process::id()),
|
||||
stack_a: STACK_A.to_owned(),
|
||||
stack_b: config.stack_b.clone(),
|
||||
region_a: config.region_a.clone(),
|
||||
region_b: config.region_b.clone(),
|
||||
compared_ops: compared_ops(&a, &b).len(),
|
||||
compared_cases: compared_cases(&a, &b),
|
||||
differences: compare(&a, &b),
|
||||
};
|
||||
let path = report.write_to(&report_directory()).expect("write the dual-stack report");
|
||||
Run { report, path, a }
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn both_stacks_answer_every_operation_identically() {
|
||||
init_logging();
|
||||
let config = PairConfig::from_env();
|
||||
let run = run_differential("self-comparison", &config).await;
|
||||
let report = &run.report;
|
||||
|
||||
let mut observed: Vec<_> = run.a.iter().map(|observation| observation.op).collect();
|
||||
observed.sort_unstable();
|
||||
observed.dedup();
|
||||
assert_eq!(
|
||||
observed,
|
||||
scenarios::OPERATIONS,
|
||||
"the script must drive exactly the operations of impl S3 for FS"
|
||||
);
|
||||
assert_eq!(report.compared_ops, scenarios::OPERATIONS.len(), "report: {}", run.path.display());
|
||||
assert!(
|
||||
report.differences.is_empty(),
|
||||
"stack {} and stack {} differ on {} of {} cases (report: {}):\n{}",
|
||||
report.stack_a,
|
||||
report.stack_b,
|
||||
report.differences.len(),
|
||||
report.compared_cases,
|
||||
run.path.display(),
|
||||
render(&report.differences)
|
||||
);
|
||||
|
||||
let not_4xx: Vec<_> = run
|
||||
.a
|
||||
.iter()
|
||||
.filter(|observation| observation.case == "negative" && !(400..500).contains(&observation.status))
|
||||
.collect();
|
||||
assert!(not_4xx.is_empty(), "negative cases must fail with a 4xx on {STACK_A}: {not_4xx:#?}");
|
||||
let successes = run
|
||||
.a
|
||||
.iter()
|
||||
.filter(|observation| observation.case == "normal" && observation.is_success())
|
||||
.count();
|
||||
assert!(
|
||||
successes >= MIN_NORMAL_SUCCESSES,
|
||||
"only {successes} normal cases succeeded on {STACK_A}: {:#?}",
|
||||
run.a
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn differential_detects_an_injected_difference() {
|
||||
init_logging();
|
||||
let config = PairConfig {
|
||||
stack_b: STACK_A.to_owned(),
|
||||
region_a: Some(CONTROL_REGION_A.to_owned()),
|
||||
region_b: Some(CONTROL_REGION_B.to_owned()),
|
||||
};
|
||||
let run = run_differential("injected-region", &config).await;
|
||||
let expected = Difference {
|
||||
op: "GetBucketLocation".to_owned(),
|
||||
case: "normal".to_owned(),
|
||||
field: "body.location_constraint".to_owned(),
|
||||
a: CONTROL_REGION_A.to_owned(),
|
||||
b: CONTROL_REGION_B.to_owned(),
|
||||
};
|
||||
assert!(
|
||||
run.report.differences.contains(&expected),
|
||||
"B started in {CONTROL_REGION_B} but GetBucketLocation was not reported (report: {}):\n{}",
|
||||
run.path.display(),
|
||||
render(&run.report.differences)
|
||||
);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -31,12 +31,6 @@ pub mod chaos;
|
||||
#[cfg(test)]
|
||||
pub mod fake_s3_target;
|
||||
|
||||
// Dual-stack differential (rustfs/backlog#2740): one binary, two processes on
|
||||
// different RUSTFS_S3_STACK values, the same SDK script for every S3 operation,
|
||||
// compared on status, error code, allowlisted headers and typed body fields.
|
||||
#[cfg(test)]
|
||||
mod dual_stack_differential;
|
||||
|
||||
// On-demand migration (backlog#2147): shared two-server environment, admin
|
||||
// wrappers, and the harness self-test (backlog#2151). Behavior scenarios are
|
||||
// added by later ODM tasks.
|
||||
|
||||
@@ -32,9 +32,7 @@
|
||||
//! Advisory: <https://github.com/rustfs/rustfs/security/advisories/GHSA-3p3x-734c-h5vx>
|
||||
|
||||
use crate::common::rustfs_binary_path_with_features;
|
||||
use crate::common::{
|
||||
AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, signed_s3_request,
|
||||
};
|
||||
use crate::common::{AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via};
|
||||
use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, admin_request};
|
||||
use crate::protocols::test_env::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, ProtocolTestEnvironment};
|
||||
use anyhow::Result;
|
||||
@@ -776,81 +774,6 @@ async fn test_webdav_core_operations_direct() -> Result<()> {
|
||||
test_webdav_core_operations().await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_webdav_content_type_directory_marker() -> Result<()> {
|
||||
let env = ProtocolTestEnvironment::new().map_err(|e| anyhow::anyhow!("{}", e))?;
|
||||
let admin_base_url = format!("http://{}", S3_TEST_ADDRESS);
|
||||
let binary_path = rustfs_binary_path_with_features(Some("webdav"));
|
||||
let mut server_process = Command::new(&binary_path)
|
||||
.arg("--address")
|
||||
.arg(S3_TEST_ADDRESS)
|
||||
.env("RUSTFS_CONSOLE_ENABLE", "false")
|
||||
.env("RUSTFS_WEBDAV_ENABLE", "true")
|
||||
.env("RUSTFS_WEBDAV_ADDRESS", WEBDAV_ADDRESS)
|
||||
.env("RUSTFS_WEBDAV_TLS_ENABLED", "false")
|
||||
.arg(&env.temp_dir)
|
||||
.spawn()?;
|
||||
|
||||
let result = async {
|
||||
ProtocolTestEnvironment::wait_for_port_ready(WEBDAV_PORT, 30)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("{}", e))?;
|
||||
|
||||
let client = create_client();
|
||||
let bucket_name = "webdav-content-type-bucket";
|
||||
let directory_name = "content-type-directory";
|
||||
let webdav_base_url = format!("http://{}", WEBDAV_ADDRESS);
|
||||
let mkcol_bucket = client
|
||||
.request(
|
||||
reqwest::Method::from_bytes(b"MKCOL").expect("MKCOL is a valid HTTP method"),
|
||||
format!("{}/{}", webdav_base_url, bucket_name),
|
||||
)
|
||||
.header("Authorization", basic_auth_header())
|
||||
.send()
|
||||
.await?;
|
||||
anyhow::ensure!(
|
||||
mkcol_bucket.status().is_success() || mkcol_bucket.status().as_u16() == 201,
|
||||
"MKCOL bucket should succeed, got: {}",
|
||||
mkcol_bucket.status()
|
||||
);
|
||||
|
||||
let mkcol_directory = client
|
||||
.request(
|
||||
reqwest::Method::from_bytes(b"MKCOL").expect("MKCOL is a valid HTTP method"),
|
||||
format!("{}/{}/{}", webdav_base_url, bucket_name, directory_name),
|
||||
)
|
||||
.header("Authorization", basic_auth_header())
|
||||
.send()
|
||||
.await?;
|
||||
anyhow::ensure!(
|
||||
mkcol_directory.status().is_success() || mkcol_directory.status().as_u16() == 201,
|
||||
"MKCOL directory should succeed, got: {}",
|
||||
mkcol_directory.status()
|
||||
);
|
||||
|
||||
let head_url = format!("{}/{}/{}/", admin_base_url, bucket_name, directory_name);
|
||||
let head = signed_s3_request(http::Method::HEAD, &head_url, None, None, DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY)
|
||||
.await
|
||||
.map_err(|error| anyhow::anyhow!("{error}"))?;
|
||||
anyhow::ensure!(head.status().is_success(), "HEAD directory marker should succeed, got: {}", head.status());
|
||||
let content_type = head
|
||||
.headers()
|
||||
.get("content-type")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.unwrap_or_default();
|
||||
anyhow::ensure!(
|
||||
content_type == "application/x-directory",
|
||||
"WebDAV directory marker must preserve its content type, got: {content_type}"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
.await;
|
||||
|
||||
let _ = server_process.kill().await;
|
||||
let _ = server_process.wait().await;
|
||||
result
|
||||
}
|
||||
|
||||
/// Extract the numeric value of the first occurrence of a DAV property in a
|
||||
/// PROPFIND multistatus body. Namespace-prefix agnostic; a self-closing
|
||||
/// (unreported) property yields None.
|
||||
|
||||
@@ -27,8 +27,8 @@ use hyper_util::{
|
||||
};
|
||||
use prost::Message as _;
|
||||
use rustfs_protos::proto_gen::node_service::{
|
||||
BatchGenerallyLockRequest, DeleteRequest, GenerallyLockRequest, ReadVersionRequest, RecoverMultipartSourceRequest,
|
||||
RenameDataResponse,
|
||||
BatchGenerallyLockRequest, DeleteRequest, GenerallyLockRequest, PreparePartTransactionResponse, ReadVersionRequest,
|
||||
RecoverMultipartSourceRequest, RenameDataResponse, SettlePartTransactionRequest, SettlePartTransactionResponse,
|
||||
};
|
||||
use std::{
|
||||
collections::HashSet,
|
||||
@@ -56,6 +56,35 @@ enum Disposition {
|
||||
Discard,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum MutationReply {
|
||||
None,
|
||||
Rename,
|
||||
PreparePart,
|
||||
SettlePart,
|
||||
}
|
||||
|
||||
impl MutationReply {
|
||||
fn outcome(self, bytes: &[u8]) -> io::Result<Option<(bool, bool)>> {
|
||||
let result = match self {
|
||||
Self::None => return Ok(None),
|
||||
Self::Rename => {
|
||||
let reply = RenameDataResponse::decode(unary_payload(bytes)?).map_err(io::Error::other)?;
|
||||
(reply.success, reply.error.is_some())
|
||||
}
|
||||
Self::PreparePart => {
|
||||
let reply = PreparePartTransactionResponse::decode(unary_payload(bytes)?).map_err(io::Error::other)?;
|
||||
(reply.success, reply.error.is_some())
|
||||
}
|
||||
Self::SettlePart => {
|
||||
let reply = SettlePartTransactionResponse::decode(unary_payload(bytes)?).map_err(io::Error::other)?;
|
||||
(reply.success, reply.error.is_some())
|
||||
}
|
||||
};
|
||||
Ok(Some(result))
|
||||
}
|
||||
}
|
||||
|
||||
struct PauseState {
|
||||
methods: Vec<&'static str>,
|
||||
scope: Option<Vec<u8>>,
|
||||
@@ -64,6 +93,9 @@ struct PauseState {
|
||||
limit: usize,
|
||||
retain_after_disconnect: bool,
|
||||
refused_mutations: AtomicUsize,
|
||||
successful_mutation_replies: AtomicUsize,
|
||||
settle_commits: AtomicUsize,
|
||||
settle_rollbacks: AtomicUsize,
|
||||
responses: AtomicUsize,
|
||||
matched: AtomicUsize,
|
||||
blocked: AtomicUsize,
|
||||
@@ -122,6 +154,17 @@ impl RpcPause {
|
||||
self.state.refused_mutations.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
pub(crate) fn successful_mutation_replies(&self) -> usize {
|
||||
self.state.successful_mutation_replies.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
pub(crate) fn retained_settle_actions(&self) -> (usize, usize) {
|
||||
(
|
||||
self.state.settle_commits.load(Ordering::SeqCst),
|
||||
self.state.settle_rollbacks.load(Ordering::SeqCst),
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn unsupported_source_inspection_routes(&self) -> HashSet<String> {
|
||||
self.state
|
||||
.unsupported_inspection_routes
|
||||
@@ -394,6 +437,9 @@ impl RpcFaultProxy {
|
||||
limit,
|
||||
retain_after_disconnect,
|
||||
refused_mutations: AtomicUsize::new(0),
|
||||
successful_mutation_replies: AtomicUsize::new(0),
|
||||
settle_commits: AtomicUsize::new(0),
|
||||
settle_rollbacks: AtomicUsize::new(0),
|
||||
responses: AtomicUsize::new(0),
|
||||
matched: AtomicUsize::new(0),
|
||||
blocked: AtomicUsize::new(0),
|
||||
@@ -509,6 +555,7 @@ async fn forward_request(
|
||||
if let Some(method) = method.filter(|name| pause.methods.contains(name)) {
|
||||
let mut route = None;
|
||||
let mut lock_owners = Vec::new();
|
||||
let mut settle_action = None;
|
||||
let matches = if let Some(scope) = pause.scope.as_deref() {
|
||||
// Inspect bounded control routing while preserving the exact
|
||||
// signed bytes and trailers forwarded to the backend.
|
||||
@@ -522,12 +569,19 @@ async fn forward_request(
|
||||
inspection = Some((pause.clone(), disk));
|
||||
}
|
||||
let mut matches = scope_matches(method, &bytes, scope)?;
|
||||
if matches && method == "SettlePartTransaction" {
|
||||
settle_action = Some(
|
||||
SettlePartTransactionRequest::decode(unary_payload(&bytes)?)
|
||||
.map_err(io::Error::other)?
|
||||
.rollback,
|
||||
);
|
||||
}
|
||||
if matches && (pause.record_lock_owners || pause.lock_owner.is_some()) {
|
||||
lock_owners = scoped_lock_owners(method, &bytes, scope)?;
|
||||
if let Some(owner) = &pause.lock_owner {
|
||||
matches = lock_owners.iter().any(|candidate| candidate == owner);
|
||||
if !matches && pause.record_lock_owners {
|
||||
observed = Some((pause.clone(), None, std::mem::take(&mut lock_owners), false));
|
||||
observed = Some((pause.clone(), None, std::mem::take(&mut lock_owners), MutationReply::None));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -549,6 +603,14 @@ async fn forward_request(
|
||||
true
|
||||
};
|
||||
if matches && pause.matched.fetch_add(1, Ordering::SeqCst) < pause.limit {
|
||||
if let Some(rollback) = settle_action {
|
||||
let counter = if rollback {
|
||||
&pause.settle_rollbacks
|
||||
} else {
|
||||
&pause.settle_commits
|
||||
};
|
||||
counter.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
let mut released = pause.release.subscribe();
|
||||
pause.blocked.fetch_add(1, Ordering::SeqCst);
|
||||
pause.reached.notify_one();
|
||||
@@ -568,10 +630,14 @@ async fn forward_request(
|
||||
pause,
|
||||
route,
|
||||
lock_owners,
|
||||
matches!(
|
||||
method,
|
||||
"RenameData" | "RenameMultipartData" | "RenameDataAtIncarnation" | "RenameDataWithQuotaFence"
|
||||
),
|
||||
match method {
|
||||
"RenameData" | "RenameMultipartData" | "RenameDataAtIncarnation" | "RenameDataWithQuotaFence" => {
|
||||
MutationReply::Rename
|
||||
}
|
||||
"PreparePartTransaction" => MutationReply::PreparePart,
|
||||
"SettlePartTransaction" => MutationReply::SettlePart,
|
||||
_ => MutationReply::None,
|
||||
},
|
||||
));
|
||||
}
|
||||
}
|
||||
@@ -604,7 +670,7 @@ async fn forward_request(
|
||||
.expect("inspection response routes")
|
||||
.insert(disk);
|
||||
}
|
||||
if let Some((pause, route, lock_owners, rename)) = observed {
|
||||
if let Some((pause, route, lock_owners, mutation_reply)) = observed {
|
||||
pause.responses.fetch_add(1, Ordering::SeqCst);
|
||||
if response.status().is_success() && response.headers().get("grpc-status").is_none_or(|status| status == "0") {
|
||||
pause.forwarded.fetch_add(1, Ordering::SeqCst);
|
||||
@@ -626,10 +692,14 @@ async fn forward_request(
|
||||
.headers
|
||||
.get("grpc-status")
|
||||
.or_else(|| trailers.as_ref()?.get("grpc-status"));
|
||||
if rename && parts.status.is_success() && status.is_none_or(|status| status == "0") {
|
||||
let result = RenameDataResponse::decode(unary_payload(&bytes)?).map_err(io::Error::other)?;
|
||||
if !result.success && result.error.is_some() {
|
||||
if parts.status.is_success()
|
||||
&& status.is_some_and(|status| status == "0")
|
||||
&& let Some((success, has_error)) = mutation_reply.outcome(&bytes)?
|
||||
{
|
||||
if !success && has_error {
|
||||
pause.refused_mutations.fetch_add(1, Ordering::SeqCst);
|
||||
} else if success && !has_error {
|
||||
pause.successful_mutation_replies.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
let body = Full::new(bytes)
|
||||
@@ -676,6 +746,10 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn backend_with_status(status: Option<&'static str>) -> io::Result<Backend> {
|
||||
backend_with_reply(status, None).await
|
||||
}
|
||||
|
||||
async fn backend_with_reply(status: Option<&'static str>, reply: Option<Bytes>) -> io::Result<Backend> {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await?;
|
||||
let address = listener.local_addr()?;
|
||||
let received = Arc::new(AtomicUsize::new(0));
|
||||
@@ -687,9 +761,11 @@ mod tests {
|
||||
accepted = listener.accept() => {
|
||||
let Ok((socket, _)) = accepted else { break };
|
||||
let received = received_task.clone();
|
||||
let reply = reply.clone();
|
||||
children.spawn(async move {
|
||||
let service = service_fn(move |request: Request<Incoming>| {
|
||||
let received = received.clone();
|
||||
let reply = reply.clone();
|
||||
async move {
|
||||
received.fetch_add(1, Ordering::SeqCst);
|
||||
let authority = request.headers().get(http::header::HOST).cloned().or_else(|| {
|
||||
@@ -698,6 +774,7 @@ mod tests {
|
||||
})
|
||||
});
|
||||
let body = request.into_body().collect().await?.to_bytes();
|
||||
let body = reply.unwrap_or(body);
|
||||
let mut trailers = http::HeaderMap::new();
|
||||
trailers.insert("grpc-status", http::HeaderValue::from_static(status.unwrap_or("0")));
|
||||
trailers.insert("x-proof", http::HeaderValue::from_static("unchanged-proof"));
|
||||
@@ -1434,6 +1511,93 @@ mod tests {
|
||||
.await
|
||||
.expect("scoped discard deadline");
|
||||
assert_late_packet_ownership().await;
|
||||
assert_late_part_reply_refusal().await;
|
||||
}
|
||||
|
||||
async fn assert_late_part_reply_refusal() {
|
||||
use rustfs_protos::proto_gen::node_service::{Error, PreparePartTransactionRequest};
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
// method, success, error, gRPC status, rollback, expected successes/refusals
|
||||
for (method, success, has_error, status, rollback, expected) in [
|
||||
("PreparePartTransaction", false, true, None, false, (0, 1)),
|
||||
("PreparePartTransaction", true, false, None, false, (1, 0)),
|
||||
("PreparePartTransaction", false, false, None, false, (0, 0)),
|
||||
("SettlePartTransaction", true, false, None, false, (1, 0)),
|
||||
("SettlePartTransaction", false, true, None, true, (0, 1)),
|
||||
("SettlePartTransaction", true, true, None, false, (0, 0)),
|
||||
("SettlePartTransaction", false, true, Some("16"), false, (0, 0)),
|
||||
("PreparePartTransaction", false, true, Some("16"), false, (0, 0)),
|
||||
("RenameMultipartData", false, true, None, false, (0, 1)),
|
||||
] {
|
||||
let error = has_error.then(|| Error {
|
||||
code: 1,
|
||||
error_info: "retired source".to_owned(),
|
||||
});
|
||||
let reply = match method {
|
||||
"PreparePartTransaction" => unary_frame(&PreparePartTransactionResponse { success, error }),
|
||||
"SettlePartTransaction" => unary_frame(&SettlePartTransactionResponse { success, error }),
|
||||
_ => unary_frame(&RenameDataResponse {
|
||||
success,
|
||||
error,
|
||||
..Default::default()
|
||||
}),
|
||||
};
|
||||
// An unknown field must survive typed inspection byte-for-byte.
|
||||
let mut framed = reply.to_vec();
|
||||
framed.extend_from_slice(&[0xfa, 0x03, 0x03, b'x', b'y', b'z']);
|
||||
let payload_len = u32::try_from(framed.len() - 5).expect("small reply");
|
||||
framed[1..5].copy_from_slice(&payload_len.to_be_bytes());
|
||||
let reply = Bytes::from(framed);
|
||||
let backend = backend_with_reply(status, Some(reply.clone()))
|
||||
.await
|
||||
.expect("typed reply backend");
|
||||
let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
|
||||
.await
|
||||
.expect("proxy");
|
||||
let pause = proxy.pause_late(&[method], 1, "late-upload").expect("part gate");
|
||||
let request = match method {
|
||||
"PreparePartTransaction" => unary_frame(&PreparePartTransactionRequest {
|
||||
disk: "disk".to_owned(),
|
||||
dst_path: "late-upload/part.1".to_owned(),
|
||||
..Default::default()
|
||||
}),
|
||||
"SettlePartTransaction" => unary_frame(&SettlePartTransactionRequest {
|
||||
disk: "disk".to_owned(),
|
||||
path: "late-upload/part.1".to_owned(),
|
||||
rollback,
|
||||
..Default::default()
|
||||
}),
|
||||
_ => Bytes::from_static(b"late-upload"),
|
||||
};
|
||||
let address = proxy.local_addr();
|
||||
let caller = tokio::spawn(async move {
|
||||
let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
|
||||
let response = call_body(&client, address, method, request).await.expect("retained reply");
|
||||
response.into_body().collect().await.expect("typed response frames")
|
||||
});
|
||||
pause.wait_for_hits(1).await.expect("actual retained request");
|
||||
pause.release();
|
||||
let response = caller.await.expect("caller task");
|
||||
let trailers = response.trailers().expect("reply trailers");
|
||||
assert_eq!(trailers["x-proof"], "unchanged-proof");
|
||||
assert_eq!(trailers["grpc-status"], status.unwrap_or("0"));
|
||||
assert_eq!(response.to_bytes(), reply);
|
||||
assert_eq!(pause.responses(), 1);
|
||||
assert_eq!(
|
||||
(pause.successful_mutation_replies(), pause.refused_mutations()),
|
||||
expected,
|
||||
"typed {method} outcome"
|
||||
);
|
||||
let actions = if method == "SettlePartTransaction" {
|
||||
if rollback { (0, 1) } else { (1, 0) }
|
||||
} else {
|
||||
(0, 0)
|
||||
};
|
||||
assert_eq!(pause.retained_settle_actions(), actions);
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("part reply deadline");
|
||||
}
|
||||
async fn assert_late_packet_ownership() {
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
|
||||
@@ -600,6 +600,142 @@ mod tests {
|
||||
info!("✅ PASSED: Veeam backup workflow simulation completed successfully");
|
||||
}
|
||||
|
||||
/// Veeam B&R discovers SOSAPI by GET/HEAD of two well-known keys.
|
||||
#[tokio::test]
|
||||
async fn test_veeam_sosapi_system_and_capacity() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
use crate::common::local_http_client;
|
||||
|
||||
const SYSTEM_KEY: &str = ".system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml";
|
||||
const CAPACITY_KEY: &str = ".system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/capacity.xml";
|
||||
|
||||
init_logging();
|
||||
info!("🧪 TEST: Veeam SOSAPI system.xml and capacity.xml");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
let client = create_s3_client(&env);
|
||||
let bucket = "veeam-sosapi";
|
||||
create_bucket(&client, bucket).await?;
|
||||
|
||||
let system = client.get_object().bucket(bucket).key(SYSTEM_KEY).send().await?;
|
||||
let system_etag = system.e_tag().map(str::to_string);
|
||||
let system_len = system.content_length();
|
||||
assert_eq!(system.content_type(), Some("application/xml"));
|
||||
let system_body = system.body.collect().await?.into_bytes();
|
||||
let system_xml = std::str::from_utf8(&system_body)?;
|
||||
assert!(system_xml.contains("<ProtocolVersion>\"1.0\"</ProtocolVersion>"), "{system_xml}");
|
||||
assert!(
|
||||
system_xml.contains("<ModelName>\"RustFS ") && system_xml.contains("\"</ModelName>"),
|
||||
"{system_xml}"
|
||||
);
|
||||
assert!(system_xml.contains("<CapacityInfo>true</CapacityInfo>"));
|
||||
assert!(system_xml.contains("<UploadSessions>false</UploadSessions>"));
|
||||
assert!(system_xml.contains("<IAMSTS>false</IAMSTS>"));
|
||||
assert!(!system_xml.contains("SystemRecommendations"));
|
||||
assert!(!system_xml.contains("APIEndpoints"));
|
||||
assert!(!system_xml.contains("""));
|
||||
|
||||
let head = client.head_object().bucket(bucket).key(SYSTEM_KEY).send().await?;
|
||||
assert_eq!(head.content_type(), Some("application/xml"));
|
||||
assert_eq!(head.content_length(), system_len);
|
||||
assert_eq!(head.e_tag().map(str::to_string), system_etag);
|
||||
assert!(head.last_modified().is_some());
|
||||
|
||||
let ranged = client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(SYSTEM_KEY)
|
||||
.range("bytes=0-3")
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(ranged.content_length(), Some(4));
|
||||
let ranged_header = ranged.content_range().unwrap_or("");
|
||||
assert!(ranged_header.starts_with("bytes 0-3/"), "{ranged_header}");
|
||||
let ranged_body = ranged.body.collect().await?.into_bytes();
|
||||
assert_eq!(&ranged_body[..], &system_body[..4]);
|
||||
|
||||
let capacity = client.get_object().bucket(bucket).key(CAPACITY_KEY).send().await?;
|
||||
assert_eq!(capacity.content_type(), Some("application/xml"));
|
||||
let capacity_body = capacity.body.collect().await?.into_bytes();
|
||||
let capacity_xml = std::str::from_utf8(&capacity_body)?;
|
||||
let capacity_value = xml_u64(capacity_xml, "Capacity")?;
|
||||
let available_value = xml_u64(capacity_xml, "Available")?;
|
||||
let used_value = xml_u64(capacity_xml, "Used")?;
|
||||
assert!(used_value <= capacity_value, "{capacity_xml}");
|
||||
assert!(available_value <= capacity_value.saturating_sub(used_value), "{capacity_xml}");
|
||||
|
||||
let put_err = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(SYSTEM_KEY)
|
||||
.body(ByteStream::from_static(b"overwrite"))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("SOSAPI put must fail");
|
||||
assert_eq!(put_err.code(), Some("MethodNotAllowed"));
|
||||
let delete_err = client
|
||||
.delete_object()
|
||||
.bucket(bucket)
|
||||
.key(CAPACITY_KEY)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("SOSAPI delete must fail");
|
||||
assert_eq!(delete_err.code(), Some("MethodNotAllowed"));
|
||||
|
||||
let normal_key = "Veeam/Backup/job.dat";
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(normal_key)
|
||||
.body(ByteStream::from_static(b"backup-bytes"))
|
||||
.send()
|
||||
.await?;
|
||||
let normal = client.get_object().bucket(bucket).key(normal_key).send().await?;
|
||||
let normal_body = normal.body.collect().await?.into_bytes();
|
||||
assert_eq!(&normal_body[..], b"backup-bytes");
|
||||
|
||||
let anon = local_http_client()
|
||||
.get(format!("{}/{bucket}/{SYSTEM_KEY}", env.url))
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(anon.status(), reqwest::StatusCode::FORBIDDEN);
|
||||
let anon_body = anon.text().await?;
|
||||
assert!(!anon_body.contains("ProtocolVersion"), "{anon_body}");
|
||||
|
||||
let now = time::OffsetDateTime::now_utc();
|
||||
let scope_date = format!("{:04}{:02}{:02}", now.year(), u8::from(now.month()), now.day());
|
||||
let amz_date = format!("{scope_date}T{:02}{:02}{:02}Z", now.hour(), now.minute(), now.second());
|
||||
let bad = local_http_client()
|
||||
.get(format!("{}/{bucket}/{SYSTEM_KEY}", env.url))
|
||||
.header("x-amz-date", &amz_date)
|
||||
.header("x-amz-content-sha256", "UNSIGNED-PAYLOAD")
|
||||
.header(
|
||||
"authorization",
|
||||
format!(
|
||||
"AWS4-HMAC-SHA256 Credential=bad/{scope_date}/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=0000000000000000000000000000000000000000000000000000000000000000"
|
||||
),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(bad.status(), reqwest::StatusCode::FORBIDDEN);
|
||||
let bad_body = bad.text().await?;
|
||||
assert!(!bad_body.contains("ProtocolVersion"), "{bad_body}");
|
||||
|
||||
info!("✅ PASSED: Veeam SOSAPI system.xml and capacity.xml");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn xml_u64(xml: &str, element: &str) -> Result<u64, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let start = format!("<{element}>");
|
||||
let end = format!("</{element}>");
|
||||
let start_idx = xml.find(&start).ok_or_else(|| format!("missing {element} in {xml}"))? + start.len();
|
||||
let end_idx = xml[start_idx..]
|
||||
.find(&end)
|
||||
.ok_or_else(|| format!("unclosed {element} in {xml}"))?
|
||||
+ start_idx;
|
||||
Ok(xml[start_idx..end_idx].parse()?)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_terraform_put_after_delete() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -12,6 +12,9 @@ Applies to `crates/ecstore/`.
|
||||
|
||||
- Be careful with allocations and locking in hot paths.
|
||||
- Keep network and disk operations async-friendly; avoid introducing unnecessary blocking.
|
||||
- Coalesce related `open`, `read`/`write`, `fsync`, and `close` filesystem steps into a single `blocking_queue` or `run_blocking_namespace_operation` closure per disk rather than chaining multiple `tokio::fs` or `spawn_blocking` dispatches.
|
||||
- Pin a parent directory once (`openat2(BENEATH | NO_SYMLINKS)` on Linux) and reuse the pinned guard across sibling metadata reads (`read_optional_in_pinned`, `part.N.meta` loops) instead of re-walking ancestor components per file or re-opening when `source_parent == destination_parent`.
|
||||
- After `file.metadata()`, pre-size read buffers (`Vec::with_capacity(len + 1)`) before `read_to_end` to avoid `std::io::read_to_end`'s 32-byte exponential growth (`6–7` `read()` syscalls per small metadata file).
|
||||
- Benchmark-sensitive changes should include measurable rationale.
|
||||
|
||||
## Allocation Discipline in Hot Paths
|
||||
|
||||
@@ -137,7 +137,6 @@ rustfs-lifecycle.workspace = true
|
||||
rustfs-s3-client = { workspace = true }
|
||||
rustfs-s3-types = { workspace = true }
|
||||
rustfs-data-usage.workspace = true
|
||||
rustfs-gateway-types.workspace = true
|
||||
rustfs-object-capacity.workspace = true
|
||||
rustfs-object-data-cache = { workspace = true, features = ["runtime-memory"] }
|
||||
arc-swap.workspace = true
|
||||
@@ -156,6 +155,7 @@ serde = { workspace = true, features = ["derive"] }
|
||||
time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }
|
||||
bytesize.workspace = true
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
http.workspace = true
|
||||
opentelemetry.workspace = true
|
||||
http-body = { workspace = true }
|
||||
@@ -244,7 +244,6 @@ windows-sys = { workspace = true, features = [
|
||||
windows-sys = { workspace = true, features = ["Win32_System_Ioctl"] }
|
||||
|
||||
[dev-dependencies]
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
aws-smithy-async.workspace = true
|
||||
rustfs-s3-ops.workspace = true
|
||||
tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util", "fs"] }
|
||||
|
||||
@@ -62,8 +62,6 @@ pub mod bucket {
|
||||
}
|
||||
|
||||
pub mod bucket_lifecycle_ops {
|
||||
pub use rustfs_lifecycle::{RestoreOutputLocation, RestoreRequest, RestoreS3Location, RestoreTag, RestoreTagging};
|
||||
|
||||
pub use crate::bucket::lifecycle::bucket_lifecycle_ops::{
|
||||
ExpiryState, LifecycleOps, ManualTransitionCancelCheck, ManualTransitionProgressSink,
|
||||
ManualTransitionQueueSnapshot, ManualTransitionRunExecution, ManualTransitionRunOptions,
|
||||
@@ -135,7 +133,7 @@ pub mod bucket {
|
||||
pub use crate::bucket::lifecycle::lifecycle::{
|
||||
Event, ExpirationOptions, IlmAction, LIFECYCLE_MALFORMED_XML_ERROR_KIND, Lifecycle, LifecycleCalculate,
|
||||
ObjectOpts, RuleValidate, TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, expected_expiry_time,
|
||||
format_lifecycle_date, object_opts_from_object_info, parse_lifecycle_date,
|
||||
object_opts_from_object_info,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -227,7 +225,7 @@ pub mod bucket {
|
||||
}
|
||||
|
||||
pub mod object_lock {
|
||||
pub use crate::bucket::object_lock::ObjectLockApi;
|
||||
pub use crate::bucket::object_lock::{ObjectLockApi, ObjectLockStatusExt};
|
||||
|
||||
pub mod types {
|
||||
pub use crate::bucket::object_lock::types::{
|
||||
@@ -254,7 +252,7 @@ pub mod bucket {
|
||||
pub mod quota {
|
||||
pub use crate::bucket::quota::{
|
||||
BucketQuota, QUOTA_RESERVATION_PROTOCOL_V1, QUOTA_RESERVATION_PROTOCOL_V2, QuotaCheckResult, QuotaError,
|
||||
QuotaMigrationSkipReason, QuotaOperation, QuotaProtocolMigration,
|
||||
QuotaMigrationSkipReason, QuotaOperation, QuotaProtocolMigration, QuotaType,
|
||||
};
|
||||
|
||||
pub mod checker {
|
||||
@@ -322,9 +320,9 @@ pub mod bucket {
|
||||
|
||||
pub mod utils {
|
||||
pub use crate::bucket::utils::{
|
||||
BucketConfigXml, BucketConfigXmlError, check_bucket_and_object_names, check_list_objs_args,
|
||||
check_object_name_for_length_and_slash, check_valid_bucket_name_strict, deserialize, has_bad_path_component,
|
||||
is_meta_bucketname, is_valid_object_prefix, serialize,
|
||||
check_bucket_and_object_names, check_list_objs_args, check_object_name_for_length_and_slash,
|
||||
check_valid_bucket_name_strict, deserialize, has_bad_path_component, is_meta_bucketname, is_valid_object_prefix,
|
||||
serialize,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -351,7 +349,7 @@ pub mod capacity {
|
||||
}
|
||||
|
||||
pub mod object_api_utils {
|
||||
pub use crate::object_api::object_api_utils::{ObjReaderFn, PutObjReader, get_raw_etag, new_getobjectreader};
|
||||
pub use crate::object_api::object_api_utils::{ObjReaderFn, PutObjReader, get_raw_etag, new_getobjectreader, to_s3s_etag};
|
||||
}
|
||||
|
||||
pub mod cluster {
|
||||
|
||||
@@ -87,14 +87,12 @@ use rustfs_filemeta::{
|
||||
FileInfo, FileInfoOpts, NULL_VERSION_ID, RestoreStatus, RestoreStatusOps, TRANSITION_COMPLETE, get_file_info,
|
||||
is_restored_object_on_disk,
|
||||
};
|
||||
use rustfs_gateway_types::persistence::PersistedLifecycleConfiguration;
|
||||
use rustfs_gateway_types::persistence::PersistedObjectLockConfiguration as ObjectLockConfiguration;
|
||||
use rustfs_lifecycle::{RestoreRequest, RestoreS3Location};
|
||||
use rustfs_lock::{NamespaceLockGuard, error::LockError};
|
||||
use rustfs_scanner_metrics::metrics::{
|
||||
IlmAction, Metrics, ScannerLifecycleExpiryStateUpdate, ScannerLifecycleTransitionStateUpdate, global_metrics,
|
||||
};
|
||||
use rustfs_utils::{get_env_i64, get_env_usize, path::encode_dir_object, string::parse_bool};
|
||||
use s3s::dto::{BucketLifecycleConfiguration, ExpirationStatus, ObjectLockConfiguration, RestoreRequest, RestoreRequestType};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::any::Any;
|
||||
use std::collections::{BTreeMap, HashMap, HashSet};
|
||||
@@ -300,7 +298,7 @@ impl LifecycleSys {
|
||||
Arc::new(Self)
|
||||
}
|
||||
|
||||
pub async fn get(&self, bucket: &str) -> Option<PersistedLifecycleConfiguration> {
|
||||
pub async fn get(&self, bucket: &str) -> Option<BucketLifecycleConfiguration> {
|
||||
match metadata_boundary::get_lifecycle_config(bucket).await {
|
||||
Ok((lc, _)) => Some(lc),
|
||||
Err(Error::ConfigNotFound) => None,
|
||||
@@ -3936,7 +3934,7 @@ pub fn init_background_stale_multipart_upload_cleanup(api: Arc<ECStore>) {
|
||||
});
|
||||
}
|
||||
|
||||
pub async fn validate_transition_tier(lc: &PersistedLifecycleConfiguration) -> Result<(), std::io::Error> {
|
||||
pub async fn validate_transition_tier(lc: &BucketLifecycleConfiguration) -> Result<(), std::io::Error> {
|
||||
for rule in &lc.rules {
|
||||
if let Some(transitions) = &rule.transitions {
|
||||
for transition in transitions {
|
||||
@@ -4053,9 +4051,8 @@ pub(crate) async fn enqueue_immediate_expiry(api: Arc<ECStore>, oi: &ObjectInfo,
|
||||
.map(lifecycle::object_opts_from_object_info)
|
||||
.collect::<Vec<ObjectOpts>>();
|
||||
let lock_config = configs.object_lock;
|
||||
let evaluator_lock_config = lock_config.clone();
|
||||
let events = match Evaluator::new(lifecycle)
|
||||
.with_lock_retention(evaluator_lock_config)
|
||||
.with_lock_retention(lock_config.clone())
|
||||
.eval(&object_opts)
|
||||
.await
|
||||
{
|
||||
@@ -4535,26 +4532,19 @@ pub async fn enqueue_transition_for_existing_objects_scoped(
|
||||
}
|
||||
}
|
||||
|
||||
fn lifecycle_rule_has_date_expiration(lc: &PersistedLifecycleConfiguration, rule_id: &str) -> bool {
|
||||
fn lifecycle_rule_has_date_expiration(lc: &BucketLifecycleConfiguration, rule_id: &str) -> bool {
|
||||
lc.rules.iter().any(|rule| {
|
||||
rule.status == "Enabled"
|
||||
rule.status == ExpirationStatus::from_static(ExpirationStatus::ENABLED)
|
||||
&& rule.id.as_deref() == Some(rule_id)
|
||||
&& rule.expiration.as_ref().is_some_and(|expiration| expiration.date.is_some())
|
||||
})
|
||||
}
|
||||
|
||||
fn should_defer_date_expiry_for_recent_config_update(lc: &PersistedLifecycleConfiguration, now: OffsetDateTime) -> bool {
|
||||
lc.expiry_updated_at
|
||||
.as_ref()
|
||||
.is_some_and(|updated_at| match rustfs_lifecycle::parse_lifecycle_date(updated_at) {
|
||||
Ok(updated_at) => {
|
||||
now.unix_timestamp().saturating_sub(updated_at.unix_timestamp()) < DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS
|
||||
}
|
||||
Err(error) => {
|
||||
warn!(error = %error, "Deferring date expiry with an unreadable update timestamp");
|
||||
true
|
||||
}
|
||||
})
|
||||
fn should_defer_date_expiry_for_recent_config_update(lc: &BucketLifecycleConfiguration, now: OffsetDateTime) -> bool {
|
||||
lc.expiry_updated_at.as_ref().is_some_and(|updated_at| {
|
||||
let updated_at = OffsetDateTime::from(updated_at.clone());
|
||||
now.unix_timestamp().saturating_sub(updated_at.unix_timestamp()) < DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS
|
||||
})
|
||||
}
|
||||
|
||||
async fn apply_existing_object_expiry(
|
||||
@@ -4574,7 +4564,7 @@ async fn apply_existing_object_expiry(
|
||||
struct ExistingObjectExpiryContext<'a> {
|
||||
api: Arc<ECStore>,
|
||||
bucket: &'a str,
|
||||
lc: Arc<PersistedLifecycleConfiguration>,
|
||||
lc: Arc<BucketLifecycleConfiguration>,
|
||||
lock_config: Option<Arc<ObjectLockConfiguration>>,
|
||||
bucket_incarnation_id: Uuid,
|
||||
src: &'a LcEventSrc,
|
||||
@@ -4594,9 +4584,8 @@ async fn enqueue_expiry_for_existing_object_group(
|
||||
.iter()
|
||||
.map(lifecycle::object_opts_from_object_info)
|
||||
.collect::<Vec<ObjectOpts>>();
|
||||
let evaluator_lock_config = context.lock_config.clone();
|
||||
let events = match Evaluator::new(context.lc.clone())
|
||||
.with_lock_retention(evaluator_lock_config)
|
||||
.with_lock_retention(context.lock_config.clone())
|
||||
.eval(&object_opts)
|
||||
.await
|
||||
{
|
||||
@@ -4790,7 +4779,7 @@ fn manual_transition_version_marker(oi: &ObjectInfo) -> String {
|
||||
async fn enqueue_transition_with_lifecycle_report(
|
||||
api: Option<Arc<ECStore>>,
|
||||
oi: &ObjectInfo,
|
||||
lc: &PersistedLifecycleConfiguration,
|
||||
lc: &BucketLifecycleConfiguration,
|
||||
src: &LcEventSrc,
|
||||
options: &ManualTransitionRunOptions,
|
||||
report: &mut ManualTransitionRunReport,
|
||||
@@ -4856,7 +4845,7 @@ async fn enqueue_transition_with_lifecycle_report(
|
||||
false
|
||||
}
|
||||
|
||||
async fn enqueue_transition_with_lifecycle(oi: &ObjectInfo, lc: &PersistedLifecycleConfiguration, src: &LcEventSrc) -> bool {
|
||||
async fn enqueue_transition_with_lifecycle(oi: &ObjectInfo, lc: &BucketLifecycleConfiguration, src: &LcEventSrc) -> bool {
|
||||
let event = lc.eval(&oi.to_lifecycle_opts()).await;
|
||||
match event.action {
|
||||
IlmAction::TransitionAction | IlmAction::TransitionVersionAction => {
|
||||
@@ -5370,8 +5359,12 @@ pub async fn post_restore_opts(version_id: &str, bucket: &str, object: &str) ->
|
||||
})
|
||||
}
|
||||
|
||||
fn select_restore_s3_location(rreq: &RestoreRequest) -> Result<Option<&RestoreS3Location>, std::io::Error> {
|
||||
if rreq.type_.as_ref().is_none_or(|type_| type_.as_str() != "SELECT") {
|
||||
fn select_restore_s3_location(rreq: &RestoreRequest) -> Result<Option<&s3s::dto::S3Location>, std::io::Error> {
|
||||
if rreq
|
||||
.type_
|
||||
.as_ref()
|
||||
.is_none_or(|type_| type_.as_str() != RestoreRequestType::SELECT)
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
let output_location = rreq
|
||||
@@ -5413,25 +5406,27 @@ pub async fn put_restore_opts(
|
||||
// the OutputLocation metadata instead of a write to `OutputLocation.S3`
|
||||
// (backlog#1341). RestoreObject rejects SELECT at the API boundary; this
|
||||
// is the fail-closed backstop for any other caller.
|
||||
if rreq.type_.as_ref().is_some_and(|type_| type_.as_str() == "SELECT") {
|
||||
if rreq
|
||||
.type_
|
||||
.as_ref()
|
||||
.is_some_and(|type_| type_.as_str() == RestoreRequestType::SELECT)
|
||||
{
|
||||
return Err(std::io::Error::other("SELECT restore requests are not supported"));
|
||||
}
|
||||
for (k, v) in oi.user_defined.iter() {
|
||||
meta.insert(k.to_string(), v.clone());
|
||||
}
|
||||
// The fleet capability gate guarantees physical copy-back support. Keep
|
||||
// the completed transition identity for orphan recovery and remote cleanup.
|
||||
rustfs_utils::http::metadata_compat::remove_str(&mut meta, rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_OPERATION_ID);
|
||||
rustfs_utils::http::metadata_compat::remove_str(&mut meta, rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_WORKER_LOCK);
|
||||
if !oi.user_tags.is_empty() {
|
||||
meta.insert(AMZ_OBJECT_TAGGING.to_string(), (*oi.user_tags).clone());
|
||||
}
|
||||
// Finalization publishes completion and its expiry under the object lock.
|
||||
// The local copy-back commit must not make unfinished work eligible for expiry.
|
||||
let restore_expiry = lifecycle::expected_expiry_time(OffsetDateTime::now_utc(), rreq.days.unwrap_or(1));
|
||||
meta.insert(
|
||||
metadata_keys::RESTORE.to_string(),
|
||||
RestoreStatus {
|
||||
is_restore_in_progress: Some(true),
|
||||
restore_expiry_date: None,
|
||||
is_restore_in_progress: Some(false),
|
||||
restore_expiry_date: Some(restore_expiry),
|
||||
}
|
||||
.to_string(),
|
||||
);
|
||||
@@ -5445,7 +5440,6 @@ pub async fn put_restore_opts(
|
||||
// computed MD5 is not the object's public plaintext ETag.
|
||||
preserve_etag: oi.etag.clone(),
|
||||
shard_integrity_write_mode: Some(oi.shard_integrity_write_mode()),
|
||||
restore_copyback: true,
|
||||
//expires: oi.expires,
|
||||
..Default::default()
|
||||
})
|
||||
@@ -5465,18 +5459,6 @@ impl LifecycleOps for ObjectInfo {
|
||||
if self.transitioned_object.status != lifecycle::TRANSITION_COMPLETE {
|
||||
return false;
|
||||
}
|
||||
if rustfs_utils::http::metadata_compat::get_consistent_str(
|
||||
&self.user_defined,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_COPYBACK,
|
||||
) == Some(rustfs_utils::http::metadata_compat::RESTORE_COPYBACK_PROTOCOL_V1)
|
||||
&& self
|
||||
.user_defined
|
||||
.get(metadata_keys::RESTORE)
|
||||
.and_then(|value| rustfs_filemeta::parse_restore_obj_status(value).ok())
|
||||
.is_some_and(|status| status.is_restore_in_progress == Some(true))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
!is_restored_object_on_disk(&self.user_defined)
|
||||
}
|
||||
}
|
||||
@@ -5488,25 +5470,25 @@ pub trait RestoreRequestOps {
|
||||
impl RestoreRequestOps for RestoreRequest {
|
||||
fn validate(&self, _api: Arc<ECStore>) -> Result<(), std::io::Error> {
|
||||
// SELECT type requires select_parameters, and vice versa
|
||||
if self.type_.as_ref().is_none_or(|t| t.as_str() != "SELECT") && self.select_parameters.is_some() {
|
||||
if self.type_.as_ref().is_none_or(|t| t.as_str() != RestoreRequestType::SELECT) && self.select_parameters.is_some() {
|
||||
return Err(std::io::Error::other("Select parameters can only be specified with SELECT request type"));
|
||||
}
|
||||
if let Some(type_) = &self.type_
|
||||
&& type_.as_str() == "SELECT"
|
||||
&& type_.as_str() == RestoreRequestType::SELECT
|
||||
&& self.select_parameters.is_none()
|
||||
{
|
||||
return Err(std::io::Error::other("SELECT restore request requires select parameters to be specified"));
|
||||
}
|
||||
|
||||
// OutputLocation is only valid for SELECT requests
|
||||
if self.type_.as_ref().is_none_or(|t| t.as_str() != "SELECT") && self.output_location.is_some() {
|
||||
if self.type_.as_ref().is_none_or(|t| t.as_str() != RestoreRequestType::SELECT) && self.output_location.is_some() {
|
||||
return Err(std::io::Error::other("OutputLocation can only be specified with SELECT request type"));
|
||||
}
|
||||
select_restore_s3_location(self)?;
|
||||
|
||||
// Days must not be specified with SELECT requests
|
||||
if let Some(type_) = &self.type_
|
||||
&& type_.as_str() == "SELECT"
|
||||
&& type_.as_str() == RestoreRequestType::SELECT
|
||||
&& self.days.is_some_and(|d| d > 0)
|
||||
{
|
||||
return Err(std::io::Error::other("Days cannot be specified with SELECT restore request"));
|
||||
@@ -5548,7 +5530,7 @@ pub fn lifecycle_version_delete_target(oi: &ObjectInfo) -> Option<ObjectToDelete
|
||||
}
|
||||
|
||||
pub async fn eval_action_from_lifecycle(
|
||||
lc: &PersistedLifecycleConfiguration,
|
||||
lc: &BucketLifecycleConfiguration,
|
||||
lock_config: Option<&ObjectLockConfiguration>,
|
||||
oi: &ObjectInfo,
|
||||
) -> lifecycle::Event {
|
||||
@@ -6253,19 +6235,14 @@ mod tests {
|
||||
use rustfs_data_usage::TierStats;
|
||||
use rustfs_filemeta::metadata_keys;
|
||||
use rustfs_filemeta::{FileInfo, FileMeta};
|
||||
use rustfs_gateway_types::dto::MetadataEntry;
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedDefaultRetention as DefaultRetention, PersistedObjectLockConfiguration as ObjectLockConfiguration,
|
||||
PersistedObjectLockRule as ObjectLockRule,
|
||||
};
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedLifecycleConfiguration, PersistedLifecycleExpiration, PersistedLifecycleRule,
|
||||
PersistedNoncurrentVersionExpiration, PersistedTransition,
|
||||
};
|
||||
use rustfs_lifecycle::{RestoreOutputLocation as OutputLocation, RestoreRequest, RestoreS3Location as S3Location};
|
||||
#[cfg(feature = "test-util")]
|
||||
use rustfs_s3_client::transition_api::ReaderImpl;
|
||||
use rustfs_scanner_metrics::metrics::{IlmAction, global_metrics};
|
||||
use s3s::dto::{
|
||||
BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, LifecycleExpiration, LifecycleRule, MetadataEntry,
|
||||
NoncurrentVersionExpiration, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockRetentionMode, ObjectLockRule,
|
||||
OutputLocation, RestoreRequest, RestoreRequestType, S3Location, Timestamp, Transition, TransitionStorageClass,
|
||||
};
|
||||
use serial_test::serial;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashMap;
|
||||
@@ -9272,7 +9249,7 @@ mod tests {
|
||||
output_location,
|
||||
select_parameters: None,
|
||||
tier: None,
|
||||
type_: Some("SELECT".to_owned()),
|
||||
type_: Some(RestoreRequestType::from_static(RestoreRequestType::SELECT)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9589,12 +9566,12 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn lifecycle_rule_has_date_expiration_detects_enabled_date_rule() {
|
||||
let lc = PersistedLifecycleConfiguration {
|
||||
let lc = BucketLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![PersistedLifecycleRule {
|
||||
status: "Enabled".to_owned(),
|
||||
expiration: Some(PersistedLifecycleExpiration {
|
||||
date: Some(lifecycle::format_lifecycle_date(OffsetDateTime::now_utc()).unwrap()),
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
expiration: Some(LifecycleExpiration {
|
||||
date: Some(Timestamp::from(OffsetDateTime::now_utc())),
|
||||
..Default::default()
|
||||
}),
|
||||
id: Some("rule-date".to_string()),
|
||||
@@ -10111,15 +10088,14 @@ mod tests {
|
||||
#[test]
|
||||
fn should_defer_date_expiry_for_recent_config_update_respects_grace_window() {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let recent = PersistedLifecycleConfiguration {
|
||||
expiry_updated_at: Some(lifecycle::format_lifecycle_date(now - time::Duration::seconds(1)).unwrap()),
|
||||
let recent = BucketLifecycleConfiguration {
|
||||
expiry_updated_at: Some(Timestamp::from(now - time::Duration::seconds(1))),
|
||||
rules: Vec::new(),
|
||||
};
|
||||
let stale = PersistedLifecycleConfiguration {
|
||||
expiry_updated_at: Some(
|
||||
lifecycle::format_lifecycle_date(now - time::Duration::seconds(DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS + 1))
|
||||
.unwrap(),
|
||||
),
|
||||
let stale = BucketLifecycleConfiguration {
|
||||
expiry_updated_at: Some(Timestamp::from(
|
||||
now - time::Duration::seconds(DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS + 1),
|
||||
)),
|
||||
rules: Vec::new(),
|
||||
};
|
||||
|
||||
@@ -10247,12 +10223,12 @@ mod tests {
|
||||
assert_eq!(deleted.version_id, None);
|
||||
}
|
||||
|
||||
fn expired_delete_marker_lifecycle() -> PersistedLifecycleConfiguration {
|
||||
PersistedLifecycleConfiguration {
|
||||
fn expired_delete_marker_lifecycle() -> BucketLifecycleConfiguration {
|
||||
BucketLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![PersistedLifecycleRule {
|
||||
status: "Enabled".to_owned(),
|
||||
expiration: Some(PersistedLifecycleExpiration {
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
expiration: Some(LifecycleExpiration {
|
||||
expired_object_delete_marker: Some(true),
|
||||
..Default::default()
|
||||
}),
|
||||
@@ -10268,12 +10244,12 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn latest_expiration_lifecycle() -> PersistedLifecycleConfiguration {
|
||||
PersistedLifecycleConfiguration {
|
||||
fn latest_expiration_lifecycle() -> BucketLifecycleConfiguration {
|
||||
BucketLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![PersistedLifecycleRule {
|
||||
status: "Enabled".to_owned(),
|
||||
expiration: Some(PersistedLifecycleExpiration {
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
expiration: Some(LifecycleExpiration {
|
||||
days: Some(1),
|
||||
..Default::default()
|
||||
}),
|
||||
@@ -10289,12 +10265,12 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn current_and_noncurrent_expiration_lifecycle() -> PersistedLifecycleConfiguration {
|
||||
PersistedLifecycleConfiguration {
|
||||
fn current_and_noncurrent_expiration_lifecycle() -> BucketLifecycleConfiguration {
|
||||
BucketLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![PersistedLifecycleRule {
|
||||
status: "Enabled".to_owned(),
|
||||
expiration: Some(PersistedLifecycleExpiration {
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
expiration: Some(LifecycleExpiration {
|
||||
days: Some(30),
|
||||
..Default::default()
|
||||
}),
|
||||
@@ -10302,7 +10278,7 @@ mod tests {
|
||||
del_marker_expiration: None,
|
||||
filter: None,
|
||||
id: Some("expire-current-and-noncurrent".to_string()),
|
||||
noncurrent_version_expiration: Some(PersistedNoncurrentVersionExpiration {
|
||||
noncurrent_version_expiration: Some(NoncurrentVersionExpiration {
|
||||
noncurrent_days: Some(1),
|
||||
newer_noncurrent_versions: None,
|
||||
}),
|
||||
@@ -10313,12 +10289,12 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn all_versions_expiration_lifecycle() -> PersistedLifecycleConfiguration {
|
||||
PersistedLifecycleConfiguration {
|
||||
fn all_versions_expiration_lifecycle() -> BucketLifecycleConfiguration {
|
||||
BucketLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![PersistedLifecycleRule {
|
||||
status: "Enabled".to_owned(),
|
||||
expiration: Some(PersistedLifecycleExpiration {
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
expiration: Some(LifecycleExpiration {
|
||||
days: Some(1),
|
||||
expired_object_all_versions: Some(true),
|
||||
..Default::default()
|
||||
@@ -10337,29 +10313,29 @@ mod tests {
|
||||
|
||||
fn lock_enabled_without_default_retention() -> ObjectLockConfiguration {
|
||||
ObjectLockConfiguration {
|
||||
object_lock_enabled: Some("Enabled".to_owned()),
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
rule: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn lock_enabled_with_default_retention() -> ObjectLockConfiguration {
|
||||
ObjectLockConfiguration {
|
||||
object_lock_enabled: Some("Enabled".to_owned()),
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
rule: Some(ObjectLockRule {
|
||||
default_retention: Some(DefaultRetention {
|
||||
days: Some(30),
|
||||
mode: Some("COMPLIANCE".to_owned()),
|
||||
mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE)),
|
||||
years: None,
|
||||
}),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
fn latest_transition_lifecycle() -> PersistedLifecycleConfiguration {
|
||||
PersistedLifecycleConfiguration {
|
||||
fn latest_transition_lifecycle() -> BucketLifecycleConfiguration {
|
||||
BucketLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![PersistedLifecycleRule {
|
||||
status: "Enabled".to_owned(),
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
expiration: None,
|
||||
abort_incomplete_multipart_upload: None,
|
||||
del_marker_expiration: None,
|
||||
@@ -10368,10 +10344,10 @@ mod tests {
|
||||
noncurrent_version_expiration: None,
|
||||
noncurrent_version_transitions: None,
|
||||
prefix: None,
|
||||
transitions: Some(vec![PersistedTransition {
|
||||
transitions: Some(vec![Transition {
|
||||
days: Some(1),
|
||||
date: None,
|
||||
storage_class: Some("WARM".to_owned()),
|
||||
storage_class: Some(TransitionStorageClass::from_static("WARM")),
|
||||
}]),
|
||||
}],
|
||||
}
|
||||
@@ -12908,9 +12884,8 @@ mod tests {
|
||||
|
||||
for (name, object, lock_config, expected) in cases {
|
||||
let object_opts = lifecycle::object_opts_from_object_info(&object);
|
||||
let evaluator_lock_config = lock_config.clone().map(Arc::new);
|
||||
let batch_event = Evaluator::new(Arc::new(lifecycle.clone()))
|
||||
.with_lock_retention(evaluator_lock_config)
|
||||
.with_lock_retention(lock_config.clone().map(Arc::new))
|
||||
.eval(&[object_opts])
|
||||
.await
|
||||
.expect("batch lifecycle evaluation should succeed")
|
||||
@@ -14167,7 +14142,10 @@ mod tests {
|
||||
let object = current_object_with_metadata(
|
||||
ReplicationStatusType::Completed,
|
||||
HashMap::from([
|
||||
(metadata_keys::OBJECT_LOCK_MODE.to_string(), "COMPLIANCE".to_string()),
|
||||
(
|
||||
metadata_keys::OBJECT_LOCK_MODE.to_string(),
|
||||
s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(),
|
||||
),
|
||||
(metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(), retain_until),
|
||||
]),
|
||||
);
|
||||
@@ -14179,7 +14157,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn restored_copy_expiry_is_not_blocked_by_retention() {
|
||||
let lifecycle = PersistedLifecycleConfiguration {
|
||||
let lifecycle = BucketLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: Vec::new(),
|
||||
};
|
||||
@@ -14189,7 +14167,10 @@ mod tests {
|
||||
let mut object = current_object_with_metadata(
|
||||
ReplicationStatusType::Completed,
|
||||
HashMap::from([
|
||||
(metadata_keys::OBJECT_LOCK_MODE.to_string(), "COMPLIANCE".to_string()),
|
||||
(
|
||||
metadata_keys::OBJECT_LOCK_MODE.to_string(),
|
||||
ObjectLockRetentionMode::COMPLIANCE.to_string(),
|
||||
),
|
||||
(metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(), retain_until),
|
||||
]),
|
||||
);
|
||||
@@ -14406,7 +14387,7 @@ mod tests {
|
||||
"removed" => {
|
||||
metadata_sys::delete(&bucket, BUCKET_LIFECYCLE_CONFIG).await.unwrap();
|
||||
}
|
||||
"disabled" => changed.rules[0].status = "Disabled".to_owned(),
|
||||
"disabled" => changed.rules[0].status = ExpirationStatus::from_static(ExpirationStatus::DISABLED),
|
||||
"longer-days" => {
|
||||
changed.rules[0]
|
||||
.noncurrent_version_expiration
|
||||
@@ -14522,7 +14503,7 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
async fn install_test_expiry_policy(ecstore: &Arc<ECStore>, bucket: &str) -> Arc<PersistedLifecycleConfiguration> {
|
||||
async fn install_test_expiry_policy(ecstore: &Arc<ECStore>, bucket: &str) -> Arc<BucketLifecycleConfiguration> {
|
||||
let xml = br#"<LifecycleConfiguration><Rule><ID>expire</ID><Status>Enabled</Status><Filter><Prefix></Prefix></Filter><Expiration><Days>0</Days></Expiration><NoncurrentVersionExpiration><NoncurrentDays>0</NoncurrentDays></NoncurrentVersionExpiration></Rule></LifecycleConfiguration>"#;
|
||||
metadata_sys::update_in(&ecstore.ctx, bucket, BUCKET_LIFECYCLE_CONFIG, xml.to_vec())
|
||||
.await
|
||||
@@ -14995,11 +14976,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn tier_free_version_recovery_production_entrypoint_enqueues_seeded_item() {
|
||||
let (disk_paths, ecstore) =
|
||||
temp_env::async_with_vars([(super::ENV_TIER_FREE_VERSION_RECOVERY_ENABLED, Some("false"))], async {
|
||||
setup_test_env().await
|
||||
})
|
||||
.await;
|
||||
let (disk_paths, ecstore) = setup_test_env().await;
|
||||
let tier = format!("RECOVERY{}", Uuid::new_v4().simple()).to_uppercase();
|
||||
crate::services::tier::test_util::register_mock_tier(&ecstore.tier_config_mgr(), &tier).await;
|
||||
let runtime_state = install_unconsumed_runtime_expiry_worker(&ecstore, 1).await;
|
||||
|
||||
@@ -17,7 +17,7 @@ use crate::object_api::ObjectInfo;
|
||||
pub use rustfs_lifecycle::{
|
||||
Event, ExpirationOptions, IlmAction, LIFECYCLE_MALFORMED_XML_ERROR_KIND, Lifecycle, LifecycleCalculate, ObjectOpts,
|
||||
RuleValidate, TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, abort_incomplete_multipart_upload_due,
|
||||
expected_expiry_time, expiration_action_has_valid_target, format_lifecycle_date, parse_lifecycle_date,
|
||||
expected_expiry_time, expiration_action_has_valid_target,
|
||||
};
|
||||
|
||||
pub fn object_opts_from_object_info(oi: &ObjectInfo) -> ObjectOpts {
|
||||
|
||||
@@ -14,8 +14,7 @@
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use rustfs_gateway_types::persistence::PersistedLifecycleConfiguration;
|
||||
use rustfs_gateway_types::persistence::PersistedObjectLockConfiguration as ObjectLockConfiguration;
|
||||
use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration};
|
||||
use time::OffsetDateTime;
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -25,7 +24,7 @@ use crate::error::{Error, Result};
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct LifecycleExpiryConfigs {
|
||||
pub(crate) lifecycle: Option<Arc<PersistedLifecycleConfiguration>>,
|
||||
pub(crate) lifecycle: Option<Arc<BucketLifecycleConfiguration>>,
|
||||
pub(crate) object_lock: Option<Arc<ObjectLockConfiguration>>,
|
||||
pub(crate) bucket_incarnation_id: Uuid,
|
||||
pub(crate) table_bucket_enabled: bool,
|
||||
@@ -108,7 +107,7 @@ pub(crate) async fn get_expiry_configs_for_options(
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn get_lifecycle_config(bucket: &str) -> Result<(PersistedLifecycleConfiguration, OffsetDateTime)> {
|
||||
pub(crate) async fn get_lifecycle_config(bucket: &str) -> Result<(BucketLifecycleConfiguration, OffsetDateTime)> {
|
||||
metadata_sys::get_lifecycle_config(bucket).await
|
||||
}
|
||||
|
||||
@@ -118,15 +117,15 @@ mod tests {
|
||||
use crate::bucket::metadata::BucketMetadata;
|
||||
use crate::bucket::metadata_sys::{self, test_support::isolated_store_over_temp_disks};
|
||||
use crate::storage_api_contracts::bucket::MakeBucketOptions;
|
||||
use rustfs_gateway_types::persistence::{PersistedLifecycleExpiration, PersistedLifecycleRule};
|
||||
use s3s::dto::{ExpirationStatus, LifecycleExpiration, LifecycleRule};
|
||||
use serial_test::serial;
|
||||
|
||||
fn lifecycle_config() -> PersistedLifecycleConfiguration {
|
||||
PersistedLifecycleConfiguration {
|
||||
fn lifecycle_config() -> BucketLifecycleConfiguration {
|
||||
BucketLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![PersistedLifecycleRule {
|
||||
status: "Enabled".to_owned(),
|
||||
expiration: Some(PersistedLifecycleExpiration {
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
expiration: Some(LifecycleExpiration {
|
||||
days: Some(1),
|
||||
..Default::default()
|
||||
}),
|
||||
|
||||
@@ -22,7 +22,7 @@ pub(crate) fn is_object_locked_by_metadata(user_defined: &HashMap<String, String
|
||||
}
|
||||
|
||||
pub(crate) fn check_object_lock_for_deletion_with_config(
|
||||
config: Option<&rustfs_gateway_types::persistence::PersistedObjectLockConfiguration>,
|
||||
config: Option<&s3s::dto::ObjectLockConfiguration>,
|
||||
obj_info: &ObjectInfo,
|
||||
bypass_governance: bool,
|
||||
) -> crate::error::Result<Option<ObjectLockBlockReason>> {
|
||||
|
||||
@@ -82,13 +82,12 @@ mod tests {
|
||||
use crate::bucket::replication::{DeleteReplicationConfigSnapshot, ReplicationObjectBridge};
|
||||
use crate::object_api::{ObjectInfo, ObjectOptions};
|
||||
use crate::storage_api_contracts::object::ObjectToDelete;
|
||||
use rustfs_gateway_types::dto::Status;
|
||||
use rustfs_gateway_types::persistence::PersistedVersioningConfiguration as VersioningConfiguration;
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedOptionalReplicationStatus, PersistedReplicationConfiguration, PersistedReplicationDestination,
|
||||
PersistedReplicationRule, PersistedReplicationStatus,
|
||||
};
|
||||
use rustfs_scanner_metrics::metrics::IlmAction;
|
||||
use s3s::dto::{
|
||||
BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, DeleteReplication,
|
||||
DeleteReplicationStatus, Destination, ReplicationConfiguration, ReplicationRule, ReplicationRuleStatus,
|
||||
VersioningConfiguration,
|
||||
};
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::*;
|
||||
@@ -161,27 +160,27 @@ mod tests {
|
||||
for purge_enabled in [false, true] {
|
||||
let snapshot = DeleteReplicationConfigSnapshot::from_configs_for_test(
|
||||
VersioningConfiguration {
|
||||
status: Some("Enabled".to_owned()),
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
..Default::default()
|
||||
},
|
||||
Some(PersistedReplicationConfiguration {
|
||||
Some(ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![PersistedReplicationRule {
|
||||
delete_marker_replication: Some(PersistedOptionalReplicationStatus {
|
||||
rules: vec![ReplicationRule {
|
||||
delete_marker_replication: Some(DeleteMarkerReplication {
|
||||
status: Some(if marker_enabled {
|
||||
Status::ENABLED.to_string()
|
||||
DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)
|
||||
} else {
|
||||
Status::DISABLED.to_string()
|
||||
DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::DISABLED)
|
||||
}),
|
||||
}),
|
||||
delete_replication: Some(PersistedReplicationStatus {
|
||||
delete_replication: Some(DeleteReplication {
|
||||
status: if purge_enabled {
|
||||
Status::ENABLED.to_string()
|
||||
DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED)
|
||||
} else {
|
||||
Status::DISABLED.to_string()
|
||||
DeleteReplicationStatus::from_static(DeleteReplicationStatus::DISABLED)
|
||||
},
|
||||
}),
|
||||
destination: PersistedReplicationDestination {
|
||||
destination: Destination {
|
||||
bucket: "arn:rustfs:replication:target".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
@@ -191,7 +190,7 @@ mod tests {
|
||||
prefix: Some(String::new()),
|
||||
priority: Some(1),
|
||||
source_selection_criteria: None,
|
||||
status: Status::ENABLED.to_string(),
|
||||
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
|
||||
}],
|
||||
}),
|
||||
);
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use rustfs_gateway_types::persistence::PersistedLifecycleConfiguration;
|
||||
use s3s::dto::BucketLifecycleConfiguration;
|
||||
use tokio::sync::RwLock;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
@@ -52,7 +52,7 @@ pub(crate) fn deployment_id() -> Option<String> {
|
||||
sources::deployment_id()
|
||||
}
|
||||
|
||||
pub(crate) async fn bucket_lifecycle_config(bucket: &str) -> Option<PersistedLifecycleConfiguration> {
|
||||
pub(crate) async fn bucket_lifecycle_config(bucket: &str) -> Option<BucketLifecycleConfiguration> {
|
||||
sources::bucket_lifecycle_config(bucket).await
|
||||
}
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
use super::msgp_decode::{read_msgp_ext8_time, skip_msgp_value, write_msgp_time};
|
||||
use super::object_lock::ObjectLockApi;
|
||||
use super::object_lock::{ObjectLockApi, ObjectLockStatusExt};
|
||||
use super::versioning::VersioningApi;
|
||||
use super::{quota::BucketQuota, target::BucketTargets};
|
||||
use crate::bucket::replication::invalid_replication_config_status_field;
|
||||
@@ -25,17 +25,12 @@ use crate::object_api::WriteCompletion;
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use crate::store::ECStore;
|
||||
use byteorder::{BigEndian, ByteOrder, LittleEndian};
|
||||
use rustfs_gateway_types::cors_tagging::{PersistedCorsConfiguration as CORSConfiguration, PersistedTagging as Tagging};
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedAccelerateConfiguration as AccelerateConfiguration,
|
||||
PersistedBucketEncryptionConfiguration as ServerSideEncryptionConfiguration,
|
||||
PersistedBucketLoggingStatus as BucketLoggingStatus, PersistedLifecycleConfiguration,
|
||||
PersistedNotificationConfiguration as NotificationConfiguration, PersistedObjectLockConfiguration as ObjectLockConfiguration,
|
||||
PersistedPublicAccessBlockConfiguration as PublicAccessBlockConfiguration, PersistedReplicationConfiguration,
|
||||
PersistedRequestPaymentConfiguration as RequestPaymentConfiguration,
|
||||
PersistedVersioningConfiguration as VersioningConfiguration, PersistedWebsiteConfiguration as WebsiteConfiguration,
|
||||
};
|
||||
use rustfs_policy::policy::BucketPolicy;
|
||||
use s3s::dto::{
|
||||
AccelerateConfiguration, BucketLifecycleConfiguration, BucketLoggingStatus, BucketVersioningStatus, CORSConfiguration,
|
||||
NotificationConfiguration, ObjectLockConfiguration, PublicAccessBlockConfiguration, ReplicationConfiguration,
|
||||
RequestPaymentConfiguration, ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration, WebsiteConfiguration,
|
||||
};
|
||||
use serde::Serializer;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashMap;
|
||||
@@ -45,11 +40,23 @@ use time::{Date, OffsetDateTime, PrimitiveDateTime, Time as CivilTime, UtcOffset
|
||||
use tracing::error;
|
||||
use uuid::Uuid;
|
||||
|
||||
// The persisted Object Lock configuration answers the storage-level trait
|
||||
// here because this module owns the persisted `ObjectLockConfiguration`.
|
||||
// The serving-layer DTO impls for the storage-level Object Lock traits live
|
||||
// here because this module owns the persisted `ObjectLockConfiguration`
|
||||
// during the s3s ratchet migration (rustfs/backlog#1842).
|
||||
impl ObjectLockApi for ObjectLockConfiguration {
|
||||
fn enabled(&self) -> bool {
|
||||
self.object_lock_enabled.as_deref() == Some("Enabled")
|
||||
self.object_lock_enabled
|
||||
.as_ref()
|
||||
.is_some_and(|v| v.as_str() == s3s::dto::ObjectLockEnabled::ENABLED)
|
||||
}
|
||||
}
|
||||
|
||||
impl ObjectLockStatusExt for s3s::dto::ObjectLockLegalHoldStatus {
|
||||
fn valid(&self) -> bool {
|
||||
matches!(
|
||||
self.as_str(),
|
||||
s3s::dto::ObjectLockLegalHoldStatus::ON | s3s::dto::ObjectLockLegalHoldStatus::OFF
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -432,8 +439,6 @@ pub struct BucketMetadata {
|
||||
pub bucket_incarnation_id: Uuid,
|
||||
pub(crate) bucket_incarnation_sidecar: bool,
|
||||
pub(crate) bucket_creation_committed: bool,
|
||||
// A legacy default is not proof of an interrupted creation.
|
||||
pub(crate) bucket_creation_commit_record_present: bool,
|
||||
pub policy_config_json: Vec<u8>,
|
||||
pub notification_config_xml: Vec<u8>,
|
||||
pub lifecycle_config_xml: Vec<u8>,
|
||||
@@ -482,13 +487,13 @@ pub struct BucketMetadata {
|
||||
|
||||
pub policy_config: Option<BucketPolicy>,
|
||||
pub notification_config: Option<NotificationConfiguration>,
|
||||
pub lifecycle_config: Option<PersistedLifecycleConfiguration>,
|
||||
pub lifecycle_config: Option<BucketLifecycleConfiguration>,
|
||||
pub object_lock_config: Option<ObjectLockConfiguration>,
|
||||
pub versioning_config: Option<VersioningConfiguration>,
|
||||
pub sse_config: Option<ServerSideEncryptionConfiguration>,
|
||||
pub tagging_config: Option<Tagging>,
|
||||
pub quota_config: Option<BucketQuota>,
|
||||
pub replication_config: Option<PersistedReplicationConfiguration>,
|
||||
pub replication_config: Option<ReplicationConfiguration>,
|
||||
pub bucket_target_config: Option<BucketTargets>,
|
||||
pub bucket_target_config_meta: Option<HashMap<String, String>>,
|
||||
pub cors_config: Option<CORSConfiguration>,
|
||||
@@ -509,7 +514,6 @@ impl Default for BucketMetadata {
|
||||
bucket_incarnation_id: Uuid::nil(),
|
||||
bucket_incarnation_sidecar: false,
|
||||
bucket_creation_committed: false,
|
||||
bucket_creation_commit_record_present: false,
|
||||
policy_config_json: Default::default(),
|
||||
notification_config_xml: Default::default(),
|
||||
lifecycle_config_xml: Default::default(),
|
||||
@@ -748,10 +752,7 @@ impl BucketMetadata {
|
||||
"TableBucketConfigUpdatedAt" => self.table_bucket_config_updated_at = read_msgp_time_value(rd)?,
|
||||
"DurabilityConfigUpdatedAt" => self.durability_config_updated_at = read_msgp_time_value(rd)?,
|
||||
"OnDemandMigrationConfigUpdatedAt" => self.on_demand_migration_config_updated_at = read_msgp_time_value(rd)?,
|
||||
"BucketCreationCommitted" => {
|
||||
self.bucket_creation_committed = read_msgp_bool(rd)?;
|
||||
self.bucket_creation_commit_record_present = true;
|
||||
}
|
||||
"BucketCreationCommitted" => self.bucket_creation_committed = read_msgp_bool(rd)?,
|
||||
other => {
|
||||
tracing::debug!(field = %other, "BucketMetadata decode_from: skipping unknown field");
|
||||
skip_msgp_value(rd)?;
|
||||
@@ -764,10 +765,8 @@ impl BucketMetadata {
|
||||
|
||||
/// Encode to msgp bytes. Field order follows MinIO BucketMetadata for compatibility.
|
||||
pub fn encode_to<W: Write>(&self, wr: &mut W) -> Result<()> {
|
||||
// Preserve the absence of a legacy commit record across config rewrites.
|
||||
let commit_record_present = self.bucket_creation_commit_record_present || self.bucket_creation_committed;
|
||||
// Map size: MinIO fields (25) + RustFS extensions (21) + optional commit record.
|
||||
let map_len: u32 = 46 + u32::from(commit_record_present);
|
||||
// Map size: MinIO fields (25) + RustFS extensions (22)
|
||||
let map_len: u32 = 47;
|
||||
rmp::encode::write_map_len(wr, map_len)?;
|
||||
|
||||
// MinIO field order (same as Go struct)
|
||||
@@ -848,10 +847,8 @@ impl BucketMetadata {
|
||||
write_msgp_time(wr, self.durability_config_updated_at)?;
|
||||
rmp::encode::write_str(wr, "OnDemandMigrationConfigUpdatedAt")?;
|
||||
write_msgp_time(wr, self.on_demand_migration_config_updated_at)?;
|
||||
if commit_record_present {
|
||||
rmp::encode::write_str(wr, "BucketCreationCommitted")?;
|
||||
rmp::encode::write_bool(wr, self.bucket_creation_committed)?;
|
||||
}
|
||||
rmp::encode::write_str(wr, "BucketCreationCommitted")?;
|
||||
rmp::encode::write_bool(wr, self.bucket_creation_committed)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1021,11 +1018,9 @@ impl BucketMetadata {
|
||||
None
|
||||
} else {
|
||||
let config = deserialize::<VersioningConfiguration>(&data)?;
|
||||
if config
|
||||
.status
|
||||
.as_ref()
|
||||
.is_some_and(|status| !matches!(status.as_str(), "Enabled" | "Suspended"))
|
||||
{
|
||||
if config.status.as_ref().is_some_and(|status| {
|
||||
!matches!(status.as_str(), BucketVersioningStatus::ENABLED | BucketVersioningStatus::SUSPENDED)
|
||||
}) {
|
||||
return Err(Error::other("bucket versioning configuration has an invalid status"));
|
||||
}
|
||||
Some(config)
|
||||
@@ -1038,7 +1033,7 @@ impl BucketMetadata {
|
||||
let config = if data.is_empty() {
|
||||
None
|
||||
} else {
|
||||
let config = deserialize::<PersistedReplicationConfiguration>(&data)?;
|
||||
let config = deserialize::<ReplicationConfiguration>(&data)?;
|
||||
if let Some(field) = invalid_replication_config_status_field(&config) {
|
||||
return Err(Error::other(format!("replication field {field} has an invalid status")));
|
||||
}
|
||||
@@ -1232,8 +1227,8 @@ impl BucketMetadata {
|
||||
}
|
||||
self.lifecycle_config = None;
|
||||
if !self.lifecycle_config_xml.is_empty()
|
||||
&& let Err(e) = deserialize::<PersistedLifecycleConfiguration>(&self.lifecycle_config_xml)
|
||||
.map(|c| self.lifecycle_config = Some(c))
|
||||
&& let Err(e) =
|
||||
deserialize::<BucketLifecycleConfiguration>(&self.lifecycle_config_xml).map(|c| self.lifecycle_config = Some(c))
|
||||
{
|
||||
tracing::warn!(
|
||||
event = "bucket_metadata_parse_failed",
|
||||
@@ -1319,8 +1314,8 @@ impl BucketMetadata {
|
||||
}
|
||||
self.replication_config = None;
|
||||
if !self.replication_config_xml.is_empty()
|
||||
&& let Err(e) = deserialize::<PersistedReplicationConfiguration>(&self.replication_config_xml)
|
||||
.map(|c| self.replication_config = Some(c))
|
||||
&& let Err(e) =
|
||||
deserialize::<ReplicationConfiguration>(&self.replication_config_xml).map(|c| self.replication_config = Some(c))
|
||||
{
|
||||
tracing::warn!(
|
||||
event = "bucket_metadata_parse_failed",
|
||||
@@ -1696,13 +1691,6 @@ mod test {
|
||||
let mut bm = BucketMetadata::unmarshal(&body[4..]).expect("unmarshal recovered blob");
|
||||
assert_eq!(bm.name, "interop");
|
||||
assert!(!bm.bucket_creation_committed, "legacy metadata must default to uncommitted");
|
||||
assert!(!bm.bucket_creation_commit_record_present, "legacy metadata has no creation-commit record");
|
||||
let rewritten = BucketMetadata::unmarshal(&bm.marshal_msg().expect("rewrite legacy metadata"))
|
||||
.expect("read rewritten legacy metadata");
|
||||
assert!(
|
||||
!rewritten.bucket_creation_commit_record_present,
|
||||
"rewriting legacy metadata must not fabricate a creation intent"
|
||||
);
|
||||
bm.parse_all_configs().expect("parse recovered configs");
|
||||
assert!(bm.lifecycle_config.is_some());
|
||||
}
|
||||
@@ -1720,7 +1708,6 @@ mod test {
|
||||
|
||||
assert_eq!(bm.name, new.name);
|
||||
assert!(new.bucket_creation_committed);
|
||||
assert!(new.bucket_creation_commit_record_present);
|
||||
assert!(!bm.bucket_incarnation_id.is_nil());
|
||||
assert_eq!(bm.bucket_incarnation_id, new.bucket_incarnation_id);
|
||||
}
|
||||
@@ -1825,15 +1812,17 @@ mod test {
|
||||
|
||||
#[test]
|
||||
fn object_locking_requires_lock_metadata_not_plain_versioning() {
|
||||
use s3s::dto::ObjectLockEnabled;
|
||||
|
||||
let mut bm = BucketMetadata::new("test-bucket");
|
||||
bm.versioning_config = Some(VersioningConfiguration {
|
||||
status: Some("Enabled".to_owned()),
|
||||
status: Some(s3s::dto::BucketVersioningStatus::from_static("Enabled")),
|
||||
..Default::default()
|
||||
});
|
||||
assert!(!bm.object_locking());
|
||||
|
||||
bm.object_lock_config = Some(ObjectLockConfiguration {
|
||||
object_lock_enabled: Some("Enabled".to_owned()),
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
..Default::default()
|
||||
});
|
||||
assert!(bm.object_locking());
|
||||
|
||||
@@ -30,18 +30,14 @@ use crate::storage_api_contracts::heal::HealOperations as _;
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
use crate::store::{ECStore, await_bucket_namespace_operation};
|
||||
use futures::future::join_all;
|
||||
use rustfs_gateway_types::cors_tagging::{PersistedCorsConfiguration as CORSConfiguration, PersistedTagging as Tagging};
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedAccelerateConfiguration as AccelerateConfiguration,
|
||||
PersistedBucketEncryptionConfiguration as ServerSideEncryptionConfiguration,
|
||||
PersistedBucketLoggingStatus as BucketLoggingStatus, PersistedLifecycleConfiguration,
|
||||
PersistedNotificationConfiguration as NotificationConfiguration, PersistedObjectLockConfiguration as ObjectLockConfiguration,
|
||||
PersistedPublicAccessBlockConfiguration as PublicAccessBlockConfiguration, PersistedReplicationConfiguration,
|
||||
PersistedRequestPaymentConfiguration as RequestPaymentConfiguration,
|
||||
PersistedVersioningConfiguration as VersioningConfiguration, PersistedWebsiteConfiguration as WebsiteConfiguration,
|
||||
};
|
||||
use rustfs_heal_contracts::heal_channel::HealOpts;
|
||||
use rustfs_policy::policy::BucketPolicy;
|
||||
use s3s::dto::ReplicationConfiguration;
|
||||
use s3s::dto::{
|
||||
AccelerateConfiguration, BucketLifecycleConfiguration, BucketLoggingStatus, CORSConfiguration, NotificationConfiguration,
|
||||
ObjectLockConfiguration, ObjectLockEnabled, ObjectLockRetentionMode, PublicAccessBlockConfiguration,
|
||||
RequestPaymentConfiguration, ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration, WebsiteConfiguration,
|
||||
};
|
||||
use std::collections::HashSet;
|
||||
use std::time::Duration;
|
||||
use std::{
|
||||
@@ -172,7 +168,7 @@ pub(crate) fn object_lock_config_state_from_authoritative_metadata(bm: &BucketMe
|
||||
if bm.lock_enabled {
|
||||
return Ok(ObjectLockConfigState::Configured {
|
||||
config: ObjectLockConfiguration {
|
||||
object_lock_enabled: Some("Enabled".to_owned()),
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
rule: None,
|
||||
},
|
||||
updated_at: bm.object_lock_config_updated_at,
|
||||
@@ -216,10 +212,17 @@ pub(crate) fn configured_object_lock_state_for_tests(
|
||||
) -> ObjectLockConfigState {
|
||||
ObjectLockConfigState::Configured {
|
||||
config: ObjectLockConfiguration {
|
||||
object_lock_enabled: Some("Enabled".to_owned()),
|
||||
rule: Some(rustfs_gateway_types::persistence::PersistedObjectLockRule {
|
||||
default_retention: Some(rustfs_gateway_types::persistence::PersistedDefaultRetention {
|
||||
mode: Some(mode.as_str().to_owned()),
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
rule: Some(s3s::dto::ObjectLockRule {
|
||||
default_retention: Some(s3s::dto::DefaultRetention {
|
||||
mode: Some(s3s::dto::ObjectLockRetentionMode::from_static(match mode {
|
||||
crate::bucket::object_lock::types::RetentionMode::Governance => {
|
||||
s3s::dto::ObjectLockRetentionMode::GOVERNANCE
|
||||
}
|
||||
crate::bucket::object_lock::types::RetentionMode::Compliance => {
|
||||
s3s::dto::ObjectLockRetentionMode::COMPLIANCE
|
||||
}
|
||||
})),
|
||||
days: Some(days),
|
||||
years: None,
|
||||
}),
|
||||
@@ -230,7 +233,7 @@ pub(crate) fn configured_object_lock_state_for_tests(
|
||||
}
|
||||
|
||||
fn validate_authoritative_object_lock_config(config: &ObjectLockConfiguration) -> Result<()> {
|
||||
if config.object_lock_enabled.as_deref() != Some("Enabled") {
|
||||
if config.object_lock_enabled.as_ref().map(ObjectLockEnabled::as_str) != Some(ObjectLockEnabled::ENABLED) {
|
||||
return Err(Error::other("persisted bucket Object Lock enabled state is invalid"));
|
||||
}
|
||||
|
||||
@@ -243,7 +246,7 @@ fn validate_authoritative_object_lock_config(config: &ObjectLockConfiguration) -
|
||||
if !retention
|
||||
.mode
|
||||
.as_ref()
|
||||
.is_some_and(|mode| matches!(mode.as_str(), "COMPLIANCE" | "GOVERNANCE"))
|
||||
.is_some_and(|mode| matches!(mode.as_str(), ObjectLockRetentionMode::COMPLIANCE | ObjectLockRetentionMode::GOVERNANCE))
|
||||
{
|
||||
return Err(Error::other("persisted bucket Object Lock retention mode is invalid"));
|
||||
}
|
||||
@@ -1365,7 +1368,7 @@ pub async fn get_public_access_block_config(bucket: &str) -> Result<(PublicAcces
|
||||
bucket_meta_sys.get_public_access_block_config(bucket).await
|
||||
}
|
||||
|
||||
pub async fn get_lifecycle_config(bucket: &str) -> Result<(PersistedLifecycleConfiguration, OffsetDateTime)> {
|
||||
pub async fn get_lifecycle_config(bucket: &str) -> Result<(BucketLifecycleConfiguration, OffsetDateTime)> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||
|
||||
@@ -1584,7 +1587,7 @@ pub(crate) async fn get_cached_quota_config_and_incarnation_in(
|
||||
bucket_meta_sys.cached_quota_config_and_incarnation(bucket).await
|
||||
}
|
||||
|
||||
pub async fn get_replication_config(bucket: &str) -> Result<(PersistedReplicationConfiguration, OffsetDateTime)> {
|
||||
pub async fn get_replication_config(bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||
|
||||
@@ -1594,7 +1597,7 @@ pub async fn get_replication_config(bucket: &str) -> Result<(PersistedReplicatio
|
||||
pub(crate) async fn get_replication_config_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
) -> Result<(PersistedReplicationConfiguration, OffsetDateTime)> {
|
||||
) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
|
||||
let bucket_meta_sys_lock = bucket_metadata_sys_of(ctx)?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||
|
||||
@@ -1720,27 +1723,9 @@ struct MetadataPublishGuard {
|
||||
_guard: tokio::sync::OwnedMutexGuard<MetadataPublishLockState>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct PendingBucketCreationProof {
|
||||
pub(crate) generation: Uuid,
|
||||
pub(crate) intent: Vec<u8>,
|
||||
pub(crate) versioning_requested: bool,
|
||||
pub(crate) witnesses: Vec<crate::disk::os::NativeBucketCreationWitness>,
|
||||
}
|
||||
|
||||
impl PendingBucketCreationProof {
|
||||
pub(crate) fn matches_metadata(&self, metadata: &BucketMetadata) -> Result<bool> {
|
||||
Ok(!metadata.bucket_creation_committed
|
||||
&& metadata.bucket_incarnation_sidecar
|
||||
&& metadata.bucket_incarnation_id == self.generation
|
||||
&& metadata.marshal_msg()? == self.intent)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BucketMetadataSys {
|
||||
metadata_map: Arc<RwLock<HashMap<String, Arc<BucketMetadata>>>>,
|
||||
pending_bucket_creations: Arc<RwLock<HashMap<String, PendingBucketCreationProof>>>,
|
||||
/// Serializes metadata-map commits and their derived cache updates for one
|
||||
/// bucket. Namespace locks, when present, are acquired before this lock.
|
||||
metadata_publish_locks: Arc<MetadataPublishLockRegistry>,
|
||||
@@ -1784,7 +1769,6 @@ impl BucketMetadataSys {
|
||||
pub fn new(api: Arc<ECStore>) -> Self {
|
||||
Self {
|
||||
metadata_map: Arc::new(RwLock::new(HashMap::new())),
|
||||
pending_bucket_creations: Arc::new(RwLock::new(HashMap::new())),
|
||||
metadata_publish_locks: Arc::new(MetadataPublishLockRegistry {
|
||||
locks: StdMutex::new(HashMap::new()),
|
||||
}),
|
||||
@@ -1815,81 +1799,6 @@ impl BucketMetadataSys {
|
||||
.expect("bucket metadata object store should still be live")
|
||||
}
|
||||
|
||||
pub(crate) async fn pending_bucket_creation(&self, bucket: &str) -> Option<PendingBucketCreationProof> {
|
||||
self.pending_bucket_creations.read().await.get(bucket).cloned()
|
||||
}
|
||||
|
||||
pub(crate) async fn grant_pending_bucket_creation(&self, bucket: &str, proof: PendingBucketCreationProof) {
|
||||
self.pending_bucket_creations.write().await.insert(bucket.to_owned(), proof);
|
||||
}
|
||||
|
||||
pub(crate) async fn revoke_pending_bucket_creation(&self, bucket: &str) {
|
||||
self.pending_bucket_creations.write().await.remove(bucket);
|
||||
}
|
||||
|
||||
pub(crate) async fn consume_pending_bucket_creation(
|
||||
&self,
|
||||
bucket: &str,
|
||||
metadata: &BucketMetadata,
|
||||
witnesses: &[crate::disk::os::NativeBucketCreationWitness],
|
||||
) -> Result<bool> {
|
||||
let mut pending = self.pending_bucket_creations.write().await;
|
||||
let Some(proof) = pending.get(bucket) else {
|
||||
return Ok(false);
|
||||
};
|
||||
if !proof.matches_metadata(metadata)?
|
||||
|| proof.witnesses.len() != witnesses.len()
|
||||
|| !proof
|
||||
.witnesses
|
||||
.iter()
|
||||
.zip(witnesses)
|
||||
.all(|(old, current)| old.matches(current))
|
||||
|| !crate::disk::os::consume_native_creation_witnesses(witnesses)
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
pending.remove(bucket);
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn refresh_bucket_for_test(&self, bucket: &str) -> Result<()> {
|
||||
let expected = self.metadata_map.read().await.get(bucket).cloned();
|
||||
let lock = self.object_store().new_ns_lock(bucket, bucket).await?;
|
||||
let guard = lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
|
||||
self.load_bucket_under_namespace(bucket, MetadataLoadMode::Refresh, expected.as_ref(), &guard)
|
||||
.await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn publish_persisted_metadata_for_test(&self, bucket: &str) -> Result<()> {
|
||||
let expected = self.metadata_map.read().await.get(bucket).cloned();
|
||||
let lock = self.object_store().new_ns_lock(bucket, bucket).await?;
|
||||
let guard = lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
|
||||
let (metadata, persisted) = await_bucket_namespace_operation(
|
||||
Some(&guard),
|
||||
bucket,
|
||||
"test cached metadata load",
|
||||
load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true),
|
||||
)
|
||||
.await?;
|
||||
self.publish_if_unchanged(bucket, expected.as_ref(), metadata, persisted, &guard)
|
||||
.await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn migrate_bucket_creation_commit_for_test(&self, bucket: &str) -> Result<()> {
|
||||
self.migrate_bucket_creation_commit(bucket).await.map(|_| ())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn legacy_migration_reports_missing_bucket_for_test(&self, bucket: &str) -> Result<bool> {
|
||||
Ok(matches!(
|
||||
self.migrate_legacy_metadata(bucket).await?,
|
||||
BucketMetadataAuthority::MissingBucket
|
||||
))
|
||||
}
|
||||
|
||||
fn object_store_if_live(&self) -> Option<Arc<ECStore>> {
|
||||
self.api.upgrade()
|
||||
}
|
||||
@@ -2080,7 +1989,6 @@ impl BucketMetadataSys {
|
||||
expected: Option<&Arc<BucketMetadata>>,
|
||||
namespace_guard: &rustfs_lock::NamespaceLockGuard,
|
||||
) -> Result<()> {
|
||||
self.revoke_pending_bucket_creation(bucket).await;
|
||||
if !await_bucket_namespace_operation(
|
||||
Some(namespace_guard),
|
||||
bucket,
|
||||
@@ -2164,7 +2072,6 @@ impl BucketMetadataSys {
|
||||
persisted: bool,
|
||||
namespace_guard: &rustfs_lock::NamespaceLockGuard,
|
||||
) -> Result<()> {
|
||||
self.revoke_pending_bucket_creation(bucket).await;
|
||||
if !persisted {
|
||||
let _publish_guard = self
|
||||
.lock_metadata_publish(bucket, namespace_guard, "refreshed bucket metadata absence publish")
|
||||
@@ -2226,7 +2133,6 @@ impl BucketMetadataSys {
|
||||
}
|
||||
|
||||
pub async fn set(&self, bucket: String, bm: Arc<BucketMetadata>) {
|
||||
self.revoke_pending_bucket_creation(&bucket).await;
|
||||
if !is_meta_bucketname(&bucket) {
|
||||
let publish_lock = self.metadata_publish_lock(&bucket);
|
||||
let _publish_guard = publish_lock.lock().await;
|
||||
@@ -2248,7 +2154,6 @@ impl BucketMetadataSys {
|
||||
if is_meta_bucketname(bucket) {
|
||||
return false;
|
||||
}
|
||||
self.revoke_pending_bucket_creation(bucket).await;
|
||||
let publish_lock = self.metadata_publish_lock(bucket);
|
||||
let _publish_guard = publish_lock.lock().await;
|
||||
let mut map = self.metadata_map.write().await;
|
||||
@@ -2266,7 +2171,6 @@ impl BucketMetadataSys {
|
||||
}
|
||||
|
||||
async fn _reset(&mut self) {
|
||||
self.pending_bucket_creations.write().await.clear();
|
||||
let mut map = self.metadata_map.write().await;
|
||||
map.clear();
|
||||
drop(map);
|
||||
@@ -2405,7 +2309,6 @@ impl BucketMetadataSys {
|
||||
/// server's metadata never leaks into the ambient (first) instance.
|
||||
pub(crate) async fn persist_and_set(&self, bm: BucketMetadata) -> Result<()> {
|
||||
let mut bm = bm;
|
||||
self.revoke_pending_bucket_creation(&bm.name).await;
|
||||
bm.save_with_store(self.object_store()).await?;
|
||||
|
||||
self.set(bm.name.clone(), Arc::new(bm)).await;
|
||||
@@ -2414,7 +2317,6 @@ impl BucketMetadataSys {
|
||||
}
|
||||
|
||||
async fn persist_new_and_set(&self, mut bm: BucketMetadata) -> Result<()> {
|
||||
self.revoke_pending_bucket_creation(&bm.name).await;
|
||||
bm.bucket_creation_committed = true;
|
||||
bm.save_with_store_committed(self.object_store()).await?;
|
||||
save_bucket_incarnation(self.object_store(), &bm.name, bm.bucket_incarnation_id).await?;
|
||||
@@ -2424,9 +2326,7 @@ impl BucketMetadataSys {
|
||||
}
|
||||
|
||||
async fn persist_new_bucket_metadata_intent(&self, mut bm: BucketMetadata) -> Result<()> {
|
||||
self.revoke_pending_bucket_creation(&bm.name).await;
|
||||
bm.bucket_creation_committed = false;
|
||||
bm.bucket_creation_commit_record_present = true;
|
||||
bm.save_with_store(self.object_store()).await?;
|
||||
save_bucket_incarnation(self.object_store(), &bm.name, bm.bucket_incarnation_id).await?;
|
||||
bm.bucket_incarnation_sidecar = true;
|
||||
@@ -2436,7 +2336,6 @@ impl BucketMetadataSys {
|
||||
}
|
||||
|
||||
async fn commit_bucket_metadata(&self, mut bm: BucketMetadata) -> Result<()> {
|
||||
self.revoke_pending_bucket_creation(&bm.name).await;
|
||||
bm.parse_all_configs()?;
|
||||
bm.bucket_incarnation_sidecar = true;
|
||||
// The caller holds the full bucket creation fence (lifecycle, metadata
|
||||
@@ -2485,7 +2384,6 @@ impl BucketMetadataSys {
|
||||
bucket: &str,
|
||||
namespace_guard: &rustfs_lock::NamespaceLockGuard,
|
||||
) -> Result<()> {
|
||||
self.revoke_pending_bucket_creation(bucket).await;
|
||||
let expected = self.metadata_map.read().await.get(bucket).cloned();
|
||||
if !self
|
||||
.bucket_exists(bucket, namespace_guard, "peer bucket metadata existence check")
|
||||
@@ -2542,7 +2440,6 @@ impl BucketMetadataSys {
|
||||
#[cfg(test)]
|
||||
self.lazy_disk_loads.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
|
||||
self.revoke_pending_bucket_creation(bucket).await;
|
||||
let lock = self.object_store().new_ns_lock(bucket, bucket).await?;
|
||||
let guard = lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
|
||||
#[cfg(test)]
|
||||
@@ -2868,7 +2765,6 @@ impl BucketMetadataSys {
|
||||
/// revalidated at write quorum, so a stale snapshot can neither revert an
|
||||
/// acknowledged configuration update nor outlive a delete/recreate.
|
||||
async fn migrate_bucket_creation_commit(&self, bucket: &str) -> Result<BucketMetadataAuthority> {
|
||||
self.revoke_pending_bucket_creation(bucket).await;
|
||||
let transaction_lock = self
|
||||
.object_store()
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
|
||||
@@ -2946,7 +2842,6 @@ impl BucketMetadataSys {
|
||||
&self,
|
||||
bucket: &str,
|
||||
) -> Result<BucketMetadataAuthority> {
|
||||
self.revoke_pending_bucket_creation(bucket).await;
|
||||
#[cfg(test)]
|
||||
if self.object_lock_disk_read_errors.write().await.remove(bucket) {
|
||||
return Err(Error::other(format!("injected Object Lock metadata disk read failure: {bucket}")));
|
||||
@@ -3122,7 +3017,7 @@ impl BucketMetadataSys {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_lifecycle_config(&self, bucket: &str) -> Result<(PersistedLifecycleConfiguration, OffsetDateTime)> {
|
||||
pub async fn get_lifecycle_config(&self, bucket: &str) -> Result<(BucketLifecycleConfiguration, OffsetDateTime)> {
|
||||
let (bm, _) = self.get_config(bucket).await?;
|
||||
|
||||
match ConfigState::of(&bm.lifecycle_config_xml, &bm.lifecycle_config).require(bucket, BUCKET_LIFECYCLE_CONFIG)? {
|
||||
@@ -3245,7 +3140,7 @@ impl BucketMetadataSys {
|
||||
}))
|
||||
}
|
||||
|
||||
pub async fn get_replication_config(&self, bucket: &str) -> Result<(PersistedReplicationConfiguration, OffsetDateTime)> {
|
||||
pub async fn get_replication_config(&self, bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
|
||||
let (bm, _) = self.get_config(bucket).await?;
|
||||
|
||||
if !bm.replication_config_xml.is_empty() && bm.replication_config.is_none() {
|
||||
@@ -3339,7 +3234,6 @@ mod tests {
|
||||
};
|
||||
use crate::bucket::target::{BucketTarget, BucketTargetType, Credentials};
|
||||
use crate::config::com::read_config;
|
||||
use crate::disk::{DiskAPI as _, RUSTFS_META_TMP_BUCKET};
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions};
|
||||
use byteorder::{ByteOrder as _, LittleEndian};
|
||||
use serial_test::serial;
|
||||
@@ -3394,180 +3288,6 @@ mod tests {
|
||||
),
|
||||
];
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(storage_class_env)]
|
||||
async fn persistence_entry_failures_revoke_pending_creation_before_cache_publish() {
|
||||
let (dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let sys = bucket_metadata_sys_of(&store.ctx).expect("instance metadata system");
|
||||
let sys = sys.read().await.clone();
|
||||
let options = MakeBucketOptions {
|
||||
lock_enabled: true,
|
||||
..Default::default()
|
||||
};
|
||||
for entry in ["save", "persist", "persist-new", "commit"] {
|
||||
let bucket = format!("revoke-{entry}-{}", Uuid::new_v4());
|
||||
store.fail_next_bucket_creation_after_intent_for_test(&bucket);
|
||||
store
|
||||
.make_bucket(&bucket, &options)
|
||||
.await
|
||||
.expect_err("stop the real producer before physical publication");
|
||||
sys.pending_bucket_creation(&bucket)
|
||||
.await
|
||||
.expect("the producer must leave a real live creation permission");
|
||||
let (intent, persisted) = get_config_from_disk_with_presence_in(&store.ctx, &bucket)
|
||||
.await
|
||||
.expect("read the original producer intent");
|
||||
assert!(persisted, "{entry}: the complete original intent must be persisted");
|
||||
let original_intent = intent.marshal_msg().expect("encode the original intent");
|
||||
|
||||
// Keep the creation fence order used by the real producer.
|
||||
let lifecycle_guard = store
|
||||
.acquire_bucket_lifecycle_write_lock(&bucket)
|
||||
.await
|
||||
.expect("hold the bucket lifecycle fence");
|
||||
let transaction_guard = acquire_bucket_metadata_transaction_lock_in(&store.ctx, &bucket)
|
||||
.await
|
||||
.expect("hold the metadata transaction fence");
|
||||
let namespace_lock = store
|
||||
.new_ns_lock(&bucket, &bucket)
|
||||
.await
|
||||
.expect("create the exact bucket namespace lock");
|
||||
let namespace_guard = namespace_lock
|
||||
.get_write_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await
|
||||
.expect("hold the exact bucket namespace fence");
|
||||
|
||||
// A file at each native staging root causes actual IO to fail before
|
||||
// metadata can reach the cache-publishing set call. Preserve the
|
||||
// original staging directories so no fixture data is discarded.
|
||||
for dir in &dirs {
|
||||
let staging = dir.path().join(RUSTFS_META_TMP_BUCKET);
|
||||
let saved = dir.path().join(".rustfs.sys/revocation-saved-tmp");
|
||||
std::fs::rename(&staging, &saved).expect("preserve the native staging directory");
|
||||
std::fs::write(&staging, b"block metadata staging with a non-directory")
|
||||
.expect("install an actual non-directory IO obstruction");
|
||||
}
|
||||
let result = match entry {
|
||||
"save" => sys.save(intent).await,
|
||||
"persist" => sys.persist_and_set(intent).await,
|
||||
"persist-new" => sys.persist_new_and_set(intent).await,
|
||||
"commit" => sys.commit_bucket_metadata(intent).await,
|
||||
_ => unreachable!(),
|
||||
};
|
||||
for dir in &dirs {
|
||||
let staging = dir.path().join(RUSTFS_META_TMP_BUCKET);
|
||||
let saved = dir.path().join(".rustfs.sys/revocation-saved-tmp");
|
||||
std::fs::remove_file(&staging).expect("remove only the fixture's staging obstruction");
|
||||
std::fs::rename(&saved, &staging).expect("restore the complete original staging directory");
|
||||
}
|
||||
drop((namespace_guard, transaction_guard, lifecycle_guard));
|
||||
result.expect_err("native persistence must fail before its downstream cache set");
|
||||
assert!(
|
||||
sys.pending_bucket_creation(&bucket).await.is_none(),
|
||||
"{entry}: the entry revoker must retire live permission even when persistence fails"
|
||||
);
|
||||
let (after_failure, present) = get_config_from_disk_with_presence_in(&store.ctx, &bucket)
|
||||
.await
|
||||
.expect("read the intent after the failed persistence entry");
|
||||
assert!(present, "{entry}: a failed persistence entry must retain the original metadata");
|
||||
assert_eq!(
|
||||
after_failure
|
||||
.marshal_msg()
|
||||
.expect("encode the original intent after persistence failure"),
|
||||
original_intent,
|
||||
"{entry}: native IO failure must not change the original generation or complete intent"
|
||||
);
|
||||
let error = store
|
||||
.make_bucket(&bucket, &options)
|
||||
.await
|
||||
.expect_err("a failed ordinary persistence entry cannot preserve creation permission");
|
||||
assert!(error.to_string().contains("administrator recovery"), "{entry}: {error}");
|
||||
let (after_retry, _) = get_config_from_disk_with_presence_in(&store.ctx, &bucket)
|
||||
.await
|
||||
.expect("read the original intent after the rejected retry");
|
||||
assert_eq!(
|
||||
after_retry.marshal_msg().expect("encode the intent after the rejected retry"),
|
||||
original_intent,
|
||||
"{entry}: the rejected retry must preserve the complete original intent"
|
||||
);
|
||||
for disk in store.pools[0].disk_set[0].disks.read().await.iter() {
|
||||
assert!(
|
||||
matches!(
|
||||
disk.as_ref().expect("native disk").stat_volume(&bucket).await,
|
||||
Err(crate::disk::error::DiskError::VolumeNotFound)
|
||||
),
|
||||
"{entry}: failed persistence and the rejected retry must not publish a physical volume"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(storage_class_env)]
|
||||
async fn metadata_reset_revokes_real_pending_creation_without_changing_persisted_intent() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("revoke-reset-{}", Uuid::new_v4());
|
||||
let options = MakeBucketOptions {
|
||||
lock_enabled: true,
|
||||
..Default::default()
|
||||
};
|
||||
store.fail_next_bucket_creation_after_intent_for_test(&bucket);
|
||||
store
|
||||
.make_bucket(&bucket, &options)
|
||||
.await
|
||||
.expect_err("stop the real producer before physical publication");
|
||||
let sys = bucket_metadata_sys_of(&store.ctx).expect("instance metadata system");
|
||||
let mut sys = sys.read().await.clone();
|
||||
sys.pending_bucket_creation(&bucket)
|
||||
.await
|
||||
.expect("the real producer must leave live creation permission before reset");
|
||||
let (intent, persisted) = get_config_from_disk_with_presence_in(&store.ctx, &bucket)
|
||||
.await
|
||||
.expect("read the original intent before cache reset");
|
||||
assert!(persisted, "cache reset starts from a complete real persisted intent");
|
||||
let original_intent = intent.marshal_msg().expect("encode the original reset intent");
|
||||
sys._reset().await;
|
||||
assert!(
|
||||
sys.pending_bucket_creation(&bucket).await.is_none(),
|
||||
"reset must revoke the producer-issued permission shared with the instance metadata system"
|
||||
);
|
||||
let (after_reset, present) = get_config_from_disk_with_presence_in(&store.ctx, &bucket)
|
||||
.await
|
||||
.expect("read the original intent after cache reset");
|
||||
assert!(present, "cache reset must retain the persisted intent");
|
||||
assert_eq!(
|
||||
after_reset.marshal_msg().expect("encode the intent after cache reset"),
|
||||
original_intent,
|
||||
"cache reset must leave the original generation and complete persisted intent unchanged"
|
||||
);
|
||||
let error = store
|
||||
.make_bucket(&bucket, &options)
|
||||
.await
|
||||
.expect_err("reset cannot regrant the producer's old creation permission");
|
||||
assert!(error.to_string().contains("administrator recovery"), "{error}");
|
||||
let (after_retry, _) = get_config_from_disk_with_presence_in(&store.ctx, &bucket)
|
||||
.await
|
||||
.expect("read the intent after the reset-induced retry rejection");
|
||||
assert_eq!(
|
||||
after_retry
|
||||
.marshal_msg()
|
||||
.expect("encode the intent after the rejected reset retry"),
|
||||
original_intent,
|
||||
"the reset-induced rejection must preserve the original complete intent"
|
||||
);
|
||||
for disk in store.pools[0].disk_set[0].disks.read().await.iter() {
|
||||
assert!(
|
||||
matches!(
|
||||
disk.as_ref().expect("native disk").stat_volume(&bucket).await,
|
||||
Err(crate::disk::error::DiskError::VolumeNotFound)
|
||||
),
|
||||
"cache reset and the rejected retry must not publish a physical volume"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn g_d3_003_new_writer_replication_loads_without_fail_closed_state() {
|
||||
let (dirs, store) = isolated_store_over_temp_disks().await;
|
||||
@@ -3756,7 +3476,7 @@ mod tests {
|
||||
let mut versioned = BucketMetadata::new("authoritative-versioned");
|
||||
versioned.versioning_config_xml = b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec();
|
||||
versioned.versioning_config = Some(VersioningConfiguration {
|
||||
status: Some("Enabled".to_owned()),
|
||||
status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::ENABLED)),
|
||||
..Default::default()
|
||||
});
|
||||
sys.set("authoritative-versioned".to_string(), Arc::new(versioned)).await;
|
||||
@@ -3879,24 +3599,25 @@ mod tests {
|
||||
else {
|
||||
panic!("legacy lock-enabled metadata must remain configured");
|
||||
};
|
||||
assert_eq!(config.object_lock_enabled.as_deref(), Some("Enabled"));
|
||||
assert_eq!(
|
||||
config.object_lock_enabled.as_ref().map(|value| value.as_str()),
|
||||
Some(ObjectLockEnabled::ENABLED)
|
||||
);
|
||||
assert!(config.rule.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authoritative_object_lock_state_rejects_invalid_retention_periods() {
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedDefaultRetention as DefaultRetention, PersistedObjectLockRule as ObjectLockRule,
|
||||
};
|
||||
use s3s::dto::{DefaultRetention, ObjectLockRule};
|
||||
|
||||
for (days, years) in [(Some(0), None), (Some(-1), None), (Some(1), Some(1)), (None, None)] {
|
||||
let mut metadata = BucketMetadata::new("invalid-object-lock");
|
||||
metadata.object_lock_config = Some(ObjectLockConfiguration {
|
||||
object_lock_enabled: Some("Enabled".to_owned()),
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
rule: Some(ObjectLockRule {
|
||||
default_retention: Some(DefaultRetention {
|
||||
days,
|
||||
mode: Some("COMPLIANCE".to_owned()),
|
||||
mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE)),
|
||||
years,
|
||||
}),
|
||||
}),
|
||||
@@ -3911,11 +3632,9 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn authoritative_object_lock_state_rejects_invalid_enabled_mode_and_rule() {
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedDefaultRetention as DefaultRetention, PersistedObjectLockRule as ObjectLockRule,
|
||||
};
|
||||
use s3s::dto::{DefaultRetention, ObjectLockRule};
|
||||
|
||||
for enabled in [None, Some("Disabled".to_string())] {
|
||||
for enabled in [None, Some(ObjectLockEnabled::from("Disabled".to_string()))] {
|
||||
let mut metadata = BucketMetadata::new("invalid-object-lock-enabled");
|
||||
metadata.lock_enabled = true;
|
||||
metadata.object_lock_config = Some(ObjectLockConfiguration {
|
||||
@@ -3927,11 +3646,11 @@ mod tests {
|
||||
|
||||
let mut invalid_mode = BucketMetadata::new("invalid-object-lock-mode");
|
||||
invalid_mode.object_lock_config = Some(ObjectLockConfiguration {
|
||||
object_lock_enabled: Some("Enabled".to_owned()),
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
rule: Some(ObjectLockRule {
|
||||
default_retention: Some(DefaultRetention {
|
||||
days: Some(1),
|
||||
mode: Some("invalid".to_string()),
|
||||
mode: Some(ObjectLockRetentionMode::from("invalid".to_string())),
|
||||
years: None,
|
||||
}),
|
||||
}),
|
||||
@@ -3940,7 +3659,7 @@ mod tests {
|
||||
|
||||
let mut missing_retention = BucketMetadata::new("missing-object-lock-retention");
|
||||
missing_retention.object_lock_config = Some(ObjectLockConfiguration {
|
||||
object_lock_enabled: Some("Enabled".to_owned()),
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
rule: Some(ObjectLockRule { default_retention: None }),
|
||||
});
|
||||
assert!(object_lock_config_state_from_authoritative_metadata(&missing_retention).is_err());
|
||||
@@ -3950,7 +3669,7 @@ mod tests {
|
||||
fn object_lock_ever_enabled_marker_survives_config_removal() {
|
||||
let mut metadata = BucketMetadata::new("lock-config-removed");
|
||||
let config = ObjectLockConfiguration {
|
||||
object_lock_enabled: Some("Enabled".to_owned()),
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
rule: None,
|
||||
};
|
||||
metadata
|
||||
@@ -3968,7 +3687,10 @@ mod tests {
|
||||
else {
|
||||
panic!("an ever-enabled bucket must remain Object Lock configured");
|
||||
};
|
||||
assert_eq!(config.object_lock_enabled.as_deref(), Some("Enabled"));
|
||||
assert_eq!(
|
||||
config.object_lock_enabled.as_ref().map(ObjectLockEnabled::as_str),
|
||||
Some(ObjectLockEnabled::ENABLED)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -3984,7 +3706,7 @@ mod tests {
|
||||
|
||||
let node = Arc::new(RwLock::new(BucketMetadataSys::new(ecstore.clone())));
|
||||
let config = ObjectLockConfiguration {
|
||||
object_lock_enabled: Some("Enabled".to_owned()),
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
rule: None,
|
||||
};
|
||||
update_with_sys(
|
||||
@@ -4965,7 +4687,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn update_config_with_persists_tagging_rewrite_across_disk_reload() {
|
||||
use crate::bucket::metadata::BUCKET_TAGGING_CONFIG;
|
||||
use rustfs_gateway_types::cors_tagging::PersistedTag as Tag;
|
||||
use s3s::dto::Tag;
|
||||
|
||||
let (dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||
|
||||
@@ -5129,7 +4851,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn concurrent_update_config_with_calls_do_not_lose_writes() {
|
||||
use crate::bucket::metadata::BUCKET_TAGGING_CONFIG;
|
||||
use rustfs_gateway_types::cors_tagging::PersistedTag as Tag;
|
||||
use s3s::dto::Tag;
|
||||
|
||||
let (dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||
let sys = Arc::new(RwLock::new(BucketMetadataSys::new(ecstore)));
|
||||
@@ -5229,7 +4951,7 @@ mod tests {
|
||||
let mut persisted = BucketMetadata::new("reload-bucket");
|
||||
persisted.policy_config_json = b"persisted-marker".to_vec();
|
||||
let lock_config = ObjectLockConfiguration {
|
||||
object_lock_enabled: Some("Enabled".to_owned()),
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
rule: None,
|
||||
};
|
||||
persisted
|
||||
@@ -5508,7 +5230,7 @@ mod tests {
|
||||
#[serial]
|
||||
async fn bucket_delete_waits_for_config_mutation_fence() {
|
||||
use crate::bucket::metadata::BUCKET_TAGGING_CONFIG;
|
||||
use rustfs_gateway_types::cors_tagging::PersistedTag as Tag;
|
||||
use s3s::dto::Tag;
|
||||
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
|
||||
@@ -17,8 +17,15 @@ pub mod objectlock_sys;
|
||||
pub mod types;
|
||||
|
||||
/// Whether a bucket Object Lock configuration has locking enabled. The
|
||||
/// persisted `ObjectLockConfiguration` implements this in the bucket-metadata
|
||||
/// module, which owns the persisted configuration type.
|
||||
/// serving-layer `ObjectLockConfiguration` DTO implements this in
|
||||
/// the bucket-metadata module, which owns the persisted configuration type
|
||||
/// during the s3s ratchet migration (rustfs/backlog#1842).
|
||||
pub trait ObjectLockApi {
|
||||
fn enabled(&self) -> bool;
|
||||
}
|
||||
|
||||
/// Whether a legal-hold status value is one of the two valid wire values.
|
||||
/// Implemented for the serving-layer DTO in the bucket-metadata module.
|
||||
pub trait ObjectLockStatusExt {
|
||||
fn valid(&self) -> bool;
|
||||
}
|
||||
|
||||
@@ -12,7 +12,6 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub use rustfs_replication::config_cache_serde;
|
||||
pub use rustfs_replication::{
|
||||
ObjectOpts, OperatorRuleContract, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_READ_ONLY_HISTORICAL_FIELDS,
|
||||
REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS, REPLICATION_CAPABILITY_CONTRACT_VERSION,
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use rustfs_gateway_types::persistence::PersistedReplicationConfiguration;
|
||||
use s3s::dto::ReplicationConfiguration;
|
||||
|
||||
use super::replication_config_boundary::ReplicationConfigurationExt as _;
|
||||
use super::replication_filemeta_boundary::{
|
||||
@@ -42,7 +42,7 @@ impl ReplicationLifecycleBridge {
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn new_config(config: PersistedReplicationConfiguration) -> ReplicationLifecycleConfig {
|
||||
pub(crate) fn new_config(config: ReplicationConfiguration) -> ReplicationLifecycleConfig {
|
||||
ReplicationConfig::new(Some(config), None)
|
||||
}
|
||||
|
||||
@@ -118,17 +118,16 @@ impl ReplicationLifecycleBridge {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rustfs_gateway_types::dto::Status;
|
||||
use rustfs_gateway_types::persistence::{PersistedReplicationDestination, PersistedReplicationRule};
|
||||
use s3s::dto::{Destination, ReplicationRule, ReplicationRuleStatus};
|
||||
|
||||
use super::super::replication_filemeta_boundary::{ReplicateTargetDecision, VersionPurgeStatusType};
|
||||
use super::*;
|
||||
|
||||
fn replication_rule() -> PersistedReplicationRule {
|
||||
PersistedReplicationRule {
|
||||
fn replication_rule() -> ReplicationRule {
|
||||
ReplicationRule {
|
||||
delete_marker_replication: None,
|
||||
delete_replication: None,
|
||||
destination: PersistedReplicationDestination {
|
||||
destination: Destination {
|
||||
bucket: "arn:aws:s3:::target-bucket".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
@@ -138,13 +137,13 @@ mod tests {
|
||||
prefix: Some(String::new()),
|
||||
priority: Some(1),
|
||||
source_selection_criteria: None,
|
||||
status: Status::ENABLED.to_string(),
|
||||
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn has_pending_version_purge_preserves_replication_active_rule_behavior() {
|
||||
let config = ReplicationLifecycleBridge::new_config(PersistedReplicationConfiguration {
|
||||
let config = ReplicationLifecycleBridge::new_config(ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![replication_rule()],
|
||||
});
|
||||
|
||||
@@ -17,8 +17,8 @@ use std::sync::Arc;
|
||||
use crate::bucket::{metadata::BucketMetadata, metadata_sys};
|
||||
use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
|
||||
use crate::runtime::instance::InstanceContext;
|
||||
use rustfs_gateway_types::persistence::PersistedReplicationConfiguration;
|
||||
use rustfs_utils::path::path_join_buf;
|
||||
use s3s::dto::ReplicationConfiguration;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
use super::replication_error_boundary::{Error, Result};
|
||||
@@ -36,11 +36,11 @@ impl ReplicationMetadataStore {
|
||||
pub(crate) const FORCE_DELETE_REPLICATION_FILE: &'static str = "config/replication/force-delete.bin";
|
||||
pub(crate) const FORCE_DELETE_REPLICATION_TRANSACTION_LOCK: &'static str = "config/replication/force-delete.bin.transaction";
|
||||
|
||||
pub(crate) async fn replication_config(bucket: &str) -> Result<(PersistedReplicationConfiguration, OffsetDateTime)> {
|
||||
pub(crate) async fn replication_config(bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
|
||||
metadata_sys::get_replication_config(bucket).await
|
||||
}
|
||||
|
||||
pub(crate) async fn optional_replication_config(bucket: &str) -> Result<Option<PersistedReplicationConfiguration>> {
|
||||
pub(crate) async fn optional_replication_config(bucket: &str) -> Result<Option<ReplicationConfiguration>> {
|
||||
let config = match Self::replication_config(bucket).await {
|
||||
Ok((config, _)) => Some(config),
|
||||
Err(err) => {
|
||||
|
||||
@@ -16,9 +16,7 @@ use std::{collections::HashMap, fmt, sync::Arc};
|
||||
|
||||
use super::replication_filemeta_boundary::metadata_keys;
|
||||
use crate::bucket::metadata::BucketMetadata;
|
||||
use rustfs_gateway_types::dto::Status;
|
||||
use rustfs_gateway_types::persistence::PersistedReplicationConfiguration;
|
||||
use rustfs_gateway_types::persistence::PersistedVersioningConfiguration as VersioningConfiguration;
|
||||
use s3s::dto::{BucketVersioningStatus, ReplicationConfiguration, ReplicationRuleStatus, VersioningConfiguration};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use time::OffsetDateTime;
|
||||
use tracing::error;
|
||||
@@ -41,7 +39,7 @@ use super::replication_target_boundary::{BucketTargets, ReplicationTargetStore};
|
||||
use super::replication_versioning_boundary::ReplicationVersioningStore;
|
||||
use super::runtime_boundary as runtime_sources;
|
||||
|
||||
pub(crate) async fn get_replication_config(bucket: &str) -> Result<Option<PersistedReplicationConfiguration>> {
|
||||
pub(crate) async fn get_replication_config(bucket: &str) -> Result<Option<ReplicationConfiguration>> {
|
||||
let config = ReplicationMetadataStore::optional_replication_config(bucket).await?;
|
||||
validate_delete_replication_config(&VersioningConfiguration::default(), config.as_ref())?;
|
||||
Ok(config)
|
||||
@@ -66,7 +64,7 @@ impl DeleteReplicationConfigSnapshot {
|
||||
#[cfg(test)]
|
||||
pub(crate) fn from_configs_for_test(
|
||||
versioning: VersioningConfiguration,
|
||||
replication: Option<PersistedReplicationConfiguration>,
|
||||
replication: Option<ReplicationConfiguration>,
|
||||
) -> Self {
|
||||
let metadata = replication.map(|config| {
|
||||
let mut metadata = BucketMetadata::new("test-bucket");
|
||||
@@ -80,7 +78,7 @@ impl DeleteReplicationConfigSnapshot {
|
||||
&self.versioning
|
||||
}
|
||||
|
||||
pub fn replication_config(&self) -> Option<&PersistedReplicationConfiguration> {
|
||||
pub fn replication_config(&self) -> Option<&ReplicationConfiguration> {
|
||||
self.metadata
|
||||
.as_ref()
|
||||
.and_then(|metadata| metadata.replication_config.as_ref())
|
||||
@@ -103,7 +101,7 @@ impl DeleteReplicationConfigSnapshot {
|
||||
pub(crate) fn active_delete_marker_rules_require_tags(&self, object: &str) -> bool {
|
||||
self.replication_config().is_some_and(|config| {
|
||||
config.rules.iter().any(|rule| {
|
||||
if rule.status == Status::DISABLED.as_str() {
|
||||
if rule.status == ReplicationRuleStatus::from_static(ReplicationRuleStatus::DISABLED) {
|
||||
return false;
|
||||
}
|
||||
if !object.starts_with(rule.prefix()) {
|
||||
@@ -124,12 +122,12 @@ impl DeleteReplicationConfigSnapshot {
|
||||
|
||||
fn validate_delete_replication_config(
|
||||
versioning: &VersioningConfiguration,
|
||||
config: Option<&PersistedReplicationConfiguration>,
|
||||
config: Option<&ReplicationConfiguration>,
|
||||
) -> Result<()> {
|
||||
if versioning
|
||||
.status
|
||||
.as_ref()
|
||||
.is_some_and(|status| !matches!(status.as_str(), "Enabled" | "Suspended"))
|
||||
.is_some_and(|status| !matches!(status.as_str(), BucketVersioningStatus::ENABLED | BucketVersioningStatus::SUSPENDED))
|
||||
{
|
||||
return Err(super::replication_error_boundary::Error::other(
|
||||
"bucket versioning configuration has an invalid status",
|
||||
@@ -146,7 +144,7 @@ fn validate_delete_replication_config(
|
||||
let role = config.role.trim();
|
||||
let mut role_destination = None;
|
||||
for rule in &config.rules {
|
||||
if rule.status.as_str() == Status::ENABLED.as_str() {
|
||||
if rule.status.as_str() == ReplicationRuleStatus::ENABLED {
|
||||
let destination = rule.destination.bucket.trim();
|
||||
if role.is_empty() && destination.is_empty() {
|
||||
return Err(super::replication_error_boundary::Error::other(
|
||||
@@ -171,7 +169,7 @@ fn validate_delete_replication_config(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn replication_config_from_metadata(metadata: &BucketMetadata) -> Result<Option<&PersistedReplicationConfiguration>> {
|
||||
fn replication_config_from_metadata(metadata: &BucketMetadata) -> Result<Option<&ReplicationConfiguration>> {
|
||||
if !metadata.replication_config_xml.is_empty() && metadata.replication_config.is_none() {
|
||||
return Err(super::replication_error_boundary::Error::other(
|
||||
"persisted bucket replication configuration is invalid",
|
||||
@@ -252,15 +250,12 @@ pub(crate) async fn load_delete_replication_config_in(
|
||||
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub struct ReplicationConfig {
|
||||
/// Carried in the scanner's persisted data usage cache, so its serde form
|
||||
/// is the one the s3s DTO wrote there (see `config_cache_serde` in the replication crate).
|
||||
#[serde(with = "super::replication_config_boundary::config_cache_serde")]
|
||||
pub config: Option<PersistedReplicationConfiguration>,
|
||||
pub config: Option<ReplicationConfiguration>,
|
||||
pub remotes: Option<BucketTargets>,
|
||||
}
|
||||
|
||||
impl ReplicationConfig {
|
||||
pub fn new(config: Option<PersistedReplicationConfiguration>, remotes: Option<BucketTargets>) -> Self {
|
||||
pub fn new(config: Option<ReplicationConfiguration>, remotes: Option<BucketTargets>) -> Self {
|
||||
Self { config, remotes }
|
||||
}
|
||||
|
||||
@@ -473,7 +468,7 @@ fn check_replicate_delete_with_config(
|
||||
oi: &ObjectInfo,
|
||||
del_opts: &ObjectOptions,
|
||||
source_error: bool,
|
||||
config: Option<&PersistedReplicationConfiguration>,
|
||||
config: Option<&ReplicationConfiguration>,
|
||||
existing_delete_marker: bool,
|
||||
trust_persisted_replica_status: bool,
|
||||
) -> ReplicateDecision {
|
||||
@@ -592,7 +587,7 @@ pub(crate) async fn must_replicate(bucket: &str, object: &str, mopts: MustReplic
|
||||
}
|
||||
|
||||
fn metadata_target_should_replicate(
|
||||
cfg: &PersistedReplicationConfiguration,
|
||||
cfg: &ReplicationConfiguration,
|
||||
opts: &ObjectOpts,
|
||||
mopts: &MustReplicateOptions,
|
||||
arn: &str,
|
||||
@@ -602,21 +597,20 @@ fn metadata_target_should_replicate(
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rustfs_gateway_types::dto::Status;
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedOptionalReplicationStatus, PersistedReplicationDestination, PersistedReplicationFilter,
|
||||
PersistedReplicationRule, PersistedReplicationStatus, PersistedReplicationTag, PersistedSourceSelectionCriteria,
|
||||
use s3s::dto::{
|
||||
DeleteMarkerReplication, DeleteMarkerReplicationStatus, DeleteReplication, DeleteReplicationStatus, Destination,
|
||||
ReplicaModifications, ReplicationRule, ReplicationRuleFilter, ReplicationRuleStatus, SourceSelectionCriteria, Tag,
|
||||
};
|
||||
|
||||
use super::super::replication_filemeta_boundary::VersionPurgeStatusType;
|
||||
use super::super::replication_target_boundary::BucketTarget;
|
||||
use super::*;
|
||||
|
||||
fn replication_rule() -> PersistedReplicationRule {
|
||||
PersistedReplicationRule {
|
||||
fn replication_rule() -> ReplicationRule {
|
||||
ReplicationRule {
|
||||
delete_marker_replication: None,
|
||||
delete_replication: None,
|
||||
destination: PersistedReplicationDestination {
|
||||
destination: Destination {
|
||||
bucket: "arn:aws:s3:::target-bucket".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
@@ -626,7 +620,7 @@ mod tests {
|
||||
prefix: Some(String::new()),
|
||||
priority: Some(1),
|
||||
source_selection_criteria: None,
|
||||
status: Status::ENABLED.to_string(),
|
||||
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -635,11 +629,11 @@ mod tests {
|
||||
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
|
||||
let mut rule = replication_rule();
|
||||
rule.destination.bucket = arn.to_string();
|
||||
rule.filter = Some(PersistedReplicationFilter {
|
||||
rule.filter = Some(ReplicationRuleFilter {
|
||||
prefix: Some("admitted/".to_string()),
|
||||
..Default::default()
|
||||
});
|
||||
let cfg = PersistedReplicationConfiguration {
|
||||
let cfg = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![rule],
|
||||
};
|
||||
@@ -677,7 +671,7 @@ mod tests {
|
||||
assert!(!empty.replicate(&ObjectOpts::default()));
|
||||
|
||||
let config = ReplicationConfig::new(
|
||||
Some(PersistedReplicationConfiguration {
|
||||
Some(ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![replication_rule()],
|
||||
}),
|
||||
@@ -731,7 +725,7 @@ mod tests {
|
||||
fn delete_snapshot_rejects_enabled_rules_without_a_destination() {
|
||||
let mut rule = replication_rule();
|
||||
rule.destination.bucket.clear();
|
||||
let config = PersistedReplicationConfiguration {
|
||||
let config = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![rule],
|
||||
};
|
||||
@@ -747,7 +741,7 @@ mod tests {
|
||||
let first = replication_rule();
|
||||
let mut second = replication_rule();
|
||||
second.destination.bucket = "arn:aws:s3:::other-target".to_string();
|
||||
let config = PersistedReplicationConfiguration {
|
||||
let config = ReplicationConfiguration {
|
||||
role: "arn:aws:s3:::role-target".to_string(),
|
||||
rules: vec![first, second],
|
||||
};
|
||||
@@ -758,47 +752,47 @@ mod tests {
|
||||
#[test]
|
||||
fn delete_snapshot_rejects_unknown_status_values() {
|
||||
let invalid_versioning = VersioningConfiguration {
|
||||
status: Some("Enabld".to_string()),
|
||||
status: Some("Enabld".to_string().into()),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(validate_delete_replication_config(&invalid_versioning, None).is_err());
|
||||
|
||||
let mut invalid_rule = replication_rule();
|
||||
invalid_rule.status = "Enabld".to_string();
|
||||
let config = PersistedReplicationConfiguration {
|
||||
invalid_rule.status = "Enabld".to_string().into();
|
||||
let config = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![invalid_rule],
|
||||
};
|
||||
assert!(validate_delete_replication_config(&VersioningConfiguration::default(), Some(&config)).is_err());
|
||||
|
||||
let mut invalid_delete = replication_rule();
|
||||
invalid_delete.delete_replication = Some(PersistedReplicationStatus {
|
||||
status: "Enabld".to_string(),
|
||||
invalid_delete.delete_replication = Some(DeleteReplication {
|
||||
status: "Enabld".to_string().into(),
|
||||
});
|
||||
let config = PersistedReplicationConfiguration {
|
||||
let config = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![invalid_delete],
|
||||
};
|
||||
assert!(validate_delete_replication_config(&VersioningConfiguration::default(), Some(&config)).is_err());
|
||||
|
||||
let mut invalid_delete_marker = replication_rule();
|
||||
invalid_delete_marker.delete_marker_replication = Some(PersistedOptionalReplicationStatus {
|
||||
status: Some("Enabld".to_string()),
|
||||
invalid_delete_marker.delete_marker_replication = Some(DeleteMarkerReplication {
|
||||
status: Some("Enabld".to_string().into()),
|
||||
});
|
||||
let config = PersistedReplicationConfiguration {
|
||||
let config = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![invalid_delete_marker],
|
||||
};
|
||||
assert!(validate_delete_replication_config(&VersioningConfiguration::default(), Some(&config)).is_err());
|
||||
|
||||
let mut invalid_replica_modifications = replication_rule();
|
||||
invalid_replica_modifications.source_selection_criteria = Some(PersistedSourceSelectionCriteria {
|
||||
replica_modifications: Some(PersistedReplicationStatus {
|
||||
status: "Enabld".to_string(),
|
||||
invalid_replica_modifications.source_selection_criteria = Some(SourceSelectionCriteria {
|
||||
replica_modifications: Some(ReplicaModifications {
|
||||
status: "Enabld".to_string().into(),
|
||||
}),
|
||||
sse_kms_encrypted_objects: None,
|
||||
});
|
||||
let config = PersistedReplicationConfiguration {
|
||||
let config = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![invalid_replica_modifications],
|
||||
};
|
||||
@@ -810,11 +804,11 @@ mod tests {
|
||||
let mut metadata = BucketMetadata::new("bucket");
|
||||
metadata.versioning_config_xml = b"configured".to_vec();
|
||||
metadata.versioning_config = Some(VersioningConfiguration {
|
||||
status: Some("Enabled".to_owned()),
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
..Default::default()
|
||||
});
|
||||
metadata.replication_config_xml = b"configured".to_vec();
|
||||
metadata.replication_config = Some(PersistedReplicationConfiguration {
|
||||
metadata.replication_config = Some(ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![replication_rule()],
|
||||
});
|
||||
@@ -854,7 +848,7 @@ mod tests {
|
||||
);
|
||||
|
||||
let mut inconsistent = BucketMetadata::new("bucket");
|
||||
inconsistent.replication_config = Some(PersistedReplicationConfiguration {
|
||||
inconsistent.replication_config = Some(ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![replication_rule()],
|
||||
});
|
||||
@@ -869,11 +863,11 @@ mod tests {
|
||||
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
|
||||
let mut rule = replication_rule();
|
||||
rule.destination.bucket = arn.to_string();
|
||||
rule.delete_marker_replication = Some(PersistedOptionalReplicationStatus {
|
||||
status: Some(Status::ENABLED.to_string()),
|
||||
rule.delete_marker_replication = Some(DeleteMarkerReplication {
|
||||
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)),
|
||||
});
|
||||
let mut metadata = BucketMetadata::new("bucket");
|
||||
metadata.replication_config = Some(PersistedReplicationConfiguration {
|
||||
metadata.replication_config = Some(ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![rule],
|
||||
});
|
||||
@@ -904,27 +898,27 @@ mod tests {
|
||||
fn delete_marker_source_read_is_required_only_for_tag_filtered_rules() {
|
||||
let mut prefix_rule = replication_rule();
|
||||
prefix_rule.prefix = Some("logs/".to_string());
|
||||
prefix_rule.delete_marker_replication = Some(PersistedOptionalReplicationStatus {
|
||||
status: Some(Status::ENABLED.to_string()),
|
||||
prefix_rule.delete_marker_replication = Some(DeleteMarkerReplication {
|
||||
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)),
|
||||
});
|
||||
|
||||
let prefix_snapshot = DeleteReplicationConfigSnapshot::from_configs_for_test(
|
||||
VersioningConfiguration {
|
||||
status: Some("Enabled".to_owned()),
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
..Default::default()
|
||||
},
|
||||
Some(PersistedReplicationConfiguration {
|
||||
Some(ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![prefix_rule],
|
||||
}),
|
||||
);
|
||||
|
||||
let mut tag_rule = replication_rule();
|
||||
tag_rule.delete_marker_replication = Some(PersistedOptionalReplicationStatus {
|
||||
status: Some(Status::ENABLED.to_string()),
|
||||
tag_rule.delete_marker_replication = Some(DeleteMarkerReplication {
|
||||
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)),
|
||||
});
|
||||
tag_rule.filter = Some(PersistedReplicationFilter {
|
||||
tag: Some(PersistedReplicationTag {
|
||||
tag_rule.filter = Some(ReplicationRuleFilter {
|
||||
tag: Some(Tag {
|
||||
key: Some("class".to_string()),
|
||||
value: Some("audit".to_string()),
|
||||
}),
|
||||
@@ -932,10 +926,10 @@ mod tests {
|
||||
});
|
||||
let tag_snapshot = DeleteReplicationConfigSnapshot::from_configs_for_test(
|
||||
VersioningConfiguration {
|
||||
status: Some("Enabled".to_owned()),
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
..Default::default()
|
||||
},
|
||||
Some(PersistedReplicationConfiguration {
|
||||
Some(ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![tag_rule],
|
||||
}),
|
||||
@@ -950,14 +944,14 @@ mod tests {
|
||||
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
|
||||
let mut rule = replication_rule();
|
||||
rule.destination.bucket = arn.to_string();
|
||||
rule.delete_replication = Some(PersistedReplicationStatus {
|
||||
status: Status::ENABLED.to_string(),
|
||||
rule.delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
});
|
||||
rule.delete_marker_replication = Some(PersistedOptionalReplicationStatus {
|
||||
status: Some(Status::DISABLED.to_string()),
|
||||
rule.delete_marker_replication = Some(DeleteMarkerReplication {
|
||||
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::DISABLED)),
|
||||
});
|
||||
let config = ReplicationConfig::new(
|
||||
Some(PersistedReplicationConfiguration {
|
||||
Some(ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![rule],
|
||||
}),
|
||||
@@ -991,22 +985,22 @@ mod tests {
|
||||
#[test]
|
||||
fn live_delete_does_not_trust_persisted_replica_status() {
|
||||
let mut rule = replication_rule();
|
||||
rule.delete_replication = Some(PersistedReplicationStatus {
|
||||
status: Status::ENABLED.to_string(),
|
||||
rule.delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
});
|
||||
rule.source_selection_criteria = Some(PersistedSourceSelectionCriteria {
|
||||
replica_modifications: Some(PersistedReplicationStatus {
|
||||
status: Status::DISABLED.to_string(),
|
||||
rule.source_selection_criteria = Some(SourceSelectionCriteria {
|
||||
replica_modifications: Some(ReplicaModifications {
|
||||
status: s3s::dto::ReplicaModificationsStatus::from_static(s3s::dto::ReplicaModificationsStatus::DISABLED),
|
||||
}),
|
||||
sse_kms_encrypted_objects: None,
|
||||
});
|
||||
let replication = PersistedReplicationConfiguration {
|
||||
let replication = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![rule],
|
||||
};
|
||||
let snapshot = DeleteReplicationConfigSnapshot::from_configs_for_test(
|
||||
VersioningConfiguration {
|
||||
status: Some("Enabled".to_owned()),
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
..Default::default()
|
||||
},
|
||||
Some(replication.clone()),
|
||||
@@ -1042,14 +1036,14 @@ mod tests {
|
||||
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
|
||||
let mut rule = replication_rule();
|
||||
rule.destination.bucket = arn.to_string();
|
||||
rule.delete_replication = Some(PersistedReplicationStatus {
|
||||
status: Status::ENABLED.to_string(),
|
||||
rule.delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
});
|
||||
rule.delete_marker_replication = Some(PersistedOptionalReplicationStatus {
|
||||
status: Some(Status::DISABLED.to_string()),
|
||||
rule.delete_marker_replication = Some(DeleteMarkerReplication {
|
||||
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::DISABLED)),
|
||||
});
|
||||
let config = ReplicationConfig::new(
|
||||
Some(PersistedReplicationConfiguration {
|
||||
Some(ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![rule],
|
||||
}),
|
||||
|
||||
@@ -4794,10 +4794,9 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn heal_queue_marks_missing_versioning_state_as_missed() {
|
||||
use super::super::replication_target_boundary::BucketTargets;
|
||||
use rustfs_gateway_types::dto::Status;
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedReplicationConfiguration, PersistedReplicationDestination, PersistedReplicationRule,
|
||||
PersistedReplicationStatus,
|
||||
use s3s::dto::{
|
||||
DeleteReplication, DeleteReplicationStatus, Destination, ReplicationConfiguration, ReplicationRule,
|
||||
ReplicationRuleStatus,
|
||||
};
|
||||
|
||||
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
|
||||
@@ -4812,14 +4811,14 @@ mod tests {
|
||||
..Default::default()
|
||||
},
|
||||
ReplicationConfig::new(
|
||||
Some(PersistedReplicationConfiguration {
|
||||
Some(ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![PersistedReplicationRule {
|
||||
rules: vec![ReplicationRule {
|
||||
delete_marker_replication: None,
|
||||
delete_replication: Some(PersistedReplicationStatus {
|
||||
status: Status::ENABLED.to_string(),
|
||||
delete_replication: Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
}),
|
||||
destination: PersistedReplicationDestination {
|
||||
destination: Destination {
|
||||
bucket: arn.to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
@@ -4829,7 +4828,7 @@ mod tests {
|
||||
prefix: Some(String::new()),
|
||||
priority: Some(1),
|
||||
source_selection_criteria: None,
|
||||
status: Status::ENABLED.to_string(),
|
||||
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
|
||||
}],
|
||||
}),
|
||||
Some(BucketTargets::default()),
|
||||
|
||||
@@ -85,8 +85,6 @@ use http_body_util::StreamBody;
|
||||
use metrics::counter;
|
||||
#[cfg(test)]
|
||||
use rmp_serde;
|
||||
#[cfg(test)]
|
||||
use rustfs_gateway_types::persistence::PersistedReplicationConfiguration;
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_utils::http::{
|
||||
AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_MODE, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_TAGGING_DIRECTIVE,
|
||||
@@ -94,6 +92,8 @@ use rustfs_utils::http::{
|
||||
insert_str, replication_target_versions,
|
||||
};
|
||||
use rustfs_utils::{DEFAULT_SIP_HASH_KEY, get_env_usize, sip_hash};
|
||||
#[cfg(test)]
|
||||
use s3s::dto::ReplicationConfiguration;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::fmt::Display;
|
||||
use std::pin::Pin;
|
||||
@@ -6500,10 +6500,9 @@ mod tests {
|
||||
|
||||
use super::super::replication_target_boundary::{BucketTarget, BucketTargets};
|
||||
use super::*;
|
||||
use rustfs_gateway_types::dto::Status;
|
||||
use rustfs_gateway_types::persistence::PersistedVersioningConfiguration as VersioningConfiguration;
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedReplicationDestination, PersistedReplicationRule, PersistedReplicationStatus,
|
||||
use s3s::dto::{
|
||||
BucketVersioningStatus, DeleteReplication, DeleteReplicationStatus, Destination, ExcludedPrefix, ReplicationRule,
|
||||
ReplicationRuleStatus, VersioningConfiguration,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use time::OffsetDateTime;
|
||||
@@ -8569,7 +8568,7 @@ mod tests {
|
||||
..Default::default()
|
||||
};
|
||||
let rcfg = ReplicationConfig::new(
|
||||
Some(PersistedReplicationConfiguration {
|
||||
Some(ReplicationConfiguration {
|
||||
role: role.to_string(),
|
||||
rules: vec![],
|
||||
}),
|
||||
@@ -8595,7 +8594,7 @@ mod tests {
|
||||
ReplicationVersioningStore::install_prefix_state_test_config(
|
||||
&bucket,
|
||||
VersioningConfiguration {
|
||||
status: Some("Enabled".to_owned()),
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
@@ -8628,20 +8627,22 @@ mod tests {
|
||||
ReplicationVersioningStore::install_prefix_state_test_config(
|
||||
&bucket,
|
||||
VersioningConfiguration {
|
||||
status: Some("Enabled".to_owned()),
|
||||
excluded_prefixes: Some(vec![Some("archive/".to_string())]),
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
excluded_prefixes: Some(vec![ExcludedPrefix {
|
||||
prefix: Some("archive/".to_string()),
|
||||
}]),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let rcfg = ReplicationConfig::new(
|
||||
Some(PersistedReplicationConfiguration {
|
||||
Some(ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![PersistedReplicationRule {
|
||||
rules: vec![ReplicationRule {
|
||||
delete_marker_replication: None,
|
||||
delete_replication: Some(PersistedReplicationStatus {
|
||||
status: Status::ENABLED.to_string(),
|
||||
delete_replication: Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
}),
|
||||
destination: PersistedReplicationDestination {
|
||||
destination: Destination {
|
||||
bucket: arn.to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
@@ -8651,7 +8652,7 @@ mod tests {
|
||||
prefix: Some(String::new()),
|
||||
priority: Some(1),
|
||||
source_selection_criteria: None,
|
||||
status: Status::ENABLED.to_string(),
|
||||
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
|
||||
}],
|
||||
}),
|
||||
Some(BucketTargets {
|
||||
@@ -8687,14 +8688,14 @@ mod tests {
|
||||
let admitted_arn = "arn:rustfs:replication:us-east-1:target:admitted";
|
||||
let current_role = "arn:rustfs:replication:us-east-1:target:current";
|
||||
let rcfg = ReplicationConfig::new(
|
||||
Some(PersistedReplicationConfiguration {
|
||||
Some(ReplicationConfiguration {
|
||||
role: current_role.to_string(),
|
||||
rules: vec![PersistedReplicationRule {
|
||||
rules: vec![ReplicationRule {
|
||||
delete_marker_replication: None,
|
||||
delete_replication: Some(PersistedReplicationStatus {
|
||||
status: Status::DISABLED.to_string(),
|
||||
delete_replication: Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::DISABLED),
|
||||
}),
|
||||
destination: PersistedReplicationDestination {
|
||||
destination: Destination {
|
||||
bucket: current_role.to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
@@ -8704,7 +8705,7 @@ mod tests {
|
||||
prefix: Some(String::new()),
|
||||
priority: Some(1),
|
||||
source_selection_criteria: None,
|
||||
status: Status::ENABLED.to_string(),
|
||||
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
|
||||
}],
|
||||
}),
|
||||
Some(BucketTargets {
|
||||
@@ -8752,12 +8753,12 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn heal_rejects_semantically_invalid_replication_config() {
|
||||
let rcfg = ReplicationConfig::new(
|
||||
Some(PersistedReplicationConfiguration {
|
||||
Some(ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![PersistedReplicationRule {
|
||||
rules: vec![ReplicationRule {
|
||||
delete_marker_replication: None,
|
||||
delete_replication: None,
|
||||
destination: PersistedReplicationDestination {
|
||||
destination: Destination {
|
||||
bucket: "arn:rustfs:replication:us-east-1:target:bucket".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
@@ -8767,7 +8768,7 @@ mod tests {
|
||||
prefix: Some(String::new()),
|
||||
priority: Some(1),
|
||||
source_selection_criteria: None,
|
||||
status: "Enabld".to_owned(),
|
||||
status: ReplicationRuleStatus::from_static("Enabld"),
|
||||
}],
|
||||
}),
|
||||
Some(BucketTargets::default()),
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use rustfs_gateway_types::persistence::PersistedReplicationConfiguration;
|
||||
use s3s::dto::ReplicationConfiguration;
|
||||
|
||||
use super::replication_config_boundary::{ObjectOpts, ReplicationConfigurationExt};
|
||||
use super::replication_filemeta_boundary::ReplicationType;
|
||||
@@ -20,7 +20,7 @@ use super::replication_filemeta_boundary::ReplicationType;
|
||||
pub(crate) struct ReplicationTargetConfigBridge;
|
||||
|
||||
impl ReplicationTargetConfigBridge {
|
||||
pub(crate) fn target_is_used_by_rules(config: &PersistedReplicationConfiguration, arn: &str) -> bool {
|
||||
pub(crate) fn target_is_used_by_rules(config: &ReplicationConfiguration, arn: &str) -> bool {
|
||||
config
|
||||
.filter_target_arns(&ObjectOpts {
|
||||
op_type: ReplicationType::All,
|
||||
@@ -37,7 +37,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn target_config_bridge_matches_role_target() {
|
||||
let config = PersistedReplicationConfiguration {
|
||||
let config = ReplicationConfiguration {
|
||||
role: " arn:target ".to_string(),
|
||||
rules: Vec::new(),
|
||||
};
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
use super::replication_error_boundary::Result;
|
||||
use crate::bucket::{versioning::VersioningApi as _, versioning_sys::BucketVersioningSys};
|
||||
#[cfg(test)]
|
||||
use rustfs_gateway_types::persistence::PersistedVersioningConfiguration as VersioningConfiguration;
|
||||
use s3s::dto::VersioningConfiguration;
|
||||
#[cfg(test)]
|
||||
use std::{collections::HashMap, sync::LazyLock, sync::Mutex};
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use rustfs_gateway_types::cors_tagging::PersistedTag as Tag;
|
||||
use s3s::dto::Tag;
|
||||
use std::cmp::Ordering;
|
||||
use std::collections::HashMap;
|
||||
use url::form_urlencoded;
|
||||
|
||||
@@ -15,23 +15,9 @@
|
||||
use crate::disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
|
||||
use crate::error::{Error, Result, StorageError};
|
||||
use regex::Regex;
|
||||
use rustfs_gateway_types::cors_tagging::{
|
||||
CorsTaggingCodecError, PersistedCorsConfiguration, PersistedTagging, parse_cors, parse_tagging, serialize_cors,
|
||||
serialize_tagging,
|
||||
};
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedAccelerateConfiguration, PersistedBucketEncryptionConfiguration, PersistedBucketLoggingStatus,
|
||||
PersistedLifecycleConfiguration, PersistedNotificationConfiguration, PersistedObjectLockConfiguration,
|
||||
PersistedPublicAccessBlockConfiguration, PersistedReplicationConfiguration, PersistedRequestPaymentConfiguration,
|
||||
PersistedVersioningConfiguration, PersistedWebsiteConfiguration, PersistenceCodecError, parse_accelerate,
|
||||
parse_bucket_encryption, parse_bucket_logging, parse_lifecycle_legacy_dates, parse_notification, parse_object_lock,
|
||||
parse_public_access_block, parse_replication, parse_request_payment, parse_versioning, parse_website, serialize_accelerate,
|
||||
serialize_bucket_encryption, serialize_bucket_logging, serialize_lifecycle_legacy_dates, serialize_notification,
|
||||
serialize_object_lock, serialize_public_access_block, serialize_replication, serialize_request_payment, serialize_versioning,
|
||||
serialize_website,
|
||||
};
|
||||
use rustfs_utils::WindowsFileNameError;
|
||||
use rustfs_utils::path::SLASH_SEPARATOR;
|
||||
use s3s::xml;
|
||||
use tracing::instrument;
|
||||
|
||||
pub fn is_meta_bucketname(name: &str) -> bool {
|
||||
@@ -88,173 +74,37 @@ pub fn check_valid_bucket_name_strict(bucket_name: &str) -> Result<()> {
|
||||
check_bucket_name_common(bucket_name, true)
|
||||
}
|
||||
|
||||
/// A bucket configuration persisted as XML in the bucket metadata, together
|
||||
/// with the codec that reads and writes its stored bytes.
|
||||
///
|
||||
/// Every family reads and writes its stored bytes through the gateway
|
||||
/// persistence codecs (`rustfs_gateway_types::persistence` and
|
||||
/// `rustfs_gateway_types::cors_tagging`, rustfs/backlog#2745). The codecs keep
|
||||
/// the persisted bytes the `bucket_config_goldens` pin, so a document the old
|
||||
/// s3s codec wrote reads back and re-encodes unchanged.
|
||||
pub trait BucketConfigXml: Sized {
|
||||
/// Decodes the stored bytes of one configuration document.
|
||||
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError>;
|
||||
|
||||
/// Encodes the value as RustFS stores it.
|
||||
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError>;
|
||||
// RUSTFS_COMPAT_TODO(s3gate-metadata-xml): the s3s codec reads persisted XML during migration. Remove after every supported writer uses the gateway codec and every retained metadata object and backup archive is verified or rewritten.
|
||||
pub fn deserialize<T>(input: &[u8]) -> xml::DeResult<T>
|
||||
where
|
||||
T: for<'xml> xml::Deserialize<'xml>,
|
||||
{
|
||||
let mut d = xml::Deserializer::new(input);
|
||||
let ans = T::deserialize(&mut d)?;
|
||||
d.expect_eof()?;
|
||||
Ok(ans)
|
||||
}
|
||||
|
||||
/// Why a persisted bucket configuration could not be decoded or encoded.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum BucketConfigXmlError {
|
||||
/// A gateway persistence codec refused the bytes or the value.
|
||||
#[error(transparent)]
|
||||
Persisted(#[from] PersistenceCodecError),
|
||||
/// The gateway CORS or Tagging codec refused the bytes or the value.
|
||||
#[error(transparent)]
|
||||
CorsTagging(#[from] CorsTaggingCodecError),
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "xml serialize helper with no caller in this port; the live sibling is deserialize (backlog#1823)"
|
||||
)]
|
||||
pub fn serialize_content<T: xml::SerializeContent>(val: &T) -> xml::SerResult<String> {
|
||||
let mut buf = Vec::with_capacity(256);
|
||||
{
|
||||
let mut ser = xml::Serializer::new(&mut buf);
|
||||
val.serialize_content(&mut ser)?;
|
||||
}
|
||||
Ok(String::from_utf8(buf).unwrap())
|
||||
}
|
||||
|
||||
/// Decodes one persisted bucket configuration document.
|
||||
pub fn deserialize<T: BucketConfigXml>(input: &[u8]) -> std::result::Result<T, BucketConfigXmlError> {
|
||||
T::decode_config_xml(input)
|
||||
}
|
||||
|
||||
/// Encodes one bucket configuration as RustFS persists it.
|
||||
pub fn serialize<T: BucketConfigXml>(val: &T) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
|
||||
val.encode_config_xml()
|
||||
}
|
||||
|
||||
impl BucketConfigXml for PersistedReplicationConfiguration {
|
||||
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
|
||||
Ok(parse_replication(input)?)
|
||||
}
|
||||
|
||||
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
|
||||
Ok(serialize_replication(self)?)
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketConfigXml for PersistedLifecycleConfiguration {
|
||||
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
|
||||
Ok(parse_lifecycle_legacy_dates(input)?)
|
||||
}
|
||||
|
||||
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
|
||||
Ok(serialize_lifecycle_legacy_dates(self)?)
|
||||
}
|
||||
}
|
||||
|
||||
// The gateway writers of these families are infallible: a value that exists is
|
||||
// always encodable, so only the readers can refuse bytes.
|
||||
impl BucketConfigXml for PersistedAccelerateConfiguration {
|
||||
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
|
||||
Ok(parse_accelerate(input)?)
|
||||
}
|
||||
|
||||
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
|
||||
Ok(serialize_accelerate(self))
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketConfigXml for PersistedBucketLoggingStatus {
|
||||
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
|
||||
Ok(parse_bucket_logging(input)?)
|
||||
}
|
||||
|
||||
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
|
||||
Ok(serialize_bucket_logging(self))
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketConfigXml for PersistedCorsConfiguration {
|
||||
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
|
||||
Ok(parse_cors(input)?)
|
||||
}
|
||||
|
||||
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
|
||||
Ok(serialize_cors(self))
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketConfigXml for PersistedNotificationConfiguration {
|
||||
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
|
||||
Ok(parse_notification(input)?)
|
||||
}
|
||||
|
||||
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
|
||||
Ok(serialize_notification(self))
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketConfigXml for PersistedObjectLockConfiguration {
|
||||
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
|
||||
Ok(parse_object_lock(input)?)
|
||||
}
|
||||
|
||||
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
|
||||
Ok(serialize_object_lock(self))
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketConfigXml for PersistedPublicAccessBlockConfiguration {
|
||||
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
|
||||
Ok(parse_public_access_block(input)?)
|
||||
}
|
||||
|
||||
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
|
||||
Ok(serialize_public_access_block(self))
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketConfigXml for PersistedRequestPaymentConfiguration {
|
||||
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
|
||||
Ok(parse_request_payment(input)?)
|
||||
}
|
||||
|
||||
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
|
||||
Ok(serialize_request_payment(self))
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketConfigXml for PersistedBucketEncryptionConfiguration {
|
||||
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
|
||||
Ok(parse_bucket_encryption(input)?)
|
||||
}
|
||||
|
||||
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
|
||||
Ok(serialize_bucket_encryption(self))
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketConfigXml for PersistedTagging {
|
||||
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
|
||||
Ok(parse_tagging(input)?)
|
||||
}
|
||||
|
||||
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
|
||||
Ok(serialize_tagging(self))
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketConfigXml for PersistedVersioningConfiguration {
|
||||
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
|
||||
Ok(parse_versioning(input)?)
|
||||
}
|
||||
|
||||
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
|
||||
Ok(serialize_versioning(self))
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketConfigXml for PersistedWebsiteConfiguration {
|
||||
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
|
||||
Ok(parse_website(input)?)
|
||||
}
|
||||
|
||||
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
|
||||
Ok(serialize_website(self))
|
||||
pub fn serialize<T: xml::Serialize>(val: &T) -> xml::SerResult<Vec<u8>> {
|
||||
let mut buf = Vec::with_capacity(256);
|
||||
{
|
||||
let mut ser = xml::Serializer::new(&mut buf);
|
||||
val.serialize(&mut ser)?;
|
||||
}
|
||||
Ok(buf)
|
||||
}
|
||||
|
||||
pub fn has_bad_path_component(path: &str) -> bool {
|
||||
@@ -584,40 +434,6 @@ mod tests {
|
||||
use super::*;
|
||||
use proptest::prelude::*;
|
||||
|
||||
/// Every bucket-configuration family reads and writes through its own
|
||||
/// gateway codec (rustfs/backlog#2745): the family's golden document
|
||||
/// decodes and re-encodes to the same bytes, and a document of another
|
||||
/// family is refused instead of decoding into an empty configuration.
|
||||
#[test]
|
||||
fn every_bucket_config_family_uses_its_own_gateway_codec() {
|
||||
fn assert_family<T: BucketConfigXml>(family: &str, own: &[u8], foreign: &[u8]) {
|
||||
let value = deserialize::<T>(own).unwrap_or_else(|err| panic!("{family}: own document must decode: {err}"));
|
||||
let encoded = serialize(&value).unwrap_or_else(|err| panic!("{family}: value must encode: {err}"));
|
||||
assert_eq!(encoded, own, "{family}: re-encoding must keep the persisted bytes");
|
||||
assert!(deserialize::<T>(foreign).is_err(), "{family}: a foreign root element must be refused");
|
||||
}
|
||||
|
||||
macro_rules! golden {
|
||||
($family:literal) => {
|
||||
include_bytes!(concat!("../../tests/fixtures/bucket-config-goldens/", $family, "/minio.out.xml"))
|
||||
};
|
||||
}
|
||||
let tagging = golden!("tagging");
|
||||
let versioning = golden!("versioning");
|
||||
|
||||
assert_family::<PersistedAccelerateConfiguration>("accelerate", golden!("accelerate"), tagging);
|
||||
assert_family::<PersistedBucketLoggingStatus>("logging", golden!("logging"), tagging);
|
||||
assert_family::<PersistedCorsConfiguration>("cors", golden!("cors"), tagging);
|
||||
assert_family::<PersistedNotificationConfiguration>("notification", golden!("notification"), tagging);
|
||||
assert_family::<PersistedObjectLockConfiguration>("object_lock", golden!("object_lock"), tagging);
|
||||
assert_family::<PersistedPublicAccessBlockConfiguration>("public_access_block", golden!("public_access_block"), tagging);
|
||||
assert_family::<PersistedRequestPaymentConfiguration>("request_payment", golden!("request_payment"), tagging);
|
||||
assert_family::<PersistedBucketEncryptionConfiguration>("encryption", golden!("encryption"), tagging);
|
||||
assert_family::<PersistedTagging>("tagging", tagging, versioning);
|
||||
assert_family::<PersistedVersioningConfiguration>("versioning", versioning, tagging);
|
||||
assert_family::<PersistedWebsiteConfiguration>("website", golden!("website"), tagging);
|
||||
}
|
||||
|
||||
/// rustfs#5785: keys whose path segments exceed the on-disk NAME_MAX
|
||||
/// budget must be rejected up front as ObjectNameInvalid (4xx), not leak
|
||||
/// ENAMETOOLONG as InternalError 500 from the disk layer.
|
||||
|
||||
@@ -12,8 +12,8 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use rustfs_gateway_types::persistence::PersistedVersioningConfiguration as VersioningConfiguration;
|
||||
use rustfs_utils::string::match_simple;
|
||||
use s3s::dto::{BucketVersioningStatus, VersioningConfiguration};
|
||||
|
||||
pub trait VersioningApi {
|
||||
fn enabled(&self) -> bool;
|
||||
@@ -28,10 +28,10 @@ pub trait VersioningApi {
|
||||
|
||||
impl VersioningApi for VersioningConfiguration {
|
||||
fn enabled(&self) -> bool {
|
||||
self.status.as_deref() == Some("Enabled")
|
||||
self.status == Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED))
|
||||
}
|
||||
fn prefix_enabled(&self, prefix: &str) -> bool {
|
||||
if self.status.as_deref() != Some("Enabled") {
|
||||
if self.status != Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -44,10 +44,12 @@ impl VersioningApi for VersioningConfiguration {
|
||||
}
|
||||
|
||||
if let Some(ref excluded_prefixes) = self.excluded_prefixes {
|
||||
for sprefix in excluded_prefixes.iter().flatten() {
|
||||
let pattern = format!("{sprefix}*");
|
||||
if match_simple(&pattern, prefix) {
|
||||
return false;
|
||||
for p in excluded_prefixes.iter() {
|
||||
if let Some(ref sprefix) = p.prefix {
|
||||
let pattern = format!("{sprefix}*");
|
||||
if match_simple(&pattern, prefix) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -56,11 +58,11 @@ impl VersioningApi for VersioningConfiguration {
|
||||
}
|
||||
|
||||
fn prefix_suspended(&self, prefix: &str) -> bool {
|
||||
if self.status.as_deref() == Some("Suspended") {
|
||||
if self.status == Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if self.status.as_deref() == Some("Enabled") {
|
||||
if self.status == Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)) {
|
||||
if prefix.is_empty() {
|
||||
return false;
|
||||
}
|
||||
@@ -73,10 +75,12 @@ impl VersioningApi for VersioningConfiguration {
|
||||
}
|
||||
|
||||
if let Some(ref excluded_prefixes) = self.excluded_prefixes {
|
||||
for sprefix in excluded_prefixes.iter().flatten() {
|
||||
let pattern = format!("{sprefix}*");
|
||||
if match_simple(&pattern, prefix) {
|
||||
return true;
|
||||
for p in excluded_prefixes.iter() {
|
||||
if let Some(ref sprefix) = p.prefix {
|
||||
let pattern = format!("{sprefix}*");
|
||||
if match_simple(&pattern, prefix) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -88,12 +92,14 @@ impl VersioningApi for VersioningConfiguration {
|
||||
self.prefix_enabled(prefix) || self.prefix_suspended(prefix)
|
||||
}
|
||||
fn suspended(&self) -> bool {
|
||||
self.status.as_deref() == Some("Suspended")
|
||||
self.status == Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use s3s::dto::{BucketVersioningStatus, ExcludedPrefix};
|
||||
|
||||
use super::*;
|
||||
|
||||
struct LegacyVersioning;
|
||||
@@ -128,8 +134,10 @@ mod tests {
|
||||
#[test]
|
||||
fn delete_state_treats_excluded_prefixes_as_unversioned() {
|
||||
let config = VersioningConfiguration {
|
||||
status: Some("Enabled".to_owned()),
|
||||
excluded_prefixes: Some(vec![Some("archive/".to_string())]),
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
excluded_prefixes: Some(vec![ExcludedPrefix {
|
||||
prefix: Some("archive/".to_string()),
|
||||
}]),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -138,7 +146,7 @@ mod tests {
|
||||
assert_eq!(config.delete_state("live/object"), (true, false));
|
||||
|
||||
let suspended = VersioningConfiguration {
|
||||
status: Some("Suspended".to_owned()),
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED)),
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(suspended.delete_state("archive/object"), (false, true));
|
||||
|
||||
@@ -17,7 +17,7 @@ use super::metadata::unreadable_config_refusal;
|
||||
use super::{metadata_sys::get_bucket_metadata_sys, versioning::VersioningApi};
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::Result;
|
||||
use rustfs_gateway_types::persistence::PersistedVersioningConfiguration as VersioningConfiguration;
|
||||
use s3s::dto::VersioningConfiguration;
|
||||
use tracing::{error, warn};
|
||||
|
||||
pub struct BucketVersioningSys {}
|
||||
@@ -163,7 +163,7 @@ mod tests {
|
||||
|
||||
fn enabled() -> VersioningConfiguration {
|
||||
VersioningConfiguration {
|
||||
status: Some("Enabled".to_owned()),
|
||||
status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::ENABLED)),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1965,15 +1965,6 @@ impl PeerRestClient {
|
||||
Ok((self.topology_member.clone(), epoch))
|
||||
}
|
||||
|
||||
pub async fn probe_restore_copyback_capability(&self, topology_fingerprint: String) -> Result<(String, Uuid)> {
|
||||
let probe = rustfs_protos::restore_copyback_capability_probe(Uuid::new_v4().as_bytes());
|
||||
let result = self
|
||||
.heal_control(rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION, topology_fingerprint, probe)
|
||||
.await?;
|
||||
let epoch = decode_remote_version_state_capability(&self.topology_member, &result)?;
|
||||
Ok((self.topology_member.clone(), epoch))
|
||||
}
|
||||
|
||||
pub async fn probe_cross_pool_fence(&self, topology_fingerprint: String) -> Result<(String, u32, Uuid)> {
|
||||
let mut probe = rustfs_protos::CROSS_POOL_FENCE_CAPABILITY_PROBE_PREFIX.to_vec();
|
||||
probe.extend_from_slice(Uuid::new_v4().as_bytes());
|
||||
|
||||
@@ -41,6 +41,10 @@ use tonic::service::interceptor::InterceptedService;
|
||||
use tonic::{Request, Response};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
const EVENT_NAMESPACE_LOCK_RELEASE: &str = "namespace_lock_release";
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_REMOTE_LOCK: &str = "remote_lock";
|
||||
|
||||
fn attach_lock_mutation_body_digest<T: rustfs_protos::CanonicalMutationBody>(request: &mut Request<T>) -> std::io::Result<()> {
|
||||
set_tonic_rolling_mutation_body_digest(request)
|
||||
}
|
||||
@@ -233,6 +237,20 @@ pub struct RemoteClient {
|
||||
impl RemoteClient {
|
||||
const ONLINE_CHECK_RESOURCE: &'static str = "health-lock-online";
|
||||
|
||||
fn record_release_ack(&self, lock_id: &LockId) {
|
||||
tracing::trace!(
|
||||
target: "rustfs_ecstore::object_lock_diag",
|
||||
event = EVENT_NAMESPACE_LOCK_RELEASE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
|
||||
state = "acknowledged",
|
||||
peer_kind = "remote",
|
||||
lock_id = %lock_id,
|
||||
peer = %self.addr,
|
||||
"Remote namespace lock release acknowledged"
|
||||
);
|
||||
}
|
||||
|
||||
pub fn new(endpoint: String) -> Self {
|
||||
Self { addr: endpoint }
|
||||
}
|
||||
@@ -1204,8 +1222,6 @@ impl LockClient for RemoteClient {
|
||||
}
|
||||
|
||||
async fn release(&self, lock_id: &LockId) -> Result<bool> {
|
||||
info!("remote release for {}", lock_id);
|
||||
|
||||
let unlock_request = Self::create_unlock_request(lock_id);
|
||||
let request_string = serde_json::to_string(&unlock_request)
|
||||
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?;
|
||||
@@ -1226,6 +1242,9 @@ impl LockClient for RemoteClient {
|
||||
if let Some(error_info) = resp.error_info {
|
||||
return Err(LockError::internal(error_info));
|
||||
}
|
||||
if resp.success {
|
||||
self.record_release_ack(lock_id);
|
||||
}
|
||||
Ok(resp.success)
|
||||
}
|
||||
|
||||
@@ -1261,7 +1280,13 @@ impl LockClient for RemoteClient {
|
||||
Ok(lock_ids
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(idx, _)| resp.results.get(idx).map(|result| result.success).unwrap_or(false))
|
||||
.map(|(idx, lock_id)| {
|
||||
let result = resp.results.get(idx);
|
||||
if result.is_some_and(|result| result.success && result.error_info.is_none()) {
|
||||
self.record_release_ack(lock_id);
|
||||
}
|
||||
result.map(|result| result.success).unwrap_or(false)
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
|
||||
@@ -84,13 +84,12 @@ use rand::RngExt as _;
|
||||
use rmp_serde::Deserializer;
|
||||
use rmp_serde::Serializer;
|
||||
use rustfs_filemeta::{FileInfo, FileInfoVersions, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams};
|
||||
use rustfs_gateway_types::persistence::PersistedObjectLockConfiguration as ObjectLockConfiguration;
|
||||
use rustfs_gateway_types::persistence::{PersistedLifecycleConfiguration, PersistedReplicationConfiguration};
|
||||
use rustfs_heal_contracts::heal_channel::HealOpts;
|
||||
use rustfs_utils::crypto::{hex_sha256, is_sha256_checksum};
|
||||
use rustfs_utils::path::{
|
||||
decode_dir_object, encode_dir_object, path_join, path_to_bucket_object, path_to_bucket_object_with_base_path,
|
||||
};
|
||||
use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration, ReplicationConfiguration};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
@@ -12666,7 +12665,7 @@ pub(crate) async fn should_skip_lifecycle_for_data_movement(
|
||||
store: Arc<ECStore>,
|
||||
bucket: &str,
|
||||
version: &rustfs_filemeta::FileInfo,
|
||||
lifecycle_config: Option<&PersistedLifecycleConfiguration>,
|
||||
lifecycle_config: Option<&BucketLifecycleConfiguration>,
|
||||
object_lock_config: Option<&ObjectLockConfiguration>,
|
||||
apply_actions: bool,
|
||||
event_source: &LcEventSrc,
|
||||
@@ -16529,9 +16528,9 @@ impl ECStore {
|
||||
generation: OffsetDateTime,
|
||||
bucket: String,
|
||||
set: Arc<SetDisks>,
|
||||
lifecycle_config: Option<PersistedLifecycleConfiguration>,
|
||||
lifecycle_config: Option<BucketLifecycleConfiguration>,
|
||||
object_lock_config: Option<ObjectLockConfiguration>,
|
||||
replication_config: Option<(PersistedReplicationConfiguration, OffsetDateTime)>,
|
||||
replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
|
||||
expected_bucket_incarnation_id: Option<uuid::Uuid>,
|
||||
source_changed_exhaustions: Arc<AtomicUsize>,
|
||||
entry_budget: Arc<Semaphore>,
|
||||
@@ -16703,9 +16702,9 @@ impl ECStore {
|
||||
mut deferred_entry: DeferredDecommissionEntry,
|
||||
bucket: String,
|
||||
set: Arc<SetDisks>,
|
||||
lifecycle_config: Option<PersistedLifecycleConfiguration>,
|
||||
lifecycle_config: Option<BucketLifecycleConfiguration>,
|
||||
object_lock_config: Option<ObjectLockConfiguration>,
|
||||
replication_config: Option<(PersistedReplicationConfiguration, OffsetDateTime)>,
|
||||
replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
|
||||
expected_bucket_incarnation_id: Option<uuid::Uuid>,
|
||||
source_changed_exhaustions: Arc<AtomicUsize>,
|
||||
entry_budget: Arc<Semaphore>,
|
||||
@@ -16812,9 +16811,9 @@ impl ECStore {
|
||||
generation: OffsetDateTime,
|
||||
set: Arc<SetDisks>,
|
||||
bi: DecomBucketInfo,
|
||||
lifecycle_config: Option<PersistedLifecycleConfiguration>,
|
||||
lifecycle_config: Option<BucketLifecycleConfiguration>,
|
||||
object_lock_config: Option<ObjectLockConfiguration>,
|
||||
replication_config: Option<(PersistedReplicationConfiguration, OffsetDateTime)>,
|
||||
replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
|
||||
expected_bucket_incarnation_id: Option<uuid::Uuid>,
|
||||
source_changed_exhaustions: Arc<AtomicUsize>,
|
||||
entry_budget: Arc<Semaphore>,
|
||||
@@ -17489,9 +17488,9 @@ impl ECStore {
|
||||
entry: MetaCacheEntry,
|
||||
bucket: String,
|
||||
set: Arc<SetDisks>,
|
||||
lifecycle_config: Option<PersistedLifecycleConfiguration>,
|
||||
lifecycle_config: Option<BucketLifecycleConfiguration>,
|
||||
object_lock_config: Option<ObjectLockConfiguration>,
|
||||
replication_config: Option<(PersistedReplicationConfiguration, OffsetDateTime)>,
|
||||
replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
|
||||
expected_bucket_incarnation_id: Option<uuid::Uuid>,
|
||||
source_changed_exhaustions: Arc<AtomicUsize>,
|
||||
attempts: &mut usize,
|
||||
@@ -17675,9 +17674,9 @@ impl ECStore {
|
||||
entry: MetaCacheEntry,
|
||||
bucket: String,
|
||||
set: Arc<SetDisks>,
|
||||
lifecycle_config: Option<PersistedLifecycleConfiguration>,
|
||||
lifecycle_config: Option<BucketLifecycleConfiguration>,
|
||||
object_lock_config: Option<ObjectLockConfiguration>,
|
||||
replication_config: Option<(PersistedReplicationConfiguration, OffsetDateTime)>,
|
||||
replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
|
||||
expected_bucket_incarnation_id: Option<uuid::Uuid>,
|
||||
entry_attempt: usize,
|
||||
scanner_backlog_handoff_attempt: usize,
|
||||
@@ -18965,7 +18964,7 @@ impl ECStore {
|
||||
entry: MetaCacheEntry,
|
||||
bucket: String,
|
||||
set: Arc<SetDisks>,
|
||||
lifecycle_config: Option<PersistedLifecycleConfiguration>,
|
||||
lifecycle_config: Option<BucketLifecycleConfiguration>,
|
||||
) -> Result<()> {
|
||||
let expected_bucket_incarnation_id = if is_meta_bucketname(&bucket) {
|
||||
None
|
||||
|
||||
@@ -65,38 +65,6 @@ use uuid::Uuid;
|
||||
|
||||
const LIST_MULTIPART_SETS_CONCURRENCY: usize = 4;
|
||||
|
||||
#[cfg(test)]
|
||||
type TestSetLockers = HashMap<usize, Vec<Arc<dyn rustfs_lock::LockClient>>>;
|
||||
|
||||
#[cfg(test)]
|
||||
static TEST_SET_LOCKERS: std::sync::LazyLock<std::sync::Mutex<TestSetLockers>> =
|
||||
std::sync::LazyLock::new(|| std::sync::Mutex::new(HashMap::new()));
|
||||
|
||||
/// Keep a fixture's real locker domain scoped to its owning instance.
|
||||
#[cfg(test)]
|
||||
pub(crate) struct ScopedSetLockers(Arc<InstanceContext>);
|
||||
|
||||
#[cfg(test)]
|
||||
impl ScopedSetLockers {
|
||||
pub(crate) fn install(ctx: Arc<InstanceContext>, lockers: Vec<Arc<dyn rustfs_lock::LockClient>>) -> Self {
|
||||
let key = Arc::as_ptr(&ctx) as usize;
|
||||
let mut domains = TEST_SET_LOCKERS.lock().expect("test set lockers mutex");
|
||||
assert!(!domains.contains_key(&key), "the instance already owns a test locker domain");
|
||||
domains.insert(key, lockers);
|
||||
Self(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for ScopedSetLockers {
|
||||
fn drop(&mut self) {
|
||||
TEST_SET_LOCKERS
|
||||
.lock()
|
||||
.expect("test set lockers mutex")
|
||||
.remove(&(Arc::as_ptr(&self.0) as usize));
|
||||
}
|
||||
}
|
||||
|
||||
fn is_idempotent_delete_prefix_error(err: &Error) -> bool {
|
||||
is_err_object_not_found(err) || is_err_strict_volume_not_found(err)
|
||||
}
|
||||
@@ -298,14 +266,6 @@ impl Sets {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
let pool_lockers = TEST_SET_LOCKERS
|
||||
.lock()
|
||||
.expect("test set lockers mutex")
|
||||
.get(&(Arc::as_ptr(&instance_ctx) as usize))
|
||||
.cloned()
|
||||
.unwrap_or_else(|| pool_lockers.clone());
|
||||
|
||||
let set_disks = SetDisks::new_with_instance_ctx(
|
||||
runtime_sources::local_node_name().await,
|
||||
Arc::new(RwLock::new(set_drive)),
|
||||
|
||||
@@ -19,6 +19,7 @@ use tracing::Span;
|
||||
use uuid::Uuid;
|
||||
|
||||
const EVENT_OBJECT_NAMESPACE_LOCK: &str = "object_namespace_lock";
|
||||
const EVENT_OBJECT_NAMESPACE_LOCK_PEER: &str = "object_namespace_lock_peer";
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_OBJECT_LOCK: &str = "object_lock";
|
||||
|
||||
@@ -56,6 +57,7 @@ struct AttemptContext {
|
||||
requested_version_id: Option<String>,
|
||||
namespace: String,
|
||||
owner: String,
|
||||
lock_id: Option<String>,
|
||||
mode: &'static str,
|
||||
timeout: Duration,
|
||||
started_at: Instant,
|
||||
@@ -85,6 +87,7 @@ impl ObjectLockAttempt {
|
||||
requested_version_id: requested_version_id.map(str::to_owned),
|
||||
namespace: lock.namespace().to_owned(),
|
||||
owner: lock.owner().to_owned(),
|
||||
lock_id: None,
|
||||
mode,
|
||||
timeout,
|
||||
started_at: Instant::now(),
|
||||
@@ -105,9 +108,29 @@ impl ObjectLockAttempt {
|
||||
context.finished = true;
|
||||
context.trace("disabled");
|
||||
}
|
||||
Ok(_) => {
|
||||
Ok(guard) => {
|
||||
context.lock_id = guard.lock_id().map(ToString::to_string);
|
||||
context.acquired_at = Some(Instant::now());
|
||||
context.trace("acquired");
|
||||
if let NamespaceLockGuard::Standard(guard) = guard {
|
||||
for (peer_index, peer_lock_id) in guard.granted_lock_ids().enumerate() {
|
||||
context.parent.in_scope(|| {
|
||||
tracing::trace!(
|
||||
target: "rustfs_ecstore::object_lock_diag",
|
||||
parent: &context.parent,
|
||||
event = EVENT_OBJECT_NAMESPACE_LOCK_PEER,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_OBJECT_LOCK,
|
||||
state = "granted",
|
||||
lock_attempt_id = %context.id,
|
||||
lock_id = context.lock_id.as_deref(),
|
||||
peer_lock_id = %peer_lock_id,
|
||||
peer_index,
|
||||
"Object namespace peer owner granted"
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
context.finished = true;
|
||||
@@ -171,6 +194,7 @@ impl AttemptContext {
|
||||
mode = self.mode,
|
||||
owner = %self.owner,
|
||||
lock_attempt_id = %self.id,
|
||||
lock_id = self.lock_id.as_deref(),
|
||||
acquire_ms = duration_ms(acquired_at.duration_since(self.started_at)),
|
||||
hold_ms = duration_ms(now.duration_since(acquired_at)),
|
||||
timeout_ms = duration_ms(self.timeout),
|
||||
@@ -235,15 +259,238 @@ pub(crate) mod tests {
|
||||
}
|
||||
|
||||
pub(crate) fn rows(&self) -> Vec<serde_json::Value> {
|
||||
self.all_rows()
|
||||
.into_iter()
|
||||
.filter(|row| row["fields"]["event"] == EVENT_OBJECT_NAMESPACE_LOCK)
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn all_rows(&self) -> Vec<serde_json::Value> {
|
||||
let bytes = self.0.lock().expect("lock diagnostic capture mutex");
|
||||
let text = std::str::from_utf8(&bytes).expect("UTF-8 diagnostic events");
|
||||
text.lines()
|
||||
.map(|line| serde_json::from_str::<serde_json::Value>(line).expect("JSON diagnostic event"))
|
||||
.filter(|row| row["fields"]["event"] == EVENT_OBJECT_NAMESPACE_LOCK)
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn object_lock_diagnostics_join_guard_identity_to_physical_release() {
|
||||
use rustfs_lock::{
|
||||
DisabledLockManager, FastObjectLockManager, GlobalLockManager, LocalClient, LockClient, LockRequest, LockType,
|
||||
NamespaceLock, ObjectKey,
|
||||
};
|
||||
|
||||
temp_env::with_vars([(rustfs_config::ENV_OBJECT_LOCK_DIAG_ENABLE, Some("true"))], || {
|
||||
let capture = CapturedLockEvents::default();
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("diagnostic release runtime");
|
||||
tracing::subscriber::with_default(capture.subscriber(), || {
|
||||
runtime.block_on(async {
|
||||
let clients = (0..4)
|
||||
.map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc<dyn LockClient>)
|
||||
.collect::<Vec<_>>();
|
||||
let lock = NamespaceLockWrapper::new(
|
||||
NamespaceLock::with_clients_and_quorum("diagnostics".to_owned(), clients.clone(), 4),
|
||||
ObjectKey::new("bucket", "object"),
|
||||
"coordinator".to_owned(),
|
||||
);
|
||||
let request = tracing::info_span!("request", request_id = "original-multipart-request");
|
||||
let mut attempt = request.in_scope(|| {
|
||||
ObjectLockAttempt::start(
|
||||
"abort_multipart_upload",
|
||||
"bucket",
|
||||
"object",
|
||||
None,
|
||||
&lock,
|
||||
"write",
|
||||
Duration::from_secs(5),
|
||||
)
|
||||
});
|
||||
let result = lock.get_write_lock(Duration::from_secs(5)).await;
|
||||
attempt.observe(&result);
|
||||
let guard = result.expect("four real namespace owners");
|
||||
let id = guard.lock_id().expect("distributed guard identity").clone();
|
||||
let peer_ids = match &guard {
|
||||
NamespaceLockGuard::Standard(guard) => {
|
||||
assert_eq!(guard.granted_lock_ids().count(), 4);
|
||||
guard.granted_lock_ids().cloned().collect::<std::collections::HashSet<_>>()
|
||||
}
|
||||
NamespaceLockGuard::Fast(_) => panic!("real distributed namespace required"),
|
||||
};
|
||||
let rows = capture.rows();
|
||||
assert_eq!(rows.last().expect("acquired event")["fields"]["lock_id"], id.to_string());
|
||||
let rows = capture.all_rows();
|
||||
let mappings = rows
|
||||
.iter()
|
||||
.filter(|row| row["fields"]["event"] == EVENT_OBJECT_NAMESPACE_LOCK_PEER)
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(mappings.len(), 4, "every granted peer must map to its logical guard");
|
||||
assert!(mappings.iter().all(|row| {
|
||||
row["fields"]["lock_id"] == id.to_string()
|
||||
&& peer_ids
|
||||
.iter()
|
||||
.any(|peer_id| row["fields"]["peer_lock_id"] == peer_id.to_string())
|
||||
}));
|
||||
for client in &clients {
|
||||
let mut owned = 0;
|
||||
for peer_id in &peer_ids {
|
||||
owned += usize::from(
|
||||
client
|
||||
.check_status(peer_id)
|
||||
.await
|
||||
.expect("actual peer owner status")
|
||||
.is_some(),
|
||||
);
|
||||
}
|
||||
assert_eq!(owned, 1, "each client must hold exactly one actual granted identity");
|
||||
}
|
||||
drop((guard, attempt));
|
||||
assert!(
|
||||
capture.all_rows().iter().all(|row| row["fields"]["state"] != "released"),
|
||||
"enqueue is not release acknowledgement"
|
||||
);
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while capture
|
||||
.all_rows()
|
||||
.iter()
|
||||
.filter(|row| {
|
||||
row["fields"]["event"] == "namespace_lock_release" && row["fields"]["state"] == "released"
|
||||
})
|
||||
.count()
|
||||
< 4
|
||||
{
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("all physical release callbacks finish");
|
||||
for client in &clients {
|
||||
for peer_id in &peer_ids {
|
||||
assert!(
|
||||
client
|
||||
.check_status(peer_id)
|
||||
.await
|
||||
.expect("released peer owner status")
|
||||
.is_none()
|
||||
);
|
||||
assert!(!client.release(peer_id).await.expect("duplicate release is absent"));
|
||||
}
|
||||
}
|
||||
let disabled = LocalClient::with_manager(Arc::new(GlobalLockManager::Disabled(DisabledLockManager::new())));
|
||||
let disabled_request =
|
||||
LockRequest::new(ObjectKey::new("bucket", "disabled"), LockType::Exclusive, "disabled-owner".to_owned());
|
||||
assert!(
|
||||
disabled
|
||||
.acquire_lock(&disabled_request)
|
||||
.await
|
||||
.expect("disabled acquire bypass")
|
||||
.success
|
||||
);
|
||||
assert!(
|
||||
disabled
|
||||
.release(&disabled_request.lock_id)
|
||||
.await
|
||||
.expect("disabled release bypass")
|
||||
);
|
||||
let rows = capture.all_rows();
|
||||
let releases = rows
|
||||
.iter()
|
||||
.filter(|row| row["fields"]["event"] == "namespace_lock_release" && row["fields"]["state"] == "released")
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(
|
||||
releases.len(),
|
||||
4,
|
||||
"duplicates and disabled guards cannot acknowledge another physical release"
|
||||
);
|
||||
let physical_mappings = rows
|
||||
.iter()
|
||||
.filter(|row| row["fields"]["event"] == "namespace_lock_release" && row["fields"]["state"] == "mapped")
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(physical_mappings.len(), 4);
|
||||
assert!(releases.iter().all(|row| physical_mappings.iter().any(|mapped| {
|
||||
mapped["fields"]["physical_guard_id"] == row["fields"]["physical_guard_id"]
|
||||
&& mapped["fields"]["process_id"] == row["fields"]["process_id"]
|
||||
&& peer_ids
|
||||
.iter()
|
||||
.any(|peer_id| mapped["fields"]["lock_id"] == peer_id.to_string())
|
||||
})));
|
||||
let dropped = capture.rows().pop().expect("guard drop event");
|
||||
assert_eq!(dropped["fields"]["lock_id"], id.to_string());
|
||||
assert!(
|
||||
dropped["spans"]
|
||||
.as_array()
|
||||
.expect("retained request span")
|
||||
.iter()
|
||||
.any(|span| span["request_id"] == "original-multipart-request")
|
||||
);
|
||||
let manager = Arc::new(FastObjectLockManager::new());
|
||||
let stale_client = LocalClient::with_manager(Arc::new(GlobalLockManager::Enabled(manager.clone())));
|
||||
let stale_request = LockRequest::new(
|
||||
ObjectKey::new("bucket", "externally-released"),
|
||||
LockType::Exclusive,
|
||||
"stale-owner".to_owned(),
|
||||
);
|
||||
assert!(
|
||||
stale_client
|
||||
.acquire_lock(&stale_request)
|
||||
.await
|
||||
.expect("real stale owner")
|
||||
.success
|
||||
);
|
||||
assert_eq!(manager.force_unlock(&stale_request.resource), 1);
|
||||
let mut successor = manager
|
||||
.acquire_write_lock(stale_request.resource.clone(), "successor-owner")
|
||||
.await
|
||||
.expect("real successor guard");
|
||||
assert!(
|
||||
stale_client
|
||||
.release(&stale_request.lock_id)
|
||||
.await
|
||||
.expect("retire stale client record")
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.get_lock_info(&stale_request.resource)
|
||||
.expect("successor remains locked")
|
||||
.owner
|
||||
.as_ref(),
|
||||
"successor-owner"
|
||||
);
|
||||
assert_eq!(
|
||||
capture
|
||||
.all_rows()
|
||||
.iter()
|
||||
.filter(
|
||||
|row| row["fields"]["event"] == "namespace_lock_release" && row["fields"]["state"] == "released"
|
||||
)
|
||||
.count(),
|
||||
4,
|
||||
"a stale guard's failed physical release must not produce an ACK"
|
||||
);
|
||||
let successor_id = successor.guard_id();
|
||||
assert!(successor.release(), "the actual successor release succeeds");
|
||||
assert!(!successor.release(), "a duplicate explicit release cannot complete again");
|
||||
drop(successor);
|
||||
assert_eq!(
|
||||
capture
|
||||
.all_rows()
|
||||
.iter()
|
||||
.filter(|row| row["fields"]["event"] == "namespace_lock_release"
|
||||
&& row["fields"]["state"] == "released"
|
||||
&& row["fields"]["physical_guard_id"] == successor_id)
|
||||
.count(),
|
||||
1,
|
||||
"explicit release followed by Drop records one physical completion"
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
fn namespace() -> NamespaceLockWrapper {
|
||||
let manager = rustfs_lock::GlobalLockManager::Enabled(Arc::new(rustfs_lock::fast_lock::FastObjectLockManager::new()));
|
||||
NamespaceLockWrapper::new(
|
||||
|
||||
@@ -2776,15 +2776,6 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
})
|
||||
}
|
||||
|
||||
async fn bucket_creation_witness(&self, volume: &str) -> Result<Option<super::os::NativeBucketCreationWitness>> {
|
||||
self.track_disk_health_with_op(
|
||||
"bucket_creation_witness",
|
||||
|| async { self.disk.bucket_creation_witness(volume).await },
|
||||
Duration::ZERO,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn make_volume(&self, volume: &str) -> Result<()> {
|
||||
// Scoped heal must drain directory creation before releasing its lifecycle owner.
|
||||
let timeout = if crate::store::bucket_heal_scope(volume).is_some() {
|
||||
|
||||
@@ -117,9 +117,6 @@ pub const O_APPEND: FileMode = 0x00400;
|
||||
// create_new: bool,
|
||||
|
||||
pub async fn open_file(path: impl AsRef<Path>, mode: FileMode) -> io::Result<File> {
|
||||
if mode & O_CREATE != 0 {
|
||||
super::os::record_native_publication(path.as_ref());
|
||||
}
|
||||
let base_opts = match mode & (O_RDONLY | O_WRONLY | O_RDWR) {
|
||||
O_RDONLY => get_readonly_options(),
|
||||
O_WRONLY => get_writeonly_options(),
|
||||
@@ -172,7 +169,6 @@ pub fn create_dir_all_std(path: &Path) -> io::Result<()> {
|
||||
}
|
||||
|
||||
pub async fn make_dir_all(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
super::os::record_native_publication(path.as_ref());
|
||||
let path = path.as_ref().to_path_buf();
|
||||
tokio::task::spawn_blocking(move || create_dir_all_std(&path)).await?
|
||||
}
|
||||
@@ -226,18 +222,15 @@ pub fn remove_all_std(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
|
||||
#[cfg(test)]
|
||||
pub async fn mkdir(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
super::os::record_native_publication(path.as_ref());
|
||||
fs::create_dir(path.as_ref()).await
|
||||
}
|
||||
|
||||
pub async fn rename(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()> {
|
||||
super::os::record_native_publication(to.as_ref());
|
||||
fs::rename(from, to).await
|
||||
}
|
||||
|
||||
#[cfg(any(not(windows), test))]
|
||||
pub fn rename_std(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()> {
|
||||
super::os::record_native_publication(to.as_ref());
|
||||
std::fs::rename(from, to)
|
||||
}
|
||||
|
||||
|
||||
@@ -520,7 +520,6 @@ fn snapshot_part_transaction_file(src: &Path, backup: &Path, absent: &Path) -> s
|
||||
}
|
||||
|
||||
fn restore_part_transaction_file(current: &Path, backup: &Path, absent: &Path, restore: &Path) -> std::io::Result<()> {
|
||||
os::record_native_publication(current);
|
||||
match std::fs::symlink_metadata(backup) {
|
||||
Ok(metadata) if metadata.is_file() => {
|
||||
remove_file_if_exists(restore)?;
|
||||
@@ -573,7 +572,6 @@ async fn write_delete_rollback_files(
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<Option<crate::disk::DeleteRecoveryInspection>> {
|
||||
let backup_dir = object_dir.join(rollback_dir.to_string());
|
||||
os::record_native_publication(&backup_dir);
|
||||
let mut removed_entries = Vec::with_capacity(remove_before_publish.len());
|
||||
for &name in remove_before_publish {
|
||||
let path = backup_dir.join(name);
|
||||
@@ -2669,9 +2667,6 @@ impl AsyncWrite for DirectWriter {
|
||||
fn open_direct_writer(file_path: &Path, state: &DirectIoWriteState) -> Result<Option<DirectWriter>> {
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
|
||||
#[cfg(test)]
|
||||
os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::DirectWriteOpen, file_path);
|
||||
os::record_native_publication(file_path);
|
||||
let open_result = std::fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.write(true)
|
||||
@@ -3835,9 +3830,6 @@ impl StdBackend {
|
||||
}
|
||||
|
||||
async fn open_file(&self, path: impl AsRef<Path>, mode: usize, skip_parent: impl AsRef<Path>) -> Result<File> {
|
||||
if mode == O_CREATE | O_WRONLY | O_TRUNC {
|
||||
os::record_native_publication(path.as_ref());
|
||||
}
|
||||
let mut skip_parent = skip_parent.as_ref();
|
||||
if skip_parent.as_os_str().is_empty() {
|
||||
skip_parent = self.io_root();
|
||||
@@ -4478,7 +4470,6 @@ impl LocalIoBackend for StdBackend {
|
||||
if is_direct_io_write_enabled() && self.direct_io_write.supported.load(Ordering::Relaxed) {
|
||||
let write_state = self.direct_io_write.clone();
|
||||
let direct_path = file_path.clone();
|
||||
os::record_native_publication(&direct_path);
|
||||
let direct = tokio::task::spawn_blocking(move || open_direct_writer(&direct_path, &write_state))
|
||||
.await
|
||||
.map_err(|err| DiskError::other(format!("O_DIRECT open task failed: {err}")))??;
|
||||
@@ -5731,7 +5722,6 @@ async fn build_local_io_backend(root: PathBuf) -> Arc<dyn LocalIoBackend> {
|
||||
pub struct LocalDisk {
|
||||
pub root: PathBuf,
|
||||
publication_root: os::PublicationRoot,
|
||||
publication_history: Option<os::NativePublicationRegistration>,
|
||||
multipart_proof_authority: OnceLock<crate::disk::multipart_source::ProofVerificationContext>,
|
||||
/// I/O root pinned to the mount instance that was opened while the disk
|
||||
/// was initialized. On Linux this is `/proc/self/fd/<dirfd>/.`; resolving
|
||||
@@ -6124,10 +6114,6 @@ impl LocalDisk {
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let io_root = Self::open_mount_lease(&root)?;
|
||||
|
||||
let publication_history = os::register_native_publication_root(&root, &io_root);
|
||||
|
||||
#[cfg(all(test, target_os = "linux"))]
|
||||
os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::BeforeDataUsageLayout, &root);
|
||||
ensure_data_usage_layout(&io_root)
|
||||
.await
|
||||
.map_err(|e| e.narrow_to_disk().unwrap_or_else(DiskError::other))
|
||||
@@ -6266,7 +6252,6 @@ impl LocalDisk {
|
||||
let mut disk = Self {
|
||||
root: root.clone(),
|
||||
publication_root,
|
||||
publication_history,
|
||||
multipart_proof_authority: OnceLock::new(),
|
||||
io_root: io_root.clone(),
|
||||
#[cfg(target_os = "linux")]
|
||||
@@ -8923,7 +8908,6 @@ impl LocalDisk {
|
||||
|| should_fail_after_delete_data_staged(path))
|
||||
{
|
||||
let rollback_path = object_dir.join(rollback_dir.to_string());
|
||||
os::record_native_publication(&rollback_path);
|
||||
if let Err(err) = fs::create_dir_all(&rollback_path).await {
|
||||
let err: DiskError = to_file_error(err).into();
|
||||
if reserved_version_delete {
|
||||
@@ -9484,7 +9468,6 @@ impl LocalDisk {
|
||||
};
|
||||
|
||||
let path = file_path.to_path_buf();
|
||||
os::record_native_publication(&path);
|
||||
let create_parent = if let Some(parent) = path.parent()
|
||||
&& parent != skip_parent
|
||||
{
|
||||
@@ -9499,7 +9482,6 @@ impl LocalDisk {
|
||||
if is_cancelled() {
|
||||
return Err(std::io::Error::other("blocking namespace operation cancelled"));
|
||||
}
|
||||
os::record_native_publication(&path);
|
||||
if create_parent && let Some(parent) = path.parent() {
|
||||
super::fs::create_dir_all_std(parent)?;
|
||||
}
|
||||
@@ -11457,9 +11439,6 @@ impl DiskAPI for LocalDisk {
|
||||
}
|
||||
|
||||
async fn close(&self) -> Result<()> {
|
||||
if let Some(history) = self.publication_history.as_ref() {
|
||||
history.invalidate();
|
||||
}
|
||||
// This disk instance is being retired (e.g. replaced by renew_disk on
|
||||
// reconnect). Drop its cached descriptors so a replacement instance's
|
||||
// invalidations are never defeated by this one continuing to serve stale
|
||||
@@ -11636,7 +11615,6 @@ impl DiskAPI for LocalDisk {
|
||||
let file_path = self.io_get_object_path(volume, path)?;
|
||||
let path = path.to_string();
|
||||
let sync_metadata = effective_durability(volume).syncs_commit_metadata();
|
||||
os::record_native_publication(&file_path);
|
||||
return Ok(tokio::task::spawn_blocking(move || {
|
||||
commit_allowed()?;
|
||||
// A persistent directory lock bounds metadata growth. Removing
|
||||
@@ -12053,7 +12031,6 @@ impl DiskAPI for LocalDisk {
|
||||
.map_err(to_file_error)?;
|
||||
}
|
||||
}
|
||||
os::record_native_publication(&transaction_path);
|
||||
let retained_owner = source_owner.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let _owner = retained_owner;
|
||||
@@ -12132,7 +12109,6 @@ impl DiskAPI for LocalDisk {
|
||||
}
|
||||
let durability = effective_durability(volume);
|
||||
|
||||
os::record_native_publication(¤t_data_path);
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let _owner = source_owner;
|
||||
match std::fs::symlink_metadata(&transaction_path) {
|
||||
@@ -12829,27 +12805,12 @@ impl DiskAPI for LocalDisk {
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn bucket_creation_witness(&self, volume: &str) -> Result<Option<os::NativeBucketCreationWitness>> {
|
||||
let Some(history) = self.publication_history.as_ref() else {
|
||||
return Ok(None);
|
||||
};
|
||||
let Some(disk_id) = self.get_disk_id().await? else {
|
||||
return Ok(None);
|
||||
};
|
||||
Ok(history.witness(volume, disk_id, Arc::clone(&self.io_backend)))
|
||||
}
|
||||
|
||||
async fn make_volume(&self, volume: &str) -> Result<()> {
|
||||
if !Self::is_valid_volname(volume) {
|
||||
return Err(Error::other("Invalid arguments specified"));
|
||||
}
|
||||
|
||||
let volume_dir = self.io_get_bucket_path(volume)?;
|
||||
#[cfg(all(test, not(windows)))]
|
||||
os::prepared_publication_test_hooks::run(
|
||||
os::prepared_publication_test_hooks::Stage::CreateDirectory,
|
||||
&self.root.join(volume),
|
||||
);
|
||||
|
||||
// Volume creation is a mutation boundary, so it must observe the live
|
||||
// filesystem rather than a potentially stale existence-cache entry.
|
||||
@@ -13627,254 +13588,6 @@ mod test {
|
||||
use tokio::io::{AsyncReadExt, AsyncWrite, AsyncWriteExt, ReadBuf};
|
||||
use tracing_subscriber::fmt::MakeWriter;
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn owned_direct_child_open_attempt_consumes_native_history() {
|
||||
let dir = tempfile::tempdir().expect("native disk directory");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 disk path")).expect("disk endpoint");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("native local disk");
|
||||
let bucket = format!("owned-open-attempt-{}", Uuid::new_v4());
|
||||
let witness = disk
|
||||
.publication_history
|
||||
.as_ref()
|
||||
.expect("native registration")
|
||||
.witness(&bucket, Uuid::new_v4(), Arc::clone(&disk.io_backend))
|
||||
.expect("fresh history");
|
||||
|
||||
disk.write_all_public(&bucket, "direct-child", Bytes::from_static(b"payload"))
|
||||
.await
|
||||
.expect_err("the direct child has no bucket parent");
|
||||
assert!(
|
||||
!os::validate_native_creation_witnesses(&[witness]),
|
||||
"even a failed O_CREATE in the real owned executor must permanently consume native history"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn borrowed_open_cancellation_retains_started_history_before_execution() {
|
||||
use std::sync::mpsc;
|
||||
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.max_blocking_threads(1)
|
||||
.build()
|
||||
.expect("single blocking worker runtime");
|
||||
runtime.block_on(async {
|
||||
let dir = tempfile::tempdir().expect("native disk directory");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 disk path")).expect("disk endpoint");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("native local disk");
|
||||
let history = disk.publication_history.as_ref().expect("native registration");
|
||||
let bucket = format!("borrowed-queued-open-{}", Uuid::new_v4());
|
||||
let disk_id = Uuid::new_v4();
|
||||
let witness = history
|
||||
.witness(&bucket, disk_id, Arc::clone(&disk.io_backend))
|
||||
.expect("fresh native history");
|
||||
let path = disk.io_root().join(&bucket);
|
||||
let (entered_tx, entered_rx) = mpsc::channel();
|
||||
let (release_tx, release_rx) = mpsc::channel();
|
||||
let blocker = tokio::task::spawn_blocking(move || {
|
||||
entered_tx.send(()).expect("signal occupied blocking worker");
|
||||
let _ = release_rx.recv();
|
||||
});
|
||||
entered_rx
|
||||
.recv_timeout(Duration::from_secs(30))
|
||||
.expect("the only blocking worker must be occupied");
|
||||
|
||||
let mut operation =
|
||||
Box::pin(disk.write_all_internal(&path, InternalBuf::Ref(b"payload"), SyncMode::None, disk.io_root()));
|
||||
let queued = futures::poll!(operation.as_mut()).is_pending();
|
||||
let started_before_execution = !os::validate_native_creation_witnesses(std::slice::from_ref(&witness));
|
||||
let absent_before_execution = !path.exists();
|
||||
drop(operation);
|
||||
release_tx.send(()).expect("release the blocking worker");
|
||||
blocker.await.expect("blocking worker must drain");
|
||||
tokio::task::spawn_blocking(|| ())
|
||||
.await
|
||||
.expect("queued native open must drain before assertions");
|
||||
|
||||
assert!(queued, "the borrowed write must queue behind the occupied worker");
|
||||
assert!(
|
||||
absent_before_execution,
|
||||
"the real open must not run before the blocking worker is released"
|
||||
);
|
||||
assert!(
|
||||
started_before_execution,
|
||||
"the borrowed write must consume native history before queueing its open"
|
||||
);
|
||||
// The disk blocking queue skips an operation once its waiter is gone
|
||||
// (`run_cancellable` observes the closed result channel), so the open
|
||||
// never runs. The Started record taken before queueing must survive
|
||||
// that cancellation: the proof stays conservatively consumed.
|
||||
assert!(!path.exists(), "a cancelled borrowed write must not run its queued open");
|
||||
assert!(
|
||||
history.witness(&bucket, disk_id, Arc::clone(&disk.io_backend)).is_none(),
|
||||
"executor drain must not regrant native history to a cancelled publication"
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[tokio::test]
|
||||
async fn synchronous_direct_open_attempt_consumes_native_history() {
|
||||
let dir = tempfile::tempdir().expect("native disk directory");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 disk path")).expect("disk endpoint");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("native local disk");
|
||||
let bucket = format!("direct-io-open-attempt-{}", Uuid::new_v4());
|
||||
let witness = disk
|
||||
.publication_history
|
||||
.as_ref()
|
||||
.expect("native registration")
|
||||
.witness(&bucket, Uuid::new_v4(), Arc::clone(&disk.io_backend))
|
||||
.expect("fresh history");
|
||||
let path = disk.io_get_object_path(&bucket, "direct-child").expect("direct I/O path");
|
||||
|
||||
assert!(
|
||||
open_direct_writer(&path, &DirectIoWriteState::new()).is_err(),
|
||||
"the actual synchronous O_DIRECT entry must observe the missing bucket parent"
|
||||
);
|
||||
assert!(
|
||||
!os::validate_native_creation_witnesses(&[witness]),
|
||||
"a directly callable O_CREATE helper cannot retain retry permission after a failed publication attempt"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[tokio::test]
|
||||
async fn linux_queued_direct_open_cancellation_retains_started_history() {
|
||||
use crate::disk::os::prepared_publication_test_hooks as hooks;
|
||||
use std::sync::mpsc;
|
||||
|
||||
let dir = tempfile::tempdir().expect("native disk directory");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 disk path")).expect("disk endpoint");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("native local disk");
|
||||
let backend = Arc::new(StdBackend::new_without_fd_cache(disk.io_root().to_path_buf()));
|
||||
let actual_backend: Arc<dyn LocalIoBackend> = backend.clone();
|
||||
let history = disk.publication_history.as_ref().expect("native registration");
|
||||
let bucket = format!("native-queued-dio-{}", Uuid::new_v4());
|
||||
let disk_id = Uuid::new_v4();
|
||||
let witness = history
|
||||
.witness(&bucket, disk_id, Arc::clone(&actual_backend))
|
||||
.expect("fresh native history");
|
||||
let path = local_disk_object_path(disk.io_root(), "", &bucket).expect("first-level DIO path");
|
||||
let write_state = Arc::clone(&backend.direct_io_write);
|
||||
let idle_state_owners = Arc::strong_count(&write_state);
|
||||
let (entered_tx, entered_rx) = mpsc::channel();
|
||||
let (release_tx, release_rx) = mpsc::channel();
|
||||
let _hook = hooks::install_at(hooks::Stage::DirectWriteOpen, &path, move || {
|
||||
entered_tx.send(()).expect("signal real DIO executor entry");
|
||||
let _ = release_rx.recv();
|
||||
});
|
||||
|
||||
temp_env::async_with_vars([(ENV_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE, Some("true"))], async {
|
||||
let operation_backend = Arc::clone(&backend);
|
||||
let operation_bucket = bucket.clone();
|
||||
let operation = tokio::spawn(async move {
|
||||
operation_backend
|
||||
.open_write("", &operation_bucket, WriteMode::Truncate { size_hint: 0 })
|
||||
.await
|
||||
});
|
||||
tokio::task::spawn_blocking(move || entered_rx.recv_timeout(Duration::from_secs(30)))
|
||||
.await
|
||||
.expect("DIO entry waiter")
|
||||
.expect("the real blocking executor must enter");
|
||||
assert!(
|
||||
!os::validate_native_creation_witnesses(std::slice::from_ref(&witness)),
|
||||
"Started must be recorded before the queued DIO worker performs its own history lookup"
|
||||
);
|
||||
assert!(!path.exists(), "the keyed pause must precede the actual O_DIRECT open");
|
||||
operation.abort();
|
||||
match operation.await {
|
||||
Err(error) if error.is_cancelled() => {}
|
||||
Err(error) => panic!("DIO waiter failed instead of cancelling: {error}"),
|
||||
Ok(_) => panic!("the paused DIO waiter must be cancelled"),
|
||||
}
|
||||
release_tx.send(()).expect("release the real DIO open");
|
||||
timeout(Duration::from_secs(30), async {
|
||||
while Arc::strong_count(&write_state) > idle_state_owners {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("the detached DIO executor must drain");
|
||||
})
|
||||
.await;
|
||||
|
||||
assert!(path.is_file(), "cancelling the waiter must not discard the queued native open");
|
||||
std::fs::remove_file(&path).expect("remove the file published by the cancelled DIO waiter");
|
||||
assert!(
|
||||
history.witness(&bucket, disk_id, actual_backend).is_none(),
|
||||
"executor drain and physical removal must not regrant native history"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[tokio::test]
|
||||
async fn linux_mount_lease_alias_publication_consumes_same_history() {
|
||||
let dir = tempfile::tempdir().expect("native disk directory");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 disk path")).expect("disk endpoint");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("native local disk");
|
||||
let history = disk.publication_history.as_ref().expect("native registration");
|
||||
let disk_id = Uuid::new_v4();
|
||||
for (alias, root) in [("root", disk.root.as_path()), ("io", disk.io_root())] {
|
||||
let bucket = format!("native-alias-{alias}-{}", Uuid::new_v4());
|
||||
let witness = history
|
||||
.witness(&bucket, disk_id, Arc::clone(&disk.io_backend))
|
||||
.expect("fresh native history through both real root aliases");
|
||||
super::super::fs::mkdir(root.join(&bucket))
|
||||
.await
|
||||
.expect("publish through the actual directory entry");
|
||||
assert!(
|
||||
!os::validate_native_creation_witnesses(std::slice::from_ref(&witness)),
|
||||
"{alias}: publication through either physical-root alias must consume the same history"
|
||||
);
|
||||
std::fs::remove_dir(disk.root.join(&bucket)).expect("remove the physically published directory");
|
||||
assert!(
|
||||
history.witness(&bucket, disk_id, Arc::clone(&disk.io_backend)).is_none(),
|
||||
"{alias}: removal through the other alias must not regrant native history"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[tokio::test]
|
||||
async fn linux_native_history_is_registered_before_data_usage_layout() {
|
||||
use crate::disk::os::prepared_publication_test_hooks as hooks;
|
||||
|
||||
let dir = tempfile::tempdir().expect("native disk directory");
|
||||
let root = std::fs::canonicalize(dir.path()).expect("canonical disk root");
|
||||
let bucket = format!("native-before-layout-{}", Uuid::new_v4());
|
||||
let source = root.join(format!("layout-source-{}", Uuid::new_v4()));
|
||||
std::fs::create_dir(&source).expect("prepare an actual directory for publication");
|
||||
let destination = root.join(&bucket);
|
||||
let publication_path = destination.clone();
|
||||
let _hook = hooks::install_at(hooks::Stage::BeforeDataUsageLayout, &root, move || {
|
||||
super::super::fs::rename_std(&source, &publication_path)
|
||||
.expect("publish through the native entry before data usage layout");
|
||||
});
|
||||
let endpoint = Endpoint::try_from(root.to_str().expect("UTF-8 disk path")).expect("disk endpoint");
|
||||
let disk = LocalDisk::new(&endpoint, false)
|
||||
.await
|
||||
.expect("native local disk with real startup layout");
|
||||
let history = disk.publication_history.as_ref().expect("native registration");
|
||||
let disk_id = Uuid::new_v4();
|
||||
|
||||
let untouched = format!("native-layout-untouched-{}", Uuid::new_v4());
|
||||
assert!(
|
||||
history.witness(&untouched, disk_id, Arc::clone(&disk.io_backend)).is_some(),
|
||||
"startup layout must keep an untouched bucket's native history known"
|
||||
);
|
||||
assert!(
|
||||
history.witness(&bucket, disk_id, Arc::clone(&disk.io_backend)).is_none(),
|
||||
"registration must capture real publication before startup layout finishes"
|
||||
);
|
||||
std::fs::remove_dir(destination).expect("remove the physically published directory");
|
||||
assert!(
|
||||
history.witness(&bucket, disk_id, Arc::clone(&disk.io_backend)).is_none(),
|
||||
"startup completion and physical removal must retain Started"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn read_dir_entries_skip_symlinks_without_truncating_listing() {
|
||||
@@ -17245,109 +16958,6 @@ mod test {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rename_data_pending_restore_requires_parts_only_from_multipart_staging() {
|
||||
let dir = tempfile::tempdir().expect("disk directory should be created");
|
||||
let endpoint =
|
||||
Endpoint::try_from(dir.path().to_str().expect("disk path should be UTF-8")).expect("disk endpoint should parse");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
|
||||
let bucket = "restore-rename-controls";
|
||||
ensure_test_volume(&disk, bucket).await;
|
||||
for (case, source_volume, pending, requires_parts) in [
|
||||
("remote", RUSTFS_META_TMP_BUCKET, false, false),
|
||||
("pending-remote", RUSTFS_META_TMP_BUCKET, true, false),
|
||||
("pending-multipart", crate::disk::RUSTFS_META_MULTIPART_BUCKET, true, true),
|
||||
] {
|
||||
ensure_test_volume(&disk, source_volume).await;
|
||||
let staging = dir.path().join(source_volume).join(case);
|
||||
fs::create_dir_all(&staging).await.expect("staging parent should be created");
|
||||
let data_dir = Uuid::new_v4();
|
||||
let mut fi = test_file_info(case, Uuid::nil(), Some(data_dir), None);
|
||||
fi.transition_status = rustfs_filemeta::TRANSITION_COMPLETE.to_string();
|
||||
fi.transition_tier = "WARM".to_string();
|
||||
fi.transitioned_objname = "remote/object".to_string();
|
||||
if pending {
|
||||
fi.metadata.insert(
|
||||
rustfs_filemeta::metadata_keys::RESTORE.to_string(),
|
||||
"ongoing-request=\"true\"".to_string(),
|
||||
);
|
||||
}
|
||||
let result = disk.rename_data(source_volume, case, fi, bucket, case).await;
|
||||
let destination = dir.path().join(bucket).join(case);
|
||||
if requires_parts {
|
||||
assert!(result.is_err(), "missing multipart restore parts must reject the physical commit");
|
||||
assert!(
|
||||
!destination.join(STORAGE_FORMAT_FILE).exists(),
|
||||
"failed multipart restore must not publish metadata for absent local parts"
|
||||
);
|
||||
} else {
|
||||
result.expect("remote-only metadata commits must not require local staged parts");
|
||||
let stored = fs::read(destination.join(STORAGE_FORMAT_FILE))
|
||||
.await
|
||||
.expect("remote-only metadata should be committed");
|
||||
let restored = FileMeta::load(&stored)
|
||||
.expect("committed metadata should decode")
|
||||
.into_fileinfo(bucket, case, "", false, false, false)
|
||||
.expect("committed version should decode");
|
||||
assert!(restored.is_remote(), "remote-only commits must retain remote read routing");
|
||||
assert!(
|
||||
!destination.join(data_dir.to_string()).exists(),
|
||||
"remote-only metadata commit must not create a local payload directory"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rename_data_pending_multipart_restore_rejects_absent_data_directory() {
|
||||
assert_pending_multipart_restore_rejects_directory(None).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rename_data_pending_multipart_restore_rejects_nil_data_directory() {
|
||||
assert_pending_multipart_restore_rejects_directory(Some(Uuid::nil())).await;
|
||||
}
|
||||
|
||||
async fn assert_pending_multipart_restore_rejects_directory(data_dir: Option<Uuid>) {
|
||||
let dir = tempfile::tempdir().expect("disk directory should be created");
|
||||
let endpoint =
|
||||
Endpoint::try_from(dir.path().to_str().expect("disk path should be UTF-8")).expect("disk endpoint should parse");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
|
||||
let source_volume = crate::disk::RUSTFS_META_MULTIPART_BUCKET;
|
||||
let bucket = "restore-invalid-directory";
|
||||
let object = "object.bin";
|
||||
ensure_test_volume(&disk, source_volume).await;
|
||||
ensure_test_volume(&disk, bucket).await;
|
||||
let staging = dir.path().join(source_volume).join(object);
|
||||
fs::create_dir_all(&staging).await.expect("staging parent should be created");
|
||||
if let Some(data_dir) = data_dir {
|
||||
let data_path = staging.join(data_dir.to_string());
|
||||
fs::create_dir_all(&data_path)
|
||||
.await
|
||||
.expect("invalid staging directory should be created");
|
||||
fs::write(data_path.join("part.1"), b"x")
|
||||
.await
|
||||
.expect("nil directory fixture must contain a real part");
|
||||
}
|
||||
let mut fi = test_file_info(object, Uuid::nil(), data_dir, None);
|
||||
fi.transition_status = rustfs_filemeta::TRANSITION_COMPLETE.to_string();
|
||||
fi.transition_tier = "WARM".to_string();
|
||||
fi.transitioned_objname = "remote/object".to_string();
|
||||
fi.metadata.insert(
|
||||
rustfs_filemeta::metadata_keys::RESTORE.to_string(),
|
||||
"ongoing-request=\"true\"".to_string(),
|
||||
);
|
||||
let result = disk.rename_data(source_volume, object, fi, bucket, object).await;
|
||||
assert!(
|
||||
matches!(result, Err(DiskError::FileCorrupt)),
|
||||
"pending multipart restore must reject absent or nil staging identity as corrupt: {result:?}"
|
||||
);
|
||||
assert!(
|
||||
!dir.path().join(bucket).join(object).join(STORAGE_FORMAT_FILE).exists(),
|
||||
"invalid staging identity must fail before metadata publication"
|
||||
);
|
||||
}
|
||||
|
||||
// Seed a first PUT of `object` (no prior version) through the non-inline
|
||||
// rename_data path and return (disk, tempdir). The object dir and any prefix
|
||||
// dirs are created during the commit.
|
||||
|
||||
@@ -642,20 +642,8 @@ impl LocalDisk {
|
||||
};
|
||||
let dst_file_path = self.io_get_object_open_path(dst_volume, format!("{}/{}", dst_path, STORAGE_FORMAT_FILE).as_str())?;
|
||||
|
||||
// Multipart restore completion has validated staged local parts,
|
||||
// while its public status stays pending until finalization.
|
||||
let pending_multipart_restore = src_volume == crate::disk::RUSTFS_META_MULTIPART_BUCKET
|
||||
&& fi
|
||||
.metadata
|
||||
.get(rustfs_filemeta::metadata_keys::RESTORE)
|
||||
.and_then(|value| rustfs_filemeta::parse_restore_obj_status(value).ok())
|
||||
.is_some_and(|status| status.is_restore_in_progress == Some(true));
|
||||
if pending_multipart_restore && fi.data_dir.is_none_or(|dir| dir.is_nil()) {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
|
||||
// data_dir path (only used by the non-inline commit branch)
|
||||
let has_data_dir_path = if no_inline && (!fi.is_remote() || pending_multipart_restore) {
|
||||
let has_data_dir_path = if no_inline && !fi.is_remote() {
|
||||
if let Some(data_dir) = fi
|
||||
.data_dir
|
||||
.map(|dir| rustfs_utils::path::retain_slash(dir.to_string().as_str()))
|
||||
|
||||
@@ -6,6 +6,7 @@ use super::{
|
||||
LocalDisk,
|
||||
multipart_source_state::{
|
||||
LoadedSource, SourceHead, SourceState, SourceStore, decode, encode, load_source, raw_sha, read_optional,
|
||||
read_optional_in_pinned,
|
||||
},
|
||||
};
|
||||
use crate::disk::{
|
||||
@@ -127,20 +128,6 @@ pub(super) fn validate_retired_head(
|
||||
}
|
||||
|
||||
impl SourceStore {
|
||||
pub async fn root_missing(&self) -> Result<bool> {
|
||||
let source = self.source.clone();
|
||||
let base = self.base.clone();
|
||||
let root = self.publication_root.clone();
|
||||
os::run_blocking_namespace_operation(self.lease.clone(), move || {
|
||||
match os::pin_multipart_source_directory(&source, &base, &root) {
|
||||
Ok(_) => Ok(false),
|
||||
Err(error) if error.kind() == ErrorKind::NotFound => Ok(true),
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
fn verify_abort_cohort(&self, certificate: &MultipartAbortCertificate) -> Result<()> {
|
||||
certificate.verify_with_authority(&self.authority)?;
|
||||
if certificate.disk_ids != self.disk_ids
|
||||
@@ -265,7 +252,7 @@ impl SourceStore {
|
||||
let certificate = self.read_abort_certificate(&loaded).await?;
|
||||
Ok(MultipartSourceInspection {
|
||||
disk_id: self.disk_id,
|
||||
root_missing: self.root_missing().await?,
|
||||
root_missing: loaded.root_missing,
|
||||
abort_round: loaded
|
||||
.state
|
||||
.as_ref()
|
||||
@@ -620,12 +607,17 @@ pub(super) struct OrdinaryMultipartDeleteCheck {
|
||||
}
|
||||
impl OrdinaryMultipartDeleteCheck {
|
||||
pub(super) fn validate(&self) -> Result<()> {
|
||||
if let Some(raw) = read_optional(&self.source.join(STATE), &self.base, &self.root)? {
|
||||
let pinned = match os::pin_multipart_source_directory(&self.source, &self.base, &self.root) {
|
||||
Ok(pinned) => pinned,
|
||||
Err(error) if error.kind() == ErrorKind::NotFound => return Ok(()),
|
||||
Err(error) => return Err(error.into()),
|
||||
};
|
||||
if let Some(raw) = read_optional_in_pinned(&pinned, &self.source, STATE, &self.base, &self.root)? {
|
||||
let state: SourceState = decode(&raw)?;
|
||||
if state.abort.is_some() || state.phase == MultipartSourcePhase::Aborting {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
} else if os::multipart_source_has_native_controls(&self.source, &self.base, &self.root)? {
|
||||
} else if os::multipart_source_has_native_controls_in_pinned(&self.source, &pinned)? {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
Ok(())
|
||||
|
||||
@@ -170,7 +170,7 @@ impl LocalDisk {
|
||||
recovery: loaded.state.as_ref().and_then(|state| state.recovery.clone()),
|
||||
negative_certificate,
|
||||
ready_certificate,
|
||||
root_missing: store.root_missing().await?,
|
||||
root_missing: loaded.root_missing,
|
||||
abort_round: loaded
|
||||
.state
|
||||
.as_ref()
|
||||
|
||||
@@ -106,6 +106,7 @@ impl SourceHead {
|
||||
}
|
||||
|
||||
pub(super) struct LoadedSource {
|
||||
pub root_missing: bool,
|
||||
pub state: Option<SourceState>,
|
||||
pub head: Option<SourceHead>,
|
||||
pub original: Option<Bytes>,
|
||||
@@ -224,11 +225,31 @@ pub(super) fn read_optional(path: &Path, base: &Path, root: &os::PublicationRoot
|
||||
Err(error) => Err(error.into()),
|
||||
}
|
||||
}
|
||||
pub(super) fn read_optional_in_pinned(
|
||||
pinned: &os::ExistingBaseDirectoryGuard,
|
||||
parent: &Path,
|
||||
name: &str,
|
||||
base: &Path,
|
||||
root: &os::PublicationRoot,
|
||||
) -> Result<Option<Bytes>> {
|
||||
match os::read_multipart_source_metadata_in_pinned(pinned, parent, std::ffi::OsStr::new(name), base, root) {
|
||||
Ok(bytes) => Ok(Some(Bytes::from(bytes))),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
|
||||
Err(error) => Err(error.into()),
|
||||
}
|
||||
}
|
||||
/// Canonical source mutations are admitted under SourceRW at the syscall,
|
||||
/// not on the earlier async caller that may have timed out.
|
||||
pub(super) fn validate_source_mutation(source: &Path, base: &Path, root: &os::PublicationRoot, initial: bool) -> Result<()> {
|
||||
let state = read_optional(&source.join(STATE), base, root)?;
|
||||
let head = read_optional(&source.join(STORAGE_FORMAT_FILE), base, root)?;
|
||||
let pinned = match os::pin_multipart_source_directory(source, base, root) {
|
||||
Ok(pinned) => pinned,
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
|
||||
return if initial { Ok(()) } else { Err(DiskError::FileNotFound) };
|
||||
}
|
||||
Err(error) => return Err(error.into()),
|
||||
};
|
||||
let state = read_optional_in_pinned(&pinned, source, STATE, base, root)?;
|
||||
let head = read_optional_in_pinned(&pinned, source, STORAGE_FORMAT_FILE, base, root)?;
|
||||
let Some(head) = head else {
|
||||
return if initial && state.is_none() {
|
||||
Ok(())
|
||||
@@ -348,6 +369,9 @@ impl SourceStore {
|
||||
.map_err(DiskError::from)?
|
||||
}
|
||||
pub async fn persist(&self, state: &SourceState) -> Result<()> {
|
||||
self.persist_with_source_sync(state, false).await
|
||||
}
|
||||
async fn persist_with_source_sync(&self, state: &SourceState, sync_source_before: bool) -> Result<()> {
|
||||
state.validate(self.disk_id, &self.path)?;
|
||||
let raw = encode(state)?;
|
||||
let source = self.source.clone();
|
||||
@@ -365,6 +389,9 @@ impl SourceStore {
|
||||
});
|
||||
os::run_blocking_namespace_operation(self.lease.clone(), move || {
|
||||
let _pin = os::pin_multipart_source_directory(&source, &base, &root)?;
|
||||
if sync_source_before {
|
||||
os::fsync_dir_std(&source)?;
|
||||
}
|
||||
if let Some(path) = terminal {
|
||||
os::replace_multipart_source_metadata(&path, &raw, &base, &root)?;
|
||||
}
|
||||
@@ -373,6 +400,7 @@ impl SourceStore {
|
||||
.await
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
#[cfg(test)]
|
||||
pub async fn original(&self, state: &SourceState) -> Result<Bytes> {
|
||||
let path = original_path(&self.source, state.backup_owner);
|
||||
let base = self.base.clone();
|
||||
@@ -490,7 +518,7 @@ impl SourceStore {
|
||||
if encode(actual)? != encode(state)? {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let raw = self.original(state).await?;
|
||||
let raw = loaded.original.as_ref().ok_or(DiskError::FileCorrupt)?;
|
||||
let allowed = match loaded.head.as_ref() {
|
||||
None => {
|
||||
state.phase == MultipartSourcePhase::Recovering
|
||||
@@ -551,7 +579,7 @@ impl SourceStore {
|
||||
if !allowed {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let mut info = file_info(&raw, RUSTFS_META_MULTIPART_BUCKET, &self.path, state.lineage.original_version_id)?;
|
||||
let mut info = file_info(raw, RUSTFS_META_MULTIPART_BUCKET, &self.path, state.lineage.original_version_id)?;
|
||||
strip_native_controls(&mut info.metadata);
|
||||
mark_native_source(&mut info);
|
||||
set_source_status(
|
||||
@@ -772,13 +800,32 @@ pub(super) fn load_source(
|
||||
physical_slot: usize,
|
||||
disk_ids: &[Uuid],
|
||||
) -> Result<LoadedSource> {
|
||||
let state_raw = read_optional(&source.join(STATE), base, root)?;
|
||||
let pinned = match os::pin_multipart_source_directory(source, base, root) {
|
||||
Ok(pinned) => pinned,
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
|
||||
return Ok(LoadedSource {
|
||||
root_missing: true,
|
||||
state: None,
|
||||
head: None,
|
||||
original: None,
|
||||
info: None,
|
||||
snapshot: None,
|
||||
head_matches_state: false,
|
||||
head_matches_activation: false,
|
||||
});
|
||||
}
|
||||
Err(error) => return Err(error.into()),
|
||||
};
|
||||
let state_raw = read_optional_in_pinned(&pinned, source, STATE, base, root)?;
|
||||
let state: Option<SourceState> = state_raw.as_deref().map(decode).transpose()?;
|
||||
if state.is_none() && os::multipart_source_has_native_controls(source, base, root)? {
|
||||
if state.is_none() && os::multipart_source_has_native_controls_in_pinned(source, &pinned)? {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let head_path = source.join(STORAGE_FORMAT_FILE);
|
||||
let head = if let Some(state) = state.as_ref() {
|
||||
let head = if let Some(state) = state
|
||||
.as_ref()
|
||||
.filter(|state| state.prepared_length.is_some_and(|len| len > 64 * 1024))
|
||||
{
|
||||
let head_path = source.join(STORAGE_FORMAT_FILE);
|
||||
let maximum = state
|
||||
.prepared_length
|
||||
.unwrap_or(0)
|
||||
@@ -789,7 +836,8 @@ pub(super) fn load_source(
|
||||
let raw = if is_proxy {
|
||||
None
|
||||
} else {
|
||||
let raw = read_optional(&head_path, base, root)?.ok_or(DiskError::FileCorrupt)?;
|
||||
let raw = read_optional_in_pinned(&pinned, source, STORAGE_FORMAT_FILE, base, root)?
|
||||
.ok_or(DiskError::FileCorrupt)?;
|
||||
if raw_sha(&raw) != sha256 {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
@@ -801,10 +849,17 @@ pub(super) fn load_source(
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
} else {
|
||||
read_optional(&head_path, base, root)?.map(|raw| SourceHead {
|
||||
sha256: raw_sha(&raw),
|
||||
length: raw.len() as u64,
|
||||
raw: Some(raw),
|
||||
read_optional_in_pinned(&pinned, source, STORAGE_FORMAT_FILE, base, root)?.map(|raw| {
|
||||
let sha256 = raw_sha(&raw);
|
||||
let length = raw.len() as u64;
|
||||
let is_proxy = state
|
||||
.as_ref()
|
||||
.is_some_and(|state| Some(sha256) == state.prepared_sha256 && Some(length) == state.prepared_length);
|
||||
SourceHead {
|
||||
sha256,
|
||||
length,
|
||||
raw: (!is_proxy).then_some(raw),
|
||||
}
|
||||
})
|
||||
};
|
||||
let mut original = None;
|
||||
@@ -950,6 +1005,7 @@ pub(super) fn load_source(
|
||||
head_matches_state = true;
|
||||
}
|
||||
Ok(LoadedSource {
|
||||
root_missing: false,
|
||||
state,
|
||||
head,
|
||||
original,
|
||||
@@ -1587,10 +1643,8 @@ impl NativeMultipartPublication {
|
||||
}
|
||||
}
|
||||
self.store.sync_target_with_lease(&self.publication, target_lease).await?;
|
||||
let source = self.store.source.clone();
|
||||
os::run_blocking_namespace_operation(self.store.lease.clone(), move || os::fsync_dir_std(&source)).await?;
|
||||
state.advance(MultipartSourcePhase::ConsumedUnsealed)?;
|
||||
self.store.persist(&state).await?;
|
||||
self.store.persist_with_source_sync(&state, true).await?;
|
||||
*self.state.lock() = state;
|
||||
let loaded = self.store.load().await?;
|
||||
let snapshot = loaded.snapshot.ok_or(DiskError::FileCorrupt)?;
|
||||
|
||||
@@ -411,15 +411,23 @@ impl LocalDisk {
|
||||
}
|
||||
let _attempt = os::mkdir_all_below_existing_base_std(&private_object, &base, &publication_root)?;
|
||||
if incoming.data.is_none() && !incoming.is_remote() {
|
||||
let source_data_dir = source_object.join(source_dir.to_string());
|
||||
let pinned_source_data = (!incoming.parts.is_empty())
|
||||
.then(|| os::pin_multipart_source_directory(&source_data_dir, &base, &publication_root))
|
||||
.transpose()?;
|
||||
let mut names = Vec::with_capacity(incoming.parts.len().saturating_mul(3));
|
||||
for part in &incoming.parts {
|
||||
if cancelled() {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "multipart staging cancelled"));
|
||||
}
|
||||
let path = source_object
|
||||
.join(source_dir.to_string())
|
||||
.join(format!("part.{}.meta", part.number));
|
||||
let bytes = os::read_multipart_source_metadata(&path, &base, &publication_root)?;
|
||||
let meta_name = format!("part.{}.meta", part.number);
|
||||
let bytes = os::read_multipart_source_metadata_in_pinned(
|
||||
pinned_source_data.as_ref().expect("pinned source data directory"),
|
||||
&source_data_dir,
|
||||
std::ffi::OsStr::new(&meta_name),
|
||||
&base,
|
||||
&publication_root,
|
||||
)?;
|
||||
let actual = rustfs_filemeta::ObjectPartInfo::unmarshal(&bytes).map_err(std::io::Error::other)?;
|
||||
let normalized = i64::try_from(actual.size)
|
||||
.map_err(std::io::Error::other)?
|
||||
@@ -441,13 +449,13 @@ impl LocalDisk {
|
||||
// Preserve the native completed-MPU directory format. This
|
||||
// immutable, validated generation record is distinct from
|
||||
// the upload's mutable top-level xl.meta.
|
||||
names.push(format!("part.{}.meta", part.number));
|
||||
names.push(meta_name);
|
||||
if let Some(integrity) = &part.integrity {
|
||||
names.push(integrity.file_name());
|
||||
}
|
||||
}
|
||||
os::hardlink_multipart_source_files(
|
||||
&source_object.join(source_dir.to_string()),
|
||||
&source_data_dir,
|
||||
&private_object.join(final_dir.to_string()),
|
||||
&names,
|
||||
&base,
|
||||
|
||||
@@ -613,13 +613,6 @@ impl DiskAPI for Disk {
|
||||
}
|
||||
}
|
||||
|
||||
async fn bucket_creation_witness(&self, volume: &str) -> Result<Option<os::NativeBucketCreationWitness>> {
|
||||
match self {
|
||||
Disk::Local(local_disk) => local_disk.bucket_creation_witness(volume).await,
|
||||
Disk::Remote(_) => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn make_volume(&self, volume: &str) -> Result<()> {
|
||||
match self {
|
||||
@@ -1477,11 +1470,6 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
|
||||
// NSScanner
|
||||
|
||||
// Volume operations.
|
||||
/// Unknown or remote backends cannot attest process-local publication history.
|
||||
async fn bucket_creation_witness(&self, _volume: &str) -> Result<Option<os::NativeBucketCreationWitness>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn make_volume(&self, volume: &str) -> Result<()>;
|
||||
async fn make_volumes(&self, volume: Vec<&str>) -> Result<()>;
|
||||
async fn list_volumes(&self) -> Result<Vec<VolumeInfo>>;
|
||||
@@ -2257,7 +2245,7 @@ impl DeleteRecoveryJournal {
|
||||
}
|
||||
let id = request.version_id.unwrap_or_default();
|
||||
let Some(original) = originals.get(&id) else {
|
||||
return !id.is_nil() && !locals.contains_key(&id);
|
||||
return false;
|
||||
};
|
||||
match locals.get(&id) {
|
||||
Some(local) => local == original,
|
||||
@@ -2307,10 +2295,7 @@ impl DeleteRecoveryJournal {
|
||||
};
|
||||
match (local, remote) {
|
||||
(Some(local), Some(remote)) => local == remote,
|
||||
(None, None) => {
|
||||
(!(request.deleted && request.mark_deleted) && request.version_id.is_some_and(|id| !id.is_nil()))
|
||||
|| (self.original_digest.is_none() && authority.original_digest.is_none())
|
||||
}
|
||||
(None, None) => self.original_digest.is_none() && authority.original_digest.is_none(),
|
||||
_ => false,
|
||||
}
|
||||
})
|
||||
@@ -3635,46 +3620,6 @@ mod delete_recovery_adoption_tests {
|
||||
};
|
||||
assert!(journal.matches_adoption_source(&[]), "certified non-null removal can already be absent");
|
||||
assert!(journal.matches_adoption_source(std::slice::from_ref(&source)));
|
||||
let missing = Uuid::new_v4();
|
||||
journal.requests.push(FileInfo {
|
||||
version_id: Some(missing),
|
||||
..Default::default()
|
||||
});
|
||||
journal.original_digest = Some([3; 32]);
|
||||
assert!(
|
||||
journal.matches_prepared_authority(&journal),
|
||||
"an explicit UUID absent from both prepared sources remains a no-op with original metadata"
|
||||
);
|
||||
assert!(journal.matches_adoption_source(&[]), "both non-null versions can remain absent");
|
||||
assert!(
|
||||
journal.matches_adoption_source(std::slice::from_ref(&source)),
|
||||
"a missing non-null version in the batch remains a no-op"
|
||||
);
|
||||
let mut unexpected = source.clone();
|
||||
unexpected.version_id = Some(missing);
|
||||
let mut foreign_preparation = journal.clone();
|
||||
foreign_preparation.original_versions.push(unexpected.clone());
|
||||
assert!(
|
||||
!journal.matches_prepared_authority(&foreign_preparation)
|
||||
&& !foreign_preparation.matches_prepared_authority(&journal),
|
||||
"absence from only one prepared source cannot establish a generation"
|
||||
);
|
||||
assert!(
|
||||
!journal.matches_adoption_source(&[source.clone(), unexpected]),
|
||||
"a version absent from the authority cannot be deleted locally"
|
||||
);
|
||||
for version_id in [None, Some(Uuid::nil())] {
|
||||
journal.requests.last_mut().expect("missing-version request").version_id = version_id;
|
||||
assert!(
|
||||
!journal.matches_prepared_authority(&journal),
|
||||
"a missing null version with original metadata cannot establish its generation"
|
||||
);
|
||||
assert!(
|
||||
!journal.matches_adoption_source(std::slice::from_ref(&source)),
|
||||
"a missing null version cannot establish its generation"
|
||||
);
|
||||
}
|
||||
journal.requests.pop();
|
||||
let mut foreign = source.clone();
|
||||
foreign.metadata_digest = [2; 32];
|
||||
assert!(!journal.matches_adoption_source(std::slice::from_ref(&foreign)));
|
||||
@@ -3690,25 +3635,6 @@ mod delete_recovery_adoption_tests {
|
||||
assert!(journal.matches_adoption_source(std::slice::from_ref(&journal.original_versions[0])));
|
||||
journal.requests.clear();
|
||||
assert!(!journal.matches_adoption_source(&journal.original_versions));
|
||||
|
||||
let mut marker = journal.clone();
|
||||
marker.requests = vec![FileInfo {
|
||||
version_id: Some(id),
|
||||
deleted: true,
|
||||
mark_deleted: true,
|
||||
..Default::default()
|
||||
}];
|
||||
marker.original_versions.clear();
|
||||
marker.original_digest = Some([3; 32]);
|
||||
assert!(
|
||||
!marker.matches_prepared_authority(&marker),
|
||||
"a marker with original metadata requires a source generation"
|
||||
);
|
||||
marker.original_digest = None;
|
||||
assert!(
|
||||
marker.matches_prepared_authority(&marker),
|
||||
"marker preparation without original metadata retains the existing allowance"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+99
-674
@@ -21,7 +21,7 @@ use futures::TryStreamExt;
|
||||
use parking_lot::Mutex;
|
||||
use rustfs_utils::path::SLASH_SEPARATOR;
|
||||
use std::{
|
||||
collections::{HashMap, HashSet, VecDeque},
|
||||
collections::{HashMap, VecDeque},
|
||||
io,
|
||||
path::{Component, Path, PathBuf},
|
||||
sync::{Arc, LazyLock, Weak},
|
||||
@@ -337,14 +337,6 @@ pub(crate) mod prepared_publication_test_hooks {
|
||||
Rollback,
|
||||
#[cfg(test)]
|
||||
DirFsync,
|
||||
#[cfg(test)]
|
||||
CreateDirectory,
|
||||
#[cfg(test)]
|
||||
NativeRootIdentity,
|
||||
#[cfg(all(test, target_os = "linux"))]
|
||||
DirectWriteOpen,
|
||||
#[cfg(all(test, target_os = "linux"))]
|
||||
BeforeDataUsageLayout,
|
||||
}
|
||||
|
||||
type Hook = Box<dyn FnOnce() + Send>;
|
||||
@@ -1721,226 +1713,6 @@ fn fsync_spawn_blocking<T: Send + 'static>(f: impl FnOnce() -> T + Send + 'stati
|
||||
}
|
||||
static DISK_VOLUME_MUTATION_LOCKS: LazyLock<Mutex<HashMap<PathBuf, Weak<RwLock<()>>>>> =
|
||||
LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
#[cfg(windows)]
|
||||
type NativeRootIdentity = (u64, [u8; 16]);
|
||||
#[cfg(not(windows))]
|
||||
type NativeRootIdentity = (u64, u64);
|
||||
|
||||
// Publication history outlives disk handles, cleanup, and metadata snapshots.
|
||||
// Reusing a pathname or descriptor for another physical root is not new proof.
|
||||
#[derive(Debug, Default)]
|
||||
struct NativePublicationHistory {
|
||||
roots: HashMap<NativeRootIdentity, NativeRootHistory>,
|
||||
aliases: HashMap<PathBuf, HashSet<NativeRootIdentity>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct NativeRootHistory {
|
||||
unknown: bool,
|
||||
started: HashSet<String>,
|
||||
}
|
||||
|
||||
static NATIVE_PUBLICATION_HISTORY: LazyLock<Mutex<NativePublicationHistory>> =
|
||||
LazyLock::new(|| Mutex::new(NativePublicationHistory::default()));
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct NativePublicationRegistration {
|
||||
identity: NativeRootIdentity,
|
||||
root: PathBuf,
|
||||
io_root: PathBuf,
|
||||
}
|
||||
|
||||
/// Opaque, process-local observation. Unsupported disks cannot construct it.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct NativeBucketCreationWitness {
|
||||
registration: NativePublicationRegistration,
|
||||
backend: Arc<dyn super::local::LocalIoBackend>,
|
||||
bucket: String,
|
||||
disk_id: uuid::Uuid,
|
||||
}
|
||||
|
||||
fn native_root_identity(path: &Path) -> Option<NativeRootIdentity> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
#[cfg(test)]
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::NativeRootIdentity, path);
|
||||
let metadata = std::fs::metadata(path).ok()?;
|
||||
metadata.is_dir().then(|| (metadata.dev(), metadata.ino()))
|
||||
}
|
||||
#[cfg(windows)]
|
||||
{
|
||||
// Resolve and validate the same directory anchor used for publication;
|
||||
// keep its volume and complete 128-bit file ID in the history key.
|
||||
let (_, directory) = open_windows_publication_root(path).ok()?;
|
||||
windows_file_identity(directory.handle.as_ref()).ok()
|
||||
}
|
||||
#[cfg(all(not(unix), not(windows)))]
|
||||
{
|
||||
let _ = path;
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn register_native_publication_root(root: &Path, io_root: &Path) -> Option<NativePublicationRegistration> {
|
||||
let identity = native_root_identity(root)?;
|
||||
if native_root_identity(io_root) != Some(identity) {
|
||||
return None;
|
||||
}
|
||||
let mut history = NATIVE_PUBLICATION_HISTORY.lock();
|
||||
history.roots.entry(identity).or_default();
|
||||
for alias in [root, io_root] {
|
||||
let alias: PathBuf = alias.components().collect();
|
||||
let previous = history.aliases.entry(alias).or_default();
|
||||
let collisions: Vec<_> = previous.iter().copied().filter(|old| *old != identity).collect();
|
||||
previous.insert(identity);
|
||||
if !collisions.is_empty() {
|
||||
history.roots.get_mut(&identity)?.unknown = true;
|
||||
for old in collisions {
|
||||
history.roots.get_mut(&old)?.unknown = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
Some(NativePublicationRegistration {
|
||||
identity,
|
||||
root: root.to_path_buf(),
|
||||
io_root: io_root.to_path_buf(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Record before queueing filesystem work. Cancellation never removes Started.
|
||||
pub(crate) fn record_native_publication(path: &Path) {
|
||||
let aliases: Vec<_> = {
|
||||
let history = NATIVE_PUBLICATION_HISTORY.lock();
|
||||
path.ancestors()
|
||||
.filter(|alias| history.aliases.contains_key(*alias))
|
||||
.map(Path::to_path_buf)
|
||||
.collect()
|
||||
};
|
||||
// Physical lookups must never hold the process-wide publication mutex.
|
||||
let observations: Vec<_> = aliases
|
||||
.into_iter()
|
||||
.filter_map(|alias| {
|
||||
let relative = path.strip_prefix(&alias).ok()?;
|
||||
let mut components = relative.components().filter(|part| !matches!(part, Component::CurDir));
|
||||
let first = components.next()?;
|
||||
let bucket = match first {
|
||||
Component::Normal(bucket) if components.all(|part| matches!(part, Component::Normal(_))) => {
|
||||
bucket.to_str().map(str::to_owned)
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
let physical = native_root_identity(&alias);
|
||||
Some((alias, physical, bucket))
|
||||
})
|
||||
.collect();
|
||||
let mut history = NATIVE_PUBLICATION_HISTORY.lock();
|
||||
for (alias, physical, bucket) in observations {
|
||||
// Aliases only accumulate identities. Include registration concurrent
|
||||
// with the stat, rather than using a stale captured identity list.
|
||||
let identities = history.aliases.get(&alias).cloned().unwrap_or_default();
|
||||
if physical.is_none() || identities.iter().any(|identity| Some(*identity) != physical) {
|
||||
for identity in identities {
|
||||
history.roots.entry(identity).or_default().unknown = true;
|
||||
}
|
||||
if let Some(identity) = physical {
|
||||
history.roots.entry(identity).or_default().unknown = true;
|
||||
}
|
||||
} else if let Some(identity) = physical {
|
||||
let root = history.roots.entry(identity).or_default();
|
||||
match bucket {
|
||||
Some(bucket) => {
|
||||
root.started.insert(bucket);
|
||||
}
|
||||
None => root.unknown = true,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl NativePublicationRegistration {
|
||||
pub(crate) fn invalidate(&self) {
|
||||
if let Some(root) = NATIVE_PUBLICATION_HISTORY.lock().roots.get_mut(&self.identity) {
|
||||
root.unknown = true;
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn witness(
|
||||
&self,
|
||||
bucket: &str,
|
||||
disk_id: uuid::Uuid,
|
||||
backend: Arc<dyn super::local::LocalIoBackend>,
|
||||
) -> Option<NativeBucketCreationWitness> {
|
||||
if disk_id.is_nil()
|
||||
|| !matches!(Path::new(bucket).components().next(), Some(Component::Normal(_)))
|
||||
|| Path::new(bucket).components().count() != 1
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let witness = NativeBucketCreationWitness {
|
||||
registration: self.clone(),
|
||||
backend,
|
||||
bucket: bucket.to_owned(),
|
||||
disk_id,
|
||||
};
|
||||
validate_native_creation_witnesses(std::slice::from_ref(&witness)).then_some(witness)
|
||||
}
|
||||
}
|
||||
|
||||
impl NativeBucketCreationWitness {
|
||||
pub(crate) fn matches(&self, other: &Self) -> bool {
|
||||
self.registration.identity == other.registration.identity
|
||||
&& Arc::ptr_eq(&self.backend, &other.backend)
|
||||
&& self.disk_id == other.disk_id
|
||||
&& self.bucket == other.bucket
|
||||
}
|
||||
|
||||
fn physical_root_is_current(&self) -> bool {
|
||||
let root = native_root_identity(&self.registration.root);
|
||||
let io_root = native_root_identity(&self.registration.io_root);
|
||||
if root == Some(self.registration.identity) && io_root == Some(self.registration.identity) {
|
||||
return true;
|
||||
}
|
||||
let mut history = NATIVE_PUBLICATION_HISTORY.lock();
|
||||
history.roots.entry(self.registration.identity).or_default().unknown = true;
|
||||
for identity in [root, io_root].into_iter().flatten() {
|
||||
history.roots.entry(identity).or_default().unknown = true;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn valid_in(&self, history: &NativePublicationHistory) -> bool {
|
||||
history
|
||||
.roots
|
||||
.get(&self.registration.identity)
|
||||
.is_some_and(|root| !root.unknown && !root.started.contains(&self.bucket))
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn validate_native_creation_witnesses(witnesses: &[NativeBucketCreationWitness]) -> bool {
|
||||
if witnesses.is_empty() || !witnesses.iter().all(NativeBucketCreationWitness::physical_root_is_current) {
|
||||
return false;
|
||||
}
|
||||
let history = NATIVE_PUBLICATION_HISTORY.lock();
|
||||
witnesses.iter().all(|witness| witness.valid_in(&history))
|
||||
}
|
||||
|
||||
pub(crate) fn consume_native_creation_witnesses(witnesses: &[NativeBucketCreationWitness]) -> bool {
|
||||
if witnesses.is_empty() || !witnesses.iter().all(NativeBucketCreationWitness::physical_root_is_current) {
|
||||
return false;
|
||||
}
|
||||
let mut history = NATIVE_PUBLICATION_HISTORY.lock();
|
||||
if !witnesses.iter().all(|witness| witness.valid_in(&history)) {
|
||||
return false;
|
||||
}
|
||||
for witness in witnesses {
|
||||
if let Some(root) = history.roots.get_mut(&witness.registration.identity) {
|
||||
root.started.insert(witness.bucket.clone());
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
type NamespaceMutationLock = AsyncMutex<()>;
|
||||
type NamespaceMutationLockRegistry = HashMap<PathBuf, Weak<NamespaceMutationLock>>;
|
||||
static DISK_NAMESPACE_MUTATION_LOCKS: LazyLock<Mutex<NamespaceMutationLockRegistry>> =
|
||||
@@ -3212,7 +2984,6 @@ pub(in crate::disk) async fn rename_with_namespace_owner(
|
||||
dst: &Path,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> io::Result<()> {
|
||||
record_native_publication(dst);
|
||||
if namespace_owner.is_none() {
|
||||
return tokio::fs::rename(src, dst).await;
|
||||
}
|
||||
@@ -3234,19 +3005,12 @@ pub(in crate::disk) async fn create_dir_all_with_namespace_owner(
|
||||
path: &Path,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> io::Result<()> {
|
||||
record_native_publication(path);
|
||||
if namespace_owner.is_none() {
|
||||
return tokio::fs::create_dir_all(path).await;
|
||||
}
|
||||
let path = path.to_path_buf();
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(&path, namespace_owner).await;
|
||||
run_blocking_namespace_operation(lease, move || {
|
||||
#[cfg(all(test, not(windows)))]
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::CreateDirectory, &path);
|
||||
record_native_publication(&path);
|
||||
std::fs::create_dir_all(path)
|
||||
})
|
||||
.await
|
||||
run_blocking_namespace_operation(lease, move || std::fs::create_dir_all(path)).await
|
||||
}
|
||||
|
||||
#[tracing::instrument(name = "rename_all", level = "debug", skip_all)]
|
||||
@@ -3257,7 +3021,6 @@ pub(crate) async fn rename_all_with_owner(
|
||||
publication_root: &PublicationRoot,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
record_native_publication(dst_file_path.as_ref());
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(dst_file_path.as_ref(), namespace_owner).await;
|
||||
rename_all_with_lease(src_file_path, dst_file_path, base_dir, publication_root, lease).await
|
||||
}
|
||||
@@ -3269,7 +3032,6 @@ pub(crate) async fn rename_all_with_lease(
|
||||
publication_root: &PublicationRoot,
|
||||
lease: Arc<NamespaceMutationLease>,
|
||||
) -> Result<()> {
|
||||
record_native_publication(dst_file_path.as_ref());
|
||||
reliable_rename_inner_with_lease(
|
||||
src_file_path.as_ref().to_path_buf(),
|
||||
dst_file_path.as_ref().to_path_buf(),
|
||||
@@ -3363,7 +3125,6 @@ pub(crate) async fn rename_reconciled_metadata(
|
||||
owner: Option<Arc<dyn Send + Sync>>,
|
||||
authority: Arc<crate::bucket::lifecycle::legacy_transition_state_reconcile::TransitionStateReconcileAuthority>,
|
||||
) -> Result<()> {
|
||||
record_native_publication(&destination);
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(&destination, owner).await;
|
||||
run_blocking_namespace_operation(lease, move || {
|
||||
let preparation = prepare_rename_with_retry(&source, &destination, &base_dir, &publication_root)?;
|
||||
@@ -3611,6 +3372,29 @@ fn open_existing_directory_below_base_std(
|
||||
use rustix::fs::{Mode, OFlags, openat};
|
||||
let relative = existing_base_directory_suffix(directory, base)?;
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
use rustix::fs::{ResolveFlags, openat2};
|
||||
use std::os::fd::AsFd;
|
||||
let flags = OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC;
|
||||
let owned_base;
|
||||
let base_fd = if base == publication_root.io_path.as_path() || base == publication_root.path.as_path() {
|
||||
publication_root.directory.as_fd()
|
||||
} else {
|
||||
owned_base = std::os::fd::OwnedFd::from(publication_root.open_directory(base)?);
|
||||
owned_base.as_fd()
|
||||
};
|
||||
let target = if relative.as_os_str().is_empty() {
|
||||
Path::new(".")
|
||||
} else {
|
||||
relative
|
||||
};
|
||||
match openat2(base_fd, target, flags, Mode::empty(), ResolveFlags::BENEATH | ResolveFlags::NO_SYMLINKS) {
|
||||
Ok(leaf) => return Ok(vec![leaf]),
|
||||
Err(rustix::io::Errno::NOSYS) => {}
|
||||
Err(err) => return Err(err.into()),
|
||||
}
|
||||
}
|
||||
#[cfg(target_os = "linux")]
|
||||
let base = publication_root.open_directory(base)?.into();
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let base = {
|
||||
@@ -3983,7 +3767,6 @@ pub(crate) async fn rename_all_with_prepared_source(
|
||||
commit_guard: &RenameCommitGuard,
|
||||
lease: Arc<NamespaceMutationLease>,
|
||||
) -> Result<()> {
|
||||
record_native_publication(dst_file_path.as_ref());
|
||||
let src_file_path = src_file_path.as_ref().to_path_buf();
|
||||
let dst_file_path = dst_file_path.as_ref().to_path_buf();
|
||||
let base_dir = base_dir.as_ref().to_path_buf();
|
||||
@@ -4013,7 +3796,6 @@ pub(crate) async fn rename_all_with_commit_guard(
|
||||
commit_guard: &RenameCommitGuard,
|
||||
lease: Arc<NamespaceMutationLease>,
|
||||
) -> Result<()> {
|
||||
record_native_publication(dst_file_path.as_ref());
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
let source = src_file_path.as_ref().to_path_buf();
|
||||
@@ -4067,7 +3849,6 @@ pub(crate) async fn rename_all_ignore_missing_source_with_owner(
|
||||
publication_root: &PublicationRoot,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
record_native_publication(dst_file_path.as_ref());
|
||||
let src_file_path = src_file_path.as_ref();
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(dst_file_path.as_ref(), namespace_owner).await;
|
||||
match reliable_rename_inner_with_lease(
|
||||
@@ -4122,7 +3903,6 @@ async fn reliable_rename_inner(
|
||||
publication_root: &PublicationRoot,
|
||||
warn_on_missing_source: bool,
|
||||
) -> io::Result<()> {
|
||||
record_native_publication(dst_file_path.as_ref());
|
||||
let src_file_path = src_file_path.as_ref().to_path_buf();
|
||||
let dst_file_path = dst_file_path.as_ref().to_path_buf();
|
||||
let base_dir = base_dir.as_ref().to_path_buf();
|
||||
@@ -4146,7 +3926,6 @@ async fn reliable_rename_inner_with_lease(
|
||||
warn_on_missing_source: bool,
|
||||
lease: Arc<NamespaceMutationLease>,
|
||||
) -> io::Result<()> {
|
||||
record_native_publication(&dst_file_path);
|
||||
let operation = {
|
||||
let src_file_path = src_file_path.clone();
|
||||
let dst_file_path = dst_file_path.clone();
|
||||
@@ -4616,7 +4395,6 @@ fn rename_into_existing_parent(
|
||||
parent_guard: Option<&ExistingBaseDirectoryGuard>,
|
||||
#[cfg(target_os = "linux")] source_parent: &std::fs::File,
|
||||
) -> io::Result<()> {
|
||||
record_native_publication(dst_file_path);
|
||||
use rustix::fs::renameat;
|
||||
|
||||
let Some(parent_guard) = parent_guard else {
|
||||
@@ -4670,7 +4448,6 @@ fn rename_into_existing_parent(
|
||||
parent_guard: Option<&ExistingBaseDirectoryGuard>,
|
||||
source: &winapi_util::Handle,
|
||||
) -> io::Result<()> {
|
||||
record_native_publication(dst_file_path);
|
||||
use std::{
|
||||
mem::size_of,
|
||||
os::windows::{ffi::OsStrExt, io::AsRawHandle},
|
||||
@@ -4914,7 +4691,6 @@ impl PublicationRoot {
|
||||
}
|
||||
|
||||
pub(crate) fn create_truncate(&self, path: &Path) -> io::Result<std::fs::File> {
|
||||
record_native_publication(path);
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
use rustix::fs::{Mode, OFlags};
|
||||
@@ -4925,7 +4701,6 @@ impl PublicationRoot {
|
||||
}
|
||||
|
||||
pub(crate) fn create_truncate_with_parent(&self, path: &Path) -> io::Result<(std::fs::File, Option<std::fs::File>)> {
|
||||
record_native_publication(path);
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
use rustix::fs::{Mode, OFlags};
|
||||
@@ -5173,12 +4948,16 @@ impl LinuxCommitDirectories {
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
// A pinned directory may have been unlinked or replaced. Do not report a
|
||||
// successful publication into a detached object tree.
|
||||
for (path, directory) in [
|
||||
(&self.source_path, &self.source),
|
||||
(&self.destination_path, self.prepare_destination()?),
|
||||
] {
|
||||
let current = self.root.open_directory(path)?.metadata()?;
|
||||
if (current.dev(), current.ino()) != (directory.device, directory.inode) {
|
||||
let current = self.root.open_directory(&self.source_path)?.metadata()?;
|
||||
if (current.dev(), current.ino()) != (self.source.device, self.source.inode) {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "commit directory identity changed"));
|
||||
}
|
||||
let destination = self.prepare_destination()?;
|
||||
if self.source_path != self.destination_path
|
||||
|| (self.source.device, self.source.inode) != (destination.device, destination.inode)
|
||||
{
|
||||
let current = self.root.open_directory(&self.destination_path)?.metadata()?;
|
||||
if (current.dev(), current.ino()) != (destination.device, destination.inode) {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "commit directory identity changed"));
|
||||
}
|
||||
}
|
||||
@@ -5186,7 +4965,6 @@ impl LinuxCommitDirectories {
|
||||
}
|
||||
|
||||
fn rename(&self, source: &Path, destination: &Path, prepared: Option<&PreparedRenameSource>) -> io::Result<()> {
|
||||
record_native_publication(destination);
|
||||
let source_name = self.file_name(source, true)?;
|
||||
let destination_name = self.file_name(destination, false)?;
|
||||
let destination_parent = self.prepare_destination()?;
|
||||
@@ -5472,12 +5250,20 @@ pub(crate) fn prepare_rename_commit_guard(
|
||||
existing_base_directory_suffix(destination_parent, destination_base)?;
|
||||
let source = LinuxCommitDirectory::new(publication_root.open_directory(source_parent)?)?;
|
||||
let destination = std::sync::OnceLock::new();
|
||||
match publication_root.open_directory(destination_parent) {
|
||||
Ok(directory) => {
|
||||
let _ = destination.set(LinuxCommitDirectory::new(directory)?);
|
||||
if source_parent == destination_parent {
|
||||
let _ = destination.set(LinuxCommitDirectory {
|
||||
file: Arc::clone(&source.file),
|
||||
device: source.device,
|
||||
inode: source.inode,
|
||||
});
|
||||
} else {
|
||||
match publication_root.open_directory(destination_parent) {
|
||||
Ok(directory) => {
|
||||
let _ = destination.set(LinuxCommitDirectory::new(directory)?);
|
||||
}
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
Ok(RenameCommitGuard {
|
||||
directories: Arc::new(LinuxCommitDirectories {
|
||||
@@ -6337,12 +6123,20 @@ fn existing_base_directory_suffix<'a>(dir_path: &'a Path, base_dir: &Path) -> io
|
||||
|
||||
/// Missing state never downgrades surviving native control records to legacy.
|
||||
/// Inspect only this pinned upload root; parts and proof trees are not traversed.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn multipart_source_has_native_controls(source: &Path, base: &Path, root: &PublicationRoot) -> io::Result<bool> {
|
||||
let _pin = match pin_multipart_source_directory(source, base, root) {
|
||||
let pin = match pin_multipart_source_directory(source, base, root) {
|
||||
Ok(pin) => pin,
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(false),
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
multipart_source_has_native_controls_in_pinned(source, &pin)
|
||||
}
|
||||
|
||||
pub(crate) fn multipart_source_has_native_controls_in_pinned(
|
||||
source: &Path,
|
||||
_pin: &ExistingBaseDirectoryGuard,
|
||||
) -> io::Result<bool> {
|
||||
let mut entries = std::fs::read_dir(source)?;
|
||||
for _ in 0..READ_DIR_PROBE_RAW_LIMIT {
|
||||
let Some(entry) = entries.next() else {
|
||||
@@ -6646,15 +6440,10 @@ pub(crate) fn sync_multipart_owned_record_parents(
|
||||
if !directory.starts_with(source) {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidInput, "native record barrier escaped source"));
|
||||
}
|
||||
let pinned = pin_multipart_source_directory(directory, base, publication_root)?;
|
||||
let mut pinned = pin_multipart_source_directory(directory, base, publication_root)?;
|
||||
#[cfg(unix)]
|
||||
{
|
||||
let opened = std::fs::File::from(
|
||||
pinned
|
||||
.last()
|
||||
.ok_or_else(|| io::Error::other("empty native directory guard"))?
|
||||
.try_clone()?,
|
||||
);
|
||||
let opened = std::fs::File::from(pinned.pop().ok_or_else(|| io::Error::other("empty native directory guard"))?);
|
||||
fsync_directory_handle_std(directory, Some(&opened))?;
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
@@ -6766,6 +6555,16 @@ pub(crate) fn read_multipart_source_metadata(
|
||||
read_multipart_metadata(path, base, publication_root, Some(16 * 1024 * 1024))
|
||||
}
|
||||
|
||||
pub(crate) fn read_multipart_source_metadata_in_pinned(
|
||||
pinned: &ExistingBaseDirectoryGuard,
|
||||
parent: &Path,
|
||||
name: &std::ffi::OsStr,
|
||||
base: &Path,
|
||||
publication_root: &PublicationRoot,
|
||||
) -> io::Result<Vec<u8>> {
|
||||
read_multipart_metadata_in_pinned(pinned, parent, name, base, publication_root, Some(16 * 1024 * 1024))
|
||||
}
|
||||
|
||||
/// Target FI histories have no source-control size limit. Keep the same
|
||||
/// pinned-parent and no-follow checks when inspecting every referenced DD.
|
||||
pub(crate) fn read_multipart_reclaim_target_metadata(
|
||||
@@ -6807,8 +6606,10 @@ fn read_multipart_metadata_bytes(
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
fn read_multipart_metadata(
|
||||
path: &Path,
|
||||
fn read_multipart_metadata_in_pinned(
|
||||
pinned: &ExistingBaseDirectoryGuard,
|
||||
parent: &Path,
|
||||
name: &std::ffi::OsStr,
|
||||
base: &Path,
|
||||
publication_root: &PublicationRoot,
|
||||
maximum: Option<u64>,
|
||||
@@ -6816,16 +6617,10 @@ fn read_multipart_metadata(
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use rustix::fs::{Mode, OFlags, openat};
|
||||
let parent = path
|
||||
.parent()
|
||||
.ok_or_else(|| io::Error::other("multipart metadata has no parent"))?;
|
||||
let guards = open_existing_directory_below_base_std(parent, base, publication_root)?;
|
||||
let directory = guards
|
||||
let _ = (parent, base, publication_root);
|
||||
let directory = pinned
|
||||
.last()
|
||||
.ok_or_else(|| io::Error::other("missing multipart source directory"))?;
|
||||
let name = path
|
||||
.file_name()
|
||||
.ok_or_else(|| io::Error::other("multipart metadata has no name"))?;
|
||||
let mut file = std::fs::File::from(openat(
|
||||
directory,
|
||||
name,
|
||||
@@ -6838,6 +6633,30 @@ fn read_multipart_metadata(
|
||||
}
|
||||
read_multipart_metadata_bytes(&mut file, Some(metadata.len()), maximum)
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
let _ = pinned;
|
||||
read_multipart_metadata(&parent.join(name), base, publication_root, maximum)
|
||||
}
|
||||
}
|
||||
|
||||
fn read_multipart_metadata(
|
||||
path: &Path,
|
||||
base: &Path,
|
||||
publication_root: &PublicationRoot,
|
||||
maximum: Option<u64>,
|
||||
) -> io::Result<Vec<u8>> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
let parent = path
|
||||
.parent()
|
||||
.ok_or_else(|| io::Error::other("multipart metadata has no parent"))?;
|
||||
let name = path
|
||||
.file_name()
|
||||
.ok_or_else(|| io::Error::other("multipart metadata has no name"))?;
|
||||
let guards = open_existing_directory_below_base_std(parent, base, publication_root)?;
|
||||
read_multipart_metadata_in_pinned(&guards, parent, name, base, publication_root, maximum)
|
||||
}
|
||||
#[cfg(windows)]
|
||||
{
|
||||
use windows_sys::{
|
||||
@@ -7284,7 +7103,6 @@ pub(crate) fn mkdir_all_below_existing_base_std(
|
||||
base_dir: &Path,
|
||||
publication_root: &PublicationRoot,
|
||||
) -> io::Result<ExistingBaseDirectoryGuard> {
|
||||
record_native_publication(dir_path);
|
||||
let relative = existing_base_directory_suffix(dir_path, base_dir)?;
|
||||
|
||||
#[cfg(unix)]
|
||||
@@ -8974,77 +8792,6 @@ mod tests {
|
||||
assert_eq!(err.raw_os_error(), Some(disk_full));
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
fn windows_native_root_identity_matches_directory_handles() {
|
||||
let temp = tempdir().expect("create native identity fixture");
|
||||
let first = temp.path().join("first");
|
||||
let second = temp.path().join("second");
|
||||
std::fs::create_dir(&first).expect("create first root");
|
||||
std::fs::create_dir(&second).expect("create second root");
|
||||
let (_, first_anchor) = open_windows_publication_root(&first).expect("open first directory anchor");
|
||||
let (_, second_anchor) = open_windows_publication_root(&second).expect("open second directory anchor");
|
||||
let first_identity = windows_file_identity(first_anchor.handle.as_ref()).expect("query full first identity");
|
||||
let second_identity = windows_file_identity(second_anchor.handle.as_ref()).expect("query full second identity");
|
||||
|
||||
assert_eq!(native_root_identity(&first), Some(first_identity));
|
||||
assert_eq!(native_root_identity(&first.join(".")), Some(first_identity));
|
||||
assert_eq!(native_root_identity(&second), Some(second_identity));
|
||||
assert_ne!(first_identity, second_identity, "distinct physical roots must have distinct identities");
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[tokio::test]
|
||||
async fn windows_native_creation_witness_retains_started_history() {
|
||||
let temp = tempdir().expect("create native witness fixture");
|
||||
let root = std::fs::canonicalize(temp.path()).expect("canonical native root");
|
||||
let _publication_root = PublicationRoot::new(&root).expect("retain native directory anchor");
|
||||
let registration = register_native_publication_root(&root, &root.join(".")).expect("register same physical root");
|
||||
let backend: Arc<dyn super::super::local::LocalIoBackend> = Arc::new(super::super::local::StdBackend::new(root.clone()));
|
||||
let disk_id = uuid::Uuid::new_v4();
|
||||
let witness = registration
|
||||
.witness("published", disk_id, Arc::clone(&backend))
|
||||
.expect("fresh native witness");
|
||||
let bucket = root.join("published");
|
||||
|
||||
super::super::fs::mkdir(&bucket)
|
||||
.await
|
||||
.expect("publish through the actual native entry");
|
||||
assert!(bucket.is_dir(), "the production entry must create the directory");
|
||||
assert!(!validate_native_creation_witnesses(std::slice::from_ref(&witness)));
|
||||
std::fs::remove_dir(&bucket).expect("remove published directory");
|
||||
assert!(registration.witness("published", disk_id, Arc::clone(&backend)).is_none());
|
||||
let reconnected = register_native_publication_root(&root, &root).expect("reconnect same physical root");
|
||||
assert!(reconnected.witness("published", disk_id, Arc::clone(&backend)).is_none());
|
||||
|
||||
let unstarted = reconnected
|
||||
.witness("unstarted", disk_id, Arc::clone(&backend))
|
||||
.expect("untouched bucket keeps its witness");
|
||||
assert!(consume_native_creation_witnesses(std::slice::from_ref(&unstarted)));
|
||||
assert!(!validate_native_creation_witnesses(std::slice::from_ref(&unstarted)));
|
||||
assert!(reconnected.witness("unstarted", disk_id, backend).is_none());
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
fn windows_native_creation_witness_rejects_unknown_roots() {
|
||||
let temp = tempdir().expect("create unknown identity fixture");
|
||||
let root = temp.path().join("root");
|
||||
let other = temp.path().join("other");
|
||||
let file = temp.path().join("file");
|
||||
std::fs::create_dir(&root).expect("create native root");
|
||||
std::fs::create_dir(&other).expect("create distinct native root");
|
||||
std::fs::write(&file, b"not a directory").expect("create regular file");
|
||||
|
||||
assert!(native_root_identity(&temp.path().join("missing")).is_none());
|
||||
assert!(native_root_identity(&file).is_none());
|
||||
assert!(register_native_publication_root(&root, &other).is_none());
|
||||
let registration = register_native_publication_root(&root, &root).expect("register actual native root");
|
||||
let backend: Arc<dyn super::super::local::LocalIoBackend> = Arc::new(super::super::local::StdBackend::new(root));
|
||||
registration.invalidate();
|
||||
assert!(registration.witness("unstarted", uuid::Uuid::new_v4(), backend).is_none());
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
fn windows_rename_all_legacy_file_identity_distinguishes_sources() {
|
||||
@@ -11728,328 +11475,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, unix))]
|
||||
mod native_creation_history_tests {
|
||||
use super::*;
|
||||
|
||||
struct NativeFixture {
|
||||
_temp: tempfile::TempDir,
|
||||
root: PathBuf,
|
||||
registration: NativePublicationRegistration,
|
||||
backend: Arc<dyn super::super::local::LocalIoBackend>,
|
||||
disk_id: uuid::Uuid,
|
||||
}
|
||||
|
||||
impl NativeFixture {
|
||||
fn new() -> Self {
|
||||
let temp = tempfile::tempdir().expect("native root");
|
||||
let root = std::fs::canonicalize(temp.path()).expect("canonical root");
|
||||
let registration = register_native_publication_root(&root, &root).expect("native registration");
|
||||
let backend = Arc::new(super::super::local::StdBackend::new(root.clone()));
|
||||
Self {
|
||||
_temp: temp,
|
||||
root,
|
||||
registration,
|
||||
backend,
|
||||
disk_id: uuid::Uuid::new_v4(),
|
||||
}
|
||||
}
|
||||
|
||||
fn witness(&self, bucket: &str) -> Option<NativeBucketCreationWitness> {
|
||||
self.registration.witness(bucket, self.disk_id, Arc::clone(&self.backend))
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn every_native_directory_publication_entry_invalidates_retry() {
|
||||
let fixture = NativeFixture::new();
|
||||
for operation in [
|
||||
"recursive",
|
||||
"single",
|
||||
"open",
|
||||
"rename",
|
||||
"rename_std",
|
||||
"fallback",
|
||||
"owned_mkdir",
|
||||
"owned_rename",
|
||||
"handle_relative",
|
||||
"root_create",
|
||||
"root_create_with_parent",
|
||||
"guarded_rename",
|
||||
"guarded_prepared_rename",
|
||||
] {
|
||||
let bucket = format!("native-{operation}-{}", uuid::Uuid::new_v4());
|
||||
let path = fixture.root.join(&bucket);
|
||||
let witness = fixture.witness(&bucket).expect("fresh native witness");
|
||||
match operation {
|
||||
"recursive" => super::super::fs::make_dir_all(path.join("object/child"))
|
||||
.await
|
||||
.expect("recursive mkdir"),
|
||||
"single" => super::super::fs::mkdir(&path).await.expect("single mkdir"),
|
||||
"open" => {
|
||||
let file = super::super::fs::open_file(&path, super::super::fs::O_CREATE | super::super::fs::O_WRONLY)
|
||||
.await
|
||||
.expect("create file");
|
||||
drop(file);
|
||||
}
|
||||
"rename" | "rename_std" | "owned_rename" => {
|
||||
let source = fixture.root.join(format!("source-{}", uuid::Uuid::new_v4()));
|
||||
std::fs::create_dir(&source).expect("rename source");
|
||||
match operation {
|
||||
"rename" => super::super::fs::rename(&source, &path).await.expect("async rename"),
|
||||
"rename_std" => super::super::fs::rename_std(&source, &path).expect("sync rename"),
|
||||
_ => rename_with_namespace_owner(&source, &path, Some(Arc::new(())))
|
||||
.await
|
||||
.expect("owned rename"),
|
||||
}
|
||||
}
|
||||
"fallback" => make_dir_all(path.join("object/child"), &path)
|
||||
.await
|
||||
.expect("implicit volume fallback"),
|
||||
"owned_mkdir" => create_dir_all_with_namespace_owner(&path.join("child"), Some(Arc::new(())))
|
||||
.await
|
||||
.expect("owned recursive mkdir"),
|
||||
"handle_relative" => {
|
||||
let publication_root = PublicationRoot::new(&fixture.root).expect("publication root");
|
||||
let guard = mkdir_all_below_existing_base_std(&path.join("child"), &fixture.root, &publication_root)
|
||||
.expect("first-level handle-relative mkdir");
|
||||
drop(guard);
|
||||
}
|
||||
"root_create" | "root_create_with_parent" => {
|
||||
let publication_root = PublicationRoot::new(&fixture.root).expect("publication root");
|
||||
if operation == "root_create" {
|
||||
drop(publication_root.create_truncate(&path).expect("root-relative creation"));
|
||||
} else {
|
||||
drop(
|
||||
publication_root
|
||||
.create_truncate_with_parent(&path)
|
||||
.expect("root-relative creation with parent"),
|
||||
);
|
||||
}
|
||||
}
|
||||
"guarded_rename" | "guarded_prepared_rename" => {
|
||||
let publication_root = PublicationRoot::new(&fixture.root).expect("publication root");
|
||||
let source_parent = fixture.root.join(format!("source-{}", uuid::Uuid::new_v4()));
|
||||
std::fs::create_dir(&source_parent).expect("source directory");
|
||||
std::fs::create_dir(&path).expect("existing destination directory");
|
||||
let guard = prepare_rename_commit_guard(&source_parent, &path, &fixture.root, &publication_root)
|
||||
.expect("existing commit directories");
|
||||
let source = source_parent.join("xl.meta");
|
||||
let destination = path.join("xl.meta");
|
||||
assert!(
|
||||
validate_native_creation_witnesses(std::slice::from_ref(&witness)),
|
||||
"preparing existing directories must not mask the publication entry"
|
||||
);
|
||||
let lease = acquire_namespace_mutation_lease(&destination).await;
|
||||
if operation == "guarded_rename" {
|
||||
std::fs::write(&source, b"metadata").expect("source metadata");
|
||||
rename_all_with_commit_guard(&source, &destination, &fixture.root, &publication_root, &guard, lease)
|
||||
.await
|
||||
.expect("guarded publication");
|
||||
} else {
|
||||
let mut staged = create_prepared_rename_source_with_commit_guard(&source, &destination, &guard)
|
||||
.expect("prepared source");
|
||||
staged.write_all(b"metadata", false).expect("source metadata");
|
||||
rename_all_with_prepared_source(
|
||||
staged,
|
||||
&source,
|
||||
&destination,
|
||||
&fixture.root,
|
||||
&publication_root,
|
||||
&guard,
|
||||
lease,
|
||||
)
|
||||
.await
|
||||
.expect("prepared guarded publication");
|
||||
}
|
||||
assert_eq!(std::fs::read(&destination).expect("published metadata"), b"metadata");
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
assert!(
|
||||
!validate_native_creation_witnesses(std::slice::from_ref(&witness)),
|
||||
"{operation}: old proof remains live"
|
||||
);
|
||||
if path.is_dir() {
|
||||
std::fs::remove_dir_all(&path).expect("remove published directory");
|
||||
} else {
|
||||
std::fs::remove_file(&path).expect("remove published file");
|
||||
}
|
||||
assert!(fixture.witness(&bucket).is_none(), "{operation}: removal must not regrant proof");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelled_namespace_waiter_cannot_restore_native_proof() {
|
||||
let fixture = NativeFixture::new();
|
||||
let bucket = format!("native-cancel-{}", uuid::Uuid::new_v4());
|
||||
let path = fixture.root.join(&bucket);
|
||||
let witness = fixture.witness(&bucket).expect("fresh witness");
|
||||
let lease = acquire_namespace_mutation_lease(&path).await;
|
||||
let mut attempt = Box::pin(create_dir_all_with_namespace_owner(&path, Some(Arc::new(()))));
|
||||
assert!(matches!(futures::poll!(&mut attempt), std::task::Poll::Pending));
|
||||
assert!(
|
||||
!validate_native_creation_witnesses(std::slice::from_ref(&witness)),
|
||||
"Started must precede admission wait"
|
||||
);
|
||||
assert!(!path.exists(), "the namespace wait must still precede the syscall");
|
||||
drop(attempt);
|
||||
drop(lease);
|
||||
assert!(fixture.witness(&bucket).is_none(), "cancellation cannot undo an admitted attempt");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn queued_mkdir_cancellation_retains_lease_and_consumed_history() {
|
||||
use futures::FutureExt;
|
||||
use std::sync::mpsc;
|
||||
|
||||
let fixture = NativeFixture::new();
|
||||
let bucket = format!("native-queued-cancel-{}", uuid::Uuid::new_v4());
|
||||
let path = fixture.root.join(&bucket);
|
||||
let witness = fixture.witness(&bucket).expect("fresh witness");
|
||||
let (entered_tx, entered_rx) = mpsc::channel();
|
||||
let (release_tx, release_rx) = mpsc::channel();
|
||||
let _hook = prepared_publication_test_hooks::install_at(
|
||||
prepared_publication_test_hooks::Stage::CreateDirectory,
|
||||
&path,
|
||||
move || {
|
||||
entered_tx.send(()).expect("signal real executor entry");
|
||||
let _ = release_rx.recv();
|
||||
},
|
||||
);
|
||||
let operation_path = path.clone();
|
||||
let operation =
|
||||
tokio::spawn(async move { create_dir_all_with_namespace_owner(&operation_path, Some(Arc::new(()))).await });
|
||||
tokio::task::spawn_blocking(move || entered_rx.recv_timeout(std::time::Duration::from_secs(30)))
|
||||
.await
|
||||
.expect("entry waiter")
|
||||
.expect("the real blocking executor must enter");
|
||||
operation.abort();
|
||||
assert!(operation.await.expect_err("cancel the async waiter").is_cancelled());
|
||||
assert!(
|
||||
!validate_native_creation_witnesses(std::slice::from_ref(&witness)),
|
||||
"queued publication is permanently Started"
|
||||
);
|
||||
assert!(
|
||||
acquire_namespace_mutation_lease(&path).now_or_never().is_none(),
|
||||
"the actual blocked executor must still own the namespace lease"
|
||||
);
|
||||
release_tx.send(()).expect("release the syscall");
|
||||
let drained = tokio::time::timeout(std::time::Duration::from_secs(30), acquire_namespace_mutation_lease(&path))
|
||||
.await
|
||||
.expect("the lease must drain with the syscall");
|
||||
drop(drained);
|
||||
assert!(path.is_dir(), "the cancelled waiter cannot discard the already queued syscall");
|
||||
std::fs::remove_dir_all(&path).expect("remove the published directory");
|
||||
assert!(fixture.witness(&bucket).is_none(), "drain and removal cannot restore proof");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn publication_detects_retargeted_physical_alias_without_registration() {
|
||||
let source = NativeFixture::new();
|
||||
let destination = NativeFixture::new();
|
||||
let bucket = format!("native-retarget-{}", uuid::Uuid::new_v4());
|
||||
let witness = destination.witness(&bucket).expect("destination witness");
|
||||
let retired = source.root.with_extension("retired");
|
||||
std::fs::rename(&source.root, &retired).expect("move the original physical root");
|
||||
std::os::unix::fs::symlink(&destination.root, &source.root).expect("retarget the registered alias");
|
||||
super::super::fs::mkdir(source.root.join(&bucket))
|
||||
.await
|
||||
.expect("publish through the changed alias");
|
||||
assert!(
|
||||
!validate_native_creation_witnesses(std::slice::from_ref(&witness)),
|
||||
"publication-time alias drift must invalidate the actual destination root"
|
||||
);
|
||||
std::fs::remove_dir(destination.root.join(&bucket)).expect("remove the published bucket");
|
||||
assert!(destination.witness(&bucket).is_none(), "removal must not undo alias uncertainty");
|
||||
std::fs::remove_file(&source.root).expect("remove the symlink");
|
||||
std::fs::rename(retired, &source.root).expect("restore the test root");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blocked_physical_identity_lookup_does_not_block_other_publication() {
|
||||
use std::sync::mpsc;
|
||||
let stalled = NativeFixture::new();
|
||||
let healthy = NativeFixture::new();
|
||||
let witness = stalled.witness("unpublished").expect("stalled root witness");
|
||||
let (entered_tx, entered_rx) = mpsc::channel();
|
||||
let (release_tx, release_rx) = mpsc::channel();
|
||||
let _hook = prepared_publication_test_hooks::install_at(
|
||||
prepared_publication_test_hooks::Stage::NativeRootIdentity,
|
||||
&stalled.root,
|
||||
move || {
|
||||
entered_tx.send(()).expect("signal physical lookup");
|
||||
let _ = release_rx.recv();
|
||||
},
|
||||
);
|
||||
let validator = std::thread::spawn(move || validate_native_creation_witnesses(&[witness]));
|
||||
entered_rx
|
||||
.recv_timeout(std::time::Duration::from_secs(30))
|
||||
.expect("enter the actual identity lookup");
|
||||
let (published_tx, published_rx) = mpsc::channel();
|
||||
let path = healthy.root.join("published");
|
||||
let publisher = std::thread::spawn(move || {
|
||||
record_native_publication(&path);
|
||||
published_tx.send(()).expect("signal independent publication");
|
||||
});
|
||||
let published = published_rx.recv_timeout(std::time::Duration::from_secs(5));
|
||||
release_tx.send(()).expect("release the identity lookup");
|
||||
validator.join().expect("identity lookup thread");
|
||||
publisher.join().expect("publication thread");
|
||||
assert!(
|
||||
published.is_ok(),
|
||||
"a slow physical lookup must not retain the process-wide publication mutex"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_consume_is_atomic_and_history_survives_reconnect() {
|
||||
let fixture = NativeFixture::new();
|
||||
let first = format!("native-first-{}", uuid::Uuid::new_v4());
|
||||
let second = format!("native-second-{}", uuid::Uuid::new_v4());
|
||||
let first_witness = fixture.witness(&first).expect("first witness");
|
||||
let second_witness = fixture.witness(&second).expect("second witness");
|
||||
record_native_publication(&fixture.root.join(&second));
|
||||
assert!(!consume_native_creation_witnesses(&[first_witness.clone(), second_witness]));
|
||||
assert!(
|
||||
validate_native_creation_witnesses(std::slice::from_ref(&first_witness)),
|
||||
"failed cohort consume must be atomic"
|
||||
);
|
||||
assert!(consume_native_creation_witnesses(std::slice::from_ref(&first_witness)));
|
||||
assert!(!validate_native_creation_witnesses(std::slice::from_ref(&first_witness)));
|
||||
let reconnected = register_native_publication_root(&fixture.root, &fixture.root).expect("reconnected registration");
|
||||
let backend: Arc<dyn super::super::local::LocalIoBackend> =
|
||||
Arc::new(super::super::local::StdBackend::new(fixture.root.clone()));
|
||||
assert!(
|
||||
reconnected.witness(&first, fixture.disk_id, backend).is_none(),
|
||||
"reconnect must retain Started"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn physical_alias_reuse_is_unknown_and_backend_identity_is_real() {
|
||||
let temp = tempfile::tempdir().expect("alias parent");
|
||||
let root = temp.path().join("root");
|
||||
std::fs::create_dir(&root).expect("first physical root");
|
||||
let old = register_native_publication_root(&root, &root).expect("first registration");
|
||||
let backend: Arc<dyn super::super::local::LocalIoBackend> = Arc::new(super::super::local::StdBackend::new(root.clone()));
|
||||
let id = uuid::Uuid::new_v4();
|
||||
let witness = old.witness("same-name", id, Arc::clone(&backend)).expect("old witness");
|
||||
let other_backend: Arc<dyn super::super::local::LocalIoBackend> =
|
||||
Arc::new(super::super::local::StdBackend::new(root.clone()));
|
||||
let other = old.witness("same-name", id, other_backend).expect("new backend witness");
|
||||
assert!(!witness.matches(&other), "a different actual backend cannot reuse a pending observation");
|
||||
std::fs::rename(&root, temp.path().join("retired-root")).expect("retire physical root");
|
||||
std::fs::create_dir(&root).expect("replacement root at the same alias");
|
||||
let replacement = register_native_publication_root(&root, &root).expect("replacement registration");
|
||||
assert!(replacement.witness("same-name", id, backend).is_none(), "alias reuse must remain Unknown");
|
||||
assert!(
|
||||
!validate_native_creation_witnesses(&[witness]),
|
||||
"old physical identity must not validate at a new root"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, target_os = "linux"))]
|
||||
mod fd_relative_tests {
|
||||
use super::*;
|
||||
|
||||
@@ -17,8 +17,8 @@
|
||||
use crate::bucket::error::BucketMetadataError;
|
||||
use crate::disk::error::DiskError;
|
||||
use crate::storage_api_contracts::{error::StorageErrorCode, range::HTTPRangeError};
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
use rustfs_utils::path::decode_dir_object;
|
||||
use s3s::S3ErrorCode;
|
||||
|
||||
pub type Error = StorageError;
|
||||
pub type Result<T> = core::result::Result<T, Error>;
|
||||
@@ -1086,8 +1086,14 @@ impl From<rmp_serde::decode::Error> for Error {
|
||||
}
|
||||
}
|
||||
|
||||
impl From<crate::bucket::utils::BucketConfigXmlError> for Error {
|
||||
fn from(e: crate::bucket::utils::BucketConfigXmlError) -> Self {
|
||||
impl From<s3s::xml::SerError> for Error {
|
||||
fn from(e: s3s::xml::SerError) -> Self {
|
||||
Error::other(e)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<s3s::xml::DeError> for Error {
|
||||
fn from(e: s3s::xml::DeError) -> Self {
|
||||
Error::other(e)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use http::HeaderMap;
|
||||
use s3s::dto::ETag;
|
||||
use std::{collections::HashMap, io::Cursor, sync::Arc};
|
||||
use tokio::io::BufReader;
|
||||
|
||||
@@ -28,7 +29,7 @@ use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions};
|
||||
use crate::storage_api_contracts::range::HTTPRangeSpec;
|
||||
use rustfs_filemeta::ObjectPartInfo;
|
||||
use rustfs_rio::HashReader;
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
use s3s::S3ErrorCode;
|
||||
|
||||
//#[derive(Clone)]
|
||||
pub struct PutObjReader {
|
||||
@@ -149,6 +150,24 @@ pub fn new_getobjectreader<'a>(
|
||||
})
|
||||
}
|
||||
|
||||
/// Convert a raw stored ETag into the strongly-typed `s3s::dto::ETag`.
|
||||
///
|
||||
/// Supports already quoted (`"abc"`), weak (`W/"abc"`), or plain (`abc`) values.
|
||||
pub fn to_s3s_etag(etag: &str) -> ETag {
|
||||
if let Some(rest) = etag.strip_prefix("W/\"") {
|
||||
if let Some(body) = rest.strip_suffix('"') {
|
||||
return ETag::Weak(body.to_string());
|
||||
}
|
||||
return ETag::Weak(rest.to_string());
|
||||
}
|
||||
|
||||
if let Some(body) = etag.strip_prefix('"').and_then(|rest| rest.strip_suffix('"')) {
|
||||
return ETag::Strong(body.to_string());
|
||||
}
|
||||
|
||||
ETag::Strong(etag.to_string())
|
||||
}
|
||||
|
||||
pub fn get_raw_etag(metadata: &HashMap<String, String>) -> String {
|
||||
metadata
|
||||
.get("etag")
|
||||
@@ -213,42 +232,28 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_new_getobjectreader_rejects_a_range_past_the_object_as_invalid_range() {
|
||||
let oi = multipart_object_info();
|
||||
let past_the_end = Some(HTTPRangeSpec {
|
||||
is_suffix_length: false,
|
||||
start: oi.size,
|
||||
end: oi.size + 10,
|
||||
});
|
||||
fn test_to_s3s_etag() {
|
||||
// Test unquoted ETag - should become strong etag
|
||||
assert_eq!(
|
||||
to_s3s_etag("6af8d12c0c74b78094884349f3c8a079"),
|
||||
ETag::Strong("6af8d12c0c74b78094884349f3c8a079".to_string())
|
||||
);
|
||||
|
||||
let err = new_getobjectreader(&past_the_end, &oi, &ObjectOptions::default(), &HeaderMap::new())
|
||||
.err()
|
||||
.expect("a range starting at the object size must be rejected");
|
||||
assert_eq!(
|
||||
to_s3s_etag("\"6af8d12c0c74b78094884349f3c8a079\""),
|
||||
ETag::Strong("6af8d12c0c74b78094884349f3c8a079".to_string())
|
||||
);
|
||||
|
||||
assert_eq!(err.code, S3ErrorCode::InvalidRange);
|
||||
assert_eq!(err.code.as_str(), "InvalidRange");
|
||||
assert_eq!(err.key, None);
|
||||
assert_eq!(err.bucket_name, None);
|
||||
}
|
||||
assert_eq!(
|
||||
to_s3s_etag("W/\"6af8d12c0c74b78094884349f3c8a079\""),
|
||||
ETag::Weak("6af8d12c0c74b78094884349f3c8a079".to_string())
|
||||
);
|
||||
|
||||
#[test]
|
||||
fn test_new_getobjectreader_rejects_an_unknown_size_without_a_range_as_invalid_range() {
|
||||
let oi = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
size: -1,
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(to_s3s_etag(""), ETag::Strong(String::new()));
|
||||
|
||||
let err = new_getobjectreader(&None, &oi, &ObjectOptions::default(), &HeaderMap::new())
|
||||
.err()
|
||||
.expect("an object of unknown size cannot be read without a range");
|
||||
assert_eq!(to_s3s_etag("\"incomplete"), ETag::Strong("\"incomplete".to_string()));
|
||||
|
||||
assert_eq!(err.code, S3ErrorCode::InvalidRange);
|
||||
assert_eq!(err.code.as_str(), "InvalidRange");
|
||||
assert_eq!(err.message, "Invalid range");
|
||||
assert_eq!(err.key.as_deref(), Some("object"));
|
||||
assert_eq!(err.bucket_name.as_deref(), Some("bucket"));
|
||||
assert_eq!(to_s3s_etag("incomplete\""), ETag::Strong("incomplete\"".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -1013,10 +1013,6 @@ pub struct ObjectOptions {
|
||||
/// `ecstore` must leave it unset.
|
||||
#[doc(hidden)]
|
||||
pub data_movement_transferred_tier_versions: Option<crate::data_movement::TransferredTierVersions>,
|
||||
/// Marks the physical copy-back write of a RestoreObject operation. This
|
||||
/// is in-memory request context and must not be copied into object metadata.
|
||||
#[doc(hidden)]
|
||||
pub restore_copyback: bool,
|
||||
/// Internal, in-memory protection context. Never populated from S3 metadata.
|
||||
/// None selects the rollout default only for a new write, not for a rewrite.
|
||||
#[doc(hidden)]
|
||||
|
||||
@@ -275,16 +275,16 @@ pub(crate) type TypeLocalDiskSetDrives = Vec<Vec<Vec<Option<DiskStore>>>>;
|
||||
///
|
||||
/// # Returns
|
||||
/// * None
|
||||
pub fn set_global_region(region: rustfs_s3_types::Region) {
|
||||
pub fn set_global_region(region: s3s::region::Region) {
|
||||
current_ctx().set_region(region);
|
||||
}
|
||||
|
||||
/// Get the global region
|
||||
///
|
||||
/// # Returns
|
||||
/// * `Option<rustfs_s3_types::Region>` - The global region, if set
|
||||
/// * `Option<s3s::region::Region>` - The global region, if set
|
||||
///
|
||||
pub fn get_global_region() -> Option<rustfs_s3_types::Region> {
|
||||
pub fn get_global_region() -> Option<s3s::region::Region> {
|
||||
current_ctx().region()
|
||||
}
|
||||
|
||||
|
||||
@@ -52,7 +52,7 @@ use crate::services::event_notification::EventNotifier;
|
||||
use crate::services::tier::tier::TierConfigMgr;
|
||||
use crate::store::delete_recovery_runtime::DeleteRecoveryHints;
|
||||
use rustfs_lock::{GlobalLockManager, get_global_lock_manager};
|
||||
use rustfs_s3_types::Region;
|
||||
use s3s::region::Region;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::sync::{
|
||||
Arc, OnceLock,
|
||||
|
||||
@@ -44,10 +44,10 @@ use crate::{
|
||||
};
|
||||
use rustfs_concurrency::WorkloadAdmissionSnapshotProvider;
|
||||
use rustfs_config::server_config::{Config, get_global_server_config, set_global_server_config};
|
||||
use rustfs_gateway_types::persistence::PersistedLifecycleConfiguration;
|
||||
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
|
||||
use rustfs_lock::client::LockClient;
|
||||
use rustfs_s3_types::Region;
|
||||
use s3s::dto::BucketLifecycleConfiguration;
|
||||
use s3s::region::Region;
|
||||
use tokio::sync::{OwnedRwLockReadGuard, RwLock};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
#[cfg(test)]
|
||||
@@ -371,7 +371,7 @@ pub fn global_tier_config_mgr() -> Arc<RwLock<TierConfigMgr>> {
|
||||
get_global_tier_config_mgr()
|
||||
}
|
||||
|
||||
pub(crate) async fn bucket_lifecycle_config(bucket: &str) -> Option<PersistedLifecycleConfiguration> {
|
||||
pub(crate) async fn bucket_lifecycle_config(bucket: &str) -> Option<BucketLifecycleConfiguration> {
|
||||
GLOBAL_LIFECYCLE_SYS.get(bucket).await
|
||||
}
|
||||
|
||||
|
||||
@@ -703,28 +703,6 @@ pub async fn acquire_legacy_transition_state_reconcile_fleet_proof() -> Option<L
|
||||
.then_some(proof)
|
||||
}
|
||||
|
||||
pub async fn require_restore_copyback_fleet_capability() -> Result<()> {
|
||||
let Some(topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
return Ok(());
|
||||
#[cfg(not(any(test, feature = "test-util")))]
|
||||
return Err(Error::other("restore copy-back capability topology is unavailable"));
|
||||
};
|
||||
let Some(notification_sys) = get_global_notification_sys() else {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
return Ok(());
|
||||
#[cfg(not(any(test, feature = "test-util")))]
|
||||
return Err(Error::other("restore copy-back capability probe is unavailable"));
|
||||
};
|
||||
timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_restore_copyback_fleet(topology),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| Error::other("restore copy-back capability fleet probe timed out"))??;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn acquire_legacy_transition_state_reconcile_fleet_proof_from(
|
||||
state: &FleetCapabilityProofState,
|
||||
expected_topology: &str,
|
||||
@@ -1782,30 +1760,6 @@ impl NotificationSys {
|
||||
Ok(peer_epochs)
|
||||
}
|
||||
|
||||
async fn probe_restore_copyback_fleet(&self, topology_fingerprint: &str) -> Result<BTreeMap<String, Uuid>> {
|
||||
if self.peer_clients.len() != self.peer_topology_hosts.len() {
|
||||
return Err(Error::other("restore copy-back capability fleet membership is incomplete"));
|
||||
}
|
||||
let probes = self.peer_clients.iter().map(|client| async {
|
||||
let client = client
|
||||
.as_ref()
|
||||
.ok_or_else(|| Error::other("restore copy-back capability peer is unreachable"))?;
|
||||
client
|
||||
.probe_restore_copyback_capability(topology_fingerprint.to_string())
|
||||
.await
|
||||
});
|
||||
let mut peer_epochs = BTreeMap::new();
|
||||
for result in join_all(probes).await {
|
||||
let (peer, epoch) = result?;
|
||||
insert_remote_version_state_peer(&mut peer_epochs, peer, epoch)?;
|
||||
}
|
||||
let expected = self.peer_topology_hosts.iter().cloned().collect::<BTreeSet<_>>();
|
||||
if expected.len() != self.peer_topology_hosts.len() || peer_epochs.keys().ne(expected.iter()) {
|
||||
return Err(Error::other("restore copy-back capability fleet membership does not match topology"));
|
||||
}
|
||||
Ok(peer_epochs)
|
||||
}
|
||||
|
||||
async fn probe_transition_transaction_compaction_fleet(&self, topology_fingerprint: &str) -> Result<BTreeMap<String, Uuid>> {
|
||||
if self.peer_clients.len() != self.peer_topology_hosts.len() {
|
||||
return Err(Error::other(
|
||||
|
||||
@@ -1032,9 +1032,8 @@ mod tests {
|
||||
use crate::storage_api_contracts::object::ObjectIO as _;
|
||||
use http::HeaderMap;
|
||||
use rustfs_filemeta::{FileInfo, FileMeta, ObjectPartInfo, TransitionVersionState};
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedLifecycleConfiguration, PersistedLifecycleExpiration, PersistedLifecycleRule,
|
||||
PersistedNoncurrentVersionExpiration,
|
||||
use s3s::dto::{
|
||||
BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, NoncurrentVersionExpiration,
|
||||
};
|
||||
use std::time::Duration as StdDuration;
|
||||
use time::OffsetDateTime;
|
||||
@@ -1078,12 +1077,12 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn expired_delete_marker_lifecycle() -> PersistedLifecycleConfiguration {
|
||||
PersistedLifecycleConfiguration {
|
||||
fn expired_delete_marker_lifecycle() -> BucketLifecycleConfiguration {
|
||||
BucketLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![PersistedLifecycleRule {
|
||||
status: "Enabled".to_owned(),
|
||||
expiration: Some(PersistedLifecycleExpiration {
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
expiration: Some(LifecycleExpiration {
|
||||
expired_object_delete_marker: Some(true),
|
||||
..Default::default()
|
||||
}),
|
||||
@@ -1832,11 +1831,11 @@ mod tests {
|
||||
))
|
||||
.await
|
||||
.expect("seed the expired source version");
|
||||
let lifecycle = PersistedLifecycleConfiguration {
|
||||
let lifecycle = BucketLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![PersistedLifecycleRule {
|
||||
status: "Enabled".to_owned(),
|
||||
expiration: Some(PersistedLifecycleExpiration {
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
expiration: Some(LifecycleExpiration {
|
||||
days: Some(1),
|
||||
expired_object_all_versions: Some(true),
|
||||
..Default::default()
|
||||
@@ -2085,7 +2084,7 @@ mod tests {
|
||||
versioned,
|
||||
version_id,
|
||||
transition: crate::bucket::lifecycle::lifecycle::TransitionOptions {
|
||||
restore_request: rustfs_lifecycle::RestoreRequest {
|
||||
restore_request: s3s::dto::RestoreRequest {
|
||||
days: Some(1),
|
||||
..Default::default()
|
||||
},
|
||||
@@ -2165,7 +2164,7 @@ mod tests {
|
||||
async fn run_entry_with(
|
||||
&self,
|
||||
entry: MetaCacheEntry,
|
||||
lifecycle_config: Option<PersistedLifecycleConfiguration>,
|
||||
lifecycle_config: Option<BucketLifecycleConfiguration>,
|
||||
) -> RebalanceEntryOutcome {
|
||||
Box::pin(Arc::clone(&self.store).rebalance_entry(
|
||||
RebalanceEntryTarget {
|
||||
@@ -2501,17 +2500,17 @@ mod tests {
|
||||
}
|
||||
|
||||
/// A rule that expires every noncurrent version at once.
|
||||
fn noncurrent_expire_all_lifecycle() -> PersistedLifecycleConfiguration {
|
||||
PersistedLifecycleConfiguration {
|
||||
fn noncurrent_expire_all_lifecycle() -> BucketLifecycleConfiguration {
|
||||
BucketLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![PersistedLifecycleRule {
|
||||
status: "Enabled".to_owned(),
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
expiration: None,
|
||||
abort_incomplete_multipart_upload: None,
|
||||
del_marker_expiration: None,
|
||||
filter: None,
|
||||
id: Some("noncurrent-expire-all".to_string()),
|
||||
noncurrent_version_expiration: Some(PersistedNoncurrentVersionExpiration {
|
||||
noncurrent_version_expiration: Some(NoncurrentVersionExpiration {
|
||||
noncurrent_days: Some(0),
|
||||
newer_noncurrent_versions: None,
|
||||
}),
|
||||
|
||||
@@ -67,8 +67,8 @@ use crate::error::{Error, Result};
|
||||
use crate::storage_api_contracts::range::HTTPRangeSpec;
|
||||
use rustfs_filemeta::TRANSITION_COMPLETE;
|
||||
use rustfs_filemeta::{FileInfo, MetaCacheEntry};
|
||||
use rustfs_gateway_types::persistence::PersistedReplicationConfiguration;
|
||||
use rustfs_rio::Index;
|
||||
use s3s::dto::ReplicationConfiguration;
|
||||
use serde::Serialize;
|
||||
use std::io::Cursor;
|
||||
use std::sync::Arc;
|
||||
@@ -2676,7 +2676,7 @@ fn test_resolve_rebalance_optional_bucket_config_result_passthrough() {
|
||||
let result = resolve_rebalance_optional_bucket_config_result(
|
||||
"bucket-a",
|
||||
"replication",
|
||||
Ok((PersistedReplicationConfiguration::default(), OffsetDateTime::UNIX_EPOCH)),
|
||||
Ok((ReplicationConfiguration::default(), OffsetDateTime::UNIX_EPOCH)),
|
||||
)
|
||||
.expect("bucket config should pass through");
|
||||
assert!(result.is_some());
|
||||
|
||||
@@ -34,9 +34,9 @@ pub struct RebalanceStats {
|
||||
#[derive(Debug, Default)]
|
||||
pub(super) struct RebalanceBucketConfigs {
|
||||
pub(super) bucket_incarnation_id: Option<uuid::Uuid>,
|
||||
pub(super) lifecycle_config: Option<rustfs_gateway_types::persistence::PersistedLifecycleConfiguration>,
|
||||
pub(super) object_lock_config: Option<rustfs_gateway_types::persistence::PersistedObjectLockConfiguration>,
|
||||
pub(super) replication_config: Option<(rustfs_gateway_types::persistence::PersistedReplicationConfiguration, OffsetDateTime)>,
|
||||
pub(super) lifecycle_config: Option<s3s::dto::BucketLifecycleConfiguration>,
|
||||
pub(super) object_lock_config: Option<s3s::dto::ObjectLockConfiguration>,
|
||||
pub(super) replication_config: Option<(s3s::dto::ReplicationConfiguration, OffsetDateTime)>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
|
||||
@@ -80,11 +80,10 @@ use crate::{
|
||||
store::ECStore,
|
||||
};
|
||||
use rustfs_filemeta::FileInfo;
|
||||
use rustfs_gateway_types::persistence::PersistedLifecycleConfiguration;
|
||||
use rustfs_rio::HashReader;
|
||||
use rustfs_s3_client::{admin_handler_utils::AdminError, provider_versions::ProviderVersionCapabilities};
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
use rustfs_utils::path::{SLASH_SEPARATOR, path_join};
|
||||
use s3s::{S3ErrorCode, dto::BucketLifecycleConfiguration};
|
||||
|
||||
use super::{
|
||||
tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_PERM_ERR},
|
||||
@@ -1037,7 +1036,7 @@ trait TierReferenceProofStore:
|
||||
.await
|
||||
}
|
||||
|
||||
async fn lifecycle_config_for_reference_proof(&self, bucket: &str) -> Result<Option<PersistedLifecycleConfiguration>>;
|
||||
async fn lifecycle_config_for_reference_proof(&self, bucket: &str) -> Result<Option<BucketLifecycleConfiguration>>;
|
||||
|
||||
async fn authoritative_bucket_tier_reference(
|
||||
self: Arc<Self>,
|
||||
@@ -1077,7 +1076,7 @@ impl TierReferenceProofStore for ECStore {
|
||||
Ok(buckets.into_values().collect())
|
||||
}
|
||||
|
||||
async fn lifecycle_config_for_reference_proof(&self, bucket: &str) -> Result<Option<PersistedLifecycleConfiguration>> {
|
||||
async fn lifecycle_config_for_reference_proof(&self, bucket: &str) -> Result<Option<BucketLifecycleConfiguration>> {
|
||||
match get_lifecycle_config(bucket).await {
|
||||
Ok((config, _updated_at)) => Ok(Some(config)),
|
||||
Err(Error::ConfigNotFound) => Ok(None),
|
||||
@@ -1339,7 +1338,7 @@ struct TierLifecycleReference<'a> {
|
||||
}
|
||||
|
||||
fn lifecycle_config_target_reference<'a>(
|
||||
config: &'a PersistedLifecycleConfiguration,
|
||||
config: &'a BucketLifecycleConfiguration,
|
||||
targets: &[TierMutationIntentTarget],
|
||||
) -> Option<TierLifecycleReference<'a>> {
|
||||
for rule in &config.rules {
|
||||
@@ -6928,8 +6927,9 @@ mod tests {
|
||||
endpoints::{Endpoints, PoolEndpoints, SetupType},
|
||||
};
|
||||
use crate::services::tier::tier_mutation_intent::TIER_MUTATION_INTENT_RECORD_PREFIX;
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedLifecycleConfiguration, PersistedLifecycleRule, PersistedNoncurrentVersionTransition, PersistedTransition,
|
||||
use s3s::dto::{
|
||||
BucketLifecycleConfiguration, ExpirationStatus, LifecycleRule, NoncurrentVersionTransition, Transition,
|
||||
TransitionStorageClass,
|
||||
};
|
||||
|
||||
#[test]
|
||||
@@ -7756,7 +7756,7 @@ mod tests {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl TierReferenceProofStore for LockingTierConfigStore {
|
||||
async fn lifecycle_config_for_reference_proof(&self, _bucket: &str) -> Result<Option<PersistedLifecycleConfiguration>> {
|
||||
async fn lifecycle_config_for_reference_proof(&self, _bucket: &str) -> Result<Option<BucketLifecycleConfiguration>> {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
@@ -10396,42 +10396,6 @@ mod tests {
|
||||
assert_eq!(observed.committed_config_etag.as_deref(), Some("etag-new"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn intent_advance_retries_cas_conflict_reload_lock_timeout() {
|
||||
let store = Arc::new(CasConfigStore::default());
|
||||
let mutation_id = uuid::Uuid::from_u128(39);
|
||||
let prepared = prepared_remove_intent("COLD-A", mutation_id);
|
||||
crate::services::tier::tier_mutation_intent::save_tier_mutation_intent_record(store.clone(), &prepared)
|
||||
.await
|
||||
.expect("prepared intent fixture should persist");
|
||||
|
||||
let committed = committed_remove_intent("COLD-A", mutation_id, "etag-new");
|
||||
let committed_bytes = committed.encode().expect("committed intent fixture should encode");
|
||||
let object = crate::services::tier::tier_mutation_intent::tier_mutation_intent_record_object_name(mutation_id)
|
||||
.expect("record object should build");
|
||||
let winning_etag = format!("etag-{}", store.next_etag.load(Ordering::SeqCst) + 1);
|
||||
store.rewrite_on_next_if_match(object.clone(), committed_bytes.clone()).await;
|
||||
*store.read_timeout_after_if_match_race.lock().await = Some(object.clone());
|
||||
|
||||
let (observed, applied) = crate::services::tier::tier_mutation_intent::advance_tier_mutation_intent_record_idempotent(
|
||||
store.clone(),
|
||||
mutation_id,
|
||||
TierMutationIntentState::Committed,
|
||||
Some("etag-new".to_string()),
|
||||
)
|
||||
.await
|
||||
.expect("CAS conflict reload should retry its typed lock timeout");
|
||||
|
||||
assert!(!applied);
|
||||
assert_eq!(observed, committed);
|
||||
assert!(store.read_timeout_after_if_match_race.lock().await.is_none());
|
||||
assert_eq!(
|
||||
store.objects.lock().await.get(&object).cloned(),
|
||||
Some((committed_bytes, winning_etag)),
|
||||
"retry must preserve the complete winning record and its object ETag"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn intent_advance_reconciles_matching_commit_after_final_cas_race() {
|
||||
let store = Arc::new(CasConfigStore::default());
|
||||
@@ -14903,7 +14867,6 @@ mod tests {
|
||||
objects: tokio::sync::Mutex<HashMap<String, (Vec<u8>, String)>>,
|
||||
legacy_state: tokio::sync::Mutex<Option<Vec<u8>>>,
|
||||
if_match_race_rewrite: tokio::sync::Mutex<std::collections::VecDeque<(String, Vec<u8>)>>,
|
||||
read_timeout_after_if_match_race: tokio::sync::Mutex<Option<String>>,
|
||||
next_etag: AtomicUsize,
|
||||
content_derived_etags: AtomicBool,
|
||||
fail_put: AtomicBool,
|
||||
@@ -14918,7 +14881,7 @@ mod tests {
|
||||
lock_manager: Arc<rustfs_lock::GlobalLockManager>,
|
||||
lock_requests: Mutex<Vec<(String, String)>>,
|
||||
listed_versions: Mutex<Vec<ObjectInfo>>,
|
||||
lifecycle_configs: Mutex<HashMap<String, PersistedLifecycleConfiguration>>,
|
||||
lifecycle_configs: Mutex<HashMap<String, BucketLifecycleConfiguration>>,
|
||||
}
|
||||
|
||||
impl Default for CasConfigStore {
|
||||
@@ -14929,7 +14892,6 @@ mod tests {
|
||||
objects: tokio::sync::Mutex::new(HashMap::new()),
|
||||
legacy_state: tokio::sync::Mutex::new(None),
|
||||
if_match_race_rewrite: tokio::sync::Mutex::new(std::collections::VecDeque::new()),
|
||||
read_timeout_after_if_match_race: tokio::sync::Mutex::new(None),
|
||||
next_etag: AtomicUsize::new(0),
|
||||
content_derived_etags: AtomicBool::new(false),
|
||||
fail_put: AtomicBool::new(false),
|
||||
@@ -14978,7 +14940,7 @@ mod tests {
|
||||
.retain(|version| version.bucket != bucket || version.name != object);
|
||||
}
|
||||
|
||||
fn add_lifecycle_config(&self, bucket: &str, config: PersistedLifecycleConfiguration) {
|
||||
fn add_lifecycle_config(&self, bucket: &str, config: BucketLifecycleConfiguration) {
|
||||
self.lifecycle_configs
|
||||
.lock()
|
||||
.expect("tier reference fixture should not poison")
|
||||
@@ -15087,19 +15049,6 @@ mod tests {
|
||||
crate::api::diagnostic::current_read_version_entry_source(),
|
||||
rustfs_io_metrics::s3_http_metrics::current_s3_http_operation_name(),
|
||||
));
|
||||
{
|
||||
// The queued rewrite keeps the initial read successful. Only
|
||||
// the first target read after that CAS race consumes the fault.
|
||||
let race_completed = self.if_match_race_rewrite.lock().await.is_empty();
|
||||
let mut timeout = self.read_timeout_after_if_match_race.lock().await;
|
||||
if race_completed && timeout.as_deref() == Some(object) {
|
||||
timeout.take();
|
||||
return Err(Error::Lock(rustfs_lock::LockError::Timeout {
|
||||
resource: format!("{bucket}/{object}@latest"),
|
||||
timeout: std::time::Duration::from_secs(5),
|
||||
}));
|
||||
}
|
||||
}
|
||||
let legacy_config_path = tier_config_path(TIER_CONFIG_LEGACY_FILE);
|
||||
let (data, etag) = if object == legacy_config_path {
|
||||
let state = self.legacy_state.lock().await;
|
||||
@@ -15681,7 +15630,7 @@ mod tests {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl TierReferenceProofStore for CasConfigStore {
|
||||
async fn lifecycle_config_for_reference_proof(&self, bucket: &str) -> Result<Option<PersistedLifecycleConfiguration>> {
|
||||
async fn lifecycle_config_for_reference_proof(&self, bucket: &str) -> Result<Option<BucketLifecycleConfiguration>> {
|
||||
Ok(self
|
||||
.lifecycle_configs
|
||||
.lock()
|
||||
@@ -15813,10 +15762,10 @@ mod tests {
|
||||
});
|
||||
store.add_lifecycle_config(
|
||||
"photos",
|
||||
PersistedLifecycleConfiguration {
|
||||
BucketLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![PersistedLifecycleRule {
|
||||
status: "Enabled".to_owned(),
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
expiration: None,
|
||||
abort_incomplete_multipart_upload: None,
|
||||
del_marker_expiration: None,
|
||||
@@ -15825,10 +15774,10 @@ mod tests {
|
||||
noncurrent_version_expiration: None,
|
||||
noncurrent_version_transitions: None,
|
||||
prefix: None,
|
||||
transitions: Some(vec![PersistedTransition {
|
||||
transitions: Some(vec![Transition {
|
||||
days: Some(1),
|
||||
date: None,
|
||||
storage_class: Some("COLD-A".to_owned()),
|
||||
storage_class: Some(TransitionStorageClass::from_static("COLD-A")),
|
||||
}]),
|
||||
}],
|
||||
},
|
||||
@@ -16481,10 +16430,10 @@ mod tests {
|
||||
.save_tiering_config_if_current(store.clone(), None)
|
||||
.await
|
||||
.expect("reference proof fixture should persist");
|
||||
let config = PersistedLifecycleConfiguration {
|
||||
let config = BucketLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![PersistedLifecycleRule {
|
||||
status: "Enabled".to_owned(),
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
expiration: None,
|
||||
abort_incomplete_multipart_upload: None,
|
||||
del_marker_expiration: None,
|
||||
@@ -16493,10 +16442,10 @@ mod tests {
|
||||
noncurrent_version_expiration: None,
|
||||
noncurrent_version_transitions: None,
|
||||
prefix: None,
|
||||
transitions: Some(vec![PersistedTransition {
|
||||
transitions: Some(vec![Transition {
|
||||
days: Some(1),
|
||||
date: None,
|
||||
storage_class: Some("COLD-A".to_owned()),
|
||||
storage_class: Some(TransitionStorageClass::from_static("COLD-A")),
|
||||
}]),
|
||||
}],
|
||||
};
|
||||
@@ -16653,10 +16602,10 @@ mod tests {
|
||||
old_backend_identity: Some(current_identity),
|
||||
new_backend_identity: None,
|
||||
};
|
||||
let config = PersistedLifecycleConfiguration {
|
||||
let config = BucketLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![PersistedLifecycleRule {
|
||||
status: "Enabled".to_owned(),
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
expiration: None,
|
||||
abort_incomplete_multipart_upload: None,
|
||||
del_marker_expiration: None,
|
||||
@@ -16665,10 +16614,10 @@ mod tests {
|
||||
noncurrent_version_expiration: None,
|
||||
noncurrent_version_transitions: None,
|
||||
prefix: None,
|
||||
transitions: Some(vec![PersistedTransition {
|
||||
transitions: Some(vec![Transition {
|
||||
days: Some(1),
|
||||
date: None,
|
||||
storage_class: Some("COLD-A".to_owned()),
|
||||
storage_class: Some(TransitionStorageClass::from_static("COLD-A")),
|
||||
}]),
|
||||
}],
|
||||
};
|
||||
@@ -16698,20 +16647,20 @@ mod tests {
|
||||
old_backend_identity: Some(current_identity),
|
||||
new_backend_identity: None,
|
||||
};
|
||||
let config = PersistedLifecycleConfiguration {
|
||||
let config = BucketLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![PersistedLifecycleRule {
|
||||
status: "Enabled".to_owned(),
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
expiration: None,
|
||||
abort_incomplete_multipart_upload: None,
|
||||
del_marker_expiration: None,
|
||||
filter: None,
|
||||
id: Some("move-noncurrent".to_string()),
|
||||
noncurrent_version_expiration: None,
|
||||
noncurrent_version_transitions: Some(vec![PersistedNoncurrentVersionTransition {
|
||||
noncurrent_version_transitions: Some(vec![NoncurrentVersionTransition {
|
||||
noncurrent_days: Some(1),
|
||||
newer_noncurrent_versions: None,
|
||||
storage_class: Some("COLD-A".to_owned()),
|
||||
storage_class: Some(TransitionStorageClass::from_static("COLD-A")),
|
||||
}]),
|
||||
prefix: None,
|
||||
transitions: None,
|
||||
@@ -16747,10 +16696,10 @@ mod tests {
|
||||
old_backend_identity: Some(current_identity),
|
||||
new_backend_identity: None,
|
||||
};
|
||||
let config = PersistedLifecycleConfiguration {
|
||||
let config = BucketLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![PersistedLifecycleRule {
|
||||
status: "Enabled".to_owned(),
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
expiration: None,
|
||||
abort_incomplete_multipart_upload: None,
|
||||
del_marker_expiration: None,
|
||||
@@ -16759,10 +16708,10 @@ mod tests {
|
||||
noncurrent_version_expiration: None,
|
||||
noncurrent_version_transitions: None,
|
||||
prefix: None,
|
||||
transitions: Some(vec![PersistedTransition {
|
||||
transitions: Some(vec![Transition {
|
||||
days: Some(1),
|
||||
date: None,
|
||||
storage_class: Some("COLD-A".to_owned()),
|
||||
storage_class: Some(TransitionStorageClass::from_static("COLD-A")),
|
||||
}]),
|
||||
}],
|
||||
};
|
||||
|
||||
@@ -700,25 +700,9 @@ async fn advance_tier_mutation_intent_record_idempotent_at_prefix<S>(
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
let mut original_intent: Option<TierMutationIntent> = None;
|
||||
for attempt in 0..TIER_MUTATION_INTENT_ADVANCE_CAS_ATTEMPTS {
|
||||
let (mut intent, current_etag) =
|
||||
match load_tier_mutation_intent_record_with_etag_at_prefix(api.clone(), prefix, mutation_id).await {
|
||||
Ok(current) => current,
|
||||
Err(Error::Lock(rustfs_lock::LockError::Timeout { .. }))
|
||||
if attempt + 1 < TIER_MUTATION_INTENT_ADVANCE_CAS_ATTEMPTS =>
|
||||
{
|
||||
continue;
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
if let Some(original) = &original_intent {
|
||||
if !intent.same_identity_as(original) {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
} else {
|
||||
original_intent = Some(intent.clone());
|
||||
}
|
||||
load_tier_mutation_intent_record_with_etag_at_prefix(api.clone(), prefix, mutation_id).await?;
|
||||
let advanced = intent
|
||||
.advance_idempotent(next, committed_config_etag.clone())
|
||||
.map_err(tier_mutation_intent_store_error)?;
|
||||
@@ -729,15 +713,7 @@ where
|
||||
Ok(()) => return Ok((intent, true)),
|
||||
Err(Error::PreconditionFailed) => {
|
||||
let (mut current, _) =
|
||||
match load_tier_mutation_intent_record_with_etag_at_prefix(api.clone(), prefix, mutation_id).await {
|
||||
Ok(current) => current,
|
||||
Err(Error::Lock(rustfs_lock::LockError::Timeout { .. }))
|
||||
if attempt + 1 < TIER_MUTATION_INTENT_ADVANCE_CAS_ATTEMPTS =>
|
||||
{
|
||||
continue;
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
load_tier_mutation_intent_record_with_etag_at_prefix(api.clone(), prefix, mutation_id).await?;
|
||||
if !current.same_identity_as(&intent) {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
@@ -752,11 +728,6 @@ where
|
||||
}
|
||||
return Ok((current, false));
|
||||
}
|
||||
Err(Error::Lock(rustfs_lock::LockError::Timeout { .. }))
|
||||
if attempt + 1 < TIER_MUTATION_INTENT_ADVANCE_CAS_ATTEMPTS =>
|
||||
{
|
||||
continue;
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,9 +36,7 @@ use crate::services::tier::{
|
||||
use bytes::Bytes;
|
||||
use http::StatusCode;
|
||||
use rustfs_filemeta::metadata_keys;
|
||||
use rustfs_gateway_types::dto::{ObjectLockLegalHoldStatus, ObjectLockMode, ReplicationStatus};
|
||||
use rustfs_s3_client::credentials::{Credentials, SignatureType, Static, Value};
|
||||
use rustfs_s3_client::remote_error_code::RemoteErrorCode;
|
||||
use rustfs_s3_client::transition_api::{BucketLookupType, Options, TransitionClient, TransitionClientTimeouts, TransitionCore};
|
||||
use rustfs_s3_client::{
|
||||
admin_handler_utils::AdminError,
|
||||
@@ -51,6 +49,10 @@ use rustfs_utils::egress::validate_outbound_url;
|
||||
use rustfs_utils::http::headers::{
|
||||
CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE, EXPIRES, HeaderExt as _,
|
||||
};
|
||||
use s3s::{
|
||||
S3ErrorCode,
|
||||
dto::{ObjectLockLegalHoldStatus, ObjectLockRetentionMode, ReplicationStatus},
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
@@ -191,15 +193,12 @@ pub trait WarmBackend {
|
||||
.await
|
||||
{
|
||||
Ok(_) => Ok(TransitionCandidateProbe::VersionedPresent(remote_version_id.to_string())),
|
||||
Err(err) if matches!(to_error_response(&err).code, RemoteErrorCode::INVALID_RANGE) => {
|
||||
Err(err) if matches!(to_error_response(&err).code, S3ErrorCode::InvalidRange) => {
|
||||
Ok(TransitionCandidateProbe::VersionedPresent(remote_version_id.to_string()))
|
||||
}
|
||||
Err(err)
|
||||
if err.kind() == std::io::ErrorKind::NotFound
|
||||
|| matches!(
|
||||
to_error_response(&err).code,
|
||||
RemoteErrorCode::NO_SUCH_KEY | RemoteErrorCode::NO_SUCH_VERSION
|
||||
) =>
|
||||
|| matches!(to_error_response(&err).code, S3ErrorCode::NoSuchKey | S3ErrorCode::NoSuchVersion) =>
|
||||
{
|
||||
Ok(TransitionCandidateProbe::Missing)
|
||||
}
|
||||
@@ -219,9 +218,9 @@ pub fn build_transition_put_options(storage_class: String, mut metadata: HashMap
|
||||
let mut opts = PutObjectOptions {
|
||||
storage_class,
|
||||
send_content_md5: true,
|
||||
legalhold: ObjectLockLegalHoldStatus::custom(""),
|
||||
legalhold: ObjectLockLegalHoldStatus::from_static(""),
|
||||
internal: AdvancedPutOptions {
|
||||
replication_status: ReplicationStatus::custom(""),
|
||||
replication_status: ReplicationStatus::from_static(""),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
@@ -252,7 +251,7 @@ pub fn build_transition_put_options(storage_class: String, mut metadata: HashMap
|
||||
}
|
||||
|
||||
if let Some(mode) = metadata.lookup(metadata_keys::OBJECT_LOCK_MODE) {
|
||||
opts.mode = ObjectLockMode::custom(mode.to_ascii_uppercase());
|
||||
opts.mode = ObjectLockRetentionMode::from(mode.to_ascii_uppercase());
|
||||
}
|
||||
|
||||
if let Some(retain_until_date) = metadata
|
||||
@@ -263,7 +262,7 @@ pub fn build_transition_put_options(storage_class: String, mut metadata: HashMap
|
||||
}
|
||||
|
||||
if let Some(legalhold) = metadata.lookup(metadata_keys::OBJECT_LOCK_LEGAL_HOLD) {
|
||||
opts.legalhold = ObjectLockLegalHoldStatus::custom(legalhold.to_ascii_uppercase());
|
||||
opts.legalhold = ObjectLockLegalHoldStatus::from(legalhold.to_ascii_uppercase());
|
||||
}
|
||||
|
||||
// Promoted keys are read above through `lookup`, which accepts more than
|
||||
@@ -680,7 +679,7 @@ fn is_remote_tier_auth_rejection(err: &std::io::Error) -> bool {
|
||||
matches!(response.status_code, StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN)
|
||||
|| matches!(
|
||||
response.code,
|
||||
RemoteErrorCode::ACCESS_DENIED | RemoteErrorCode::INVALID_ACCESS_KEY_ID | RemoteErrorCode::SIGNATURE_DOES_NOT_MATCH
|
||||
S3ErrorCode::AccessDenied | S3ErrorCode::InvalidAccessKeyId | S3ErrorCode::SignatureDoesNotMatch
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1305,7 +1304,7 @@ mod tests {
|
||||
|
||||
struct AuthRejectingProbePutBackend {
|
||||
status_code: StatusCode,
|
||||
code: RemoteErrorCode,
|
||||
code: S3ErrorCode,
|
||||
probes: Arc<AtomicUsize>,
|
||||
removes: Arc<AtomicUsize>,
|
||||
}
|
||||
@@ -1768,10 +1767,10 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn check_warm_backend_reports_auth_rejected_probe_put_as_invalid_credentials() {
|
||||
for (status_code, code) in [
|
||||
(StatusCode::FORBIDDEN, RemoteErrorCode::ACCESS_DENIED),
|
||||
(StatusCode::UNAUTHORIZED, RemoteErrorCode::custom("")),
|
||||
(StatusCode::OK, RemoteErrorCode::INVALID_ACCESS_KEY_ID),
|
||||
(StatusCode::OK, RemoteErrorCode::SIGNATURE_DOES_NOT_MATCH),
|
||||
(StatusCode::FORBIDDEN, S3ErrorCode::AccessDenied),
|
||||
(StatusCode::UNAUTHORIZED, S3ErrorCode::Custom("".into())),
|
||||
(StatusCode::OK, S3ErrorCode::InvalidAccessKeyId),
|
||||
(StatusCode::OK, S3ErrorCode::SignatureDoesNotMatch),
|
||||
] {
|
||||
let probes = Arc::new(AtomicUsize::new(0));
|
||||
let removes = Arc::new(AtomicUsize::new(0));
|
||||
@@ -2166,31 +2165,18 @@ mod tests {
|
||||
metadata_keys::OBJECT_LOCK_LEGAL_HOLD.to_string(),
|
||||
ObjectLockLegalHoldStatus::ON.to_string(),
|
||||
);
|
||||
metadata.insert(metadata_keys::OBJECT_LOCK_MODE.to_string(), ObjectLockMode::GOVERNANCE.to_string());
|
||||
metadata.insert(
|
||||
metadata_keys::OBJECT_LOCK_MODE.to_string(),
|
||||
ObjectLockRetentionMode::GOVERNANCE.to_string(),
|
||||
);
|
||||
|
||||
let opts = build_transition_put_options("COLD".to_string(), metadata);
|
||||
|
||||
assert_eq!(opts.mode, ObjectLockMode::GOVERNANCE);
|
||||
assert_eq!(opts.legalhold, ObjectLockLegalHoldStatus::ON);
|
||||
assert_eq!(opts.mode.as_str(), ObjectLockRetentionMode::GOVERNANCE);
|
||||
assert_eq!(opts.legalhold.as_str(), ObjectLockLegalHoldStatus::ON);
|
||||
assert_ne!(opts.retain_until_date, OffsetDateTime::UNIX_EPOCH);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_transition_put_options_sends_no_object_lock_or_replication_headers_by_default() {
|
||||
let opts = build_transition_put_options("COLD".to_string(), HashMap::new());
|
||||
assert_eq!(opts.mode.as_str(), "");
|
||||
assert_eq!(opts.legalhold.as_str(), "");
|
||||
assert_eq!(opts.internal.replication_status.as_str(), "");
|
||||
let header = opts.header();
|
||||
for name in [
|
||||
"x-amz-object-lock-mode",
|
||||
"x-amz-object-lock-legal-hold",
|
||||
"x-amz-replication-status",
|
||||
] {
|
||||
assert!(header.get(name).is_none(), "{name} must not be sent: {header:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_transition_put_options_filters_promoted_headers_from_user_metadata() {
|
||||
let mut metadata = HashMap::new();
|
||||
|
||||
@@ -28,8 +28,6 @@ use crate::services::tier::{
|
||||
},
|
||||
};
|
||||
use http::HeaderMap;
|
||||
use rustfs_gateway_types::dto::Status as BucketVersioningStatus;
|
||||
use rustfs_s3_client::remote_error_code::RemoteErrorCode;
|
||||
use rustfs_s3_client::{
|
||||
api_error_response::to_error_response,
|
||||
api_get_options::GetObjectOptions,
|
||||
@@ -44,6 +42,7 @@ use rustfs_s3_client::{
|
||||
};
|
||||
use rustfs_utils::egress::validate_outbound_url_allow_private;
|
||||
use rustfs_utils::path::SLASH_SEPARATOR;
|
||||
use s3s::{S3ErrorCode, dto::BucketVersioningStatus};
|
||||
|
||||
pub struct WarmBackendS3 {
|
||||
pub client: Arc<TransitionClient>,
|
||||
@@ -62,8 +61,8 @@ enum RemoteBucketVersioning {
|
||||
|
||||
fn remote_bucket_versioning_from_status(status: Option<&str>) -> Result<RemoteBucketVersioning, std::io::Error> {
|
||||
Ok(match status {
|
||||
Some(status) if status == BucketVersioningStatus::ENABLED.as_str() => RemoteBucketVersioning::Enabled,
|
||||
Some(status) if status == BucketVersioningStatus::SUSPENDED.as_str() => RemoteBucketVersioning::Suspended,
|
||||
Some(BucketVersioningStatus::ENABLED) => RemoteBucketVersioning::Enabled,
|
||||
Some(BucketVersioningStatus::SUSPENDED) => RemoteBucketVersioning::Suspended,
|
||||
Some(status) => {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
@@ -238,7 +237,7 @@ impl WarmBackendS3 {
|
||||
}
|
||||
Err(err) => {
|
||||
let response = to_error_response(&err);
|
||||
if response.code == RemoteErrorCode::NO_SUCH_KEY {
|
||||
if response.code == S3ErrorCode::NoSuchKey {
|
||||
Ok(TransitionCandidateProbe::Missing)
|
||||
} else {
|
||||
Err(err)
|
||||
@@ -325,7 +324,7 @@ impl WarmBackendS3 {
|
||||
}
|
||||
Err(err) => {
|
||||
let response = to_error_response(&err);
|
||||
if response.code == RemoteErrorCode::NO_SUCH_KEY {
|
||||
if response.code == S3ErrorCode::NoSuchKey {
|
||||
Ok(TransitionCandidateProbe::Missing)
|
||||
} else {
|
||||
Err(err)
|
||||
@@ -689,7 +688,7 @@ mod tests {
|
||||
.probe_transition_candidate("forbidden-probe")
|
||||
.await
|
||||
.expect_err("an authorization failure must not be mistaken for a missing key");
|
||||
assert_eq!(to_error_response(&err).code, RemoteErrorCode::ACCESS_DENIED);
|
||||
assert_eq!(to_error_response(&err).code, S3ErrorCode::AccessDenied);
|
||||
assert_eq!(
|
||||
backend
|
||||
.probe_transition_candidate("delete-marker-hidden")
|
||||
@@ -1067,13 +1066,11 @@ mod tests {
|
||||
RemoteBucketVersioning::Disabled
|
||||
);
|
||||
assert_eq!(
|
||||
remote_bucket_versioning_from_status(Some(BucketVersioningStatus::ENABLED.as_str()))
|
||||
.expect("enabled status should parse"),
|
||||
remote_bucket_versioning_from_status(Some(BucketVersioningStatus::ENABLED)).expect("enabled status should parse"),
|
||||
RemoteBucketVersioning::Enabled
|
||||
);
|
||||
assert_eq!(
|
||||
remote_bucket_versioning_from_status(Some(BucketVersioningStatus::SUSPENDED.as_str()))
|
||||
.expect("suspended status should parse"),
|
||||
remote_bucket_versioning_from_status(Some(BucketVersioningStatus::SUSPENDED)).expect("suspended status should parse"),
|
||||
RemoteBucketVersioning::Suspended
|
||||
);
|
||||
let err = remote_bucket_versioning_from_status(Some("UnexpectedStatus"))
|
||||
|
||||
@@ -18,6 +18,7 @@ use std::{
|
||||
sync::atomic::{AtomicBool, Ordering},
|
||||
};
|
||||
|
||||
use s3s::header::{X_AMZ_DELETE_MARKER, X_AMZ_VERSION_ID};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::services::tier::{
|
||||
@@ -127,7 +128,7 @@ impl WarmBackend for WarmBackendWasabi {
|
||||
self.ensure_unversioned().await?;
|
||||
}
|
||||
let (headers, reader) = self.s3.get_with_headers(object, rv, opts).await?;
|
||||
if rv.is_empty() && (headers.contains_key("x-amz-version-id") || headers.contains_key("x-amz-delete-marker")) {
|
||||
if rv.is_empty() && (headers.contains_key(X_AMZ_VERSION_ID) || headers.contains_key(X_AMZ_DELETE_MARKER)) {
|
||||
self.versioning_drifted.store(true, Ordering::Release);
|
||||
return Err(io::Error::other(
|
||||
"Wasabi bucket versioning changed while reading an unversioned tier object",
|
||||
|
||||
@@ -129,7 +129,7 @@ use rustfs_filemeta::{
|
||||
};
|
||||
use rustfs_heal_contracts::heal_channel::{
|
||||
DriveState, HealAdmissionResult, HealChannelPriority, HealItemType, HealOpts, HealRequestSource, HealScanMode,
|
||||
send_heal_replacement_disk, send_heal_request_with_admission,
|
||||
send_heal_request_with_admission,
|
||||
};
|
||||
use rustfs_io_metrics::{
|
||||
record_object_lock_diag_acquire_duration, record_object_lock_diag_enabled, record_object_lock_diag_hold_duration,
|
||||
@@ -154,8 +154,8 @@ use rustfs_utils::http::headers::{
|
||||
};
|
||||
use rustfs_utils::http::{
|
||||
SUFFIX_ACTUAL_OBJECT_SIZE_CAP, SUFFIX_ACTUAL_SIZE, SUFFIX_BUCKET_INCARNATION_ID, SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE,
|
||||
SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_RESTORE_COPYBACK, SUFFIX_RESTORE_OPERATION_ID, SUFFIX_RESTORE_WORKER_LOCK,
|
||||
contains_key_str, get_header_map, get_str, insert_str, is_object_encryption_marker, remove_header_map,
|
||||
SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_RESTORE_OPERATION_ID, SUFFIX_RESTORE_WORKER_LOCK, contains_key_str, get_header_map,
|
||||
get_str, insert_str, is_object_encryption_marker, remove_header_map,
|
||||
};
|
||||
use rustfs_utils::{
|
||||
HashAlgorithm,
|
||||
@@ -13721,7 +13721,10 @@ mod tests {
|
||||
let requested_until = now + Duration::from_secs(60 * 60 * 24);
|
||||
|
||||
let mut user_defined = HashMap::new();
|
||||
user_defined.insert(metadata_keys::OBJECT_LOCK_MODE.to_string(), "COMPLIANCE".to_string());
|
||||
user_defined.insert(
|
||||
metadata_keys::OBJECT_LOCK_MODE.to_string(),
|
||||
s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(),
|
||||
);
|
||||
user_defined.insert(
|
||||
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
|
||||
existing_until.format(&time::format_description::well_known::Rfc3339).unwrap(),
|
||||
@@ -13733,7 +13736,7 @@ mod tests {
|
||||
};
|
||||
let opts = ObjectOptions {
|
||||
object_lock_retention: Some(crate::storage_api_contracts::object::ObjectLockRetentionOptions {
|
||||
mode: Some("COMPLIANCE".to_string()),
|
||||
mode: Some(s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string()),
|
||||
retain_until: Some(requested_until),
|
||||
bypass_governance: true,
|
||||
}),
|
||||
@@ -13753,7 +13756,10 @@ mod tests {
|
||||
let requested_until = now + Duration::from_secs(60 * 60 * 24);
|
||||
|
||||
let mut user_defined = HashMap::new();
|
||||
user_defined.insert(metadata_keys::OBJECT_LOCK_MODE.to_string(), "GOVERNANCE".to_string());
|
||||
user_defined.insert(
|
||||
metadata_keys::OBJECT_LOCK_MODE.to_string(),
|
||||
s3s::dto::ObjectLockRetentionMode::GOVERNANCE.to_string(),
|
||||
);
|
||||
user_defined.insert(
|
||||
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
|
||||
existing_until.format(&time::format_description::well_known::Rfc3339).unwrap(),
|
||||
@@ -13765,7 +13771,7 @@ mod tests {
|
||||
};
|
||||
let opts = ObjectOptions {
|
||||
object_lock_retention: Some(crate::storage_api_contracts::object::ObjectLockRetentionOptions {
|
||||
mode: Some("GOVERNANCE".to_string()),
|
||||
mode: Some(s3s::dto::ObjectLockRetentionMode::GOVERNANCE.to_string()),
|
||||
retain_until: Some(requested_until),
|
||||
bypass_governance: true,
|
||||
}),
|
||||
@@ -13780,7 +13786,10 @@ mod tests {
|
||||
async fn test_check_object_lock_delete_blocks_compliance_version_delete() {
|
||||
let retain_until = OffsetDateTime::now_utc() + Duration::from_secs(60 * 60 * 24 * 60);
|
||||
let mut user_defined = HashMap::new();
|
||||
user_defined.insert(metadata_keys::OBJECT_LOCK_MODE.to_string(), "COMPLIANCE".to_string());
|
||||
user_defined.insert(
|
||||
metadata_keys::OBJECT_LOCK_MODE.to_string(),
|
||||
s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(),
|
||||
);
|
||||
user_defined.insert(
|
||||
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
|
||||
retain_until.format(&time::format_description::well_known::Rfc3339).unwrap(),
|
||||
@@ -13822,7 +13831,10 @@ mod tests {
|
||||
},
|
||||
restore_expires: Some(restore_expiry),
|
||||
user_defined: Arc::new(HashMap::from([
|
||||
(metadata_keys::OBJECT_LOCK_MODE.to_string(), "COMPLIANCE".to_string()),
|
||||
(
|
||||
metadata_keys::OBJECT_LOCK_MODE.to_string(),
|
||||
s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(),
|
||||
),
|
||||
(
|
||||
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
|
||||
retain_until.format(&time::format_description::well_known::Rfc3339).unwrap(),
|
||||
@@ -13893,7 +13905,10 @@ mod tests {
|
||||
async fn test_check_object_lock_delete_allows_versioned_delete_marker_creation() {
|
||||
let retain_until = OffsetDateTime::now_utc() + Duration::from_secs(60 * 60 * 24 * 60);
|
||||
let mut user_defined = HashMap::new();
|
||||
user_defined.insert(metadata_keys::OBJECT_LOCK_MODE.to_string(), "COMPLIANCE".to_string());
|
||||
user_defined.insert(
|
||||
metadata_keys::OBJECT_LOCK_MODE.to_string(),
|
||||
s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(),
|
||||
);
|
||||
user_defined.insert(
|
||||
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
|
||||
retain_until.format(&time::format_description::well_known::Rfc3339).unwrap(),
|
||||
@@ -13918,7 +13933,10 @@ mod tests {
|
||||
fn governance_retained_obj_info() -> ObjectInfo {
|
||||
let retain_until = OffsetDateTime::now_utc() + Duration::from_secs(60 * 60 * 24 * 60);
|
||||
let mut user_defined = HashMap::new();
|
||||
user_defined.insert(metadata_keys::OBJECT_LOCK_MODE.to_string(), "GOVERNANCE".to_string());
|
||||
user_defined.insert(
|
||||
metadata_keys::OBJECT_LOCK_MODE.to_string(),
|
||||
s3s::dto::ObjectLockRetentionMode::GOVERNANCE.to_string(),
|
||||
);
|
||||
user_defined.insert(
|
||||
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
|
||||
retain_until.format(&time::format_description::well_known::Rfc3339).unwrap(),
|
||||
@@ -13968,7 +13986,10 @@ mod tests {
|
||||
async fn test_check_object_lock_delete_blocks_replicated_compliance_version_purge() {
|
||||
let retain_until = OffsetDateTime::now_utc() + Duration::from_secs(60 * 60 * 24 * 60);
|
||||
let mut user_defined = HashMap::new();
|
||||
user_defined.insert(metadata_keys::OBJECT_LOCK_MODE.to_string(), "COMPLIANCE".to_string());
|
||||
user_defined.insert(
|
||||
metadata_keys::OBJECT_LOCK_MODE.to_string(),
|
||||
s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(),
|
||||
);
|
||||
user_defined.insert(
|
||||
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
|
||||
retain_until.format(&time::format_description::well_known::Rfc3339).unwrap(),
|
||||
@@ -14023,11 +14044,11 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_object_lock_delete_check_required_keeps_object_lock_config_buckets() {
|
||||
use rustfs_gateway_types::persistence::PersistedObjectLockConfiguration as ObjectLockConfiguration;
|
||||
use s3s::dto::{ObjectLockConfiguration, ObjectLockEnabled};
|
||||
|
||||
let mut bm = crate::bucket::metadata::BucketMetadata::new("lock-config-bucket");
|
||||
bm.object_lock_config = Some(ObjectLockConfiguration {
|
||||
object_lock_enabled: Some("Enabled".to_owned()),
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
..Default::default()
|
||||
});
|
||||
assert!(object_lock_delete_check_required(Some(&bm)));
|
||||
@@ -16924,8 +16945,8 @@ mod tests {
|
||||
// otherwise `from_file_info` never synthesizes the null version id and
|
||||
// the branch under test is not reached.
|
||||
let suspended = crate::bucket::replication::DeleteReplicationConfigSnapshot::from_configs_for_test(
|
||||
rustfs_gateway_types::persistence::PersistedVersioningConfiguration {
|
||||
status: Some("Suspended".to_owned()),
|
||||
s3s::dto::VersioningConfiguration {
|
||||
status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::SUSPENDED)),
|
||||
..Default::default()
|
||||
},
|
||||
None,
|
||||
|
||||
@@ -2125,86 +2125,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_batch_with_missing_version_recovers_after_preparation_failure() {
|
||||
use crate::disk::RUSTFS_META_TMP_BUCKET;
|
||||
use crate::storage_api_contracts::object::ObjectToDelete;
|
||||
|
||||
let bucket = "recover-batch-missing-version";
|
||||
let (dirs, store, disks) = fixture(bucket).await;
|
||||
let deleted = put(&store, bucket, &vec![48; 4096], true).await.expect("version to delete");
|
||||
let remaining = vec![49; 4096];
|
||||
let retained = put(&store, bucket, &remaining, true).await.expect("version to retain");
|
||||
let missing = Uuid::new_v4();
|
||||
let temporary = dirs[3].path().join(RUSTFS_META_TMP_BUCKET);
|
||||
let saved = dirs[3].path().join("saved-delete-tmp");
|
||||
tokio::fs::rename(&temporary, &saved)
|
||||
.await
|
||||
.expect("save returning disk's staging directory");
|
||||
tokio::fs::write(&temporary, b"block preparation")
|
||||
.await
|
||||
.expect("fail staging after the recovery preflight");
|
||||
|
||||
let (objects, errors) = store
|
||||
.delete_objects(
|
||||
bucket,
|
||||
[deleted, missing]
|
||||
.into_iter()
|
||||
.map(|version| ObjectToDelete {
|
||||
object_name: "object".into(),
|
||||
version_id: Some(version),
|
||||
..Default::default()
|
||||
})
|
||||
.collect(),
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert_eq!(objects.len(), 2);
|
||||
assert!(errors.iter().all(Option::is_none), "batch deletion succeeds at quorum: {errors:?}");
|
||||
assert!(
|
||||
disks[3]
|
||||
.discover_delete_transactions(bucket, Some("object"), None, 100)
|
||||
.await
|
||||
.expect("inspect failed preparation")
|
||||
.transactions
|
||||
.is_empty(),
|
||||
"returning disk must adopt the quorum decision without a local preparation"
|
||||
);
|
||||
tokio::fs::remove_file(&temporary).await.expect("remove preparation failure");
|
||||
tokio::fs::rename(&saved, &temporary)
|
||||
.await
|
||||
.expect("restore returning disk staging");
|
||||
|
||||
let result = store
|
||||
.recover_delete_transactions(bucket, &HealOpts::default())
|
||||
.await
|
||||
.expect("recover batch containing a missing non-null version");
|
||||
assert!(result.detail.contains("blocked=0"), "{}", result.detail);
|
||||
assert_eq!(body(&store, bucket, Some(retained)).await, remaining);
|
||||
for version in [deleted, missing] {
|
||||
assert_eq!(
|
||||
disks[3]
|
||||
.read_version(bucket, bucket, "object", &version.to_string(), &ReadOptions::default())
|
||||
.await
|
||||
.expect_err("returning disk must not retain or create a deleted version"),
|
||||
DiskError::FileVersionNotFound
|
||||
);
|
||||
}
|
||||
for disk in &disks {
|
||||
assert!(
|
||||
disk.discover_delete_transactions(bucket, Some("object"), None, 100)
|
||||
.await
|
||||
.expect("inspect converged batch")
|
||||
.transactions
|
||||
.is_empty(),
|
||||
"every participant must converge and release the decision"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_dry_run_matches_adoption_of_already_absent_non_null_version() {
|
||||
let bucket = "recover-preview-non-null";
|
||||
@@ -2571,140 +2491,6 @@ mod tests {
|
||||
.expect("no retained version decision may keep the empty bucket nonempty");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_partial_batch_receipt_handles_missing_versions() {
|
||||
use crate::disk::RUSTFS_META_TMP_BUCKET;
|
||||
use crate::set_disk::ops::object::DeleteCleanupBarrier;
|
||||
use crate::storage_api_contracts::object::ObjectToDelete;
|
||||
|
||||
for include_missing in [false, true] {
|
||||
let bucket = "recover-prepared-batch-missing-versions";
|
||||
let (dirs, store, disks) = fixture(bucket).await;
|
||||
metadata_sys::update_in(
|
||||
&store.ctx,
|
||||
bucket,
|
||||
crate::bucket::metadata::BUCKET_VERSIONING_CONFIG,
|
||||
b"<VersioningConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
|
||||
)
|
||||
.await
|
||||
.expect("enable supported versioned bucket");
|
||||
let first = put(&store, bucket, &vec![81; 1024 * 1024], true)
|
||||
.await
|
||||
.expect("first version");
|
||||
let second = put(&store, bucket, &vec![82; 4096], true).await.expect("retained version");
|
||||
let missing = Uuid::new_v4();
|
||||
let barrier = DeleteCleanupBarrier::install(bucket);
|
||||
let deleting = store.clone();
|
||||
let task = tokio::spawn(async move {
|
||||
let mut versions = vec![first];
|
||||
if include_missing {
|
||||
versions.push(missing);
|
||||
}
|
||||
deleting
|
||||
.delete_objects(
|
||||
bucket,
|
||||
versions
|
||||
.into_iter()
|
||||
.map(|version| ObjectToDelete {
|
||||
object_name: "object".into(),
|
||||
version_id: Some(version),
|
||||
..Default::default()
|
||||
})
|
||||
.collect(),
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
let transactions = disks[0]
|
||||
.discover_delete_transactions(bucket, Some("object"), None, 100)
|
||||
.await
|
||||
.expect("discover batch preparation")
|
||||
.transactions;
|
||||
assert_eq!(transactions.len(), 1);
|
||||
let transaction = transactions[0].transaction;
|
||||
for disk in &disks {
|
||||
let state = disk
|
||||
.inspect_delete_transaction(bucket, "object", transaction)
|
||||
.await
|
||||
.expect("inspect prepared batch");
|
||||
assert_eq!(state.state, DeleteRecoveryState::NewPrepared);
|
||||
let journal = state.journal.expect("prepared journal");
|
||||
assert!(journal.original_digest.is_some());
|
||||
assert!(journal.original_versions.iter().all(|v| v.version_id != Some(missing)));
|
||||
}
|
||||
// Fail durable receipt staging on two participants after metadata quorum.
|
||||
for dir in dirs.iter().skip(2) {
|
||||
tokio::fs::rename(dir.path().join(RUSTFS_META_TMP_BUCKET), dir.path().join("saved-receipt-staging"))
|
||||
.await
|
||||
.expect("save receipt staging directory");
|
||||
tokio::fs::write(dir.path().join(RUSTFS_META_TMP_BUCKET), b"receipt staging I/O failure")
|
||||
.await
|
||||
.expect("inject receipt staging failure");
|
||||
}
|
||||
barrier.release();
|
||||
let (_, errors) = task.await.expect("batch deletion must join");
|
||||
drop(barrier);
|
||||
assert_eq!(errors.len(), if include_missing { 2 } else { 1 });
|
||||
assert!(errors.iter().all(Option::is_some), "partial receipts are indeterminate: {errors:?}");
|
||||
let mut receipts = 0;
|
||||
let mut prepared = 0;
|
||||
for disk in &disks {
|
||||
match disk
|
||||
.inspect_delete_transaction(bucket, "object", transaction)
|
||||
.await
|
||||
.expect("partial decision")
|
||||
.state
|
||||
{
|
||||
DeleteRecoveryState::QuorumCommitted => receipts += 1,
|
||||
DeleteRecoveryState::NewPrepared => prepared += 1,
|
||||
state => panic!("unexpected post-publication state: {state:?}"),
|
||||
}
|
||||
}
|
||||
assert_eq!((receipts, prepared), (2, 2));
|
||||
for dir in dirs.iter().skip(2) {
|
||||
tokio::fs::remove_file(dir.path().join(RUSTFS_META_TMP_BUCKET))
|
||||
.await
|
||||
.expect("remove injected staging failure");
|
||||
tokio::fs::rename(dir.path().join("saved-receipt-staging"), dir.path().join(RUSTFS_META_TMP_BUCKET))
|
||||
.await
|
||||
.expect("restore receipt staging");
|
||||
}
|
||||
store
|
||||
.delete_object(
|
||||
bucket,
|
||||
"object",
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(second.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("later version deletion succeeds");
|
||||
store
|
||||
.recover_delete_transactions(bucket, &HealOpts::default())
|
||||
.await
|
||||
.expect("certified batch must recover after later version deletion");
|
||||
for disk in &disks {
|
||||
assert!(
|
||||
disk.discover_delete_transactions(bucket, Some("object"), None, 100)
|
||||
.await
|
||||
.expect("settled transaction inventory")
|
||||
.transactions
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
store
|
||||
.delete_bucket(bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("settled receipts must not block removal of the empty bucket");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_partial_receipt_converges_after_later_version_absence() {
|
||||
let bucket = "recover-partial-chained-versions";
|
||||
|
||||
@@ -21,19 +21,22 @@
|
||||
|
||||
use super::super::{
|
||||
Arc, DiskError, DiskInfo, DiskInfoOptions, DiskOption, DiskStore, Endpoint, Error, FormatV3, HealChannelPriority, LockResult,
|
||||
NamespaceLock, NamespaceLockWrapper, ObjectKey, Result, SetDisks, StorageError, debug, disk, info, load_format_erasure,
|
||||
send_heal_replacement_disk, warn,
|
||||
NamespaceLock, NamespaceLockWrapper, ObjectKey, Result, SetDisks, StorageError, debug, disk, load_format_erasure, warn,
|
||||
};
|
||||
use crate::disk::health_state::DriveMembershipSnapshot;
|
||||
use crate::disk::{DiskAPI, new_disk};
|
||||
use crate::layout::disks_layout::MAX_ERASURE_SET_DRIVE_COUNT;
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use futures::{StreamExt, stream};
|
||||
use futures::{StreamExt, stream, stream::FuturesUnordered};
|
||||
use rand::prelude::SliceRandom;
|
||||
use rustfs_heal_contracts::heal_channel::{create_auto_replacement_disks_request, send_heal_request};
|
||||
use rustfs_utils::path::object_name_disk_identity;
|
||||
#[cfg(test)]
|
||||
use uuid::Uuid;
|
||||
|
||||
// Bound batching when another reconnect is still pending.
|
||||
const REPLACEMENT_BATCH_WAIT: std::time::Duration = std::time::Duration::from_secs(1);
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::storage_api_contracts::namespace::NamespaceLocking for SetDisks {
|
||||
type Error = Error;
|
||||
@@ -329,33 +332,82 @@ impl SetDisks {
|
||||
|
||||
async fn connect_disks_with_active_monitoring(&self, active_monitoring: bool) {
|
||||
let disks = self.get_disks_internal().await;
|
||||
// A slow failed endpoint must not hold back other returning drives.
|
||||
// Keep the fanout within one set and release the inventory lock before I/O.
|
||||
stream::iter(disks.into_iter().enumerate())
|
||||
.for_each_concurrent(MAX_ERASURE_SET_DRIVE_COUNT, |(i, opdisk)| async move {
|
||||
// Reconnect drives concurrently without holding the inventory lock.
|
||||
let mut reconnects = stream::iter(disks.into_iter().enumerate())
|
||||
.map(|(i, opdisk)| async move {
|
||||
if let Some(disk) = opdisk {
|
||||
if !active_monitoring
|
||||
&& disk.get_disk_location().set_idx.is_some()
|
||||
&& let disk::Disk::Local(local) = disk.as_ref()
|
||||
&& local.can_skip_background_probe().await
|
||||
{
|
||||
return;
|
||||
return None;
|
||||
}
|
||||
if disk.is_online().await && disk.get_disk_location().set_idx.is_some() {
|
||||
return;
|
||||
return None;
|
||||
}
|
||||
|
||||
let _ = disk.close().await;
|
||||
}
|
||||
|
||||
if let Some(endpoint) = self.set_endpoints.get(i) {
|
||||
self.renew_disk(endpoint).await;
|
||||
}
|
||||
let endpoint = self.set_endpoints.get(i)?;
|
||||
self.reconnect_disk(endpoint).await.then(|| endpoint.to_string())
|
||||
})
|
||||
.await;
|
||||
.buffer_unordered(MAX_ERASURE_SET_DRIVE_COUNT);
|
||||
let mut submissions = FuturesUnordered::new();
|
||||
let mut batch = Vec::new();
|
||||
let mut deadline = None;
|
||||
let mut reconnects_done = false;
|
||||
loop {
|
||||
if reconnects_done && submissions.is_empty() {
|
||||
break;
|
||||
}
|
||||
tokio::select! {
|
||||
result = reconnects.next(), if !reconnects_done => match result {
|
||||
Some(Some(endpoint)) => {
|
||||
batch.push(endpoint);
|
||||
deadline.get_or_insert_with(|| tokio::time::Instant::now() + REPLACEMENT_BATCH_WAIT);
|
||||
}
|
||||
Some(None) => {}
|
||||
None => {
|
||||
reconnects_done = true;
|
||||
if !batch.is_empty() {
|
||||
submissions.push(self.submit_replacement_heal(std::mem::take(&mut batch)));
|
||||
}
|
||||
deadline = None;
|
||||
}
|
||||
},
|
||||
_ = tokio::time::sleep_until(deadline.unwrap_or_else(tokio::time::Instant::now)), if deadline.is_some() => {
|
||||
submissions.push(self.submit_replacement_heal(std::mem::take(&mut batch)));
|
||||
deadline = None;
|
||||
}
|
||||
_ = submissions.next(), if !submissions.is_empty() => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn submit_replacement_heal(&self, replacements: Vec<String>) {
|
||||
if replacements.is_empty() {
|
||||
return;
|
||||
}
|
||||
// Queue admission deduplicates by set, so publish ready targets together.
|
||||
let request = create_auto_replacement_disks_request(
|
||||
self.pool_index,
|
||||
self.set_index,
|
||||
replacements,
|
||||
Some(HealChannelPriority::Normal),
|
||||
);
|
||||
let _ = send_heal_request(request).await;
|
||||
}
|
||||
|
||||
pub async fn renew_disk(&self, ep: &Endpoint) {
|
||||
if self.reconnect_disk(ep).await {
|
||||
self.submit_replacement_heal(vec![ep.to_string()]).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Return whether this reconnect attached a verified unformatted target.
|
||||
async fn reconnect_disk(&self, ep: &Endpoint) -> bool {
|
||||
debug!("renew_disk: start {:?}", ep);
|
||||
|
||||
let previous_disk = {
|
||||
@@ -375,28 +427,17 @@ impl SetDisks {
|
||||
Err(e) => {
|
||||
warn!("renew_disk: connect_endpoint err {:?}", &e);
|
||||
if !matches!(e, DiskError::UnformattedDisk | DiskError::Io(_)) {
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
|
||||
let attached = match self.attach_unformatted_replacement_disk(ep).await {
|
||||
Ok(attached) => attached,
|
||||
Err(err) => {
|
||||
warn!(endpoint = %ep, error = ?err, "renew_disk: unformatted replacement probe failed");
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
};
|
||||
if !attached {
|
||||
return;
|
||||
}
|
||||
|
||||
info!("renew_disk attached unformatted replacement and will trigger heal_disk, {:?}", ep);
|
||||
let (Ok(pool_index), Ok(set_index)) = (usize::try_from(ep.pool_idx), usize::try_from(ep.set_idx)) else {
|
||||
warn!("renew_disk: replacement target has invalid pool or set index, {:?}", ep);
|
||||
return;
|
||||
};
|
||||
let _ =
|
||||
send_heal_replacement_disk(pool_index, set_index, ep.to_string(), Some(HealChannelPriority::Normal)).await;
|
||||
return;
|
||||
return attached;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -404,7 +445,7 @@ impl SetDisks {
|
||||
Ok(res) => res,
|
||||
Err(e) => {
|
||||
warn!("renew_disk: find_disk_index err {:?}", e);
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -422,7 +463,7 @@ impl SetDisks {
|
||||
expected_set_index = self.set_index,
|
||||
"renew_disk rejected a drive whose endpoint and format do not identify the same topology slot"
|
||||
);
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
|
||||
let _ = new_disk.set_disk_id(Some(fm.erasure.this)).await;
|
||||
@@ -442,7 +483,7 @@ impl SetDisks {
|
||||
new_disk.enable_health_check();
|
||||
*slot = Some(new_disk);
|
||||
}
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
|
||||
new_disk.enable_health_check();
|
||||
@@ -464,6 +505,7 @@ impl SetDisks {
|
||||
|
||||
let mut disk_lock = self.disks.write().await;
|
||||
disk_lock[disk_idx] = Some(new_disk);
|
||||
false
|
||||
}
|
||||
|
||||
/// Attach a replacement target only after proving that the exact local slot
|
||||
@@ -1102,6 +1144,162 @@ mod tests {
|
||||
drop(temp_dirs);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reconnect_sweep_collects_all_blank_targets_and_excludes_original_disks() {
|
||||
use rustfs_heal_contracts::heal_channel::{HealAdmissionResult, HealChannelCommand, init_heal_channel};
|
||||
|
||||
const CHILD: &str = "RUSTFS_TEST_GROUPED_RECONNECT_CHILD";
|
||||
if std::env::var_os(CHILD).is_none() {
|
||||
// The heal sender is process-global; keep this receiver isolated.
|
||||
let output = std::process::Command::new(std::env::current_exe().expect("test executable"))
|
||||
.args([
|
||||
"--exact",
|
||||
"set_disk::ops::locking::tests::reconnect_sweep_collects_all_blank_targets_and_excludes_original_disks",
|
||||
"--nocapture",
|
||||
])
|
||||
.env(CHILD, "1")
|
||||
.output()
|
||||
.expect("isolated reconnect test should run");
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
assert!(output.status.success(), "isolated reconnect test failed: {stdout} {stderr}");
|
||||
assert!(stdout.contains("1 passed"), "child must execute the reconnect test: {stdout}");
|
||||
return;
|
||||
}
|
||||
let mut heal_rx = init_heal_channel().expect("isolated heal receiver");
|
||||
let disk_count = 4;
|
||||
let format = FormatV3::new(1, disk_count);
|
||||
let mut directories = Vec::new();
|
||||
let mut endpoints = Vec::new();
|
||||
let mut disks = Vec::new();
|
||||
for disk_idx in 0..2 {
|
||||
let (directory, endpoint, disk) = make_formatted_local_disk(disk_idx, &format).await;
|
||||
directories.push(directory);
|
||||
endpoints.push(endpoint);
|
||||
disks.push(Some(disk));
|
||||
}
|
||||
for disk_idx in 2..disk_count {
|
||||
let directory = tempfile::tempdir().expect("replacement directory");
|
||||
let mut endpoint =
|
||||
Endpoint::try_from(directory.path().to_str().expect("UTF-8 replacement path")).expect("replacement endpoint");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(disk_idx);
|
||||
directories.push(directory);
|
||||
endpoints.push(endpoint);
|
||||
disks.push(None);
|
||||
}
|
||||
let mut ready_targets = endpoints[2..].iter().map(ToString::to_string).collect::<Vec<_>>();
|
||||
ready_targets.sort_unstable();
|
||||
// Disk 1 returns with its format and data; disks 2 and 3 are blank.
|
||||
disks[1]
|
||||
.take()
|
||||
.expect("original disk handle")
|
||||
.close()
|
||||
.await
|
||||
.expect("close original disk");
|
||||
let retained_data = directories[1].path().join("retained-data");
|
||||
tokio::fs::write(&retained_data, b"original data")
|
||||
.await
|
||||
.expect("write original data");
|
||||
let set = Arc::new(
|
||||
SetDisks::new(
|
||||
"reconnect-sweep".to_string(),
|
||||
Arc::new(RwLock::new(disks)),
|
||||
disk_count,
|
||||
2,
|
||||
0,
|
||||
0,
|
||||
endpoints.clone(),
|
||||
format,
|
||||
Vec::new(),
|
||||
)
|
||||
.await,
|
||||
);
|
||||
|
||||
// Block disk 0's format check, then make it a late blank replacement.
|
||||
let slow = set.get_disks_internal().await[0].clone().expect("original disk");
|
||||
assert!(slow.is_online().await, "initialize original disk identity");
|
||||
let disk::Disk::Local(wrapper) = slow.as_ref() else {
|
||||
panic!("fixture must use a local disk");
|
||||
};
|
||||
let slow_local = wrapper.get_disk();
|
||||
let mut slow_format_guard = slow_local.format_info.write().await;
|
||||
slow_format_guard.last_check = None;
|
||||
std::fs::remove_file(
|
||||
directories[0]
|
||||
.path()
|
||||
.join(disk::RUSTFS_META_BUCKET)
|
||||
.join(disk::FORMAT_CONFIG_FILE),
|
||||
)
|
||||
.expect("make the blocked disk a blank replacement");
|
||||
let sweep = tokio::spawn({
|
||||
let set = set.clone();
|
||||
async move { set.connect_disks_with_active_monitoring(true).await }
|
||||
});
|
||||
tokio::time::timeout(std::time::Duration::from_secs(10), async {
|
||||
loop {
|
||||
let attached = set.get_disks_internal().await;
|
||||
if attached[1..].iter().all(Option::is_some) {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("other drives must reconnect while the first drive is blocked");
|
||||
let command = tokio::time::timeout(std::time::Duration::from_secs(10), heal_rx.recv())
|
||||
.await
|
||||
.expect("ready targets must be submitted while a drive is blocked")
|
||||
.expect("heal receiver should remain open");
|
||||
let HealChannelCommand::Start { request, response_tx } = command else {
|
||||
panic!("expected a replacement heal request");
|
||||
};
|
||||
assert_eq!(request.heal_endpoints, ready_targets, "ready blank disks must share one request");
|
||||
assert!(!sweep.is_finished(), "the blocked drive must still be pending");
|
||||
drop(slow_format_guard);
|
||||
// Do not acknowledge the first request yet: a delayed admission
|
||||
// response must not stop the late drive reconnecting and submitting.
|
||||
let late_command = tokio::time::timeout(std::time::Duration::from_secs(10), heal_rx.recv())
|
||||
.await
|
||||
.expect("late target must be submitted while the first response is pending")
|
||||
.expect("heal receiver should remain open");
|
||||
let HealChannelCommand::Start {
|
||||
request: late_request,
|
||||
response_tx: late_response,
|
||||
} = late_command
|
||||
else {
|
||||
panic!("expected a late replacement heal request");
|
||||
};
|
||||
assert_eq!(late_request.heal_endpoints, [endpoints[0].to_string()]);
|
||||
response_tx
|
||||
.send(Ok(HealAdmissionResult::Accepted))
|
||||
.expect("acknowledge ready targets");
|
||||
late_response
|
||||
.send(Ok(HealAdmissionResult::Accepted))
|
||||
.expect("acknowledge late target");
|
||||
sweep.await.expect("reconnect sweep should finish");
|
||||
assert!(heal_rx.try_recv().is_err(), "no duplicate request should be sent");
|
||||
let attached = set.get_disks_internal().await;
|
||||
for (index, disk) in attached.iter().enumerate() {
|
||||
let disk = disk.as_ref().expect("each verified endpoint should be attached");
|
||||
let format = load_format_erasure(disk, false).await;
|
||||
if index == 1 {
|
||||
assert!(format.is_ok(), "original disk must retain its format");
|
||||
} else {
|
||||
assert_eq!(format.expect_err("replacement must remain unformatted"), DiskError::UnformattedDisk);
|
||||
assert!(!disk.health_check_enabled_for_test(), "format must precede replacement health checks");
|
||||
}
|
||||
disk.close().await.expect("close test disk");
|
||||
}
|
||||
assert_eq!(tokio::fs::read(&retained_data).await.expect("read original data"), b"original data");
|
||||
let local_map = runtime_sources::local_disk_map_handle();
|
||||
let mut local_map = local_map.write().await;
|
||||
for endpoint in endpoints {
|
||||
local_map.remove(&endpoint.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn renew_disk_attaches_only_a_verified_local_unformatted_replacement() {
|
||||
let disk_count = 4;
|
||||
|
||||
@@ -38,15 +38,15 @@ use super::super::{
|
||||
ObjectLockDiagGuard, ObjectOptions, ObjectPartInfo, OffsetDateTime, PartInfo, PutObjReader, RUSTFS_META_MULTIPART_BUCKET,
|
||||
RUSTFS_META_TMP_BUCKET, RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY, Result, SLASH_SEPARATOR,
|
||||
SUFFIX_ACTUAL_OBJECT_SIZE_CAP, SUFFIX_ACTUAL_SIZE, SUFFIX_BUCKET_INCARNATION_ID, SUFFIX_COMPRESSION_SIZE,
|
||||
SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_RESTORE_COPYBACK, SUFFIX_RESTORE_OPERATION_ID, SUFFIX_RESTORE_WORKER_LOCK, SetDisks,
|
||||
SmallWritePath, StorageError, Uuid, WriteLayout, check_object_lock_for_deletion_with_state,
|
||||
classify_multipart_part_write_path, coding, complete_multipart_part_error, complete_multipart_part_error_result,
|
||||
complete_part_checksum, contains_key_str, create_bitrot_writer, debug, disk, error, file_info_is_valid_for_metadata,
|
||||
get_complete_multipart_md5, get_header_map, get_str, insert_str, is_err_object_not_found, is_err_version_not_found,
|
||||
is_min_allowed_part_size, log_multipart_write_quorum_failure, parts_after_marker, path_join_buf,
|
||||
prepare_complete_multipart_parts, record_compression_total_memory, reduce_read_quorum_errs, reduce_write_quorum_errs,
|
||||
remove_header_map, resolve_write_layout, restore_commit_operation_id_from_metadata, should_persist_encryption_original_size,
|
||||
strip_internal_multipart_metadata, to_object_err, warn,
|
||||
SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_RESTORE_OPERATION_ID, SUFFIX_RESTORE_WORKER_LOCK, SetDisks, SmallWritePath, StorageError,
|
||||
Uuid, WriteLayout, check_object_lock_for_deletion_with_state, classify_multipart_part_write_path, coding,
|
||||
complete_multipart_part_error, complete_multipart_part_error_result, complete_part_checksum, contains_key_str,
|
||||
create_bitrot_writer, debug, disk, error, file_info_is_valid_for_metadata, get_complete_multipart_md5, get_header_map,
|
||||
get_str, insert_str, is_err_object_not_found, is_err_version_not_found, is_min_allowed_part_size,
|
||||
log_multipart_write_quorum_failure, parts_after_marker, path_join_buf, prepare_complete_multipart_parts,
|
||||
record_compression_total_memory, reduce_read_quorum_errs, reduce_write_quorum_errs, remove_header_map, resolve_write_layout,
|
||||
restore_commit_operation_id_from_metadata, should_persist_encryption_original_size, strip_internal_multipart_metadata,
|
||||
to_object_err, warn,
|
||||
};
|
||||
use super::bitrot_self_verify::{BitrotSelfVerifyTarget, drop_failed_writer_disks, verify_written_bitrot_shards};
|
||||
#[cfg(test)]
|
||||
@@ -3318,14 +3318,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
crate::object_api::ENCRYPTED_PART_LAYOUT_CANDIDATE_SUFFIX,
|
||||
);
|
||||
rustfs_utils::http::metadata_compat::remove_str(&mut fi.metadata, crate::object_api::ENCRYPTED_PART_LAYOUT_QUORUM_SUFFIX);
|
||||
// Pending copy-back keeps its recovery identity until finalization.
|
||||
if expected_restore_operation_id.is_some()
|
||||
&& !fi
|
||||
.metadata
|
||||
.get(metadata_keys::RESTORE)
|
||||
.and_then(|value| rustfs_filemeta::parse_restore_obj_status(value).ok())
|
||||
.is_some_and(|status| status.is_restore_in_progress == Some(true))
|
||||
{
|
||||
if expected_restore_operation_id.is_some() {
|
||||
rustfs_utils::http::metadata_compat::remove_str(&mut fi.metadata, SUFFIX_RESTORE_OPERATION_ID);
|
||||
rustfs_utils::http::metadata_compat::remove_str(&mut fi.metadata, SUFFIX_RESTORE_WORKER_LOCK);
|
||||
}
|
||||
@@ -3710,13 +3703,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
fi.metadata.remove(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM);
|
||||
fi.metadata.remove(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM_TYPE);
|
||||
strip_internal_multipart_metadata(&mut fi.metadata);
|
||||
if opts.restore_copyback {
|
||||
insert_str(
|
||||
&mut fi.metadata,
|
||||
SUFFIX_RESTORE_COPYBACK,
|
||||
rustfs_utils::http::metadata_compat::RESTORE_COPYBACK_PROTOCOL_V1.to_string(),
|
||||
);
|
||||
}
|
||||
|
||||
// A previous coordinator can stop after a minority publication without
|
||||
// undo. A fresh final directory keeps this attempt separate from old
|
||||
|
||||
@@ -41,24 +41,23 @@ use super::super::{
|
||||
ObjectKey, ObjectLockConfigSnapshot, ObjectLockConfigState, ObjectOptions, ObjectReader, ObjectToDelete, OffsetDateTime,
|
||||
Ordering, Pin, PutObjReader, RUSTFS_META_BUCKET, RUSTFS_META_TMP_BUCKET, ReadPathPlan, ReaderImpl, ReplicateDecision,
|
||||
ReplicationObjectBridge, Result, SET_DISK_COMMIT_TAIL_WARN_THRESHOLD_MS, SLASH_SEPARATOR, SUFFIX_ACTUAL_SIZE,
|
||||
SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, SUFFIX_RESTORE_COPYBACK, SUFFIX_RESTORE_OPERATION_ID,
|
||||
SUFFIX_RESTORE_WORKER_LOCK, SetDisks, SmallWritePath, StorageError, TRANSITION_COMPLETE, UpdateMetadataOpts, Uuid,
|
||||
WriteLayout, adaptive_duplex_buffer_size, build_get_object_info, build_inline_bitrot_readers,
|
||||
build_inline_bitrot_readers_from_refs, can_try_inline_data_shards_direct, check_object_lock_delete,
|
||||
check_object_lock_for_deletion_with_state, check_object_lock_retention_update, classify_get_codec_streaming_object_class,
|
||||
classify_put_write_path, classify_storage_error, collect_inline_data_shard_fileinfos_by_index, contains_key_str,
|
||||
create_bitrot_writer, debug, delete_file_info_version_id, disk, ensure_delete_commit_locks_held, error,
|
||||
explicit_delete_removed_marker, finish_set_disk_read_lock, get_codec_streaming_reader_gate_with_plan,
|
||||
get_object_body_cache_hook, get_raw_etag, get_small_object_direct_memory_decision_with_threshold_and_plan,
|
||||
get_small_object_direct_memory_threshold, get_stage_timer_if_enabled, get_str,
|
||||
get_transitioned_object_reader_with_tier_manager, inline_admission_shard_size, inline_erasure_shard_file_offset,
|
||||
inline_erasure_shard_size, insert_str, is_deadlock_detection_enabled, is_err_object_not_found, is_err_version_not_found,
|
||||
is_explicit_null_version, is_get_codec_streaming_base_enabled, is_get_small_object_direct_memory_enabled,
|
||||
is_lock_optimization_enabled, issue3031_diag_enabled, join_all, known_put_object_storage_size, path_join_buf,
|
||||
put_restore_opts, record_compression_total_memory, record_get_codec_streaming_gate_decision,
|
||||
record_get_direct_memory_decision, record_get_object_pipeline_failure, record_get_object_pipeline_failure_for_path,
|
||||
record_get_object_reader_path_observation, record_get_stage_duration_if_enabled, record_lock_acquire,
|
||||
reduce_write_quorum_errs, release_materialized_read_lock, replication_write_may_pass_worm_gate,
|
||||
SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, SUFFIX_RESTORE_OPERATION_ID, SUFFIX_RESTORE_WORKER_LOCK, SetDisks,
|
||||
SmallWritePath, StorageError, TRANSITION_COMPLETE, UpdateMetadataOpts, Uuid, WriteLayout, adaptive_duplex_buffer_size,
|
||||
build_get_object_info, build_inline_bitrot_readers, build_inline_bitrot_readers_from_refs, can_try_inline_data_shards_direct,
|
||||
check_object_lock_delete, check_object_lock_for_deletion_with_state, check_object_lock_retention_update,
|
||||
classify_get_codec_streaming_object_class, classify_put_write_path, classify_storage_error,
|
||||
collect_inline_data_shard_fileinfos_by_index, contains_key_str, create_bitrot_writer, debug, delete_file_info_version_id,
|
||||
disk, ensure_delete_commit_locks_held, error, explicit_delete_removed_marker, finish_set_disk_read_lock,
|
||||
get_codec_streaming_reader_gate_with_plan, get_object_body_cache_hook, get_raw_etag,
|
||||
get_small_object_direct_memory_decision_with_threshold_and_plan, get_small_object_direct_memory_threshold,
|
||||
get_stage_timer_if_enabled, get_str, get_transitioned_object_reader_with_tier_manager, inline_admission_shard_size,
|
||||
inline_erasure_shard_file_offset, inline_erasure_shard_size, insert_str, is_deadlock_detection_enabled,
|
||||
is_err_object_not_found, is_err_version_not_found, is_explicit_null_version, is_get_codec_streaming_base_enabled,
|
||||
is_get_small_object_direct_memory_enabled, is_lock_optimization_enabled, issue3031_diag_enabled, join_all,
|
||||
known_put_object_storage_size, path_join_buf, put_restore_opts, record_compression_total_memory,
|
||||
record_get_codec_streaming_gate_decision, record_get_direct_memory_decision, record_get_object_pipeline_failure,
|
||||
record_get_object_pipeline_failure_for_path, record_get_object_reader_path_observation, record_get_stage_duration_if_enabled,
|
||||
record_lock_acquire, reduce_write_quorum_errs, release_materialized_read_lock, replication_write_may_pass_worm_gate,
|
||||
republishes_data_movement_delete_marker, require_restore_operation_id, resolve_delete_version_state,
|
||||
resolve_tiered_decommission_write_quorum_result, resolve_write_layout, restore_commit_operation_id_from_metadata,
|
||||
restore_operation_id_from_metadata, send_event, set_disk_delete_creates_delete_marker,
|
||||
@@ -2338,8 +2337,6 @@ fn is_restore_control_metadata(key: &str) -> bool {
|
||||
.is_some_and(|remainder| remainder.is_empty())
|
||||
|| rustfs_utils::http::internal_key_strip_suffix_prefix(key, SUFFIX_RESTORE_WORKER_LOCK)
|
||||
.is_some_and(|remainder| remainder.is_empty())
|
||||
|| rustfs_utils::http::internal_key_strip_suffix_prefix(key, SUFFIX_RESTORE_COPYBACK)
|
||||
.is_some_and(|remainder| remainder.is_empty())
|
||||
}
|
||||
|
||||
fn restore_metadata_update_preserves_protected_metadata(
|
||||
@@ -4284,13 +4281,7 @@ impl SetDisks {
|
||||
// state.
|
||||
merge_replication_metadata_lww(&mut user_defined, &HashMap::new(), opts);
|
||||
}
|
||||
// Pending copy-back keeps its recovery identity until finalization.
|
||||
if expected_restore_operation_id.is_some()
|
||||
&& !user_defined
|
||||
.get(metadata_keys::RESTORE)
|
||||
.and_then(|value| rustfs_filemeta::parse_restore_obj_status(value).ok())
|
||||
.is_some_and(|status| status.is_restore_in_progress == Some(true))
|
||||
{
|
||||
if expected_restore_operation_id.is_some() {
|
||||
rustfs_utils::http::metadata_compat::remove_str(&mut user_defined, SUFFIX_RESTORE_OPERATION_ID);
|
||||
rustfs_utils::http::metadata_compat::remove_str(&mut user_defined, SUFFIX_RESTORE_WORKER_LOCK);
|
||||
}
|
||||
@@ -4674,29 +4665,19 @@ impl SetDisks {
|
||||
inline_proof.to_owned(),
|
||||
);
|
||||
}
|
||||
if opts.restore_copyback {
|
||||
insert_str(
|
||||
&mut user_defined,
|
||||
SUFFIX_RESTORE_COPYBACK,
|
||||
rustfs_utils::http::metadata_compat::RESTORE_COPYBACK_PROTOCOL_V1.to_string(),
|
||||
);
|
||||
}
|
||||
fi.metadata = user_defined;
|
||||
let put_tier_free_version_id = if fi.version_id.is_none_or(|id| id.is_nil())
|
||||
&& !opts.data_movement
|
||||
&& expected_restore_operation_id.is_none()
|
||||
&& !opts.restore_copyback
|
||||
{
|
||||
// Current disks publish the same cleanup owner alongside a
|
||||
// replaced null version. Keep the intent outside metadata:
|
||||
// older disks persist unknown metadata keys, which splits
|
||||
// the replacement's read-quorum identity during an upgrade.
|
||||
let free_version_id = Uuid::new_v4();
|
||||
fi.overwrite_tier_free_version_id = Some(free_version_id);
|
||||
Some(free_version_id)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let put_tier_free_version_id =
|
||||
if fi.version_id.is_none_or(|id| id.is_nil()) && !opts.data_movement && expected_restore_operation_id.is_none() {
|
||||
// Current disks publish the same cleanup owner alongside a
|
||||
// replaced null version. Keep the intent outside metadata:
|
||||
// older disks persist unknown metadata keys, which splits
|
||||
// the replacement's read-quorum identity during an upgrade.
|
||||
let free_version_id = Uuid::new_v4();
|
||||
fi.overwrite_tier_free_version_id = Some(free_version_id);
|
||||
Some(free_version_id)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
fi.mod_time = mod_time;
|
||||
fi.size = w_size as i64;
|
||||
fi.versioned = opts.versioned || opts.version_suspended;
|
||||
@@ -11337,14 +11318,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
if fi.transition_status == TRANSITION_COMPLETE {
|
||||
return Ok(());
|
||||
}
|
||||
if fi
|
||||
.metadata
|
||||
.get(metadata_keys::RESTORE)
|
||||
.and_then(|value| rustfs_filemeta::parse_restore_obj_status(value).ok())
|
||||
.is_some_and(|status| status.is_restore_in_progress == Some(true))
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
/*if fi.xlv1 {
|
||||
if let Err(err) = self.heal_object(bucket, object, "", &HealOpts {no_lock: true, ..Default::default()}) {
|
||||
return err.expect("err");
|
||||
@@ -11849,9 +11823,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
let actual_fi = actual.fi();
|
||||
|
||||
let oi = ObjectInfo::from_file_info(actual_fi, bucket, object, opts.versioned || opts.version_suspended);
|
||||
let mut restore_cleanup_info = oi.clone();
|
||||
let restore_result: Result<()> = async {
|
||||
crate::services::notification_sys::require_restore_copyback_fleet_capability().await?;
|
||||
// The tier reader releases its own lease at EOF, before PUT or
|
||||
// CompleteMultipartUpload necessarily reaches metadata quorum. Keep a
|
||||
// second exact-generation lease for the whole restore so the final
|
||||
@@ -11930,7 +11902,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
.put_object(bucket, object, &mut p_reader, &ropts)
|
||||
.await
|
||||
.map_err(|err| to_object_err(err, vec![bucket, object]))?;
|
||||
restore_cleanup_info = restored_info.clone();
|
||||
let restored_info = self_.finalize_restore_metadata(bucket, object, &restored_info, &opts).await?;
|
||||
send_event(EventArgs {
|
||||
event_name: EventName::ObjectRestoreCompleted.as_str().to_string(),
|
||||
@@ -12050,7 +12021,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
bucket_lifecycle_lock_fence: opts.bucket_lifecycle_lock_fence.clone(),
|
||||
user_defined: restore_commit_metadata,
|
||||
no_lock: false,
|
||||
restore_copyback: true,
|
||||
decommission_capacity_admission: opts.decommission_capacity_admission.clone(),
|
||||
// The composite ETag would otherwise be recomputed from the
|
||||
// parts as they were written back, which for an encrypted or
|
||||
@@ -12075,7 +12045,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
restore_cleanup_info = restored_info.clone();
|
||||
let restored_info = self_.finalize_restore_metadata(bucket, object, &restored_info, opts).await?;
|
||||
send_event(EventArgs {
|
||||
event_name: EventName::ObjectRestoreCompleted.as_str().to_string(),
|
||||
@@ -12089,10 +12058,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
}
|
||||
.await;
|
||||
if let Err(primary_error) = restore_result {
|
||||
if let Err(cleanup_error) = self_
|
||||
.update_restore_metadata(bucket, object, &restore_cleanup_info, opts)
|
||||
.await
|
||||
{
|
||||
if let Err(cleanup_error) = self_.update_restore_metadata(bucket, object, &oi, opts).await {
|
||||
warn!(
|
||||
event = EVENT_LIFECYCLE_RESTORE_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -13065,8 +13031,8 @@ mod replication_quota_safety_tests {
|
||||
let suspended_opts = ObjectOptions {
|
||||
version_suspended: true,
|
||||
delete_replication_config_snapshot: Some(Arc::new(DeleteReplicationConfigSnapshot::from_configs_for_test(
|
||||
rustfs_gateway_types::persistence::PersistedVersioningConfiguration {
|
||||
status: Some("Suspended".to_owned()),
|
||||
s3s::dto::VersioningConfiguration {
|
||||
status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::SUSPENDED)),
|
||||
..Default::default()
|
||||
},
|
||||
None,
|
||||
@@ -13699,11 +13665,11 @@ mod replication_lww_tests {
|
||||
|
||||
fn default_retention_snapshot(mode: &'static str) -> Arc<ObjectLockConfigSnapshot> {
|
||||
Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::Configured {
|
||||
config: rustfs_gateway_types::persistence::PersistedObjectLockConfiguration {
|
||||
object_lock_enabled: Some("Enabled".to_owned()),
|
||||
rule: Some(rustfs_gateway_types::persistence::PersistedObjectLockRule {
|
||||
default_retention: Some(rustfs_gateway_types::persistence::PersistedDefaultRetention {
|
||||
mode: Some(mode.to_owned()),
|
||||
config: s3s::dto::ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(s3s::dto::ObjectLockEnabled::from_static(s3s::dto::ObjectLockEnabled::ENABLED)),
|
||||
rule: Some(s3s::dto::ObjectLockRule {
|
||||
default_retention: Some(s3s::dto::DefaultRetention {
|
||||
mode: Some(s3s::dto::ObjectLockRetentionMode::from_static(mode)),
|
||||
days: Some(1),
|
||||
years: None,
|
||||
}),
|
||||
@@ -13720,7 +13686,10 @@ mod replication_lww_tests {
|
||||
/// timestamp the write passes and LWW judges it.
|
||||
#[tokio::test]
|
||||
async fn inbound_without_retention_timestamp_stays_rejected_under_bucket_default_retention() {
|
||||
for mode in ["COMPLIANCE", "GOVERNANCE"] {
|
||||
for mode in [
|
||||
s3s::dto::ObjectLockRetentionMode::COMPLIANCE,
|
||||
s3s::dto::ObjectLockRetentionMode::GOVERNANCE,
|
||||
] {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "lww-locked-default-retention";
|
||||
let object = "object";
|
||||
@@ -15987,7 +15956,7 @@ mod transition_commit_failure_tests {
|
||||
use crate::services::tier::tier::TierConfigMgr;
|
||||
use http::HeaderMap;
|
||||
use rustfs_filemeta::{RestoreStatusOps as _, parse_restore_obj_status};
|
||||
use rustfs_lifecycle::RestoreRequest;
|
||||
use s3s::dto::RestoreRequest;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
pub(super) fn restore_operation_id_metadata(operation_id: Uuid) -> HashMap<String, String> {
|
||||
@@ -18526,7 +18495,7 @@ mod transition_upload_integrity_tests {
|
||||
for disk in disk_stores {
|
||||
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
||||
}
|
||||
let mut source = PutObjReader::from_vec(b"restore source body".repeat(64 * 1024));
|
||||
let mut source = PutObjReader::from_vec(b"restore source body".repeat(1024));
|
||||
set_disks
|
||||
.put_object(bucket, object, &mut source, &ObjectOptions::default())
|
||||
.await
|
||||
@@ -18543,7 +18512,7 @@ mod transition_upload_integrity_tests {
|
||||
.await
|
||||
.expect("pending restore metadata should be installed");
|
||||
|
||||
let mut restored_reader = PutObjReader::from_vec(b"restored body".repeat(64 * 1024));
|
||||
let mut restored_reader = PutObjReader::from_vec(b"restored body".repeat(1024));
|
||||
set_disks
|
||||
.put_object(
|
||||
bucket,
|
||||
@@ -18551,19 +18520,18 @@ mod transition_upload_integrity_tests {
|
||||
&mut restored_reader,
|
||||
&ObjectOptions {
|
||||
user_defined: restore_metadata(operation_id, true),
|
||||
restore_copyback: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("matching restore copy-back should retain its pending operation id")
|
||||
.expect("matching restore commit should consume its operation id")
|
||||
}
|
||||
|
||||
fn restore_finalize_options(operation_id: Uuid) -> ObjectOptions {
|
||||
ObjectOptions {
|
||||
user_defined: restore_operation_id_metadata(operation_id),
|
||||
transition: TransitionOptions {
|
||||
restore_request: rustfs_lifecycle::RestoreRequest {
|
||||
restore_request: s3s::dto::RestoreRequest {
|
||||
days: Some(1),
|
||||
..Default::default()
|
||||
},
|
||||
@@ -18573,87 +18541,17 @@ mod transition_upload_integrity_tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn restore_cleanup_reclaims_committed_copyback_data_dir() {
|
||||
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "restore-cleanup-copyback-data-dir-bucket";
|
||||
let object = "object.bin";
|
||||
let operation_id = Uuid::new_v4();
|
||||
let restored = write_committed_restore(&set_disks, &disk_stores, bucket, object, operation_id).await;
|
||||
let data_dir = restored
|
||||
.data_dir
|
||||
.expect("committed copy-back must have a local data directory");
|
||||
for temp_dir in &temp_dirs {
|
||||
assert!(
|
||||
temp_dir.path().join(bucket).join(object).join(data_dir.to_string()).exists(),
|
||||
"committed copy-back must retain its local data directory before cleanup"
|
||||
);
|
||||
}
|
||||
|
||||
set_disks
|
||||
.update_restore_metadata(
|
||||
bucket,
|
||||
object,
|
||||
&restored,
|
||||
&ObjectOptions {
|
||||
user_defined: restore_operation_id_metadata(operation_id),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("matching restore cleanup should remove the failed copy-back");
|
||||
|
||||
for temp_dir in &temp_dirs {
|
||||
assert!(
|
||||
!temp_dir.path().join(bucket).join(object).join(data_dir.to_string()).exists(),
|
||||
"failed copy-back cleanup must reclaim its committed local data directory"
|
||||
);
|
||||
}
|
||||
let cleaned = set_disks
|
||||
.get_object_info(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("cleaned object metadata should remain readable");
|
||||
assert!(!cleaned.user_defined.contains_key(rustfs_filemeta::metadata_keys::RESTORE));
|
||||
assert!(
|
||||
rustfs_utils::http::metadata_compat::get_consistent_str(
|
||||
cleaned.user_defined.as_ref(),
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_COPYBACK,
|
||||
)
|
||||
.is_none(),
|
||||
"failed copy-back cleanup must remove the local routing marker"
|
||||
);
|
||||
}
|
||||
|
||||
async fn assert_committed_restore_remains_pending(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, operation_id: Uuid) {
|
||||
async fn assert_committed_restore_remains_pending(set_disks: &Arc<SetDisks>, bucket: &str, object: &str) {
|
||||
let current = set_disks
|
||||
.get_object_info(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("pending restore metadata should remain readable");
|
||||
assert_eq!(
|
||||
restore_operation_id_from_metadata(current.user_defined.as_ref()).expect("operation id metadata should parse"),
|
||||
Some(operation_id),
|
||||
"unfinished restore copy-back must retain its exact operation id"
|
||||
assert!(
|
||||
restore_operation_id_from_metadata(current.user_defined.as_ref())
|
||||
.expect("operation id metadata should parse")
|
||||
.is_none(),
|
||||
"successful restore commit must have consumed the operation id"
|
||||
);
|
||||
let operation_id = operation_id.to_string();
|
||||
for (suffix, expected) in [
|
||||
(rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_OPERATION_ID, operation_id.as_str()),
|
||||
(
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_WORKER_LOCK,
|
||||
rustfs_utils::http::metadata_compat::RESTORE_WORKER_LOCK_PROTOCOL_V1,
|
||||
),
|
||||
] {
|
||||
for prefix in [
|
||||
rustfs_utils::http::RUSTFS_INTERNAL_PREFIX,
|
||||
rustfs_utils::http::MINIO_INTERNAL_PREFIX,
|
||||
] {
|
||||
assert_eq!(
|
||||
current.user_defined.get(&format!("{prefix}{suffix}")).map(String::as_str),
|
||||
Some(expected),
|
||||
"unfinished restore copy-back must retain each recovery marker alias"
|
||||
);
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
rustfs_filemeta::parse_restore_obj_status(
|
||||
current
|
||||
@@ -18711,7 +18609,7 @@ mod transition_upload_integrity_tests {
|
||||
if error.kind() == std::io::ErrorKind::Other
|
||||
&& error.to_string() == "restore finalization lock lost before metadata update"
|
||||
));
|
||||
assert_committed_restore_remains_pending(&set_disks, bucket, object, operation_id).await;
|
||||
assert_committed_restore_remains_pending(&set_disks, bucket, object).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -18748,7 +18646,7 @@ mod transition_upload_integrity_tests {
|
||||
..
|
||||
}
|
||||
));
|
||||
assert_committed_restore_remains_pending(&set_disks, bucket, object, operation_id).await;
|
||||
assert_committed_restore_remains_pending(&set_disks, bucket, object).await;
|
||||
}
|
||||
|
||||
async fn assert_local_source_intact(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, payload: &[u8]) {
|
||||
@@ -19298,7 +19196,7 @@ mod transition_upload_integrity_tests {
|
||||
no_lock: true,
|
||||
part_number: Some(1),
|
||||
transition: TransitionOptions {
|
||||
restore_request: rustfs_lifecycle::RestoreRequest {
|
||||
restore_request: s3s::dto::RestoreRequest {
|
||||
days: Some(1),
|
||||
..Default::default()
|
||||
},
|
||||
@@ -23771,20 +23669,29 @@ mod put_object_tmp_cleanup_tests {
|
||||
(
|
||||
"compliance",
|
||||
HashMap::from([
|
||||
(metadata_keys::OBJECT_LOCK_MODE.to_string(), "COMPLIANCE".to_string()),
|
||||
(
|
||||
metadata_keys::OBJECT_LOCK_MODE.to_string(),
|
||||
s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(),
|
||||
),
|
||||
(metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(), retain_until.clone()),
|
||||
]),
|
||||
),
|
||||
(
|
||||
"governance",
|
||||
HashMap::from([
|
||||
(metadata_keys::OBJECT_LOCK_MODE.to_string(), "GOVERNANCE".to_string()),
|
||||
(
|
||||
metadata_keys::OBJECT_LOCK_MODE.to_string(),
|
||||
s3s::dto::ObjectLockRetentionMode::GOVERNANCE.to_string(),
|
||||
),
|
||||
(metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(), retain_until),
|
||||
]),
|
||||
),
|
||||
(
|
||||
"legal-hold",
|
||||
HashMap::from([(metadata_keys::OBJECT_LOCK_LEGAL_HOLD.to_string(), "ON".to_string())]),
|
||||
HashMap::from([(
|
||||
metadata_keys::OBJECT_LOCK_LEGAL_HOLD.to_string(),
|
||||
s3s::dto::ObjectLockLegalHoldStatus::ON.to_string(),
|
||||
)]),
|
||||
),
|
||||
];
|
||||
|
||||
@@ -23985,11 +23892,13 @@ mod put_object_tmp_cleanup_tests {
|
||||
.expect("versioned PUT should return a version ID")
|
||||
.to_string();
|
||||
let default_compliance = ObjectLockConfigState::Configured {
|
||||
config: rustfs_gateway_types::persistence::PersistedObjectLockConfiguration {
|
||||
object_lock_enabled: Some("Enabled".to_owned()),
|
||||
rule: Some(rustfs_gateway_types::persistence::PersistedObjectLockRule {
|
||||
default_retention: Some(rustfs_gateway_types::persistence::PersistedDefaultRetention {
|
||||
mode: Some("COMPLIANCE".to_owned()),
|
||||
config: s3s::dto::ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(s3s::dto::ObjectLockEnabled::from_static(s3s::dto::ObjectLockEnabled::ENABLED)),
|
||||
rule: Some(s3s::dto::ObjectLockRule {
|
||||
default_retention: Some(s3s::dto::DefaultRetention {
|
||||
mode: Some(s3s::dto::ObjectLockRetentionMode::from_static(
|
||||
s3s::dto::ObjectLockRetentionMode::COMPLIANCE,
|
||||
)),
|
||||
days: Some(1),
|
||||
years: None,
|
||||
}),
|
||||
@@ -24149,7 +24058,10 @@ mod put_object_tmp_cleanup_tests {
|
||||
&ObjectOptions {
|
||||
version_id: Some(destination_version.clone()),
|
||||
versioned: true,
|
||||
eval_metadata: Some(HashMap::from([(metadata_keys::OBJECT_LOCK_LEGAL_HOLD.to_string(), "ON".to_string())])),
|
||||
eval_metadata: Some(HashMap::from([(
|
||||
metadata_keys::OBJECT_LOCK_LEGAL_HOLD.to_string(),
|
||||
s3s::dto::ObjectLockLegalHoldStatus::ON.to_string(),
|
||||
)])),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
@@ -25057,7 +24969,7 @@ mod delete_objects_lock_gating_tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_objects_derives_per_object_versioning_from_the_request_snapshot() {
|
||||
use rustfs_gateway_types::persistence::PersistedVersioningConfiguration as VersioningConfiguration;
|
||||
use s3s::dto::{BucketVersioningStatus, ExcludedPrefix, VersioningConfiguration};
|
||||
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "batch-versioning-snapshot-bucket";
|
||||
@@ -25069,8 +24981,10 @@ mod delete_objects_lock_gating_tests {
|
||||
put_plain_object(&set_disks, bucket, "archive/unversioned-object").await;
|
||||
let snapshot = Arc::new(DeleteReplicationConfigSnapshot::from_configs_for_test(
|
||||
VersioningConfiguration {
|
||||
status: Some("Enabled".to_owned()),
|
||||
excluded_prefixes: Some(vec![Some("archive/".to_string())]),
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
excluded_prefixes: Some(vec![ExcludedPrefix {
|
||||
prefix: Some("archive/".to_string()),
|
||||
}]),
|
||||
..Default::default()
|
||||
},
|
||||
None,
|
||||
@@ -25116,13 +25030,11 @@ mod delete_objects_lock_gating_tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn batch_version_delete_uses_tags_read_under_the_delete_lock() {
|
||||
use rustfs_gateway_types::dto::Status;
|
||||
use rustfs_gateway_types::persistence::PersistedVersioningConfiguration as VersioningConfiguration;
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedReplicationConfiguration, PersistedReplicationDestination, PersistedReplicationFilter,
|
||||
PersistedReplicationRule, PersistedReplicationStatus, PersistedReplicationTag,
|
||||
};
|
||||
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
|
||||
use s3s::dto::{
|
||||
BucketVersioningStatus, DeleteReplication, DeleteReplicationStatus, Destination, ReplicationConfiguration,
|
||||
ReplicationRule, ReplicationRuleFilter, ReplicationRuleStatus, Tag, VersioningConfiguration,
|
||||
};
|
||||
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "batch-delete-locked-tags";
|
||||
@@ -25158,23 +25070,23 @@ mod delete_objects_lock_gating_tests {
|
||||
|
||||
let snapshot = Arc::new(DeleteReplicationConfigSnapshot::from_configs_for_test(
|
||||
VersioningConfiguration {
|
||||
status: Some("Enabled".to_owned()),
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
..Default::default()
|
||||
},
|
||||
Some(PersistedReplicationConfiguration {
|
||||
Some(ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![PersistedReplicationRule {
|
||||
rules: vec![ReplicationRule {
|
||||
delete_marker_replication: None,
|
||||
delete_replication: Some(PersistedReplicationStatus {
|
||||
status: Status::ENABLED.to_string(),
|
||||
delete_replication: Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
}),
|
||||
destination: PersistedReplicationDestination {
|
||||
destination: Destination {
|
||||
bucket: arn.to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
existing_object_replication: None,
|
||||
filter: Some(PersistedReplicationFilter {
|
||||
tag: Some(PersistedReplicationTag {
|
||||
filter: Some(ReplicationRuleFilter {
|
||||
tag: Some(Tag {
|
||||
key: Some("generation".to_string()),
|
||||
value: Some("locked".to_string()),
|
||||
}),
|
||||
@@ -25184,7 +25096,7 @@ mod delete_objects_lock_gating_tests {
|
||||
prefix: Some(String::new()),
|
||||
priority: Some(1),
|
||||
source_selection_criteria: None,
|
||||
status: Status::ENABLED.to_string(),
|
||||
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
|
||||
}],
|
||||
}),
|
||||
));
|
||||
@@ -25222,11 +25134,9 @@ mod delete_objects_lock_gating_tests {
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn lifecycle_delete_all_history_records_exact_replication_purge() {
|
||||
use rustfs_gateway_types::dto::Status;
|
||||
use rustfs_gateway_types::persistence::PersistedVersioningConfiguration as VersioningConfiguration;
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedReplicationConfiguration, PersistedReplicationDestination, PersistedReplicationRule,
|
||||
PersistedReplicationStatus,
|
||||
use s3s::dto::{
|
||||
BucketVersioningStatus, DeleteReplication, DeleteReplicationStatus, Destination, ReplicationConfiguration,
|
||||
ReplicationRule, ReplicationRuleStatus, VersioningConfiguration,
|
||||
};
|
||||
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
@@ -25268,17 +25178,17 @@ mod delete_objects_lock_gating_tests {
|
||||
|
||||
let snapshot = Arc::new(DeleteReplicationConfigSnapshot::from_configs_for_test(
|
||||
VersioningConfiguration {
|
||||
status: Some("Enabled".to_owned()),
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
..Default::default()
|
||||
},
|
||||
Some(PersistedReplicationConfiguration {
|
||||
Some(ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![PersistedReplicationRule {
|
||||
rules: vec![ReplicationRule {
|
||||
delete_marker_replication: None,
|
||||
delete_replication: Some(PersistedReplicationStatus {
|
||||
status: Status::ENABLED.to_string(),
|
||||
delete_replication: Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
}),
|
||||
destination: PersistedReplicationDestination {
|
||||
destination: Destination {
|
||||
bucket: arn.to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
@@ -25288,7 +25198,7 @@ mod delete_objects_lock_gating_tests {
|
||||
prefix: Some(String::new()),
|
||||
priority: Some(1),
|
||||
source_selection_criteria: None,
|
||||
status: Status::ENABLED.to_string(),
|
||||
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
|
||||
}],
|
||||
}),
|
||||
));
|
||||
@@ -25330,11 +25240,10 @@ mod delete_objects_lock_gating_tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn synthetic_directory_delete_uses_decoded_prefix_and_marker_switch() {
|
||||
use rustfs_gateway_types::dto::Status;
|
||||
use rustfs_gateway_types::persistence::PersistedVersioningConfiguration as VersioningConfiguration;
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedOptionalReplicationStatus, PersistedReplicationConfiguration, PersistedReplicationDestination,
|
||||
PersistedReplicationFilter, PersistedReplicationRule, PersistedReplicationStatus,
|
||||
use s3s::dto::{
|
||||
BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, DeleteReplication,
|
||||
DeleteReplicationStatus, Destination, ReplicationConfiguration, ReplicationRule, ReplicationRuleFilter,
|
||||
ReplicationRuleStatus, VersioningConfiguration,
|
||||
};
|
||||
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
@@ -25348,24 +25257,24 @@ mod delete_objects_lock_gating_tests {
|
||||
|
||||
let snapshot = Arc::new(DeleteReplicationConfigSnapshot::from_configs_for_test(
|
||||
VersioningConfiguration {
|
||||
status: Some("Enabled".to_owned()),
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
..Default::default()
|
||||
},
|
||||
Some(PersistedReplicationConfiguration {
|
||||
Some(ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![PersistedReplicationRule {
|
||||
delete_marker_replication: Some(PersistedOptionalReplicationStatus {
|
||||
status: Some(Status::ENABLED.to_string()),
|
||||
rules: vec![ReplicationRule {
|
||||
delete_marker_replication: Some(DeleteMarkerReplication {
|
||||
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)),
|
||||
}),
|
||||
delete_replication: Some(PersistedReplicationStatus {
|
||||
status: Status::DISABLED.to_string(),
|
||||
delete_replication: Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::DISABLED),
|
||||
}),
|
||||
destination: PersistedReplicationDestination {
|
||||
destination: Destination {
|
||||
bucket: arn.to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
existing_object_replication: None,
|
||||
filter: Some(PersistedReplicationFilter {
|
||||
filter: Some(ReplicationRuleFilter {
|
||||
prefix: Some("photos/".to_string()),
|
||||
..Default::default()
|
||||
}),
|
||||
@@ -25373,7 +25282,7 @@ mod delete_objects_lock_gating_tests {
|
||||
prefix: None,
|
||||
priority: Some(1),
|
||||
source_selection_criteria: None,
|
||||
status: Status::ENABLED.to_string(),
|
||||
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
|
||||
}],
|
||||
}),
|
||||
));
|
||||
@@ -25504,7 +25413,10 @@ mod delete_objects_lock_gating_tests {
|
||||
// stat and the #4297 rejection.
|
||||
let retain_until = OffsetDateTime::now_utc() + Duration::from_secs(60 * 60 * 24 * 30);
|
||||
let mut user_defined = HashMap::new();
|
||||
user_defined.insert(metadata_keys::OBJECT_LOCK_MODE.to_string(), "COMPLIANCE".to_string());
|
||||
user_defined.insert(
|
||||
metadata_keys::OBJECT_LOCK_MODE.to_string(),
|
||||
s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(),
|
||||
);
|
||||
user_defined.insert(
|
||||
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
|
||||
retain_until
|
||||
@@ -27107,10 +27019,9 @@ mod batch_delete_marker_tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn batch_delete_explicit_marker_does_not_wait_for_pending_replication() {
|
||||
use rustfs_gateway_types::dto::Status;
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedReplicationConfiguration, PersistedReplicationDestination, PersistedReplicationRule,
|
||||
PersistedReplicationStatus, PersistedVersioningConfiguration,
|
||||
use s3s::dto::{
|
||||
BucketVersioningStatus, DeleteReplication, DeleteReplicationStatus, Destination, ReplicationConfiguration,
|
||||
ReplicationRule, ReplicationRuleStatus, VersioningConfiguration,
|
||||
};
|
||||
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
@@ -27118,18 +27029,18 @@ mod batch_delete_marker_tests {
|
||||
let (data_versions, markers) = marker_history(&store, bucket).await;
|
||||
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
|
||||
let snapshot = Arc::new(DeleteReplicationConfigSnapshot::from_configs_for_test(
|
||||
PersistedVersioningConfiguration {
|
||||
status: Some("Enabled".to_owned()),
|
||||
VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
..Default::default()
|
||||
},
|
||||
Some(PersistedReplicationConfiguration {
|
||||
Some(ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![PersistedReplicationRule {
|
||||
rules: vec![ReplicationRule {
|
||||
delete_marker_replication: None,
|
||||
delete_replication: Some(PersistedReplicationStatus {
|
||||
status: Status::ENABLED.to_string(),
|
||||
delete_replication: Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
}),
|
||||
destination: PersistedReplicationDestination {
|
||||
destination: Destination {
|
||||
bucket: arn.to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
@@ -27139,7 +27050,7 @@ mod batch_delete_marker_tests {
|
||||
prefix: Some(String::new()),
|
||||
priority: Some(1),
|
||||
source_selection_criteria: None,
|
||||
status: Status::ENABLED.to_string(),
|
||||
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
|
||||
}],
|
||||
}),
|
||||
));
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user