mirror of
https://github.com/rustfs/rustfs.git
synced 2026-10-30 15:55:55 +00:00
Compare commits
68 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| cfe75fddd9 | |||
| fe7854500c | |||
| 75ab3052a9 | |||
| 368cdfd214 | |||
| 90a1eb814e | |||
| b97a952cda | |||
| 0a8e78238b | |||
| fe04c79894 | |||
| 5c4a285a7b | |||
| 9a8faf7c86 | |||
| 0a2f58c7a9 | |||
| 4d97501327 | |||
| 3b2af12c64 | |||
| c73ddde848 | |||
| ef72a37fd7 | |||
| e46cf750dd | |||
| fe955fa3ee | |||
| 64578c067e | |||
| 78715f0921 | |||
| 3284aafd73 | |||
| 9286442f87 | |||
| b8aad1c168 | |||
| 5fcb229760 | |||
| 94baef7462 | |||
| 60d90f085a | |||
| dfa5d50565 | |||
| a902886737 | |||
| 8e53b87421 | |||
| f7631e91b1 | |||
| f9bc4e921f | |||
| 781584307a | |||
| 59d641a5fe | |||
| c618b57870 | |||
| bbe1d54098 | |||
| 9e8ea4dcdb | |||
| 63cf8e2f2f | |||
| b7fe40f4b3 | |||
| 5b872b8e56 | |||
| ca5be1ef95 | |||
| 9373e8fa3e | |||
| cb32965522 | |||
| fb6cf49a67 | |||
| 8c97049e68 | |||
| 17d4fe27cc | |||
| fb0c7972b2 | |||
| eba7967755 | |||
| 451c1c31ad | |||
| 8780dd670e | |||
| f29bf2c902 | |||
| fd03436090 | |||
| 1f1e7b1c97 | |||
| 45a952609e | |||
| 323190a95a | |||
| 38cdda9292 | |||
| 85c84fa73d | |||
| 51402087c5 | |||
| a994984fe2 | |||
| 48eb0e402c | |||
| 446479791e | |||
| 99af59a914 | |||
| b77bd26874 | |||
| 956c1cf027 | |||
| 8b9d44110e | |||
| 44ce8d114f | |||
| 769b0d7578 | |||
| 03e775947d | |||
| 25e73d3bd6 | |||
| a464b45367 |
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=b9438fd1b82558f1fcd7e5a19dc0ceb0b34b5357d4ce99b7033de3ef019333ab
|
||||
sha256-linux=d4622517c13283247a022f49962c2cdb69e5ed0e13874fb55e0ab34fb923c044
|
||||
sha256-darwin=98dff0961c6eb21d6128bfd2d4a6100a624ddf1fb7db05dba360a5d3d0a310c2
|
||||
sha256-linux=890f5925b87e1bf3710c356253f3b2276d901e156df9ccc04ec9315374de6d22
|
||||
|
||||
@@ -1 +1 @@
|
||||
sha256=58df0d240cd74b916ce8f094ed1d8ad3e136f2f9bac157a155192a8786dae65c
|
||||
sha256=ba8f392b617cf9f2338142233a417e43927b28096635e628233ce8a60cf49d10
|
||||
|
||||
@@ -70,11 +70,6 @@ s3s-footprint-check: ## Check the s3s dependency footprint ratchet stays frozen
|
||||
@echo "📦 Checking s3s footprint ratchet..."
|
||||
./scripts/check_s3s_footprint.sh
|
||||
|
||||
.PHONY: test-count-ratchet-check
|
||||
test-count-ratchet-check: ## Check no workspace crate lost tests or gained #[ignore] against the committed baseline
|
||||
@echo "🧪 Checking per-crate test-count ratchet..."
|
||||
./scripts/check_test_count_ratchet.sh
|
||||
|
||||
.PHONY: fips-wording-check
|
||||
fips-wording-check: ## Check docs and crates/kms do not over-claim crypto capabilities
|
||||
@echo "📣 Checking cryptographic capability wording guard..."
|
||||
|
||||
@@ -20,11 +20,11 @@ planning-docs-check: ## Check that no planning-type documents are committed
|
||||
./scripts/check_no_planning_docs.sh
|
||||
|
||||
.PHONY: pre-commit
|
||||
pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check test-count-ratchet-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
|
||||
pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
|
||||
@echo "✅ All pre-commit checks passed!"
|
||||
|
||||
.PHONY: pre-pr
|
||||
pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check test-count-ratchet-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check log-analyzer-rules-check offline-enrollment-e2e-check clippy-check test ## Run full pre-PR checks with clippy and tests
|
||||
pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check log-analyzer-rules-check offline-enrollment-e2e-check clippy-check test ## Run full pre-PR checks with clippy and tests
|
||||
@echo "✅ All pre-PR checks passed!"
|
||||
|
||||
.PHONY: dev-check
|
||||
|
||||
@@ -50,9 +50,7 @@ script-tests: ## Run shell script tests
|
||||
./scripts/test_fuzz_runner.sh
|
||||
./scripts/test_python_bin.sh
|
||||
./scripts/check_embedded_secrets.sh --self-test
|
||||
./scripts/check_s3s_footprint.sh --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/codemods/s3_error_to_s3_types.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_e2e_binary.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_migration_gate_evidence.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/ci_gate.py --self-test
|
||||
|
||||
+10
-9
@@ -92,6 +92,12 @@ setup = 'ecstore-large-stack'
|
||||
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
# These handler/store fixtures and the relocated-pool GET compose deep storage
|
||||
# futures in libtest threads; on Linux they overflow the default 2 MiB stack.
|
||||
[[profile.default.scripts]]
|
||||
filter = 'package(rustfs) & (test(/^admin::handlers::table_catalog::tests::/) | test(/^table_catalog::store::tests::/) | test(=app::object::get::tests::execute_get_object_resumes_from_relocated_pool_without_splicing_body))'
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
[[profile.default.scripts]]
|
||||
filter = 'binary(lifecycle_integration_test) | (package(rustfs) & test(/^app::lifecycle_transition_api_test::/))'
|
||||
setup = 'lifecycle-large-stack'
|
||||
@@ -337,6 +343,10 @@ setup = 'ecstore-large-stack'
|
||||
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
[[profile.ci.scripts]]
|
||||
filter = 'package(rustfs) & (test(/^admin::handlers::table_catalog::tests::/) | test(/^table_catalog::store::tests::/) | test(=app::object::get::tests::execute_get_object_resumes_from_relocated_pool_without_splicing_body))'
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
[[profile.ci.scripts]]
|
||||
filter = 'binary(lifecycle_integration_test) | (package(rustfs) & test(/^app::lifecycle_transition_api_test::/))'
|
||||
setup = 'lifecycle-large-stack'
|
||||
@@ -579,14 +589,6 @@ test-group = 'ecstore-serial-flaky'
|
||||
# the PR-lane e2e coverage for scanner usage snapshots consumed by quota and
|
||||
# admin surfaces. It uses the same single-node, random-port, isolated-temp-dir
|
||||
# fixture as the existing smoke modules.
|
||||
#
|
||||
# Dual-stack differential (rustfs/backlog#2740): `dual_stack_differential::`
|
||||
# starts two single-node servers from the same binary, one per
|
||||
# RUSTFS_S3_STACK value, on random ports with isolated temp dirs, and runs one
|
||||
# SDK script covering every operation of `impl S3 for FS` against both. Its
|
||||
# comparator unit tests need no server. It meets the admission criteria
|
||||
# unchanged; the module clause admits a new differential test without another
|
||||
# filter edit.
|
||||
[profile.e2e-smoke]
|
||||
default-filter = """
|
||||
package(e2e_test) & (
|
||||
@@ -594,7 +596,6 @@ default-filter = """
|
||||
| test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/)
|
||||
| test(/^reliant::lifecycle::/)
|
||||
| test(/^reliant::tiering::/)
|
||||
| test(/^dual_stack_differential::/)
|
||||
| test(/^upgrade_compatibility_test::upgrade_write_readiness_tests::/)
|
||||
| test(/^on_demand_migration::(get_basic_test::(get_miss_pulls_inline_and_serves_locally_afterwards|head_miss_answers_from_the_source_without_persisting)|interaction_test::test_odm_admin_config_is_redacted_and_status_counts_match_the_source)$/)
|
||||
)
|
||||
|
||||
@@ -1,37 +0,0 @@
|
||||
# Files that may still reference s3s (`s3s::` or `s3s_sigv4`) when Phase 1 of
|
||||
# the gateway migration ends (rustfs/backlog#2734 task T0.6, rustfs/backlog#2741).
|
||||
# Consumed only by `scripts/check_s3s_footprint.sh --mode allowlist`.
|
||||
#
|
||||
# One entry per line: a repository-relative file path or a git `:(glob)`
|
||||
# pathspec. `**` matches across directories, `*` stays within one path
|
||||
# component, and a directory entry matches everything under it. Blank lines
|
||||
# and lines starting with `#` are ignored; an entry may not contain whitespace,
|
||||
# so a note belongs on its own comment line.
|
||||
#
|
||||
# Every Rust file listed by `git grep -l '\bs3s\(::\|_sigv4\)' -- '*.rs'` that
|
||||
# no entry matches fails the allowlist mode. Task T1.9 switches CI to that
|
||||
# mode; until then `--dry-run` prints the files still outside this list, which
|
||||
# is the Phase 1 tracking list.
|
||||
#
|
||||
# Never add crates/** other than crates/e2e_test: the engine crates leave s3s
|
||||
# entirely, and the script rejects such an entry. Only the s3s-facing edge of
|
||||
# the rustfs binary and the e2e oracle keep it.
|
||||
|
||||
# rustfs binary: the s3s-facing edge that survives Phase 1
|
||||
rustfs/src/storage/ecfs.rs
|
||||
rustfs/src/storage/s3_compat/**
|
||||
rustfs/src/storage/access.rs
|
||||
rustfs/src/auth.rs
|
||||
rustfs/src/error.rs
|
||||
rustfs/src/storage_api.rs
|
||||
rustfs/src/app/storage_api.rs
|
||||
rustfs/src/admin/router.rs
|
||||
rustfs/src/app/metadata_route.rs
|
||||
rustfs/src/server/http.rs
|
||||
rustfs/src/server/layer.rs
|
||||
rustfs/src/server/hybrid.rs
|
||||
rustfs/src/protocols/client.rs
|
||||
|
||||
# e2e oracle: the test target that verifies S3 behaviour against s3s
|
||||
crates/e2e_test/src/fake_s3_target/**
|
||||
crates/e2e_test/src/**
|
||||
@@ -1,17 +0,0 @@
|
||||
# Exact baseline for the direct s3s footprint, consumed only by
|
||||
# scripts/check_s3s_footprint.sh (rustfs/backlog#1677; rustfs/backlog#2734 T0.4).
|
||||
#
|
||||
# Each value is the exact measurement of the tree, not a ceiling. CI fails when a
|
||||
# measured counter differs from its line here in either direction:
|
||||
# - a larger measurement means new code widened the s3s surface: route it
|
||||
# through the gateway abstractions instead; never raise a value here;
|
||||
# - a smaller measurement means the PR shrank the surface: lower the matching
|
||||
# value to the measured number in the same PR.
|
||||
#
|
||||
# files Rust files referencing s3s paths, repo-wide minus crates/e2e_test
|
||||
# s3_error_lines lines holding s3_error!, same scope, minus those the script
|
||||
# header's rule clears (rustfs_s3_types's macro); fails closed
|
||||
# ecstore_files Rust files referencing s3s paths under crates/ecstore/src
|
||||
files=194
|
||||
s3_error_lines=1584
|
||||
ecstore_files=36
|
||||
@@ -1,63 +0,0 @@
|
||||
# Per-crate test-count baseline consumed by scripts/check_test_count_ratchet.sh
|
||||
# (rustfs/backlog#2734 section 1 rule 3; T0.3 in rustfs/backlog#2738).
|
||||
#
|
||||
# One row per workspace member: <package>, tests, ignored, tab-separated.
|
||||
# tests counts #[test], #[tokio::test], #[tokio::test(...)] and #[test_case(...)]
|
||||
# attribute lines under the package directory; ignored counts #[ignore] and
|
||||
# #[ignore = "..."]. Both columns are exact: CI fails when a crate's tests fall
|
||||
# below or rise above its row, or when its ignored count changes. Raise a row in
|
||||
# the PR that adds tests; lowering one is a reviewed statement that tests were
|
||||
# intentionally removed. Regenerate with:
|
||||
# scripts/check_test_count_ratchet.sh --update
|
||||
e2e_test 890 18
|
||||
rustfs 5540 62
|
||||
rustfs-audit 70 0
|
||||
rustfs-checksums 26 0
|
||||
rustfs-common 23 0
|
||||
rustfs-concurrency 27 0
|
||||
rustfs-config 43 0
|
||||
rustfs-connect-symbol-catalog 6 0
|
||||
rustfs-credentials 24 0
|
||||
rustfs-crypto 67 0
|
||||
rustfs-data-usage 54 0
|
||||
rustfs-ecstore 6307 1
|
||||
rustfs-extension-schema 23 0
|
||||
rustfs-filemeta 306 0
|
||||
rustfs-heal 839 3
|
||||
rustfs-heal-contracts 6 0
|
||||
rustfs-iam 371 0
|
||||
rustfs-io-core 49 0
|
||||
rustfs-io-metrics 125 0
|
||||
rustfs-keystone 29 0
|
||||
rustfs-kms 715 25
|
||||
rustfs-license 2 0
|
||||
rustfs-lifecycle 152 0
|
||||
rustfs-lock 135 0
|
||||
rustfs-log-analyzer 87 0
|
||||
rustfs-madmin 209 0
|
||||
rustfs-notify 136 0
|
||||
rustfs-object-capacity 92 0
|
||||
rustfs-object-data-cache 140 0
|
||||
rustfs-obs 365 0
|
||||
rustfs-policy 677 0
|
||||
rustfs-protocols 736 1
|
||||
rustfs-protos 52 0
|
||||
rustfs-replication 200 0
|
||||
rustfs-rio 198 0
|
||||
rustfs-rio-v2 34 6
|
||||
rustfs-s3-client 65 0
|
||||
rustfs-s3-ops 8 0
|
||||
rustfs-s3-types 97 0
|
||||
rustfs-s3select-api 206 0
|
||||
rustfs-s3select-query 188 2
|
||||
rustfs-scanner 971 20
|
||||
rustfs-scanner-metrics 75 0
|
||||
rustfs-security-governance 20 0
|
||||
rustfs-signer 31 0
|
||||
rustfs-storage-api 31 0
|
||||
rustfs-targets 603 26
|
||||
rustfs-test-utils 1 0
|
||||
rustfs-tls-runtime 20 0
|
||||
rustfs-trusted-proxies 68 0
|
||||
rustfs-utils 216 0
|
||||
rustfs-zip 30 0
|
||||
@@ -86,10 +86,6 @@ runs:
|
||||
shell: bash
|
||||
run: ./scripts/check_s3s_footprint.sh
|
||||
|
||||
- name: Check per-crate test-count ratchet
|
||||
shell: bash
|
||||
run: ./scripts/check_test_count_ratchet.sh
|
||||
|
||||
- name: Check cryptographic capability wording
|
||||
shell: bash
|
||||
run: ./scripts/check_fips_wording.sh
|
||||
|
||||
@@ -440,6 +440,17 @@ jobs:
|
||||
fi
|
||||
done
|
||||
|
||||
- name: Verify macOS binary has no dynamic liblzma dependency
|
||||
if: matrix.platform == 'macos'
|
||||
shell: bash
|
||||
run: |
|
||||
binary="target/${{ matrix.target }}/release/rustfs"
|
||||
if otool -L "$binary" | grep -E 'liblzma.*\.dylib'; then
|
||||
echo "macOS release binary must not depend on a dynamic liblzma library" >&2
|
||||
exit 1
|
||||
fi
|
||||
"$binary" --version
|
||||
|
||||
- name: Generate the CPU symbol catalog
|
||||
if: endsWith(matrix.target, '-linux-gnu') && matrix.features == 'pyroscope'
|
||||
shell: bash
|
||||
|
||||
@@ -16,9 +16,7 @@ name: Continuous Integration
|
||||
|
||||
on:
|
||||
push:
|
||||
# gateway/integration receives the daily merge of main as a direct push
|
||||
# (rustfs/backlog#2734), so its pushes must prove the result like main's.
|
||||
branches: [ main, gateway/integration ]
|
||||
branches: [ main ]
|
||||
paths-ignore:
|
||||
- "**.md"
|
||||
- "docs/**"
|
||||
@@ -782,8 +780,6 @@ jobs:
|
||||
needs: [ build-rustfs-debug-binary ]
|
||||
runs-on: sm-standard-2
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
RUSTFS_S3_STACK: legacy
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -902,105 +898,18 @@ jobs:
|
||||
path: ${{ runner.temp }}/rustfs-e2e-*/rustfs.log
|
||||
retention-days: 3
|
||||
|
||||
# The same e2e-smoke suite against the same debug binary with the gateway
|
||||
# S3 stack selected (rustfs/backlog#2734). Report-only: it is outside the
|
||||
# Test and Lint aggregate and becomes required at T2.10. The archive and run
|
||||
# steps must stay identical to e2e-tests; scripts/ci_gate.py enforces that.
|
||||
e2e-tests-gateway:
|
||||
name: e2e-smoke (gateway, report-only until T2.10)
|
||||
needs: [ build-rustfs-debug-binary ]
|
||||
runs-on: sm-standard-2
|
||||
timeout-minutes: 30
|
||||
continue-on-error: true
|
||||
env:
|
||||
RUSTFS_S3_STACK: gateway
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: ci-dev
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install awscurl
|
||||
run: |
|
||||
python3 -m pip install --user --upgrade pip "awscurl==0.44"
|
||||
echo "AWSCURL_PATH=$HOME/.local/bin/awscurl" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Verify awscurl
|
||||
run: test -x "$AWSCURL_PATH"
|
||||
|
||||
- name: Download debug binary
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: rustfs-debug-binary
|
||||
path: target/debug
|
||||
|
||||
- name: Make binary executable
|
||||
run: chmod +x ./target/debug/rustfs
|
||||
|
||||
- name: Archive e2e smoke test binaries
|
||||
env:
|
||||
NEXTEST_ARCHIVE: ${{ runner.temp }}/rustfs-e2e-smoke.tar.zst
|
||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-smoke-list.json
|
||||
run: |
|
||||
cargo nextest archive --profile e2e-smoke -p e2e_test --archive-file "${NEXTEST_ARCHIVE}"
|
||||
cargo nextest list --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" --message-format json > "${NEXTEST_LISTING}"
|
||||
python3 ./scripts/check_test_wiring.py --check-profile e2e-smoke "${NEXTEST_LISTING}"
|
||||
./scripts/check_security_smoke_count.sh check "${NEXTEST_LISTING}"
|
||||
|
||||
- name: Run e2e smoke suite
|
||||
env:
|
||||
NEXTEST_ARCHIVE: ${{ runner.temp }}/rustfs-e2e-smoke.tar.zst
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-smoke-logs
|
||||
SAMPLER_INTERVAL_SECS: "15"
|
||||
run: |
|
||||
./scripts/ci/resource_sampler.sh start e2e-smoke
|
||||
trap './scripts/ci/resource_sampler.sh stop' EXIT
|
||||
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" \
|
||||
--status-level all --final-status-level all --failure-output final
|
||||
|
||||
- name: Upload e2e smoke diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-smoke-gateway-diagnostics-${{ github.run_number }}
|
||||
path: |
|
||||
${{ runner.temp }}/rustfs-e2e-smoke-logs/
|
||||
${{ runner.temp }}/rustfs-e2e-smoke-list.json
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Upload e2e smoke JUnit report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-smoke-gateway-junit-${{ github.run_number }}
|
||||
path: target/nextest/e2e-smoke/junit.xml
|
||||
if-no-files-found: warn
|
||||
|
||||
e2e-full:
|
||||
name: End-to-End Tests (full merge gate)
|
||||
# Merge gate only (backlog#1149 ci-5): the never-automated user-visible
|
||||
# suites — KMS, object_lock, multipart_auth, quota, checksum, encryption,
|
||||
# security-boundary, ... — via the e2e-full nextest profile. Too heavy for
|
||||
# every PR, so it is gated to main and gateway/integration pushes, the merge
|
||||
# queue, and manual dispatch. protocols / the 7 cluster suites / replication
|
||||
# / #[ignore] are owned by other lanes (see .config/nextest.toml profile.e2e-full).
|
||||
# every PR, so it is gated to main pushes, the merge queue, and manual
|
||||
# dispatch. protocols / the 7 cluster suites / replication / #[ignore] are
|
||||
# owned by other lanes (see .config/nextest.toml profile.e2e-full).
|
||||
if: >-
|
||||
github.event_name == 'workflow_dispatch' ||
|
||||
github.event_name == 'merge_group' ||
|
||||
(github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/gateway/integration'))
|
||||
(github.event_name == 'push' && github.ref == 'refs/heads/main')
|
||||
needs: [ build-rustfs-debug-binary ]
|
||||
runs-on: sm-standard-2
|
||||
timeout-minutes: 55
|
||||
|
||||
+1
-1
@@ -155,7 +155,7 @@ default build (lifecycle:
|
||||
- ⚠️ PARTIALLY VIOLATED: serving-side `s3s` references remain in ecstore
|
||||
(bucket metadata/replication/lifecycle DTOs and error mapping). The
|
||||
count is ratcheted shrink-only by `scripts/check_s3s_footprint.sh`
|
||||
(the `ecstore_files` line of `.config/s3s-footprint-baseline.txt`; the `object_lock` module was converted to
|
||||
(`S3S_ECSTORE_FILES_BASELINE`; the `object_lock` module was converted to
|
||||
storage-level types as the first ratchet step). Target state: the
|
||||
baseline reaches zero and ecstore's `Cargo.toml` drops `s3s`.
|
||||
|
||||
|
||||
Generated
+188
-19
@@ -275,6 +275,15 @@ dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anyerror"
|
||||
version = "0.1.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "71add24cc141a1e8326f249b74c41cfd217aeb2a67c9c6cf9134d175469afd49"
|
||||
dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anyhow"
|
||||
version = "1.0.104"
|
||||
@@ -1706,6 +1715,30 @@ dependencies = [
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "borsh"
|
||||
version = "1.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "553c5d846a6ba5150c65e3b1b8ec073bcf1abc20f9b7220de384a4443ea4e20a"
|
||||
dependencies = [
|
||||
"borsh-derive",
|
||||
"bytes",
|
||||
"cfg_aliases",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "borsh-derive"
|
||||
version = "1.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "12cdfe656708a01f89b451a7d36466e6fe6c414de0aa18fc54f864f6f9ca9f56"
|
||||
dependencies = [
|
||||
"once_cell",
|
||||
"proc-macro-crate",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "brotli"
|
||||
version = "8.0.4"
|
||||
@@ -1772,6 +1805,18 @@ version = "3.20.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
|
||||
|
||||
[[package]]
|
||||
name = "byte-unit"
|
||||
version = "5.2.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c719d56f7e96194cfc53460976d3ba51c85719747c9c62ed99981847b551152b"
|
||||
dependencies = [
|
||||
"rust_decimal",
|
||||
"schemars 1.2.2",
|
||||
"serde",
|
||||
"utf8-width",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bytemuck"
|
||||
version = "1.25.2"
|
||||
@@ -3724,13 +3769,34 @@ dependencies = [
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "derive_more"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4a9b99b9cbbe49445b21764dc0625032a89b145a2642e67603e1c936f5458d05"
|
||||
dependencies = [
|
||||
"derive_more-impl 1.0.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "derive_more"
|
||||
version = "2.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134"
|
||||
dependencies = [
|
||||
"derive_more-impl",
|
||||
"derive_more-impl 2.1.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "derive_more-impl"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
"unicode-xid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -6234,7 +6300,7 @@ dependencies = [
|
||||
"bytes",
|
||||
"chrono",
|
||||
"dashmap",
|
||||
"derive_more",
|
||||
"derive_more 2.1.1",
|
||||
"futures-util",
|
||||
"getrandom 0.4.3",
|
||||
"lazy_static",
|
||||
@@ -6388,6 +6454,12 @@ dependencies = [
|
||||
"quote",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "maplit"
|
||||
version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d"
|
||||
|
||||
[[package]]
|
||||
name = "matchers"
|
||||
version = "0.2.0"
|
||||
@@ -7329,6 +7401,42 @@ dependencies = [
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "openraft"
|
||||
version = "0.9.25"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a97014fb78acb77be3a40ac2da305f6dd3a6b243f3a908ace87d29b3972eaafd"
|
||||
dependencies = [
|
||||
"anyerror",
|
||||
"byte-unit",
|
||||
"chrono",
|
||||
"clap",
|
||||
"derive_more 1.0.0",
|
||||
"futures",
|
||||
"maplit",
|
||||
"openraft-macros",
|
||||
"rand 0.8.8",
|
||||
"serde",
|
||||
"thiserror 1.0.69",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"tracing-futures",
|
||||
"validit",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "openraft-macros"
|
||||
version = "0.9.25"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "55417d79f46a787dd88c043d831d8b7a0713f7c69d1925f9de51cc42c437250f"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"semver",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "openssl-probe"
|
||||
version = "0.2.1"
|
||||
@@ -8958,6 +9066,15 @@ dependencies = [
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redb"
|
||||
version = "4.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fb338a6c67830a61bed824b78c2bb034ab1ff401a20616bd7957524f4fdc2f22"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redis"
|
||||
version = "1.7.1"
|
||||
@@ -9464,6 +9581,23 @@ dependencies = [
|
||||
"walkdir",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rust_decimal"
|
||||
version = "1.43.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7653272e75dcac41dc199fbea6f5797633994fafd339943c06c9af16bf29cd3a"
|
||||
dependencies = [
|
||||
"arrayvec",
|
||||
"borsh",
|
||||
"bytes",
|
||||
"num-traits",
|
||||
"rand 0.8.8",
|
||||
"rand 0.9.5",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc-demangle"
|
||||
version = "0.1.28"
|
||||
@@ -9538,6 +9672,7 @@ dependencies = [
|
||||
"mime_guess",
|
||||
"moka",
|
||||
"object 0.40.0",
|
||||
"openraft",
|
||||
"opentelemetry",
|
||||
"opentelemetry-proto",
|
||||
"opentelemetry_sdk",
|
||||
@@ -9558,6 +9693,7 @@ dependencies = [
|
||||
"rsa 0.10.0-rc.18",
|
||||
"rust-embed",
|
||||
"rustfs-audit",
|
||||
"rustfs-capture-authority",
|
||||
"rustfs-common",
|
||||
"rustfs-concurrency",
|
||||
"rustfs-config",
|
||||
@@ -9666,6 +9802,22 @@ dependencies = [
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-capture-authority"
|
||||
version = "1.0.1"
|
||||
dependencies = [
|
||||
"openraft",
|
||||
"redb",
|
||||
"rustix",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.11.0",
|
||||
"tempfile",
|
||||
"thiserror 2.0.21",
|
||||
"tokio",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-checksums"
|
||||
version = "1.0.1"
|
||||
@@ -9821,6 +9973,7 @@ dependencies = [
|
||||
"metrics-util",
|
||||
"moka",
|
||||
"num_cpus",
|
||||
"openraft",
|
||||
"opentelemetry",
|
||||
"opentelemetry_sdk",
|
||||
"parking_lot",
|
||||
@@ -9835,6 +9988,7 @@ dependencies = [
|
||||
"reqwest",
|
||||
"rmp",
|
||||
"rmp-serde",
|
||||
"rustfs-capture-authority",
|
||||
"rustfs-common",
|
||||
"rustfs-concurrency",
|
||||
"rustfs-config",
|
||||
@@ -10284,6 +10438,7 @@ dependencies = [
|
||||
"reqwest",
|
||||
"rmp-serde",
|
||||
"rustfs-signer",
|
||||
"s3s",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sysinfo",
|
||||
@@ -10472,6 +10627,7 @@ dependencies = [
|
||||
"tokio",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -10508,7 +10664,6 @@ dependencies = [
|
||||
"rustfs-keystone",
|
||||
"rustfs-policy",
|
||||
"rustfs-rio",
|
||||
"rustfs-s3-types",
|
||||
"rustfs-storage-api",
|
||||
"rustfs-test-utils",
|
||||
"rustfs-tls-runtime",
|
||||
@@ -10523,7 +10678,6 @@ dependencies = [
|
||||
"sha2 0.11.0",
|
||||
"socket2",
|
||||
"subtle",
|
||||
"sync_wrapper",
|
||||
"tempfile",
|
||||
"thiserror 2.0.21",
|
||||
"time",
|
||||
@@ -10595,6 +10749,7 @@ dependencies = [
|
||||
"faster-hex",
|
||||
"futures",
|
||||
"hex-simd",
|
||||
"hmac 0.13.0",
|
||||
"hotpath",
|
||||
"http 1.5.0",
|
||||
"http-body-util",
|
||||
@@ -10670,7 +10825,6 @@ dependencies = [
|
||||
"rustfs-checksums",
|
||||
"rustfs-config",
|
||||
"rustfs-rio",
|
||||
"rustfs-s3-types",
|
||||
"rustfs-signer",
|
||||
"rustfs-storage-api",
|
||||
"rustfs-tls-runtime",
|
||||
@@ -10704,19 +10858,9 @@ dependencies = [
|
||||
name = "rustfs-s3-types"
|
||||
version = "1.0.1"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"hotpath",
|
||||
"http 1.5.0",
|
||||
"http-body",
|
||||
"http-body-util",
|
||||
"hyper",
|
||||
"hyper-util",
|
||||
"pin-project-lite",
|
||||
"s3s",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -10740,7 +10884,6 @@ dependencies = [
|
||||
"parking_lot",
|
||||
"rustfs-common",
|
||||
"rustfs-ecstore",
|
||||
"rustfs-s3-types",
|
||||
"rustfs-storage-api",
|
||||
"rustfs-test-utils",
|
||||
"s3s",
|
||||
@@ -10854,10 +10997,9 @@ dependencies = [
|
||||
"bytes",
|
||||
"hotpath",
|
||||
"http 1.5.0",
|
||||
"http-body",
|
||||
"hyper",
|
||||
"rustfs-s3-types",
|
||||
"rustfs-utils",
|
||||
"s3s",
|
||||
"serde_urlencoded",
|
||||
"thiserror 2.0.21",
|
||||
"time",
|
||||
@@ -10915,6 +11057,7 @@ dependencies = [
|
||||
"rustfs-utils",
|
||||
"rustls",
|
||||
"rustls-native-certs",
|
||||
"s3s",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.11.0",
|
||||
@@ -11073,6 +11216,7 @@ dependencies = [
|
||||
"async-compression",
|
||||
"futures",
|
||||
"hotpath",
|
||||
"liblzma",
|
||||
"rustfs-rio",
|
||||
"rustfs-tokio-tar",
|
||||
"serde",
|
||||
@@ -13057,6 +13201,16 @@ dependencies = [
|
||||
"tracing-subscriber",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing-futures"
|
||||
version = "0.2.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2"
|
||||
dependencies = [
|
||||
"pin-project",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing-log"
|
||||
version = "0.2.0"
|
||||
@@ -13207,7 +13361,7 @@ checksum = "79acf4b257aa56dd7f00466a81693d6c79c362c55092b3137909d6e76249ad80"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"chrono",
|
||||
"derive_more",
|
||||
"derive_more 2.1.1",
|
||||
"libc",
|
||||
"md-5 0.10.6",
|
||||
"thiserror 2.0.21",
|
||||
@@ -13307,6 +13461,12 @@ version = "2.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da"
|
||||
|
||||
[[package]]
|
||||
name = "utf8-width"
|
||||
version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "159a7cadce548703edd50d24069bc294c5415ecab0a480e0cd1ca06d112dc94a"
|
||||
|
||||
[[package]]
|
||||
name = "utf8_iter"
|
||||
version = "1.0.4"
|
||||
@@ -13333,6 +13493,15 @@ dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "validit"
|
||||
version = "0.2.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4417835826b78aa6265724624d424df2b1e6e71a33329227d8b04a88950780dc"
|
||||
dependencies = [
|
||||
"anyerror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "valuable"
|
||||
version = "0.1.1"
|
||||
|
||||
+5
-6
@@ -18,6 +18,7 @@ members = [
|
||||
"crates/audit", # Audit target management system with multi-target fan-out
|
||||
"crates/checksums", # client checksums
|
||||
"crates/common", # Shared utilities and data structures
|
||||
"crates/capture-authority", # Durable capture authority storage
|
||||
"crates/data-usage", # Shared data usage models and algorithms
|
||||
"crates/config", # Configuration management
|
||||
"crates/connect-symbol-catalog", # Release-only CPU profile symbol catalog generator
|
||||
@@ -48,7 +49,7 @@ members = [
|
||||
"crates/replication", # Replication contracts and wire formats
|
||||
"crates/concurrency", # Concurrency management for RustFS - timeout, locking, backpressure, and I/O scheduling
|
||||
"crates/s3-client", # S3 client for engine-side consumption of remote S3 endpoints (tiering, transition targets)
|
||||
"crates/s3-types", # S3 event and error type definitions
|
||||
"crates/s3-types", # S3 event type definitions
|
||||
"crates/s3-ops", # S3 operation definitions and mapping
|
||||
"crates/s3select-api", # S3 Select API interface
|
||||
"crates/s3select-query", # S3 Select query engine
|
||||
@@ -103,6 +104,7 @@ rustfs-config = { path = "./crates/config", version = "1.0.1" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.1" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.1" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.1" }
|
||||
rustfs-capture-authority = { path = "crates/capture-authority", version = "1.0.1" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.1" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.1" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.1" }
|
||||
@@ -314,6 +316,8 @@ ratelimit = "2.0.0"
|
||||
rayon = "1.12.0"
|
||||
rustfs-erasure-codec = { version = "9.0.0" }
|
||||
reed-solomon-simd = "3.1.0"
|
||||
openraft = { version = "=0.9.25", default-features = false, features = ["serde"] }
|
||||
redb = "=4.3.0"
|
||||
regex = { version = "1.13.1" }
|
||||
rumqttc = { package = "rumqttc-next", version = "0.34.0" }
|
||||
redis = { version = "1.7.1" }
|
||||
@@ -323,10 +327,6 @@ rust-embed = { version = "8.12.0" }
|
||||
rustc-hash = { version = "2.1.3" }
|
||||
s3s = { git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fcac87e3b99fd5bd9e176b442c0", features = ["minio"] }
|
||||
s3s-sigv4 = { git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fcac87e3b99fd5bd9e176b442c0" }
|
||||
# The same s3s pin under the name the legacy edge of the gateway migration uses
|
||||
# (crates/s3-types, feature `compat-s3s`). Keep `rev` identical to `s3s` above.
|
||||
# DELETE BY T4.2 (rustfs/backlog#2784).
|
||||
legacy-s3s = { package = "s3s", git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fcac87e3b99fd5bd9e176b442c0", features = ["minio"] }
|
||||
serial_test = "4.0.1"
|
||||
shadow-rs = { default-features = false, version = "2.0.0" }
|
||||
siphasher = "1.0.4"
|
||||
@@ -335,7 +335,6 @@ compact_str = "0.10.0"
|
||||
snap = "1.1.2"
|
||||
starshard = { version = "2.4.0" }
|
||||
strum = { version = "0.28.0" }
|
||||
sync_wrapper = "1.0.2"
|
||||
sysinfo = "0.39.6"
|
||||
temp-env = "0.3.6"
|
||||
tempfile = "3.27.0"
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
[package]
|
||||
name = "rustfs-capture-authority"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
repository.workspace = true
|
||||
description = "Durable local storage for capture authority"
|
||||
|
||||
[dependencies]
|
||||
openraft = { workspace = true, features = ["storage-v2"] }
|
||||
redb = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
thiserror = { workspace = true }
|
||||
tokio = { workspace = true, features = ["net", "io-util", "time", "sync", "rt"] }
|
||||
url = { workspace = true }
|
||||
tempfile = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
|
||||
[target.'cfg(any(target_os = "linux", target_os = "macos"))'.dependencies]
|
||||
rustix = { workspace = true, features = ["process"] }
|
||||
|
||||
[dev-dependencies]
|
||||
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,531 @@
|
||||
use super::*;
|
||||
use crate::application::{APPLY_RESULTS, EVENTS, OPERATIONS, VERSIONS};
|
||||
use crate::tests::CrashBackend;
|
||||
use redb::ReadableTableMetadata;
|
||||
use std::{
|
||||
io::{BufRead, BufReader, Write},
|
||||
process::{Command, Stdio},
|
||||
sync::mpsc,
|
||||
time::Duration,
|
||||
};
|
||||
fn identity() -> NodeIdentity {
|
||||
NodeIdentity {
|
||||
cluster_id: [7; 16],
|
||||
node_id: 0,
|
||||
}
|
||||
}
|
||||
fn binding() -> CaptureBinding {
|
||||
CaptureBinding {
|
||||
revision: 9,
|
||||
binding_id: [8; 16],
|
||||
target_id: [9; 16],
|
||||
}
|
||||
}
|
||||
fn command() -> DecideCreated {
|
||||
DecideCreated {
|
||||
operation_id: [1; 16],
|
||||
object: ObjectIdentity {
|
||||
bucket_incarnation: [2; 16],
|
||||
key: vec![0, 255, 3],
|
||||
},
|
||||
expected_head: None,
|
||||
prepared: PreparedIdentity {
|
||||
version_id: [3; 16],
|
||||
preparation_id: [4; 16],
|
||||
content_digest: [5; 32],
|
||||
content_length: 123,
|
||||
},
|
||||
binding_revision: 9,
|
||||
}
|
||||
}
|
||||
fn id(index: u64) -> LogId {
|
||||
LogId {
|
||||
term: 8,
|
||||
leader_node: 0,
|
||||
index,
|
||||
}
|
||||
}
|
||||
fn append(store: &mut Store, index: u64, c: &DecideCreated) {
|
||||
store
|
||||
.append(&[LogEntry {
|
||||
id: id(index),
|
||||
payload: c.encode().expect("encode command"),
|
||||
}])
|
||||
.expect("append command");
|
||||
}
|
||||
fn apply(store: &mut Store, index: u64, c: &DecideCreated) -> CreatedResult {
|
||||
append(store, index, c);
|
||||
store.save_committed(id(index)).expect("commit log");
|
||||
store.apply_next_created(id(index)).expect("apply command")
|
||||
}
|
||||
fn success(c: &DecideCreated, index: u64) -> CreatedResult {
|
||||
CreatedResult::Created {
|
||||
version_id: c.prepared.version_id,
|
||||
event_id: c.operation_id,
|
||||
decision_log_id: id(index),
|
||||
}
|
||||
}
|
||||
fn complete(store: &Store) {
|
||||
let c = command();
|
||||
assert_eq!(store.capture_binding().expect("binding"), Some(binding()));
|
||||
assert_eq!(
|
||||
store.created_head(&c.object).expect("head"),
|
||||
Some(c.prepared.version_id),
|
||||
"U11 acknowledged head lost"
|
||||
);
|
||||
assert_eq!(
|
||||
store.created_version(&c.object, c.prepared.version_id).expect("version"),
|
||||
Some(CreatedVersion {
|
||||
prepared: c.prepared.clone(),
|
||||
decision_log_id: id(0),
|
||||
binding_revision: 9
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
store.created_event(c.operation_id).expect("event"),
|
||||
Some(CreatedEvent {
|
||||
object: c.object.clone(),
|
||||
prepared: c.prepared.clone(),
|
||||
decision_log_id: id(0),
|
||||
binding: binding()
|
||||
}),
|
||||
"U11 complete event missing"
|
||||
);
|
||||
assert_eq!(
|
||||
store.created_operation(c.operation_id).expect("operation"),
|
||||
Some(CreatedOperation {
|
||||
canonical_command: c.encode().expect("encode"),
|
||||
result: success(&c, 0)
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
store.created_apply_result(0).expect("result"),
|
||||
Some(CreatedApplyResult {
|
||||
log_id: id(0),
|
||||
result: success(&c, 0)
|
||||
})
|
||||
);
|
||||
assert_eq!(store.last_applied().expect("last applied"), Some(id(0)));
|
||||
}
|
||||
fn fixture() -> (tempfile::TempDir, Store) {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let mut store = Store::initialize(dir.path().join("db"), identity()).expect("initialize");
|
||||
store.pin_capture_binding(binding()).expect("pin");
|
||||
(dir, store)
|
||||
}
|
||||
fn reopen(path: &std::path::Path) -> Store {
|
||||
let result = Store::open(path, identity());
|
||||
assert!(result.is_ok(), "U11 complete reopen: {:?}", result.as_ref().err());
|
||||
result.expect("asserted successful reopen")
|
||||
}
|
||||
#[test]
|
||||
fn created_codec_is_canonical_and_strict() {
|
||||
let c = command();
|
||||
let b = c.encode().expect("encode");
|
||||
let mut golden = b"CRTD\x01".to_vec();
|
||||
golden.extend([1; 16]);
|
||||
golden.extend([2; 16]);
|
||||
golden.extend(3u32.to_be_bytes());
|
||||
golden.extend([0, 255, 3]);
|
||||
golden.push(0);
|
||||
golden.extend([3; 16]);
|
||||
golden.extend([4; 16]);
|
||||
golden.extend([5; 32]);
|
||||
golden.extend(123u64.to_be_bytes());
|
||||
golden.extend(9u64.to_be_bytes());
|
||||
assert_eq!(b, golden);
|
||||
assert_eq!(DecideCreated::decode(&b).expect("decode"), c);
|
||||
let mut variants = Vec::new();
|
||||
let mut v = c.clone();
|
||||
v.operation_id = [6; 16];
|
||||
variants.push(v);
|
||||
let mut v = c.clone();
|
||||
v.object.bucket_incarnation = [6; 16];
|
||||
variants.push(v);
|
||||
let mut v = c.clone();
|
||||
v.object.key.push(4);
|
||||
variants.push(v);
|
||||
let mut v = c.clone();
|
||||
v.expected_head = Some([6; 16]);
|
||||
variants.push(v);
|
||||
let mut v = c.clone();
|
||||
v.prepared.version_id = [6; 16];
|
||||
variants.push(v);
|
||||
let mut v = c.clone();
|
||||
v.prepared.preparation_id = [6; 16];
|
||||
variants.push(v);
|
||||
let mut v = c.clone();
|
||||
v.prepared.content_digest = [6; 32];
|
||||
variants.push(v);
|
||||
let mut v = c.clone();
|
||||
v.prepared.content_length += 1;
|
||||
variants.push(v);
|
||||
let mut v = c.clone();
|
||||
v.binding_revision += 1;
|
||||
variants.push(v);
|
||||
for v in variants {
|
||||
assert_ne!(v.encode().expect("variant"), b)
|
||||
}
|
||||
for n in 0..b.len() {
|
||||
assert!(DecideCreated::decode(&b[..n]).is_err())
|
||||
}
|
||||
let mut bad = b.clone();
|
||||
bad[4] = 2;
|
||||
assert!(DecideCreated::decode(&bad).is_err());
|
||||
let mut bad = b.clone();
|
||||
bad[44] = 2;
|
||||
assert!(DecideCreated::decode(&bad).is_err());
|
||||
let mut bad = b.clone();
|
||||
bad.push(0);
|
||||
assert!(DecideCreated::decode(&bad).is_err());
|
||||
let mut bad = b;
|
||||
bad[37..41].copy_from_slice(&4097u32.to_be_bytes());
|
||||
assert!(DecideCreated::decode(&bad).is_err());
|
||||
let mut bad = c.clone();
|
||||
bad.operation_id = [0; 16];
|
||||
assert!(bad.encode().is_err());
|
||||
bad = c;
|
||||
bad.object.key = vec![0; MAX_OBJECT_KEY_LENGTH + 1];
|
||||
assert!(bad.encode().is_err());
|
||||
}
|
||||
#[test]
|
||||
fn created_committed_order_and_identity_are_enforced() {
|
||||
let (_dir, mut s) = fixture();
|
||||
let c = command();
|
||||
append(&mut s, 0, &c);
|
||||
append(&mut s, 1, &c);
|
||||
assert!(s.apply_next_created(id(0)).is_err());
|
||||
assert_eq!(s.last_applied().expect("no apply"), None);
|
||||
s.save_committed(id(1)).expect("committed");
|
||||
assert!(matches!(s.apply_next_created(id(1)), Err(StoreError::InvalidApplySequence)));
|
||||
let mut wrong = id(0);
|
||||
wrong.term += 1;
|
||||
assert!(s.apply_next_created(wrong).is_err());
|
||||
wrong = id(0);
|
||||
wrong.leader_node += 1;
|
||||
assert!(s.apply_next_created(wrong).is_err());
|
||||
assert_eq!(s.apply_next_created(id(0)).expect("first"), success(&c, 0));
|
||||
complete(&s);
|
||||
assert_eq!(s.apply_next_created(id(0)).expect("retry"), success(&c, 0));
|
||||
complete(&s);
|
||||
assert!(s.apply_next_created(wrong).is_err());
|
||||
assert_eq!(s.apply_next_created(id(1)).expect("duplicate log"), success(&c, 0));
|
||||
assert_eq!(s.last_applied().expect("last"), Some(id(1)));
|
||||
assert_eq!(
|
||||
s.created_apply_result(1).expect("result"),
|
||||
Some(CreatedApplyResult {
|
||||
log_id: id(1),
|
||||
result: success(&c, 0)
|
||||
})
|
||||
);
|
||||
let tx = s.database.begin_read().expect("read");
|
||||
assert_eq!(tx.open_table(EVENTS).expect("events").len().expect("len"), 1);
|
||||
}
|
||||
#[test]
|
||||
fn created_head_version_event_and_dedup_reopen_together() {
|
||||
let (dir, mut s) = fixture();
|
||||
let c = command();
|
||||
assert_eq!(apply(&mut s, 0, &c), success(&c, 0));
|
||||
drop(s);
|
||||
let mut s = reopen(&dir.path().join("db"));
|
||||
complete(&s);
|
||||
assert_eq!(apply(&mut s, 1, &c), success(&c, 0));
|
||||
let mut changed = c.clone();
|
||||
changed.prepared.content_digest = [6; 32];
|
||||
assert_eq!(
|
||||
apply(&mut s, 2, &changed),
|
||||
CreatedResult::OperationConflict,
|
||||
"U11 full canonical operation conflict"
|
||||
);
|
||||
assert_eq!(
|
||||
s.created_operation(c.operation_id)
|
||||
.expect("original")
|
||||
.expect("operation")
|
||||
.result,
|
||||
success(&c, 0)
|
||||
);
|
||||
let mut stale = c.clone();
|
||||
stale.operation_id = [6; 16];
|
||||
stale.prepared.version_id = [6; 16];
|
||||
assert_eq!(
|
||||
apply(&mut s, 3, &stale),
|
||||
CreatedResult::HeadMismatch {
|
||||
actual_head: Some(c.prepared.version_id)
|
||||
}
|
||||
);
|
||||
let mut reuse = c.clone();
|
||||
reuse.operation_id = [7; 16];
|
||||
reuse.expected_head = Some(c.prepared.version_id);
|
||||
assert_eq!(apply(&mut s, 4, &reuse), CreatedResult::VersionConflict);
|
||||
let mut replacement = stale.clone();
|
||||
replacement.operation_id = [10; 16];
|
||||
replacement.expected_head = Some(c.prepared.version_id);
|
||||
assert_eq!(apply(&mut s, 5, &replacement), success(&replacement, 5));
|
||||
assert_eq!(s.created_head(&c.object).expect("head"), Some(replacement.prepared.version_id));
|
||||
assert_eq!(
|
||||
s.created_version(&c.object, c.prepared.version_id)
|
||||
.expect("old version")
|
||||
.expect("version")
|
||||
.decision_log_id,
|
||||
id(0)
|
||||
);
|
||||
assert!(s.created_event(c.operation_id).expect("old event").is_some());
|
||||
assert!(s.created_event(replacement.operation_id).expect("new event").is_some());
|
||||
assert!(s.created_event(stale.operation_id).expect("rejection event").is_none());
|
||||
assert_eq!(
|
||||
apply(&mut s, 6, &stale),
|
||||
CreatedResult::HeadMismatch {
|
||||
actual_head: Some(c.prepared.version_id)
|
||||
}
|
||||
);
|
||||
drop(s);
|
||||
let s = reopen(&dir.path().join("db"));
|
||||
assert_eq!(s.last_applied().expect("last"), Some(id(6)));
|
||||
let tx = s.database.begin_read().expect("read");
|
||||
assert_eq!(tx.open_table(EVENTS).expect("events").len().expect("len"), 2);
|
||||
}
|
||||
#[test]
|
||||
fn created_binding_is_pinned_and_not_caller_selected() {
|
||||
let (dir, mut s) = fixture();
|
||||
let mut bad = command();
|
||||
bad.binding_revision = 10;
|
||||
assert_eq!(apply(&mut s, 0, &bad), CreatedResult::BindingMismatch);
|
||||
assert!(s.created_head(&bad.object).expect("head").is_none());
|
||||
assert!(s.created_event(bad.operation_id).expect("event").is_none());
|
||||
assert_eq!(
|
||||
s.created_operation(bad.operation_id)
|
||||
.expect("operation")
|
||||
.expect("saved")
|
||||
.result,
|
||||
CreatedResult::BindingMismatch
|
||||
);
|
||||
s.pin_capture_binding(binding()).expect("idempotent pin");
|
||||
let mut changed = binding();
|
||||
changed.target_id = [10; 16];
|
||||
assert!(matches!(s.pin_capture_binding(changed), Err(StoreError::BindingConflict)));
|
||||
drop(s);
|
||||
let s = reopen(&dir.path().join("db"));
|
||||
assert_eq!(s.capture_binding().expect("binding"), Some(binding()));
|
||||
}
|
||||
#[test]
|
||||
fn created_schema_compatibility_and_corruption_are_checked() {
|
||||
let dir = tempfile::tempdir().expect("dir");
|
||||
let path = dir.path().join("db");
|
||||
let s = Store::initialize(&path, identity()).expect("schema1");
|
||||
assert_eq!(s.capture_binding().expect("unmigrated"), None);
|
||||
drop(s);
|
||||
let mut s = reopen(&path);
|
||||
assert_eq!(s.last_applied().expect("no apply"), None);
|
||||
append(&mut s, 0, &command());
|
||||
s.save_committed(id(0)).expect("schema2");
|
||||
drop(s);
|
||||
let mut s = reopen(&path);
|
||||
assert_eq!(s.capture_binding().expect("schema2 unchanged"), None);
|
||||
s.pin_capture_binding(binding()).expect("explicit upgrade");
|
||||
drop(s);
|
||||
let mut s = reopen(&path);
|
||||
assert_eq!(s.capture_binding().expect("schema3"), Some(binding()));
|
||||
s.apply_next_created(id(0)).expect("apply");
|
||||
drop(s);
|
||||
let db = Database::open(&path).expect("raw db");
|
||||
let tx = db.begin_write().expect("write corruption");
|
||||
tx.open_table(EVENTS)
|
||||
.expect("events")
|
||||
.remove(command().operation_id.as_slice())
|
||||
.expect("remove reference");
|
||||
tx.commit().expect("commit corruption");
|
||||
drop(db);
|
||||
let result = Store::open(&path, identity());
|
||||
assert!(
|
||||
matches!(result, Err(StoreError::CorruptRecord)),
|
||||
"U11 missing event rejected before expect"
|
||||
);
|
||||
let (dir, mut s) = fixture();
|
||||
assert_eq!(s.read_committed().expect("empty committed"), None);
|
||||
append(&mut s, 0, &command());
|
||||
s.save_committed(id(0)).expect("committed schema3");
|
||||
assert_eq!(s.capture_binding().expect("no downgrade"), Some(binding()));
|
||||
drop(s);
|
||||
let path = dir.path().join("db");
|
||||
let db = Database::open(&path).expect("raw db");
|
||||
let tx = db.begin_write().expect("write");
|
||||
let mut marker = vec![1];
|
||||
marker.extend(encode_id(id(0)));
|
||||
tx.open_table(META)
|
||||
.expect("meta")
|
||||
.insert("last_applied", marker.as_slice())
|
||||
.expect("corrupt applied");
|
||||
tx.commit().expect("commit");
|
||||
drop(db);
|
||||
assert!(
|
||||
matches!(Store::open(&path, identity()), Err(StoreError::CorruptRecord)),
|
||||
"U11 last applied missing result rejected"
|
||||
);
|
||||
let (dir, s) = fixture();
|
||||
drop(s);
|
||||
let path = dir.path().join("db");
|
||||
let db = Database::open(&path).expect("raw pinned db");
|
||||
let tx = db.begin_write().expect("missing binding write");
|
||||
tx.open_table(META)
|
||||
.expect("meta")
|
||||
.remove("capture_binding")
|
||||
.expect("remove binding");
|
||||
tx.commit().expect("binding corruption commit");
|
||||
drop(db);
|
||||
assert!(
|
||||
matches!(Store::open(&path, identity()), Err(StoreError::CorruptRecord)),
|
||||
"U11 missing pinned binding rejected"
|
||||
);
|
||||
|
||||
let (dir, mut s) = fixture();
|
||||
let c = command();
|
||||
assert_eq!(apply(&mut s, 0, &c), success(&c, 0));
|
||||
assert_eq!(apply(&mut s, 1, &c), success(&c, 0));
|
||||
drop(s);
|
||||
let path = dir.path().join("db");
|
||||
let db = Database::open(&path).expect("raw duplicate db");
|
||||
let tx = db.begin_write().expect("retarget decision write");
|
||||
let mut key = created::object_bytes(&c.object).expect("object key");
|
||||
key.extend(c.prepared.version_id);
|
||||
let mut version = Vec::new();
|
||||
created::put_prepared(&mut version, &c.prepared);
|
||||
version.extend(encode_id(id(1)));
|
||||
version.extend(binding().revision.to_be_bytes());
|
||||
tx.open_table(VERSIONS)
|
||||
.expect("versions")
|
||||
.insert(key.as_slice(), version.as_slice())
|
||||
.expect("retarget version");
|
||||
let mut event = created::object_bytes(&c.object).expect("event object");
|
||||
created::put_prepared(&mut event, &c.prepared);
|
||||
event.extend(encode_id(id(1)));
|
||||
event.extend(created::binding_bytes(&binding()).expect("binding bytes"));
|
||||
tx.open_table(EVENTS)
|
||||
.expect("events")
|
||||
.insert(c.operation_id.as_slice(), event.as_slice())
|
||||
.expect("retarget event");
|
||||
let canonical = c.encode().expect("canonical");
|
||||
let mut operation = u64::try_from(canonical.len()).expect("length").to_be_bytes().to_vec();
|
||||
operation.extend(canonical);
|
||||
operation.extend(created::result_bytes(&success(&c, 1)));
|
||||
tx.open_table(OPERATIONS)
|
||||
.expect("operations")
|
||||
.insert(c.operation_id.as_slice(), operation.as_slice())
|
||||
.expect("retarget original response");
|
||||
for index in 0..=1 {
|
||||
let mut result = encode_id(id(index));
|
||||
result.extend(created::result_bytes(&success(&c, 1)));
|
||||
tx.open_table(APPLY_RESULTS)
|
||||
.expect("results")
|
||||
.insert(index, result.as_slice())
|
||||
.expect("retarget duplicate response");
|
||||
}
|
||||
tx.commit().expect("coherent retarget commit");
|
||||
drop(db);
|
||||
assert!(
|
||||
matches!(Store::open(&path, identity()), Err(StoreError::CorruptRecord)),
|
||||
"U11 original decision cannot retarget duplicate log"
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn created_process_child() {
|
||||
if let Some(path) = std::env::var_os("U11_CHILD_DB") {
|
||||
let mut s = Store::initialize(path, identity()).expect("child init");
|
||||
s.pin_capture_binding(binding()).expect("child pin");
|
||||
assert_eq!(apply(&mut s, 0, &command()), success(&command(), 0));
|
||||
complete(&s);
|
||||
println!("U11_ACK");
|
||||
std::io::stdout().flush().expect("flush ACK");
|
||||
loop {
|
||||
std::thread::park()
|
||||
}
|
||||
}
|
||||
let (_dir, mut s) = fixture();
|
||||
assert_eq!(apply(&mut s, 0, &command()), success(&command(), 0));
|
||||
complete(&s);
|
||||
}
|
||||
#[test]
|
||||
fn created_acknowledged_apply_survives_child_kill() {
|
||||
let dir = tempfile::tempdir().expect("dir");
|
||||
let path = dir.path().join("db");
|
||||
let mut child = Command::new(std::env::current_exe().expect("test executable"))
|
||||
.args(["--exact", "application_tests::created_process_child", "--nocapture"])
|
||||
.env("U11_CHILD_DB", &path)
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::inherit())
|
||||
.spawn()
|
||||
.expect("spawn");
|
||||
let stdout = child.stdout.take().expect("stdout");
|
||||
let (tx, rx) = mpsc::channel();
|
||||
let reader = std::thread::spawn(move || {
|
||||
for line in BufReader::new(stdout).lines() {
|
||||
let line = line.expect("child line");
|
||||
if line.ends_with("U11_ACK") {
|
||||
let _ = tx.send(());
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
let acknowledged = rx.recv_timeout(Duration::from_secs(20));
|
||||
child.kill().expect("kill child");
|
||||
child.wait().expect("wait child");
|
||||
reader.join().expect("reader cleanup");
|
||||
assert!(acknowledged.is_ok(), "U11 bounded child ACK missing");
|
||||
complete(&reopen(&path));
|
||||
}
|
||||
#[test]
|
||||
fn created_power_cut_before_and_after_apply_flush() {
|
||||
let (dir, mut s) = fixture();
|
||||
append(&mut s, 0, &command());
|
||||
s.save_committed(id(0)).expect("committed");
|
||||
drop(s);
|
||||
let path = dir.path().join("db");
|
||||
let backend = CrashBackend::new(&path);
|
||||
let mut s = Store::from_database(backend.database(), identity()).expect("backend");
|
||||
assert_eq!(s.apply_next_created(id(0)).expect("apply ACK"), success(&command(), 0));
|
||||
backend.cut();
|
||||
drop(s);
|
||||
complete(&reopen(&path));
|
||||
for sync in 1..=2 {
|
||||
for after in [false, true] {
|
||||
let (dir, mut s) = fixture();
|
||||
append(&mut s, 0, &command());
|
||||
s.save_committed(id(0)).expect("committed");
|
||||
drop(s);
|
||||
let path = dir.path().join("db");
|
||||
let backend = CrashBackend::new(&path);
|
||||
let mut s = Store::from_database(backend.database(), identity()).expect("backend");
|
||||
let (entered, resume) = backend.pause_sync(sync, after);
|
||||
let writer = std::thread::spawn(move || {
|
||||
let result = s.apply_next_created(id(0));
|
||||
(s, result)
|
||||
});
|
||||
entered.wait();
|
||||
backend.cut();
|
||||
resume.wait();
|
||||
let (s, result) = writer.join().expect("writer");
|
||||
drop(s);
|
||||
let reopened = Store::open(&path, identity());
|
||||
assert!(
|
||||
reopened.is_ok(),
|
||||
"U11 apply cut must reopen whole old/new state sync={sync} after={after}: {:?}",
|
||||
reopened.as_ref().err()
|
||||
);
|
||||
let s = reopened.expect("asserted reopen");
|
||||
if s.last_applied().expect("last").is_some() {
|
||||
complete(&s)
|
||||
} else {
|
||||
assert!(result.is_err());
|
||||
assert_eq!(s.created_head(&command().object).expect("old head"), None);
|
||||
assert_eq!(
|
||||
s.created_version(&command().object, command().prepared.version_id)
|
||||
.expect("old version"),
|
||||
None
|
||||
);
|
||||
assert_eq!(s.created_event(command().operation_id).expect("old event"), None);
|
||||
assert_eq!(s.created_operation(command().operation_id).expect("old op"), None);
|
||||
assert_eq!(s.created_apply_result(0).expect("old result"), None)
|
||||
}
|
||||
eprintln!("U11 cut boundary sync={sync} after={after}");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,579 @@
|
||||
//! Immutable checkpoint state and disk-backed validation of the retained suffix.
|
||||
use crate::{
|
||||
CaptureMembership, CreatedApplyResult, CreatedEvent, CreatedResult, CreatedVersion, LogId, StoreError, application,
|
||||
created::{DecideCreated, Decoder, object_bytes, result_bytes},
|
||||
native_checkpoint::{CheckpointLimits, recovery_scratch},
|
||||
snapshot,
|
||||
};
|
||||
use redb::{ReadTransaction, ReadableDatabase, ReadableTable, ReadableTableMetadata, TableDefinition};
|
||||
|
||||
pub(crate) const META_KEY: &str = "checkpoint_baseline";
|
||||
pub(crate) const PURGED_KEY: &str = "checkpoint_purged";
|
||||
pub(crate) const HEADS: TableDefinition<&[u8], &[u8]> = TableDefinition::new("checkpoint_heads_v1");
|
||||
pub(crate) const VERSIONS: TableDefinition<&[u8], &[u8]> = TableDefinition::new("checkpoint_versions_v1");
|
||||
pub(crate) const EVENTS: TableDefinition<&[u8], &[u8]> = TableDefinition::new("checkpoint_events_v1");
|
||||
pub(crate) const OPERATIONS: TableDefinition<&[u8], &[u8]> = TableDefinition::new("checkpoint_operations_v1");
|
||||
const FIRST_DECISIONS: TableDefinition<u64, &[u8]> = TableDefinition::new("checkpoint_validation_decisions");
|
||||
const VERSION_OPERATIONS: TableDefinition<&[u8], &[u8]> = TableDefinition::new("checkpoint_validation_versions");
|
||||
const METADATA_BYTES: usize = 1024 * 1024;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct BaselineMetadata {
|
||||
pub(crate) generation: u64,
|
||||
pub(crate) header: snapshot::Header,
|
||||
pub(crate) limits: CheckpointLimits,
|
||||
}
|
||||
|
||||
struct EmptySink;
|
||||
impl snapshot::StagingSink for EmptySink {
|
||||
fn row(&mut self, _: snapshot::Section, _: &[u8], _: &[u8]) -> std::io::Result<()> {
|
||||
Err(std::io::Error::other("checkpoint metadata contains rows"))
|
||||
}
|
||||
}
|
||||
fn metadata_limits() -> snapshot::Limits {
|
||||
snapshot::Limits {
|
||||
total_bytes: 1024 * 1024,
|
||||
membership_bytes: 512 * 1024,
|
||||
key_bytes: 0,
|
||||
value_bytes: 0,
|
||||
records: 0,
|
||||
}
|
||||
}
|
||||
pub(crate) fn encode_metadata(metadata: &BaselineMetadata) -> Result<Vec<u8>, StoreError> {
|
||||
metadata.limits.validate()?;
|
||||
if metadata.generation == 0 {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let mut bytes = b"CBAS\x01".to_vec();
|
||||
for value in [
|
||||
metadata.generation,
|
||||
metadata.limits.snapshot.total_bytes,
|
||||
metadata.limits.snapshot.membership_bytes,
|
||||
metadata.limits.snapshot.key_bytes,
|
||||
metadata.limits.snapshot.value_bytes,
|
||||
metadata.limits.snapshot.records,
|
||||
metadata.limits.staging_bytes,
|
||||
] {
|
||||
bytes.extend(value.to_be_bytes());
|
||||
}
|
||||
let frame = snapshot::Encoder::new(Vec::new(), &metadata.header, [0; 4], metadata_limits())
|
||||
.and_then(snapshot::Encoder::finish)
|
||||
.map_err(|_| StoreError::CorruptRecord)?;
|
||||
bytes.extend(frame);
|
||||
if bytes.len() > METADATA_BYTES {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
pub(crate) fn read_metadata(
|
||||
meta: &impl ReadableTable<&'static str, &'static [u8]>,
|
||||
) -> Result<Option<BaselineMetadata>, StoreError> {
|
||||
let schema = crate::number(required(meta.get("schema")?)?.value())?;
|
||||
let stored = meta.get(META_KEY)?;
|
||||
if schema != 7 {
|
||||
if stored.is_some() || meta.get(PURGED_KEY)?.is_some() {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
return Ok(None);
|
||||
}
|
||||
let value = required(stored)?;
|
||||
let bytes = value.value();
|
||||
if bytes.len() > METADATA_BYTES || bytes.get(..5) != Some(b"CBAS\x01") {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let mut numbers = [0u64; 7];
|
||||
for (index, number) in numbers.iter_mut().enumerate() {
|
||||
let start = 5 + index * 8;
|
||||
*number = crate::number(bytes.get(start..start + 8).ok_or(StoreError::CorruptRecord)?)?;
|
||||
}
|
||||
let limits = CheckpointLimits {
|
||||
snapshot: snapshot::Limits {
|
||||
total_bytes: numbers[1],
|
||||
membership_bytes: numbers[2],
|
||||
key_bytes: numbers[3],
|
||||
value_bytes: numbers[4],
|
||||
records: numbers[5],
|
||||
},
|
||||
staging_bytes: numbers[6],
|
||||
};
|
||||
limits.validate()?;
|
||||
let header = snapshot::decode(bytes.get(61..).ok_or(StoreError::CorruptRecord)?, &mut EmptySink, metadata_limits())
|
||||
.map_err(|_| StoreError::CorruptRecord)?;
|
||||
let metadata = BaselineMetadata {
|
||||
generation: numbers[0],
|
||||
header,
|
||||
limits,
|
||||
};
|
||||
if encode_metadata(&metadata)?.as_slice() != bytes {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
Ok(Some(metadata))
|
||||
}
|
||||
pub(crate) fn read_baseline_cut(meta: &impl ReadableTable<&'static str, &'static [u8]>) -> Result<Option<LogId>, StoreError> {
|
||||
Ok(read_metadata(meta)?.and_then(|metadata| metadata.header.applied))
|
||||
}
|
||||
pub(crate) fn encode_purged(id: LogId) -> Vec<u8> {
|
||||
let mut bytes = b"CPUR\x01".to_vec();
|
||||
bytes.extend(crate::encode_id(id));
|
||||
bytes
|
||||
}
|
||||
pub(crate) fn read_purged(meta: &impl ReadableTable<&'static str, &'static [u8]>) -> Result<Option<LogId>, StoreError> {
|
||||
let cut = read_baseline_cut(meta)?;
|
||||
let Some(value) = meta.get(PURGED_KEY)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
let bytes = value.value();
|
||||
if bytes.len() != 29 || bytes.get(..5) != Some(b"CPUR\x01") {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let id = crate::decode_id(&bytes[5..])?;
|
||||
at_or_before(id, required(cut)?)?;
|
||||
Ok(Some(id))
|
||||
}
|
||||
fn at_or_before(id: LogId, cut: LogId) -> Result<(), StoreError> {
|
||||
if id.index > cut.index || (id.index == cut.index && id != cut) {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
fn required<T>(value: Option<T>) -> Result<T, StoreError> {
|
||||
value.ok_or(StoreError::CorruptRecord)
|
||||
}
|
||||
struct Budget {
|
||||
limits: snapshot::Limits,
|
||||
bytes: u64,
|
||||
records: u64,
|
||||
}
|
||||
impl Budget {
|
||||
fn new(limits: snapshot::Limits) -> Self {
|
||||
Self {
|
||||
limits,
|
||||
bytes: 0,
|
||||
records: 0,
|
||||
}
|
||||
}
|
||||
fn row(&mut self, key: &[u8], value: &[u8]) -> Result<(), StoreError> {
|
||||
let key = u64::try_from(key.len()).map_err(|_| StoreError::CorruptRecord)?;
|
||||
let value = u64::try_from(value.len()).map_err(|_| StoreError::CorruptRecord)?;
|
||||
if key > 1024 * 1024 || value > 1024 * 1024 {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
self.bytes = self
|
||||
.bytes
|
||||
.checked_add(key)
|
||||
.and_then(|n| n.checked_add(value))
|
||||
.ok_or(StoreError::ResourceLimit)?;
|
||||
self.records = self.records.checked_add(1).ok_or(StoreError::ResourceLimit)?;
|
||||
if key > self.limits.key_bytes
|
||||
|| value > self.limits.value_bytes
|
||||
|| self.bytes > self.limits.total_bytes
|
||||
|| self.records > self.limits.records
|
||||
{
|
||||
return Err(StoreError::ResourceLimit);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn resource_error(error: StoreError) -> StoreError {
|
||||
use std::error::Error;
|
||||
let mut cause: Option<&(dyn Error + 'static)> = Some(&error);
|
||||
while let Some(current) = cause {
|
||||
if current
|
||||
.downcast_ref::<std::io::Error>()
|
||||
.is_some_and(|io| io.kind() == std::io::ErrorKind::StorageFull)
|
||||
{
|
||||
return StoreError::ResourceLimit;
|
||||
}
|
||||
cause = current.source();
|
||||
}
|
||||
error
|
||||
}
|
||||
|
||||
/// Validate relationships, preserving negative history rather than re-evaluating it.
|
||||
/// The caller separately establishes local capture or authenticated installation authority.
|
||||
pub(crate) fn validate_baseline_with_budget(
|
||||
tx: &ReadTransaction,
|
||||
metadata: &BaselineMetadata,
|
||||
index_max: u64,
|
||||
) -> Result<(), StoreError> {
|
||||
validate_baseline_inner(tx, metadata, index_max).map_err(resource_error)
|
||||
}
|
||||
|
||||
fn validate_baseline_inner(tx: &ReadTransaction, metadata: &BaselineMetadata, index_max: u64) -> Result<(), StoreError> {
|
||||
encode_metadata(metadata)?;
|
||||
let Some(cut) = metadata.header.applied else {
|
||||
if metadata.header.membership != CaptureMembership::new(None, vec![], std::collections::BTreeMap::new())? {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
for definition in [HEADS, VERSIONS, EVENTS, OPERATIONS] {
|
||||
if tx.open_table(definition)?.len()? != 0 {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
}
|
||||
return Ok(());
|
||||
};
|
||||
let heads = tx.open_table(HEADS)?;
|
||||
let versions = tx.open_table(VERSIONS)?;
|
||||
let events = tx.open_table(EVENTS)?;
|
||||
let operations = tx.open_table(OPERATIONS)?;
|
||||
let (_directory, index) = recovery_scratch(index_max)?;
|
||||
let mut index_write = index.begin_write()?;
|
||||
index_write.open_table(FIRST_DECISIONS)?;
|
||||
index_write.open_table(VERSION_OPERATIONS)?;
|
||||
let mut budget = Budget::new(metadata.limits.snapshot);
|
||||
let mut pending = 0;
|
||||
for row in operations.iter()? {
|
||||
let (key, value) = row?;
|
||||
budget.row(key.value(), value.value())?;
|
||||
let provenance = application::decode_provenance(value.value())?;
|
||||
let command = DecideCreated::decode(&provenance.operation.canonical_command)?;
|
||||
let canonical = command.encode()?;
|
||||
if key.value() != command.operation_id || canonical != provenance.operation.canonical_command {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
at_or_before(provenance.first_decision_log_id, cut)?;
|
||||
let mut original = u64::try_from(canonical.len())
|
||||
.map_err(|_| StoreError::CorruptRecord)?
|
||||
.to_be_bytes()
|
||||
.to_vec();
|
||||
original.extend(&canonical);
|
||||
original.extend(result_bytes(&provenance.operation.result));
|
||||
if application::encode_provenance(provenance.first_decision_log_id, &original)?.as_slice() != value.value() {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
if index_write
|
||||
.open_table(FIRST_DECISIONS)?
|
||||
.insert(provenance.first_decision_log_id.index, key.value())?
|
||||
.is_some()
|
||||
{
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
if provenance.operation.result == CreatedResult::OperationConflict {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
if let CreatedResult::Created {
|
||||
version_id,
|
||||
event_id,
|
||||
decision_log_id,
|
||||
} = provenance.operation.result
|
||||
{
|
||||
if version_id != command.prepared.version_id
|
||||
|| event_id != command.operation_id
|
||||
|| decision_log_id != provenance.first_decision_log_id
|
||||
|| command.binding_revision != metadata.header.binding.revision
|
||||
{
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let event = application::decode_event(required(events.get(key.value())?)?.value())?;
|
||||
if event
|
||||
!= (CreatedEvent {
|
||||
object: command.object.clone(),
|
||||
prepared: command.prepared.clone(),
|
||||
decision_log_id,
|
||||
binding: metadata.header.binding.clone(),
|
||||
})
|
||||
{
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let version_key = application::version_key(&command.object, version_id)?;
|
||||
let version = application::decode_version(required(versions.get(version_key.as_slice())?)?.value())?;
|
||||
if version
|
||||
!= (CreatedVersion {
|
||||
prepared: command.prepared.clone(),
|
||||
decision_log_id,
|
||||
binding_revision: metadata.header.binding.revision,
|
||||
})
|
||||
{
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
if index_write
|
||||
.open_table(VERSION_OPERATIONS)?
|
||||
.insert(version_key.as_slice(), key.value())?
|
||||
.is_some()
|
||||
{
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
} else if events.get(key.value())?.is_some() {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
pending += 1;
|
||||
if pending == 128 {
|
||||
index_write.commit().map_err(StoreError::StagingCommit)?;
|
||||
index_write = index.begin_write()?;
|
||||
pending = 0;
|
||||
}
|
||||
}
|
||||
index_write.commit().map_err(StoreError::StagingCommit)?;
|
||||
let index_read = index.begin_read()?;
|
||||
let version_operations = index_read.open_table(VERSION_OPERATIONS)?;
|
||||
if version_operations.len()? != versions.len()? {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
for row in versions.iter()? {
|
||||
let (key, value) = row?;
|
||||
budget.row(key.value(), value.value())?;
|
||||
required(version_operations.get(key.value())?)?;
|
||||
let mut decoder = Decoder::new(key.value());
|
||||
let object = decoder.object()?;
|
||||
let id = decoder.uuid()?;
|
||||
decoder.finish()?;
|
||||
let version = application::decode_version(value.value())?;
|
||||
let head_bytes = required(heads.get(object_bytes(&object)?.as_slice())?)?;
|
||||
let mut decoder = Decoder::new(head_bytes.value());
|
||||
let head = decoder.uuid()?;
|
||||
decoder.finish()?;
|
||||
let current =
|
||||
application::decode_version(required(versions.get(application::version_key(&object, head)?.as_slice())?)?.value())?;
|
||||
if id != version.prepared.version_id
|
||||
|| current.decision_log_id.index < version.decision_log_id.index
|
||||
|| (current.decision_log_id.index == version.decision_log_id.index
|
||||
&& current.decision_log_id != version.decision_log_id)
|
||||
{
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
}
|
||||
for row in events.iter()? {
|
||||
let (key, value) = row?;
|
||||
budget.row(key.value(), value.value())?;
|
||||
let event = application::decode_event(value.value())?;
|
||||
let operation = application::decode_provenance(required(operations.get(key.value())?)?.value())?;
|
||||
if operation.operation.result
|
||||
!= (CreatedResult::Created {
|
||||
version_id: event.prepared.version_id,
|
||||
event_id: key.value().try_into().map_err(|_| StoreError::CorruptRecord)?,
|
||||
decision_log_id: event.decision_log_id,
|
||||
})
|
||||
{
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
}
|
||||
for row in heads.iter()? {
|
||||
let (key, value) = row?;
|
||||
budget.row(key.value(), value.value())?;
|
||||
let mut decoder = Decoder::new(key.value());
|
||||
let object = decoder.object()?;
|
||||
decoder.finish()?;
|
||||
let mut decoder = Decoder::new(value.value());
|
||||
let id = decoder.uuid()?;
|
||||
decoder.finish()?;
|
||||
required(versions.get(application::version_key(&object, id)?.as_slice())?)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn validate_recovery(tx: &ReadTransaction) -> Result<(), StoreError> {
|
||||
validate_recovery_with_limits(tx, None)
|
||||
}
|
||||
|
||||
pub(crate) fn validate_recovery_with_limits(
|
||||
tx: &ReadTransaction,
|
||||
override_limits: Option<CheckpointLimits>,
|
||||
) -> Result<(), StoreError> {
|
||||
validate_recovery_inner(tx, override_limits).map_err(resource_error)
|
||||
}
|
||||
|
||||
fn validate_recovery_inner(tx: &ReadTransaction, override_limits: Option<CheckpointLimits>) -> Result<(), StoreError> {
|
||||
let meta = tx.open_table(crate::META)?;
|
||||
let metadata = required(read_metadata(&meta)?)?;
|
||||
crate::raft_state_machine::selected_meta(&meta, &metadata)?;
|
||||
let cut = metadata.header.applied;
|
||||
let recovery_limits = override_limits.unwrap_or(metadata.limits);
|
||||
recovery_limits.validate()?;
|
||||
let identity = crate::decode_identity(required(meta.get("identity")?)?.value())?;
|
||||
let binding = application::read_binding(&meta)?;
|
||||
if identity.cluster_id != metadata.header.cluster_id || binding != metadata.header.binding {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let purged = read_purged(&meta)?;
|
||||
validate_baseline_with_budget(tx, &metadata, recovery_limits.staging_bytes)?;
|
||||
let last = application::read_applied(&meta)?;
|
||||
optional_at_or_before(cut, last)?;
|
||||
let logs = tx.open_table(crate::LOGS)?;
|
||||
let committed = crate::committed::validate_committed(&meta, &logs)?;
|
||||
optional_at_or_before(last, committed)?;
|
||||
let declared_tail = meta.get("tail")?.map(|v| crate::decode_id(v.value())).transpose()?;
|
||||
optional_at_or_before(committed, declared_tail)?;
|
||||
let (_directory, scratch) = recovery_scratch(recovery_limits.staging_bytes)?;
|
||||
for (source, target) in [
|
||||
(HEADS, application::HEADS),
|
||||
(VERSIONS, application::VERSIONS),
|
||||
(EVENTS, application::EVENTS),
|
||||
(OPERATIONS, application::OPERATIONS),
|
||||
] {
|
||||
let table = tx.open_table(source)?;
|
||||
let mut write = scratch.begin_write()?;
|
||||
write.open_table(target)?;
|
||||
let mut pending = 0;
|
||||
for row in table.iter()? {
|
||||
let (key, value) = row?;
|
||||
write.open_table(target)?.insert(key.value(), value.value())?;
|
||||
pending += 1;
|
||||
if pending == 128 {
|
||||
write.commit().map_err(StoreError::StagingCommit)?;
|
||||
write = scratch.begin_write()?;
|
||||
pending = 0;
|
||||
}
|
||||
}
|
||||
write.commit().map_err(StoreError::StagingCommit)?;
|
||||
}
|
||||
let mut membership: CaptureMembership = metadata.header.membership.clone();
|
||||
if let Some(cut) = cut
|
||||
&& let Some(retained_cut) = logs.get(cut.index)?
|
||||
&& crate::decode_log(retained_cut.value())?.id != cut
|
||||
{
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let results = tx.open_table(application::APPLY_RESULTS)?;
|
||||
// Covered entries may remain until purge. Check their exact typed responses
|
||||
// without rebuilding the historical state or allocating a prefix-sized map.
|
||||
let operations = tx.open_table(OPERATIONS)?;
|
||||
let mut actual_tail = purged;
|
||||
let mut prefix_next = purged.map_or(Some(0), |id| id.index.checked_add(1));
|
||||
if let Some(cut) = cut {
|
||||
for row in logs.range(..=cut.index)? {
|
||||
let (key, bytes) = row?;
|
||||
if purged.is_some_and(|purged| key.value() <= purged.index) {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
if bytes.value().len() > 1024 * 1024 {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let entry = crate::decode_log(bytes.value())?;
|
||||
if entry.id.index != key.value() || prefix_next != Some(key.value()) {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
actual_tail = Some(entry.id);
|
||||
prefix_next = key.value().checked_add(1);
|
||||
let native = application::decode_native_entry(entry)?;
|
||||
let bytes = required(results.get(key.value())?)?;
|
||||
if bytes.value().len() > 1024 * 1024 {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let response = application::decode_response(bytes.value())?;
|
||||
application::validate_response_kind(&native, &response)?;
|
||||
if let openraft::EntryPayload::Normal(command) = native.payload {
|
||||
let operation = application::decode_provenance(
|
||||
required(operations.get(command.created().operation_id.as_slice())?)?.value(),
|
||||
)?;
|
||||
let canonical = command.created().encode()?;
|
||||
let expected = if operation.operation.canonical_command == canonical {
|
||||
operation.operation.result
|
||||
} else {
|
||||
CreatedResult::OperationConflict
|
||||
};
|
||||
if response.created_result() != Some(&expected) {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
if operation.first_decision_log_id.index == key.value()
|
||||
&& operation.first_decision_log_id != LogId::from(native.log_id)
|
||||
{
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
}
|
||||
}
|
||||
for row in results.range(..=cut.index)? {
|
||||
let (key, _) = row?;
|
||||
if purged.is_some_and(|purged| key.value() <= purged.index) || logs.get(key.value())?.is_none() {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
}
|
||||
if purged.is_none_or(|purged| purged.index < cut.index) && logs.get(cut.index)?.is_none() {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
}
|
||||
if actual_tail != cut {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
|
||||
let mut observed = cut;
|
||||
let mut next = cut.map_or(Some(0), |cut| cut.index.checked_add(1));
|
||||
let mut budget = Budget::new(recovery_limits.snapshot);
|
||||
if let Some(start) = next {
|
||||
for row in logs.range(start..)? {
|
||||
let (key, bytes) = row?;
|
||||
budget.row(&key.value().to_be_bytes(), bytes.value())?;
|
||||
let entry = crate::decode_log(bytes.value())?;
|
||||
if next != Some(key.value()) || entry.id.index != key.value() {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
next = key.value().checked_add(1);
|
||||
actual_tail = Some(entry.id);
|
||||
let native = application::decode_native_entry(entry.clone())?;
|
||||
if last.is_none_or(|last| key.value() > last.index) {
|
||||
continue;
|
||||
}
|
||||
let response_bytes = required(results.get(key.value())?)?;
|
||||
if response_bytes.value().len() > 1024 * 1024 {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
if u64::try_from(response_bytes.value().len()).map_err(|_| StoreError::CorruptRecord)?
|
||||
> recovery_limits.snapshot.value_bytes
|
||||
{
|
||||
return Err(StoreError::ResourceLimit);
|
||||
}
|
||||
let response = application::decode_response(response_bytes.value())?;
|
||||
application::validate_response_kind(&native, &response)?;
|
||||
let write = scratch.begin_write()?;
|
||||
let expected = match native.payload {
|
||||
openraft::EntryPayload::Blank => crate::CaptureResponse::blank(entry.id)?,
|
||||
openraft::EntryPayload::Membership(value) => {
|
||||
membership = CaptureMembership::try_from(&openraft::StoredMembership::new(Some(entry.id.into()), value))?;
|
||||
crate::CaptureResponse::membership(entry.id)?
|
||||
}
|
||||
openraft::EntryPayload::Normal(command) => {
|
||||
let canonical = command.created().encode()?;
|
||||
let result = application::apply_created(&write, entry.id, command.created(), &canonical, &binding, 6)?;
|
||||
crate::CaptureResponse::created(CreatedApplyResult {
|
||||
log_id: entry.id,
|
||||
result,
|
||||
})?
|
||||
}
|
||||
};
|
||||
if response != expected {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
write.commit().map_err(StoreError::StagingCommit)?;
|
||||
observed = Some(entry.id);
|
||||
}
|
||||
for row in results.range(start..)? {
|
||||
let (key, _) = row?;
|
||||
if last.is_none_or(|last| key.value() > last.index) || logs.get(key.value())?.is_none() {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
}
|
||||
}
|
||||
if actual_tail != declared_tail || observed != last || application::native_application_state(tx)?.1 != membership {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let recovered = scratch.begin_read()?;
|
||||
let mut live_budget = Budget::new(recovery_limits.snapshot);
|
||||
for definition in [
|
||||
application::HEADS,
|
||||
application::VERSIONS,
|
||||
application::EVENTS,
|
||||
application::OPERATIONS,
|
||||
] {
|
||||
let expected = recovered.open_table(definition)?;
|
||||
let actual = tx.open_table(definition)?;
|
||||
if expected.len()? != actual.len()? {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
for row in actual.iter()? {
|
||||
let (key, value) = row?;
|
||||
live_budget.row(key.value(), value.value())?;
|
||||
if required(expected.get(key.value())?)?.value() != value.value() {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn optional_at_or_before(id: Option<LogId>, cut: Option<LogId>) -> Result<(), StoreError> {
|
||||
match (id, cut) {
|
||||
(None, _) => Ok(()),
|
||||
(Some(id), Some(cut)) => at_or_before(id, cut),
|
||||
_ => Err(StoreError::CorruptRecord),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
use crate::{LOGS, LogId, META, Store, StoreError, decode_id, decode_log, encode_id, number};
|
||||
use redb::{ReadableDatabase, ReadableTable};
|
||||
|
||||
pub(crate) fn validate_committed(
|
||||
meta: &impl ReadableTable<&'static str, &'static [u8]>,
|
||||
logs: &impl ReadableTable<u64, &'static [u8]>,
|
||||
) -> Result<Option<LogId>, StoreError> {
|
||||
let schema = number(meta.get("schema")?.ok_or(StoreError::CorruptRecord)?.value())?;
|
||||
let marker = meta.get("committed")?;
|
||||
match (schema, marker) {
|
||||
(1 | 3 | 4 | 5 | 6 | 7, None) => {
|
||||
if matches!(schema, 3..=7) && meta.get("last_applied")?.is_none_or(|v| v.value() != [0]) {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
(2 | 3 | 4 | 5 | 6 | 7, Some(marker)) => {
|
||||
let id = decode_id(marker.value())?;
|
||||
match logs.get(id.index)? {
|
||||
Some(record) if decode_log(record.value())?.id == id => {}
|
||||
None if schema == 7 && crate::checkpoint_baseline::read_baseline_cut(meta)? == Some(id) => {}
|
||||
_ => return Err(StoreError::CorruptRecord),
|
||||
}
|
||||
Ok(Some(id))
|
||||
}
|
||||
(1 | 2, _) => Err(StoreError::CorruptRecord),
|
||||
_ => Err(StoreError::UnsupportedSchema),
|
||||
}
|
||||
}
|
||||
|
||||
impl Store {
|
||||
/// Read the exact committed boundary, validating it against the stored log.
|
||||
pub fn read_committed(&self) -> Result<Option<LogId>, StoreError> {
|
||||
let tx = self.database.begin_read()?;
|
||||
validate_committed(&tx.open_table(META)?, &tx.open_table(LOGS)?)
|
||||
}
|
||||
|
||||
/// Persist a monotonic committed boundary supplied by the authority owner.
|
||||
pub fn save_committed(&mut self, id: LogId) -> Result<(), StoreError> {
|
||||
let tx = self.write_transaction()?;
|
||||
{
|
||||
let mut meta = tx.open_table(META)?;
|
||||
let logs = tx.open_table(LOGS)?;
|
||||
let previous = validate_committed(&meta, &logs)?;
|
||||
if previous == Some(id) {
|
||||
return Ok(());
|
||||
}
|
||||
let record = logs.get(id.index)?.ok_or(StoreError::InvalidCommittedLog)?;
|
||||
if decode_log(record.value())?.id != id {
|
||||
return Err(StoreError::InvalidCommittedLog);
|
||||
}
|
||||
if let Some(previous) = previous {
|
||||
if id.index < previous.index {
|
||||
return Err(StoreError::CommittedRegression);
|
||||
}
|
||||
}
|
||||
meta.insert("committed", encode_id(id).as_slice())?;
|
||||
if previous.is_none() && !matches!(number(meta.get("schema")?.ok_or(StoreError::CorruptRecord)?.value())?, 3..=7) {
|
||||
meta.insert("schema", 2u64.to_be_bytes().as_slice())?;
|
||||
}
|
||||
}
|
||||
self.commit(tx)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,278 @@
|
||||
//! Reuse the existing actual Raft owner fixture, never a caller-constructed result proof.
|
||||
use super::*;
|
||||
|
||||
async fn decide(cluster: &Cluster, value: u8) -> (DecideCreated, LogId) {
|
||||
let input = command(value);
|
||||
let expected = input.created().clone();
|
||||
let response = cluster
|
||||
.node(0)
|
||||
.owner()
|
||||
.client_write(input)
|
||||
.await
|
||||
.expect("actual Raft Created decision");
|
||||
let Some(CreatedResult::Created { decision_log_id, .. }) = response.data.created_result() else {
|
||||
panic!("successful Created result required")
|
||||
};
|
||||
(expected, *decision_log_id)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn committed_created_authorization_exact() {
|
||||
let cluster = Cluster::start(&[0], 100).await;
|
||||
let (expected, first) = decide(&cluster, 1).await;
|
||||
let authorization = cluster
|
||||
.node(0)
|
||||
.owner()
|
||||
.authorize_created(expected.clone(), first)
|
||||
.await
|
||||
.expect("durable exact authorization");
|
||||
assert_eq!(authorization.identity(), identity(0));
|
||||
assert_eq!(authorization.operation_id(), expected.operation_id);
|
||||
assert_eq!(authorization.object(), &expected.object);
|
||||
assert_eq!(authorization.prepared(), &expected.prepared);
|
||||
assert_eq!(authorization.binding(), &binding());
|
||||
assert_eq!(authorization.decision_log_id(), first);
|
||||
assert_eq!(cluster.node(0).page().await.events[0].captured.decision_log_id, first);
|
||||
cluster.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn committed_created_authorization_retry_first_decision() {
|
||||
let cluster = Cluster::start(&[0], 100).await;
|
||||
let (expected, first) = decide(&cluster, 2).await;
|
||||
let before = cluster
|
||||
.node(0)
|
||||
.owner()
|
||||
.authorize_created(expected.clone(), first)
|
||||
.await
|
||||
.expect("first authorization");
|
||||
let response = cluster
|
||||
.node(0)
|
||||
.owner()
|
||||
.client_write(CaptureCommand::try_created(expected.clone()).expect("retry command"))
|
||||
.await
|
||||
.expect("real retry");
|
||||
assert_ne!(LogId::from(response.log_id), first);
|
||||
assert!(
|
||||
matches!(response.data.created_result(), Some(CreatedResult::Created{decision_log_id,..}) if *decision_log_id==first)
|
||||
);
|
||||
let after = cluster
|
||||
.node(0)
|
||||
.owner()
|
||||
.authorize_created(expected.clone(), first)
|
||||
.await
|
||||
.expect("first decision retained");
|
||||
assert_eq!(before, after);
|
||||
assert!(
|
||||
cluster
|
||||
.node(0)
|
||||
.owner()
|
||||
.authorize_created(expected, response.log_id.into())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(cluster.node(0).page().await.events.len(), 1);
|
||||
cluster.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn committed_created_authorization_rejects_mismatch() {
|
||||
let cluster = Cluster::start(&[0], 100).await;
|
||||
let (expected, first) = decide(&cluster, 3).await;
|
||||
let before = cluster.node(0).page().await;
|
||||
let durable_before = cluster
|
||||
.node(0)
|
||||
.writer
|
||||
.write(|store| {
|
||||
Ok((
|
||||
store.read_vote()?,
|
||||
store.read_committed()?,
|
||||
store.read_entries(0, u64::MAX)?,
|
||||
store.native_application_state()?,
|
||||
store.created_events_page(None, 256)?,
|
||||
))
|
||||
})
|
||||
.await
|
||||
.expect("durable state before rejected authorization");
|
||||
let mut variants = Vec::new();
|
||||
let mut altered = expected.clone();
|
||||
altered.operation_id = [4; 16];
|
||||
variants.push(altered);
|
||||
let mut altered = expected.clone();
|
||||
altered.object.key.push(9);
|
||||
variants.push(altered);
|
||||
let mut altered = expected.clone();
|
||||
altered.object.bucket_incarnation = [9; 16];
|
||||
variants.push(altered);
|
||||
let mut altered = expected.clone();
|
||||
altered.expected_head = Some([9; 16]);
|
||||
variants.push(altered);
|
||||
let mut altered = expected.clone();
|
||||
altered.prepared.preparation_id = [9; 16];
|
||||
variants.push(altered);
|
||||
let mut altered = expected.clone();
|
||||
altered.prepared.version_id = [9; 16];
|
||||
variants.push(altered);
|
||||
let mut altered = expected.clone();
|
||||
altered.prepared.content_digest = [9; 32];
|
||||
variants.push(altered);
|
||||
let mut altered = expected.clone();
|
||||
altered.prepared.content_length += 1;
|
||||
variants.push(altered);
|
||||
let mut altered = expected.clone();
|
||||
altered.binding_revision += 1;
|
||||
variants.push(altered);
|
||||
for altered in variants {
|
||||
assert!(
|
||||
cluster.node(0).owner().authorize_created(altered, first).await.is_err(),
|
||||
"AUTHORIZATION_REQUIRES_EXACT_PREPARED_IDENTITY_AND_COMMAND"
|
||||
);
|
||||
}
|
||||
for wrong in [
|
||||
LogId {
|
||||
term: first.term + 1,
|
||||
..first
|
||||
},
|
||||
LogId {
|
||||
leader_node: first.leader_node + 1,
|
||||
..first
|
||||
},
|
||||
LogId {
|
||||
index: first.index + 1,
|
||||
..first
|
||||
},
|
||||
] {
|
||||
assert!(
|
||||
cluster
|
||||
.node(0)
|
||||
.owner()
|
||||
.authorize_created(expected.clone(), wrong)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
let durable_after = cluster
|
||||
.node(0)
|
||||
.writer
|
||||
.write(|store| {
|
||||
Ok((
|
||||
store.read_vote()?,
|
||||
store.read_committed()?,
|
||||
store.read_entries(0, u64::MAX)?,
|
||||
store.native_application_state()?,
|
||||
store.created_events_page(None, 256)?,
|
||||
))
|
||||
})
|
||||
.await
|
||||
.expect("durable state after rejected authorization");
|
||||
assert_eq!(durable_before, durable_after, "REJECTED_AUTHORIZATION_DOES_NOT_WRITE");
|
||||
let mut failed = expected.clone();
|
||||
failed.operation_id = [5; 16];
|
||||
failed.prepared.version_id = [5; 16];
|
||||
let response = cluster
|
||||
.node(0)
|
||||
.owner()
|
||||
.client_write(CaptureCommand::try_created(failed.clone()).expect("failed command"))
|
||||
.await
|
||||
.expect("real head mismatch decision");
|
||||
assert!(matches!(response.data.created_result(), Some(CreatedResult::HeadMismatch { .. })));
|
||||
assert!(
|
||||
cluster
|
||||
.node(0)
|
||||
.owner()
|
||||
.authorize_created(failed, response.log_id.into())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(cluster.node(0).page().await, before);
|
||||
cluster.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn committed_created_authorization_restart() {
|
||||
let routes = Arc::new(Routes::default());
|
||||
let mut node = Node::start(0, routes.clone(), config(100)).await;
|
||||
node.owner().bootstrap(members(&[0])).await.expect("bootstrap actual owner");
|
||||
node.owner()
|
||||
.raft()
|
||||
.wait(Some(Duration::from_secs(15)))
|
||||
.state(ServerState::Leader, "leader")
|
||||
.await
|
||||
.expect("leader ready");
|
||||
let expected = command(6).created().clone();
|
||||
let response = node.owner().client_write(command(6)).await.expect("real first write");
|
||||
let Some(CreatedResult::Created {
|
||||
decision_log_id: first, ..
|
||||
}) = response.data.created_result()
|
||||
else {
|
||||
panic!("Created")
|
||||
};
|
||||
let first = *first;
|
||||
let before = node
|
||||
.owner()
|
||||
.authorize_created(expected.clone(), first)
|
||||
.await
|
||||
.expect("before restart");
|
||||
node.stop(&routes).await;
|
||||
let Node {
|
||||
directory,
|
||||
writer,
|
||||
owner: _,
|
||||
id,
|
||||
} = node;
|
||||
drop(writer);
|
||||
let writer = NativeLogWriter::new(Store::open(directory.path().join("db"), identity(id)).expect("retained redb"));
|
||||
// A retained commit-before-apply crash fixture is not a client-provided proof.
|
||||
// Only real owner initialization may replay it before authorization can succeed.
|
||||
use openraft::storage::{RaftLogStorage, RaftLogStorageExt};
|
||||
let pending = command(7).created().clone();
|
||||
let mut logs = NativeRaftLogStore::new(writer.clone());
|
||||
let last = logs.get_log_state().await.expect("retained tail").last_log_id.expect("tail");
|
||||
let pending_id = LogId {
|
||||
term: last.leader_id.term,
|
||||
leader_node: last.leader_id.node_id,
|
||||
index: last.index + 1,
|
||||
};
|
||||
logs.blocking_append(vec![CaptureRaftEntry {
|
||||
log_id: pending_id.into(),
|
||||
payload: openraft::EntryPayload::Normal(CaptureCommand::try_created(pending.clone()).expect("pending command")),
|
||||
}])
|
||||
.await
|
||||
.expect("retained crash suffix");
|
||||
logs.save_committed(Some(pending_id.into()))
|
||||
.await
|
||||
.expect("retained commit marker");
|
||||
assert!(
|
||||
writer.authorize_created(pending.clone(), pending_id).await.is_err(),
|
||||
"UNAPPLIED_COMMIT_IS_NOT_AUTHORIZATION"
|
||||
);
|
||||
drop(logs);
|
||||
let owner = NativeRaftOwner::start(
|
||||
writer.clone(),
|
||||
id,
|
||||
config(100),
|
||||
CheckpointLimits::default(),
|
||||
DirectFactory {
|
||||
source: id,
|
||||
routes: routes.clone(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("real recovery");
|
||||
routes.register(id, owner.raft().clone());
|
||||
let after = owner
|
||||
.authorize_created(expected, first)
|
||||
.await
|
||||
.expect("durable authorization after restart");
|
||||
assert_eq!(before, after);
|
||||
let replayed = owner
|
||||
.authorize_created(pending.clone(), pending_id)
|
||||
.await
|
||||
.expect("actual owner committed recovery authorizes");
|
||||
assert_eq!(replayed.prepared(), &pending.prepared);
|
||||
assert_eq!(replayed.decision_log_id(), pending_id);
|
||||
owner.shutdown().await.expect("drain recovered owner");
|
||||
routes.remove(id);
|
||||
drop(owner);
|
||||
drop(writer);
|
||||
}
|
||||
@@ -0,0 +1,297 @@
|
||||
use crate::{LogId, StoreError, decode_id, encode_id};
|
||||
|
||||
pub const MAX_OBJECT_KEY_LENGTH: usize = 4096;
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ObjectIdentity {
|
||||
pub bucket_incarnation: [u8; 16],
|
||||
pub key: Vec<u8>,
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct PreparedIdentity {
|
||||
pub version_id: [u8; 16],
|
||||
pub preparation_id: [u8; 16],
|
||||
pub content_digest: [u8; 32],
|
||||
pub content_length: u64,
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct DecideCreated {
|
||||
pub operation_id: [u8; 16],
|
||||
pub object: ObjectIdentity,
|
||||
pub expected_head: Option<[u8; 16]>,
|
||||
pub prepared: PreparedIdentity,
|
||||
pub binding_revision: u64,
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct CaptureBinding {
|
||||
pub revision: u64,
|
||||
pub binding_id: [u8; 16],
|
||||
pub target_id: [u8; 16],
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum CreatedResult {
|
||||
Created {
|
||||
version_id: [u8; 16],
|
||||
event_id: [u8; 16],
|
||||
decision_log_id: LogId,
|
||||
},
|
||||
HeadMismatch {
|
||||
actual_head: Option<[u8; 16]>,
|
||||
},
|
||||
OperationConflict,
|
||||
BindingMismatch,
|
||||
VersionConflict,
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CreatedVersion {
|
||||
pub prepared: PreparedIdentity,
|
||||
pub decision_log_id: LogId,
|
||||
pub binding_revision: u64,
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct CreatedEvent {
|
||||
pub object: ObjectIdentity,
|
||||
pub prepared: PreparedIdentity,
|
||||
pub decision_log_id: LogId,
|
||||
pub binding: CaptureBinding,
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CreatedOperation {
|
||||
pub canonical_command: Vec<u8>,
|
||||
pub result: CreatedResult,
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CreatedOperationProvenance {
|
||||
pub operation: CreatedOperation,
|
||||
pub first_decision_log_id: LogId,
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CreatedApplyResult {
|
||||
pub log_id: LogId,
|
||||
pub result: CreatedResult,
|
||||
}
|
||||
|
||||
/// Local publication authority re-read from the actual owner's durable Created decision.
|
||||
/// This is not a serializable remote proof and does not attest physical preparation durability.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CommittedCreatedAuthorization {
|
||||
identity: crate::NodeIdentity,
|
||||
command: DecideCreated,
|
||||
binding: CaptureBinding,
|
||||
decision_log_id: LogId,
|
||||
}
|
||||
impl CommittedCreatedAuthorization {
|
||||
pub(crate) fn verified(
|
||||
identity: crate::NodeIdentity,
|
||||
command: DecideCreated,
|
||||
binding: CaptureBinding,
|
||||
decision_log_id: LogId,
|
||||
) -> Self {
|
||||
Self {
|
||||
identity,
|
||||
command,
|
||||
binding,
|
||||
decision_log_id,
|
||||
}
|
||||
}
|
||||
pub fn identity(&self) -> crate::NodeIdentity {
|
||||
self.identity
|
||||
}
|
||||
pub fn operation_id(&self) -> [u8; 16] {
|
||||
self.command.operation_id
|
||||
}
|
||||
pub fn object(&self) -> &ObjectIdentity {
|
||||
&self.command.object
|
||||
}
|
||||
pub fn expected_head(&self) -> Option<[u8; 16]> {
|
||||
self.command.expected_head
|
||||
}
|
||||
pub fn prepared(&self) -> &PreparedIdentity {
|
||||
&self.command.prepared
|
||||
}
|
||||
pub fn binding(&self) -> &CaptureBinding {
|
||||
&self.binding
|
||||
}
|
||||
pub fn decision_log_id(&self) -> LogId {
|
||||
self.decision_log_id
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct Decoder<'a> {
|
||||
bytes: &'a [u8],
|
||||
}
|
||||
impl<'a> Decoder<'a> {
|
||||
pub(crate) fn new(bytes: &'a [u8]) -> Self {
|
||||
Self { bytes }
|
||||
}
|
||||
pub(crate) fn take<const N: usize>(&mut self) -> Result<[u8; N], StoreError> {
|
||||
let a = self
|
||||
.bytes
|
||||
.get(..N)
|
||||
.ok_or(StoreError::CorruptRecord)?
|
||||
.try_into()
|
||||
.map_err(|_| StoreError::CorruptRecord)?;
|
||||
self.bytes = &self.bytes[N..];
|
||||
Ok(a)
|
||||
}
|
||||
pub(crate) fn u64(&mut self) -> Result<u64, StoreError> {
|
||||
Ok(u64::from_be_bytes(self.take()?))
|
||||
}
|
||||
pub(crate) fn uuid(&mut self) -> Result<[u8; 16], StoreError> {
|
||||
let id = self.take()?;
|
||||
if id == [0; 16] {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
Ok(id)
|
||||
}
|
||||
pub(crate) fn option(&mut self) -> Result<Option<[u8; 16]>, StoreError> {
|
||||
match self.take::<1>()?[0] {
|
||||
0 => Ok(None),
|
||||
1 => Ok(Some(self.uuid()?)),
|
||||
_ => Err(StoreError::CorruptRecord),
|
||||
}
|
||||
}
|
||||
pub(crate) fn object(&mut self) -> Result<ObjectIdentity, StoreError> {
|
||||
let bucket_incarnation = self.uuid()?;
|
||||
let len = usize::try_from(u32::from_be_bytes(self.take()?)).map_err(|_| StoreError::CorruptRecord)?;
|
||||
if len > MAX_OBJECT_KEY_LENGTH {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let key = self.bytes.get(..len).ok_or(StoreError::CorruptRecord)?.to_vec();
|
||||
self.bytes = &self.bytes[len..];
|
||||
Ok(ObjectIdentity { bucket_incarnation, key })
|
||||
}
|
||||
pub(crate) fn prepared(&mut self) -> Result<PreparedIdentity, StoreError> {
|
||||
Ok(PreparedIdentity {
|
||||
version_id: self.uuid()?,
|
||||
preparation_id: self.uuid()?,
|
||||
content_digest: self.take()?,
|
||||
content_length: self.u64()?,
|
||||
})
|
||||
}
|
||||
pub(crate) fn id(&mut self) -> Result<LogId, StoreError> {
|
||||
decode_id(&self.take::<24>()?)
|
||||
}
|
||||
pub(crate) fn binding(&mut self) -> Result<CaptureBinding, StoreError> {
|
||||
Ok(CaptureBinding {
|
||||
revision: self.u64()?,
|
||||
binding_id: self.uuid()?,
|
||||
target_id: self.uuid()?,
|
||||
})
|
||||
}
|
||||
pub(crate) fn finish(self) -> Result<(), StoreError> {
|
||||
if self.bytes.is_empty() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(StoreError::CorruptRecord)
|
||||
}
|
||||
}
|
||||
pub(crate) fn result(&mut self) -> Result<CreatedResult, StoreError> {
|
||||
match self.take::<1>()?[0] {
|
||||
0 => Ok(CreatedResult::Created {
|
||||
version_id: self.uuid()?,
|
||||
event_id: self.uuid()?,
|
||||
decision_log_id: self.id()?,
|
||||
}),
|
||||
1 => Ok(CreatedResult::HeadMismatch {
|
||||
actual_head: self.option()?,
|
||||
}),
|
||||
2 => Ok(CreatedResult::OperationConflict),
|
||||
3 => Ok(CreatedResult::BindingMismatch),
|
||||
4 => Ok(CreatedResult::VersionConflict),
|
||||
_ => Err(StoreError::CorruptRecord),
|
||||
}
|
||||
}
|
||||
}
|
||||
pub(crate) fn put_option(b: &mut Vec<u8>, id: Option<[u8; 16]>) {
|
||||
b.push(u8::from(id.is_some()));
|
||||
if let Some(id) = id {
|
||||
b.extend(id)
|
||||
}
|
||||
}
|
||||
pub(crate) fn object_bytes(o: &ObjectIdentity) -> Result<Vec<u8>, StoreError> {
|
||||
if o.bucket_incarnation == [0; 16] || o.key.len() > MAX_OBJECT_KEY_LENGTH {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let capacity = 20usize.checked_add(o.key.len()).ok_or(StoreError::CorruptRecord)?;
|
||||
let mut b = Vec::with_capacity(capacity);
|
||||
b.extend(o.bucket_incarnation);
|
||||
b.extend(
|
||||
u32::try_from(o.key.len())
|
||||
.map_err(|_| StoreError::CorruptRecord)?
|
||||
.to_be_bytes(),
|
||||
);
|
||||
b.extend(&o.key);
|
||||
Ok(b)
|
||||
}
|
||||
pub(crate) fn put_prepared(b: &mut Vec<u8>, p: &PreparedIdentity) {
|
||||
b.extend(p.version_id);
|
||||
b.extend(p.preparation_id);
|
||||
b.extend(p.content_digest);
|
||||
b.extend(p.content_length.to_be_bytes())
|
||||
}
|
||||
pub(crate) fn binding_bytes(v: &CaptureBinding) -> Result<Vec<u8>, StoreError> {
|
||||
let mut b = v.revision.to_be_bytes().to_vec();
|
||||
b.extend(v.binding_id);
|
||||
b.extend(v.target_id);
|
||||
let mut d = Decoder::new(&b);
|
||||
d.binding()?;
|
||||
d.finish()?;
|
||||
Ok(b)
|
||||
}
|
||||
pub(crate) fn result_bytes(r: &CreatedResult) -> Vec<u8> {
|
||||
let mut b = Vec::new();
|
||||
match r {
|
||||
CreatedResult::Created {
|
||||
version_id,
|
||||
event_id,
|
||||
decision_log_id,
|
||||
} => {
|
||||
b.push(0);
|
||||
b.extend(version_id);
|
||||
b.extend(event_id);
|
||||
b.extend(encode_id(*decision_log_id))
|
||||
}
|
||||
CreatedResult::HeadMismatch { actual_head } => {
|
||||
b.push(1);
|
||||
put_option(&mut b, *actual_head)
|
||||
}
|
||||
CreatedResult::OperationConflict => b.push(2),
|
||||
CreatedResult::BindingMismatch => b.push(3),
|
||||
CreatedResult::VersionConflict => b.push(4),
|
||||
}
|
||||
b
|
||||
}
|
||||
impl DecideCreated {
|
||||
pub fn encode(&self) -> Result<Vec<u8>, StoreError> {
|
||||
let object = object_bytes(&self.object)?;
|
||||
let capacity = object.len().checked_add(134).ok_or(StoreError::CorruptRecord)?;
|
||||
let mut b = Vec::with_capacity(capacity);
|
||||
b.extend(b"CRTD\x01");
|
||||
b.extend(self.operation_id);
|
||||
b.extend(object);
|
||||
put_option(&mut b, self.expected_head);
|
||||
put_prepared(&mut b, &self.prepared);
|
||||
b.extend(self.binding_revision.to_be_bytes());
|
||||
Self::decode(&b)?;
|
||||
Ok(b)
|
||||
}
|
||||
pub fn decode(bytes: &[u8]) -> Result<Self, StoreError> {
|
||||
let mut d = Decoder::new(bytes);
|
||||
if d.take::<5>()? != *b"CRTD\x01" {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let c = Self {
|
||||
operation_id: d.uuid()?,
|
||||
object: d.object()?,
|
||||
expected_head: d.option()?,
|
||||
prepared: d.prepared()?,
|
||||
binding_revision: d.u64()?,
|
||||
};
|
||||
d.finish()?;
|
||||
Ok(c)
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,588 @@
|
||||
//! Local checkpoints are captured from this writer's owned transaction only.
|
||||
use crate::{
|
||||
LogId, NativeLogWriter, NativeSnapshotError, NativeSnapshotView, StoreError,
|
||||
checkpoint_baseline::{self, BaselineMetadata},
|
||||
snapshot,
|
||||
};
|
||||
use redb::{ReadableDatabase, ReadableTable, StorageBackend};
|
||||
use std::{
|
||||
fs::File,
|
||||
io::{self, Seek, Write},
|
||||
ops::Bound,
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
},
|
||||
};
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct CheckpointLimits {
|
||||
pub snapshot: snapshot::Limits,
|
||||
pub staging_bytes: u64,
|
||||
}
|
||||
|
||||
impl CheckpointLimits {
|
||||
pub(crate) fn validate(&self) -> Result<(), StoreError> {
|
||||
let rows = self.snapshot;
|
||||
if rows.total_bytes == 0
|
||||
|| rows.membership_bytes == 0
|
||||
|| rows.membership_bytes > 512 * 1024
|
||||
|| rows.key_bytes == 0
|
||||
|| rows.key_bytes > 1024 * 1024
|
||||
|| rows.value_bytes == 0
|
||||
|| rows.value_bytes > 1024 * 1024
|
||||
|| rows.records == 0
|
||||
|| self.staging_bytes < 4 * 1024 * 1024
|
||||
{
|
||||
return Err(StoreError::ResourceLimit);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for CheckpointLimits {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
snapshot: snapshot::Limits {
|
||||
total_bytes: 256 * 1024 * 1024,
|
||||
membership_bytes: 512 * 1024,
|
||||
key_bytes: 8192,
|
||||
value_bytes: 8192,
|
||||
records: 1_000_000,
|
||||
},
|
||||
staging_bytes: 1024 * 1024 * 1024,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct CheckpointReceipt {
|
||||
generation: u64,
|
||||
cut: LogId,
|
||||
}
|
||||
|
||||
impl CheckpointReceipt {
|
||||
pub fn generation(&self) -> u64 {
|
||||
self.generation
|
||||
}
|
||||
pub fn cut(&self) -> LogId {
|
||||
self.cut
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum CheckpointError {
|
||||
#[error(transparent)]
|
||||
Store(#[from] StoreError),
|
||||
#[error(transparent)]
|
||||
Capture(#[from] NativeSnapshotError),
|
||||
#[error(transparent)]
|
||||
Snapshot(#[from] snapshot::SnapshotError),
|
||||
#[error("checkpoint filesystem failure")]
|
||||
Io(#[from] io::Error),
|
||||
#[error("checkpoint worker failed")]
|
||||
Worker(#[source] tokio::task::JoinError),
|
||||
#[error("checkpoint store lock poisoned")]
|
||||
Poisoned,
|
||||
#[error("checkpoint conflict: {0}")]
|
||||
Conflict(&'static str),
|
||||
#[error("checkpoint cancelled before publication")]
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
/// Enforces the file-length budget before redb can allocate or write a page.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct BoundedBackend {
|
||||
inner: redb::backends::FileBackend,
|
||||
maximum: u64,
|
||||
}
|
||||
|
||||
impl BoundedBackend {
|
||||
fn new(file: File, maximum: u64) -> Result<Self, redb::DatabaseError> {
|
||||
Ok(Self {
|
||||
inner: redb::backends::FileBackend::new(file)?,
|
||||
maximum,
|
||||
})
|
||||
}
|
||||
|
||||
fn check(&self, end: u64) -> io::Result<()> {
|
||||
if end > self.maximum {
|
||||
return Err(io::Error::new(io::ErrorKind::StorageFull, "checkpoint staging budget"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl StorageBackend for BoundedBackend {
|
||||
fn len(&self) -> io::Result<u64> {
|
||||
self.inner.len()
|
||||
}
|
||||
fn read(&self, offset: u64, out: &mut [u8]) -> io::Result<()> {
|
||||
self.inner.read(offset, out)
|
||||
}
|
||||
fn set_len(&self, len: u64) -> io::Result<()> {
|
||||
self.check(len)?;
|
||||
self.inner.set_len(len)
|
||||
}
|
||||
fn sync_data(&self) -> io::Result<()> {
|
||||
self.inner.sync_data()
|
||||
}
|
||||
fn write(&self, offset: u64, data: &[u8]) -> io::Result<()> {
|
||||
let length = u64::try_from(data.len()).map_err(io::Error::other)?;
|
||||
self.check(
|
||||
offset
|
||||
.checked_add(length)
|
||||
.ok_or_else(|| io::Error::other("checkpoint write overflow"))?,
|
||||
)?;
|
||||
self.inner.write(offset, data)
|
||||
}
|
||||
fn close(&self) -> io::Result<()> {
|
||||
self.inner.close()
|
||||
}
|
||||
fn try_lock_range(&self, start: Bound<u64>, end: Bound<u64>) -> Result<bool, redb::BackendError> {
|
||||
self.inner.try_lock_range(start, end)
|
||||
}
|
||||
fn try_lock_shared_range(&self, start: Bound<u64>, end: Bound<u64>) -> Result<bool, redb::BackendError> {
|
||||
self.inner.try_lock_shared_range(start, end)
|
||||
}
|
||||
fn lock_range(&self, start: Bound<u64>, end: Bound<u64>) -> Result<(), redb::BackendError> {
|
||||
self.inner.lock_range(start, end)
|
||||
}
|
||||
fn lock_shared_range(&self, start: Bound<u64>, end: Bound<u64>) -> Result<(), redb::BackendError> {
|
||||
self.inner.lock_shared_range(start, end)
|
||||
}
|
||||
fn unlock_range(&self, start: Bound<u64>, end: Bound<u64>) -> Result<(), redb::BackendError> {
|
||||
self.inner.unlock_range(start, end)
|
||||
}
|
||||
fn query_lock_range(&self, start: Bound<u64>, end: Bound<u64>) -> Result<bool, redb::BackendError> {
|
||||
self.inner.query_lock_range(start, end)
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn recovery_scratch(maximum: u64) -> Result<(tempfile::TempDir, redb::Database), StoreError> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let database = stage_database(directory.path().join("stage.redb"), maximum)?;
|
||||
Ok((directory, database))
|
||||
}
|
||||
|
||||
pub(crate) fn stage_database(path: std::path::PathBuf, maximum: u64) -> Result<redb::Database, StoreError> {
|
||||
let file = File::options().read(true).write(true).create_new(true).open(path)?;
|
||||
let backend = BoundedBackend::new(file, maximum)?;
|
||||
Ok(redb::Database::builder()
|
||||
.set_cache_size(8 * 1024 * 1024)
|
||||
.create_with_backend(backend)?)
|
||||
}
|
||||
|
||||
struct PayloadFile {
|
||||
file: File,
|
||||
maximum: u64,
|
||||
written: u64,
|
||||
}
|
||||
|
||||
impl Write for PayloadFile {
|
||||
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
|
||||
let length = u64::try_from(bytes.len()).map_err(io::Error::other)?;
|
||||
let end = self
|
||||
.written
|
||||
.checked_add(length)
|
||||
.ok_or_else(|| io::Error::other("checkpoint payload overflow"))?;
|
||||
if end > self.maximum {
|
||||
return Err(io::Error::new(io::ErrorKind::StorageFull, "checkpoint payload budget"));
|
||||
}
|
||||
let count = self.file.write(bytes)?;
|
||||
self.written = self
|
||||
.written
|
||||
.checked_add(u64::try_from(count).map_err(io::Error::other)?)
|
||||
.ok_or_else(|| io::Error::other("checkpoint payload overflow"))?;
|
||||
Ok(count)
|
||||
}
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
self.file.flush()
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct StageSink<'a>(pub(crate) &'a redb::WriteTransaction);
|
||||
|
||||
impl snapshot::StagingSink for StageSink<'_> {
|
||||
fn row(&mut self, section: snapshot::Section, key: &[u8], value: &[u8]) -> io::Result<()> {
|
||||
let definition = match section {
|
||||
snapshot::Section::Heads => checkpoint_baseline::HEADS,
|
||||
snapshot::Section::Versions => checkpoint_baseline::VERSIONS,
|
||||
snapshot::Section::Events => checkpoint_baseline::EVENTS,
|
||||
snapshot::Section::Operations => checkpoint_baseline::OPERATIONS,
|
||||
};
|
||||
let mut table = self.0.open_table(definition).map_err(io::Error::other)?;
|
||||
if table.insert(key, value).map_err(io::Error::other)?.is_some() {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "duplicate checkpoint key"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// This type has no external constructors, deserialization, or path-based import.
|
||||
// Its source remains the owned local read transaction, not the decoded artifact.
|
||||
struct LocalStage {
|
||||
source: NativeSnapshotView,
|
||||
database: redb::Database,
|
||||
_directory: tempfile::TempDir,
|
||||
metadata: BaselineMetadata,
|
||||
}
|
||||
|
||||
fn capture_stage(source: NativeSnapshotView, limits: CheckpointLimits) -> Result<LocalStage, CheckpointError> {
|
||||
limits.validate()?;
|
||||
source
|
||||
.header()
|
||||
.applied
|
||||
.ok_or(CheckpointError::Conflict("empty applied cut"))?;
|
||||
let directory = tempfile::tempdir()?;
|
||||
let payload = File::options()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(directory.path().join("capture.csnp"))?;
|
||||
let artifact = source.stream_local_capability(
|
||||
PayloadFile {
|
||||
file: payload,
|
||||
maximum: limits.staging_bytes.min(limits.snapshot.total_bytes),
|
||||
written: 0,
|
||||
},
|
||||
limits.snapshot,
|
||||
)?;
|
||||
let remaining = limits
|
||||
.staging_bytes
|
||||
.checked_sub(artifact.payload.written)
|
||||
.ok_or(CheckpointError::Conflict("staging budget"))?;
|
||||
let mut payload = artifact.payload.file;
|
||||
payload.sync_all()?;
|
||||
payload.rewind()?;
|
||||
let database = stage_database(directory.path().join("stage.redb"), remaining)?;
|
||||
let transaction = database.begin_write().map_err(StoreError::from)?;
|
||||
for definition in [
|
||||
checkpoint_baseline::HEADS,
|
||||
checkpoint_baseline::VERSIONS,
|
||||
checkpoint_baseline::EVENTS,
|
||||
checkpoint_baseline::OPERATIONS,
|
||||
] {
|
||||
transaction.open_table(definition).map_err(StoreError::from)?;
|
||||
}
|
||||
let header = snapshot::decode_capability_artifact(
|
||||
snapshot::CapabilityArtifact {
|
||||
manifest: artifact.manifest,
|
||||
payload,
|
||||
},
|
||||
&mut StageSink(&transaction),
|
||||
limits.snapshot,
|
||||
)?;
|
||||
std::fs::remove_file(directory.path().join("capture.csnp"))?;
|
||||
if header != *source.header() {
|
||||
return Err(CheckpointError::Conflict("capture header"));
|
||||
}
|
||||
let metadata = BaselineMetadata {
|
||||
generation: 1,
|
||||
header,
|
||||
limits,
|
||||
};
|
||||
let encoded = checkpoint_baseline::encode_metadata(&metadata)?;
|
||||
{
|
||||
let mut meta = transaction.open_table(crate::META).map_err(StoreError::from)?;
|
||||
meta.insert("schema", 7u64.to_be_bytes().as_slice())
|
||||
.map_err(StoreError::from)?;
|
||||
meta.insert("checkpoint_baseline", encoded.as_slice())
|
||||
.map_err(StoreError::from)?;
|
||||
}
|
||||
transaction.commit().map_err(StoreError::StagingCommit)?;
|
||||
let stage = LocalStage {
|
||||
source,
|
||||
database,
|
||||
_directory: directory,
|
||||
metadata,
|
||||
};
|
||||
Ok(stage)
|
||||
}
|
||||
|
||||
fn verify_local_stage(stage: &LocalStage) -> Result<(), CheckpointError> {
|
||||
let transaction = stage.database.begin_read().map_err(StoreError::from)?;
|
||||
for (source_definition, stage_definition) in [
|
||||
(crate::application::HEADS, checkpoint_baseline::HEADS),
|
||||
(crate::application::VERSIONS, checkpoint_baseline::VERSIONS),
|
||||
(crate::application::EVENTS, checkpoint_baseline::EVENTS),
|
||||
(crate::application::OPERATIONS, checkpoint_baseline::OPERATIONS),
|
||||
] {
|
||||
let source = stage
|
||||
.source
|
||||
.transaction()
|
||||
.open_table(source_definition)
|
||||
.map_err(StoreError::from)?;
|
||||
let staged = transaction.open_table(stage_definition).map_err(StoreError::from)?;
|
||||
let mut source_rows = source.iter().map_err(StoreError::from)?;
|
||||
let mut staged_rows = staged.iter().map_err(StoreError::from)?;
|
||||
loop {
|
||||
match (source_rows.next(), staged_rows.next()) {
|
||||
(None, None) => break,
|
||||
(Some(source), Some(staged)) => {
|
||||
let (source_key, source_value) = source.map_err(StoreError::from)?;
|
||||
let (stage_key, stage_value) = staged.map_err(StoreError::from)?;
|
||||
if source_key.value() != stage_key.value() || source_value.value() != stage_value.value() {
|
||||
return Err(CheckpointError::Conflict("staging differs from owned local capture"));
|
||||
}
|
||||
}
|
||||
_ => return Err(CheckpointError::Conflict("staging row count")),
|
||||
}
|
||||
}
|
||||
}
|
||||
let staged_meta = transaction.open_table(crate::META).map_err(StoreError::from)?;
|
||||
let staged_metadata =
|
||||
checkpoint_baseline::read_metadata(&staged_meta)?.ok_or(CheckpointError::Conflict("missing staging metadata"))?;
|
||||
if checkpoint_baseline::encode_metadata(&staged_metadata)? != checkpoint_baseline::encode_metadata(&stage.metadata)? {
|
||||
return Err(CheckpointError::Conflict("staging metadata differs from local capture"));
|
||||
}
|
||||
let stage_bytes = std::fs::metadata(stage._directory.path().join("stage.redb"))?.len();
|
||||
let index_budget = stage
|
||||
.metadata
|
||||
.limits
|
||||
.staging_bytes
|
||||
.checked_sub(stage_bytes)
|
||||
.ok_or(CheckpointError::Conflict("staging disk budget"))?;
|
||||
checkpoint_baseline::validate_baseline_with_budget(&transaction, &stage.metadata, index_budget)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn resource_error(error: CheckpointError) -> CheckpointError {
|
||||
use std::error::Error;
|
||||
if matches!(&error, CheckpointError::Store(StoreError::CommitIndeterminate(_))) {
|
||||
return error;
|
||||
}
|
||||
let mut source: Option<&(dyn Error + 'static)> = Some(&error);
|
||||
while let Some(current) = source {
|
||||
if current
|
||||
.downcast_ref::<io::Error>()
|
||||
.is_some_and(|error| error.kind() == io::ErrorKind::StorageFull)
|
||||
{
|
||||
return CheckpointError::Store(StoreError::ResourceLimit);
|
||||
}
|
||||
source = current.source();
|
||||
}
|
||||
error
|
||||
}
|
||||
|
||||
struct CancelOnDrop(Arc<AtomicBool>);
|
||||
impl Drop for CancelOnDrop {
|
||||
fn drop(&mut self) {
|
||||
self.0.store(true, Ordering::Release);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
type CaptureHook = Box<dyn FnOnce() + Send>;
|
||||
#[cfg(test)]
|
||||
type StageHook = Box<dyn FnOnce(&redb::Database) + Send>;
|
||||
#[cfg(test)]
|
||||
static CAPTURE_HOOKS: std::sync::Mutex<std::collections::BTreeMap<usize, CaptureHook>> =
|
||||
std::sync::Mutex::new(std::collections::BTreeMap::new());
|
||||
#[cfg(test)]
|
||||
static STAGE_HOOKS: std::sync::Mutex<std::collections::BTreeMap<usize, StageHook>> =
|
||||
std::sync::Mutex::new(std::collections::BTreeMap::new());
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) struct CheckpointHook {
|
||||
owner: usize,
|
||||
stage: bool,
|
||||
}
|
||||
#[cfg(test)]
|
||||
impl Drop for CheckpointHook {
|
||||
fn drop(&mut self) {
|
||||
if self.stage {
|
||||
STAGE_HOOKS.lock().expect("stage hooks").remove(&self.owner);
|
||||
} else {
|
||||
CAPTURE_HOOKS.lock().expect("capture hooks").remove(&self.owner);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn after_local_capture(writer: &NativeLogWriter, hook: impl FnOnce() + Send + 'static) -> CheckpointHook {
|
||||
let owner = Arc::as_ptr(&writer.store).addr();
|
||||
assert!(
|
||||
CAPTURE_HOOKS
|
||||
.lock()
|
||||
.expect("capture hooks")
|
||||
.insert(owner, Box::new(hook))
|
||||
.is_none()
|
||||
);
|
||||
CheckpointHook { owner, stage: false }
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn after_local_stage(writer: &NativeLogWriter, hook: impl FnOnce(&redb::Database) + Send + 'static) -> CheckpointHook {
|
||||
let owner = Arc::as_ptr(&writer.store).addr();
|
||||
assert!(
|
||||
STAGE_HOOKS
|
||||
.lock()
|
||||
.expect("stage hooks")
|
||||
.insert(owner, Box::new(hook))
|
||||
.is_none()
|
||||
);
|
||||
CheckpointHook { owner, stage: true }
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn run_capture_hook(owner: &Arc<std::sync::Mutex<crate::Store>>) {
|
||||
let hook = CAPTURE_HOOKS
|
||||
.lock()
|
||||
.expect("capture hooks")
|
||||
.remove(&Arc::as_ptr(owner).addr());
|
||||
if let Some(hook) = hook {
|
||||
hook();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn run_stage_hook(owner: &Arc<std::sync::Mutex<crate::Store>>, database: &redb::Database) {
|
||||
let hook = STAGE_HOOKS.lock().expect("stage hooks").remove(&Arc::as_ptr(owner).addr());
|
||||
if let Some(hook) = hook {
|
||||
hook(database);
|
||||
}
|
||||
}
|
||||
|
||||
impl NativeLogWriter {
|
||||
pub async fn checkpoint_local(&self, limits: CheckpointLimits) -> Result<CheckpointReceipt, CheckpointError> {
|
||||
self.checkpoint_local_inner(limits).await.map_err(resource_error)
|
||||
}
|
||||
|
||||
async fn checkpoint_local_inner(&self, limits: CheckpointLimits) -> Result<CheckpointReceipt, CheckpointError> {
|
||||
limits.validate()?;
|
||||
let admission = self.checkpoint_admission.clone().lock_owned().await;
|
||||
let owner = self.store.clone();
|
||||
let stage = tokio::task::spawn_blocking(move || {
|
||||
let source = NativeSnapshotView::acquire(owner.clone())?;
|
||||
#[cfg(test)]
|
||||
run_capture_hook(&owner);
|
||||
let stage = capture_stage(source, limits)?;
|
||||
#[cfg(test)]
|
||||
run_stage_hook(&owner, &stage.database);
|
||||
verify_local_stage(&stage)?;
|
||||
Ok::<_, CheckpointError>((stage, admission))
|
||||
})
|
||||
.await
|
||||
.map_err(CheckpointError::Worker)??;
|
||||
let ordering = self.ordering.clone().lock_owned().await;
|
||||
let owner = self.store.clone();
|
||||
let cancellation = CancelOnDrop(Arc::new(AtomicBool::new(false)));
|
||||
let cancelled = cancellation.0.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
// The blocking owner retains both guards until publication finishes,
|
||||
// even if the async caller is dropped while redb is committing.
|
||||
let _ordering = ordering;
|
||||
let (stage, _admission) = stage;
|
||||
if cancelled.load(Ordering::Acquire) {
|
||||
return Err(CheckpointError::Cancelled);
|
||||
}
|
||||
if !Arc::ptr_eq(&owner, stage.source.owner()) {
|
||||
return Err(CheckpointError::Conflict("local capture owner"));
|
||||
}
|
||||
let mut store = owner.lock().map_err(|_| CheckpointError::Poisoned)?;
|
||||
let transaction = store.write_transaction()?;
|
||||
let mut meta = transaction.open_table(crate::META).map_err(StoreError::from)?;
|
||||
let schema = crate::number(
|
||||
meta.get("schema")
|
||||
.map_err(StoreError::from)?
|
||||
.ok_or(StoreError::CorruptRecord)?
|
||||
.value(),
|
||||
)?;
|
||||
if !matches!(schema, 6 | 7) {
|
||||
return Err(StoreError::UnsupportedSchema.into());
|
||||
}
|
||||
let previous = checkpoint_baseline::read_metadata(&meta)?;
|
||||
if crate::application::read_binding(&meta)? != stage.metadata.header.binding {
|
||||
return Err(CheckpointError::Conflict("live capture binding changed"));
|
||||
}
|
||||
let identity = crate::decode_identity(
|
||||
meta.get("identity")
|
||||
.map_err(StoreError::from)?
|
||||
.ok_or(StoreError::CorruptRecord)?
|
||||
.value(),
|
||||
)?;
|
||||
if identity.cluster_id != stage.metadata.header.cluster_id {
|
||||
return Err(CheckpointError::Conflict("live capture cluster changed"));
|
||||
}
|
||||
let cut = stage
|
||||
.metadata
|
||||
.header
|
||||
.applied
|
||||
.ok_or(CheckpointError::Conflict("empty applied cut"))?;
|
||||
let applied = crate::application::read_applied(&meta)?.ok_or(StoreError::CorruptRecord)?;
|
||||
let logs = transaction.open_table(crate::LOGS).map_err(StoreError::from)?;
|
||||
let committed = crate::committed::validate_committed(&meta, &logs)?.ok_or(StoreError::CorruptRecord)?;
|
||||
if cut.index > applied.index
|
||||
|| (cut.index == applied.index && cut != applied)
|
||||
|| applied.index > committed.index
|
||||
|| (applied.index == committed.index && applied != committed)
|
||||
{
|
||||
return Err(CheckpointError::Conflict("live applied and committed bounds"));
|
||||
}
|
||||
drop(logs);
|
||||
if let Some(old) = previous.as_ref().and_then(|previous| previous.header.applied)
|
||||
&& (old.index > cut.index || (old.index == cut.index && old != cut))
|
||||
{
|
||||
return Err(CheckpointError::Conflict("published checkpoint cut"));
|
||||
}
|
||||
let mut metadata = stage.metadata;
|
||||
metadata.generation = previous.map_or(Ok(1), |old| {
|
||||
old.generation
|
||||
.checked_add(1)
|
||||
.ok_or(CheckpointError::Conflict("checkpoint generation exhausted"))
|
||||
})?;
|
||||
let stage_transaction = stage.database.begin_read().map_err(StoreError::from)?;
|
||||
for (source_definition, definition) in [
|
||||
(crate::application::HEADS, checkpoint_baseline::HEADS),
|
||||
(crate::application::VERSIONS, checkpoint_baseline::VERSIONS),
|
||||
(crate::application::EVENTS, checkpoint_baseline::EVENTS),
|
||||
(crate::application::OPERATIONS, checkpoint_baseline::OPERATIONS),
|
||||
] {
|
||||
let captured = stage
|
||||
.source
|
||||
.transaction()
|
||||
.open_table(source_definition)
|
||||
.map_err(StoreError::from)?;
|
||||
let staged = stage_transaction.open_table(definition).map_err(StoreError::from)?;
|
||||
let mut source_rows = captured.iter().map_err(StoreError::from)?;
|
||||
let mut stage_rows = staged.iter().map_err(StoreError::from)?;
|
||||
let mut target = transaction.open_table(definition).map_err(StoreError::from)?;
|
||||
target.retain(|_, _| false).map_err(StoreError::from)?;
|
||||
// Recheck disk bytes while copying; the earlier relationship
|
||||
// validation remains outside the writer ordering interval.
|
||||
loop {
|
||||
match (source_rows.next(), stage_rows.next()) {
|
||||
(None, None) => break,
|
||||
(Some(source), Some(staged)) => {
|
||||
let (source_key, source_value) = source.map_err(StoreError::from)?;
|
||||
let (key, value) = staged.map_err(StoreError::from)?;
|
||||
if source_key.value() != key.value() || source_value.value() != value.value() {
|
||||
return Err(CheckpointError::Conflict("publication differs from local capture"));
|
||||
}
|
||||
target.insert(key.value(), value.value()).map_err(StoreError::from)?;
|
||||
}
|
||||
_ => return Err(CheckpointError::Conflict("publication row count")),
|
||||
}
|
||||
}
|
||||
}
|
||||
let encoded = checkpoint_baseline::encode_metadata(&metadata)?;
|
||||
meta.insert("checkpoint_baseline", encoded.as_slice())
|
||||
.map_err(StoreError::from)?;
|
||||
meta.insert("schema", 7u64.to_be_bytes().as_slice())
|
||||
.map_err(StoreError::from)?;
|
||||
crate::raft_state_machine::publish_local_meta(&mut meta, &metadata)?;
|
||||
drop(meta);
|
||||
if cancelled.load(Ordering::Acquire) {
|
||||
return Err(CheckpointError::Cancelled);
|
||||
}
|
||||
store.commit(transaction)?;
|
||||
Ok(CheckpointReceipt {
|
||||
generation: metadata.generation,
|
||||
cut,
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(CheckpointError::Worker)?
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,607 @@
|
||||
use super::*;
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
fn identity() -> NodeIdentity {
|
||||
NodeIdentity {
|
||||
cluster_id: [7; 16],
|
||||
node_id: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn binding() -> CaptureBinding {
|
||||
CaptureBinding {
|
||||
revision: 9,
|
||||
binding_id: [8; 16],
|
||||
target_id: [9; 16],
|
||||
}
|
||||
}
|
||||
|
||||
fn id(index: u64) -> LogId {
|
||||
LogId {
|
||||
term: 8,
|
||||
leader_node: 0,
|
||||
index,
|
||||
}
|
||||
}
|
||||
|
||||
fn command(operation: u8) -> DecideCreated {
|
||||
DecideCreated {
|
||||
operation_id: [operation; 16],
|
||||
object: ObjectIdentity {
|
||||
bucket_incarnation: [2; 16],
|
||||
key: b"checkpoint-object".to_vec(),
|
||||
},
|
||||
expected_head: None,
|
||||
prepared: PreparedIdentity {
|
||||
version_id: [3; 16],
|
||||
preparation_id: [4; 16],
|
||||
content_digest: [5; 32],
|
||||
content_length: 4097,
|
||||
},
|
||||
binding_revision: 9,
|
||||
}
|
||||
}
|
||||
|
||||
fn apply(store: &mut Store, index: u64, command: DecideCreated) -> CaptureResponse {
|
||||
let entry = CaptureRaftEntry {
|
||||
log_id: id(index).into(),
|
||||
payload: openraft::EntryPayload::Normal(CaptureCommand::try_created(command).expect("canonical command")),
|
||||
};
|
||||
store.append_native(&[entry]).expect("actual durable append");
|
||||
store.save_committed(id(index)).expect("actual durable commit");
|
||||
store.apply_next_native(id(index)).expect("actual native apply")
|
||||
}
|
||||
|
||||
fn fixture() -> (tempfile::TempDir, Store, DecideCreated, CreatedOperationProvenance) {
|
||||
let dir = tempfile::tempdir().expect("actual native database directory");
|
||||
let mut store = Store::initialize(dir.path().join("db"), identity()).expect("initialize actual redb");
|
||||
store.pin_capture_binding(binding()).expect("bind actual store");
|
||||
store.enable_native_empty().expect("native schema4");
|
||||
store.promote_created_provenance().expect("native schema6 provenance");
|
||||
let membership = CaptureMembership::new(
|
||||
Some(id(0)),
|
||||
vec![BTreeSet::from([0])],
|
||||
BTreeMap::from([(0, CaptureNode::new("http://127.0.0.1:7000".into()).expect("native member"))]),
|
||||
)
|
||||
.expect("full native membership");
|
||||
let stored = openraft::StoredMembership::try_from(&membership).expect("OpenRaft membership");
|
||||
store
|
||||
.append_native(&[CaptureRaftEntry {
|
||||
log_id: id(0).into(),
|
||||
payload: openraft::EntryPayload::Membership(stored.membership().clone()),
|
||||
}])
|
||||
.expect("append actual membership");
|
||||
store.save_committed(id(0)).expect("commit membership");
|
||||
store.apply_next_native(id(0)).expect("apply membership");
|
||||
let negative = command(11);
|
||||
let mut original = negative.clone();
|
||||
original.expected_head = Some([12; 16]);
|
||||
apply(&mut store, 1, original.clone());
|
||||
let first = store
|
||||
.created_operation_provenance(original.operation_id)
|
||||
.expect("read first negative")
|
||||
.expect("canonical negative operation");
|
||||
assert_eq!(first.first_decision_log_id, id(1), "CHECKPOINT_FIRST_NEGATIVE_BOUND");
|
||||
assert!(
|
||||
matches!(first.operation.result, CreatedResult::HeadMismatch { .. }),
|
||||
"CHECKPOINT_FIRST_NEGATIVE_BOUND"
|
||||
);
|
||||
apply(&mut store, 2, command(1));
|
||||
(dir, store, original, first)
|
||||
}
|
||||
|
||||
fn runtime() -> tokio::runtime::Runtime {
|
||||
tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("checkpoint runtime")
|
||||
}
|
||||
|
||||
fn schema(store: &Store) -> u64 {
|
||||
let tx = store.database.begin_read().expect("read actual schema");
|
||||
let table = tx.open_table(META).expect("schema table");
|
||||
number(table.get("schema").expect("schema read").expect("schema exists").value()).expect("schema number")
|
||||
}
|
||||
|
||||
fn assert_negative(store: &Store, original: &DecideCreated, first: &CreatedOperationProvenance) {
|
||||
assert_eq!(
|
||||
store
|
||||
.created_operation_provenance(original.operation_id)
|
||||
.expect("canonical read"),
|
||||
Some(first.clone()),
|
||||
"CHECKPOINT_FIRST_NEGATIVE_BOUND: preserve original bytes/result/full first ID"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_checkpoint_negative_retry_and_purge_reopen() {
|
||||
let (dir, store, original, first) = fixture();
|
||||
assert_eq!(schema(&store), 6);
|
||||
let writer = NativeLogWriter::new(store);
|
||||
let receipt = runtime()
|
||||
.block_on(writer.checkpoint_local(CheckpointLimits::default()))
|
||||
.expect("publish local checkpoint");
|
||||
assert_eq!(receipt.cut(), id(2), "CHECKPOINT_BASELINE_BOUND");
|
||||
assert_eq!(receipt.generation(), 1);
|
||||
{
|
||||
let mut store = writer.store.lock().expect("actual writer store");
|
||||
assert_eq!(schema(&store), 7);
|
||||
assert_negative(&store, &original, &first);
|
||||
apply(&mut store, 3, original.clone());
|
||||
assert_negative(&store, &original, &first);
|
||||
let mut conflict = original.clone();
|
||||
conflict.binding_revision += 1;
|
||||
apply(&mut store, 4, conflict);
|
||||
assert_eq!(
|
||||
store
|
||||
.created_apply_result(4)
|
||||
.expect("conflicting response")
|
||||
.expect("response")
|
||||
.result,
|
||||
CreatedResult::OperationConflict,
|
||||
"CHECKPOINT_RETRY_BOUND"
|
||||
);
|
||||
assert_negative(&store, &original, &first);
|
||||
}
|
||||
runtime()
|
||||
.block_on(writer.purge_checkpoint(&receipt))
|
||||
.expect("purge exact durable baseline");
|
||||
drop(writer);
|
||||
let reopened = Store::open(dir.path().join("db"), identity()).expect("reopen purged baseline plus suffix");
|
||||
assert_negative(&reopened, &original, &first);
|
||||
assert_eq!(reopened.last_applied().expect("suffix applied"), Some(id(4)), "CHECKPOINT_SUFFIX_BOUND");
|
||||
assert!(reopened.read_entry(2).expect("purged cut").is_none());
|
||||
assert!(reopened.read_entry(3).expect("retained suffix").is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_checkpoint_capture_keeps_concurrent_suffix() {
|
||||
let (dir, store, original, first) = fixture();
|
||||
let writer = NativeLogWriter::new(store);
|
||||
let (entered_tx, entered_rx) = std::sync::mpsc::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel();
|
||||
let _hook = native_checkpoint::after_local_capture(&writer, move || {
|
||||
entered_tx.send(()).expect("capture reached");
|
||||
release_rx
|
||||
.recv_timeout(std::time::Duration::from_secs(30))
|
||||
.expect("release owned capture");
|
||||
});
|
||||
let worker = writer.clone();
|
||||
let thread = std::thread::spawn(move || runtime().block_on(worker.checkpoint_local(CheckpointLimits::default())));
|
||||
entered_rx
|
||||
.recv_timeout(std::time::Duration::from_secs(30))
|
||||
.expect("actual capture boundary");
|
||||
{
|
||||
let mut store = writer.store.lock().expect("capture releases live write owner");
|
||||
apply(&mut store, 3, original.clone());
|
||||
}
|
||||
release_tx.send(()).expect("release capture");
|
||||
let receipt = thread.join().expect("checkpoint worker").expect("concurrent checkpoint");
|
||||
assert_eq!(receipt.cut(), id(2), "CHECKPOINT_SAME_CUT_BOUND");
|
||||
runtime()
|
||||
.block_on(writer.purge_checkpoint(&receipt))
|
||||
.expect("purge captured prefix");
|
||||
drop(writer);
|
||||
let reopened = Store::open(dir.path().join("db"), identity()).expect("reopen concurrent suffix");
|
||||
assert_eq!(reopened.last_applied().expect("actual suffix"), Some(id(3)), "CHECKPOINT_SUFFIX_BOUND");
|
||||
assert_negative(&reopened, &original, &first);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_checkpoint_staging_corruption_preserves_live_generation() {
|
||||
let (dir, store, original, first) = fixture();
|
||||
let writer = NativeLogWriter::new(store);
|
||||
let before = runtime()
|
||||
.block_on(writer.checkpoint_local(CheckpointLimits::default()))
|
||||
.expect("initial baseline");
|
||||
let _hook = native_checkpoint::after_local_stage(&writer, |database| {
|
||||
let tx = database.begin_write().expect("actual isolated stage mutation");
|
||||
tx.open_table(checkpoint_baseline::OPERATIONS)
|
||||
.expect("stage operations")
|
||||
.retain(|_, _| false)
|
||||
.expect("damage cross-record relationships");
|
||||
tx.commit().expect("persist actual damaged stage");
|
||||
});
|
||||
assert!(
|
||||
runtime()
|
||||
.block_on(writer.checkpoint_local(CheckpointLimits::default()))
|
||||
.is_err(),
|
||||
"CHECKPOINT_FAILURE_ATOMIC"
|
||||
);
|
||||
let store = writer.store.lock().expect("unchanged live store");
|
||||
let tx = store.database.begin_read().expect("published generation");
|
||||
let meta = tx.open_table(META).expect("published meta");
|
||||
assert_eq!(
|
||||
checkpoint_baseline::read_metadata(&meta)
|
||||
.expect("valid old metadata")
|
||||
.expect("old baseline")
|
||||
.generation,
|
||||
before.generation(),
|
||||
"CHECKPOINT_FAILURE_ATOMIC"
|
||||
);
|
||||
assert_negative(&store, &original, &first);
|
||||
drop(meta);
|
||||
drop(tx);
|
||||
drop(store);
|
||||
drop(writer);
|
||||
let reopened = Store::open(dir.path().join("db"), identity()).expect("reopen after failed staging validation");
|
||||
assert_negative(&reopened, &original, &first);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_checkpoint_budget_failure_preserves_schema_and_history() {
|
||||
for staging in [false, true] {
|
||||
let (dir, mut store, original, first) = fixture();
|
||||
if staging {
|
||||
for index in 0u64..256 {
|
||||
let mut large = command(31);
|
||||
large.operation_id[..8].copy_from_slice(&index.to_be_bytes());
|
||||
large.prepared.version_id[..8].copy_from_slice(&index.to_be_bytes());
|
||||
large.object.key = vec![b'x'; MAX_OBJECT_KEY_LENGTH];
|
||||
large.object.key[..8].copy_from_slice(&index.to_be_bytes());
|
||||
apply(&mut store, index + 3, large);
|
||||
}
|
||||
}
|
||||
let writer = NativeLogWriter::new(store);
|
||||
let mut limits = CheckpointLimits::default();
|
||||
if staging {
|
||||
limits.staging_bytes = 4 * 1024 * 1024;
|
||||
} else {
|
||||
limits.snapshot.total_bytes = 64;
|
||||
}
|
||||
let result = runtime().block_on(writer.checkpoint_local(limits));
|
||||
if staging {
|
||||
assert!(
|
||||
matches!(&result, Err(CheckpointError::Store(StoreError::ResourceLimit))),
|
||||
"CHECKPOINT_RESOURCE_BOUND: actual 4MiB physical staging limit: {result:?}"
|
||||
);
|
||||
} else {
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"CHECKPOINT_RESOURCE_BOUND: actual bounded output must refuse: {result:?}"
|
||||
);
|
||||
}
|
||||
{
|
||||
let store = writer.store.lock().expect("live after exhausted budget");
|
||||
assert_eq!(schema(&store), 6, "CHECKPOINT_FAILURE_ATOMIC");
|
||||
assert_negative(&store, &original, &first);
|
||||
assert!(store.read_entry(0).expect("prefix intact").is_some());
|
||||
}
|
||||
drop(writer);
|
||||
assert_negative(
|
||||
&Store::open(dir.path().join("db"), identity()).expect("reopen budget failure"),
|
||||
&original,
|
||||
&first,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_checkpoint_cancelled_capture_cannot_publish_or_purge() {
|
||||
let (dir, store, original, first) = fixture();
|
||||
let writer = NativeLogWriter::new(store);
|
||||
let (entered_tx, entered_rx) = std::sync::mpsc::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel();
|
||||
let _hook = native_checkpoint::after_local_capture(&writer, move || {
|
||||
entered_tx.send(()).expect("capture cancellation boundary");
|
||||
release_rx
|
||||
.recv_timeout(std::time::Duration::from_secs(30))
|
||||
.expect("release pending actual capture");
|
||||
});
|
||||
let rt = runtime();
|
||||
let worker = writer.clone();
|
||||
rt.block_on(async {
|
||||
let task = tokio::spawn(async move { worker.checkpoint_local(CheckpointLimits::default()).await });
|
||||
tokio::task::spawn_blocking(move || entered_rx.recv_timeout(std::time::Duration::from_secs(30)))
|
||||
.await
|
||||
.expect("capture observer")
|
||||
.expect("actual capture owned transaction");
|
||||
task.abort();
|
||||
assert!(task.await.expect_err("cancelled waiter").is_cancelled(), "CHECKPOINT_CANCEL_BOUND");
|
||||
release_tx.send(()).expect("release actual worker after waiter cancellation");
|
||||
// A fresh checkpoint waits for the same production admission; completion
|
||||
// establishes that the cancelled capture worker released its real resources.
|
||||
let receipt = writer
|
||||
.checkpoint_local(CheckpointLimits::default())
|
||||
.await
|
||||
.expect("capture after cancellation");
|
||||
assert_eq!(receipt.generation(), 1, "CHECKPOINT_CANCEL_BOUND: abandoned capture never published");
|
||||
});
|
||||
drop(rt);
|
||||
drop(writer);
|
||||
let reopened = Store::open(dir.path().join("db"), identity()).expect("reopen after cancellation and successful next capture");
|
||||
assert_negative(&reopened, &original, &first);
|
||||
assert!(
|
||||
reopened.read_entry(0).expect("cancellation cannot purge").is_some(),
|
||||
"CHECKPOINT_CANCEL_BOUND"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_checkpoint_stale_receipt_cannot_remove_history() {
|
||||
let (_dir, store, original, first) = fixture();
|
||||
let writer = NativeLogWriter::new(store);
|
||||
let rt = runtime();
|
||||
let old = rt
|
||||
.block_on(writer.checkpoint_local(CheckpointLimits::default()))
|
||||
.expect("first generation");
|
||||
let new = rt
|
||||
.block_on(writer.checkpoint_local(CheckpointLimits::default()))
|
||||
.expect("next generation");
|
||||
assert!(new.generation() > old.generation());
|
||||
assert!(
|
||||
matches!(
|
||||
rt.block_on(writer.purge_checkpoint(&old)),
|
||||
Err(NativeWriteError::Store(StoreError::StaleCheckpoint))
|
||||
),
|
||||
"CHECKPOINT_PURGE_BOUND"
|
||||
);
|
||||
let store = writer.store.lock().expect("history after stale purge");
|
||||
assert_negative(&store, &original, &first);
|
||||
assert!(store.read_entry(1).expect("first decision retained").is_some());
|
||||
assert!(store.created_apply_result(1).expect("typed result retained").is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_checkpoint_reopen_rejects_conflicting_cut_and_suffix_gap() {
|
||||
for corruption in 0..4 {
|
||||
let gap = corruption == 1;
|
||||
let (dir, store, original, _) = fixture();
|
||||
let writer = NativeLogWriter::new(store);
|
||||
let rt = runtime();
|
||||
let receipt = rt
|
||||
.block_on(writer.checkpoint_local(CheckpointLimits::default()))
|
||||
.expect("valid checkpoint");
|
||||
if gap {
|
||||
{
|
||||
let mut store = writer.store.lock().expect("suffix owner");
|
||||
apply(&mut store, 3, original.clone());
|
||||
apply(&mut store, 4, original.clone());
|
||||
}
|
||||
rt.block_on(writer.purge_checkpoint(&receipt))
|
||||
.expect("purge before gap corruption");
|
||||
{
|
||||
let store = writer.store.lock().expect("actual purged metadata owner");
|
||||
let tx = store.database.begin_write().expect("actual CPUR mutation");
|
||||
let mut meta = tx.open_table(META).expect("actual CPUR table");
|
||||
let mut conflict = receipt.cut();
|
||||
conflict.term += 1;
|
||||
conflict.leader_node += 1;
|
||||
let encoded = checkpoint_baseline::encode_purged(conflict);
|
||||
meta.insert(checkpoint_baseline::PURGED_KEY, encoded.as_slice())
|
||||
.expect("persist conflicting full purge ID");
|
||||
drop(meta);
|
||||
tx.commit().expect("durable conflicting CPUR");
|
||||
let tx = store.database.begin_read().expect("actual conflicting CPUR read");
|
||||
let meta = tx.open_table(META).expect("actual persisted CPUR");
|
||||
assert!(
|
||||
matches!(checkpoint_baseline::read_purged(&meta), Err(StoreError::CorruptRecord)),
|
||||
"CHECKPOINT_FULL_CUT_BOUND: same index with different term/leader is corruption"
|
||||
);
|
||||
drop(meta);
|
||||
drop(tx);
|
||||
let tx = store
|
||||
.database
|
||||
.begin_write()
|
||||
.expect("restore valid purge boundary for gap scenario");
|
||||
let encoded = checkpoint_baseline::encode_purged(receipt.cut());
|
||||
tx.open_table(META)
|
||||
.expect("actual purge meta")
|
||||
.insert(checkpoint_baseline::PURGED_KEY, encoded.as_slice())
|
||||
.expect("restore exact cut");
|
||||
tx.commit().expect("persist exact purge boundary");
|
||||
}
|
||||
}
|
||||
if corruption >= 2 {
|
||||
let mut store = writer.store.lock().expect("actual suffix before tail corruption");
|
||||
apply(&mut store, 3, original.clone());
|
||||
}
|
||||
{
|
||||
let store = writer.store.lock().expect("actual corruption owner");
|
||||
let tx = store.database.begin_write().expect("actual redb mutation");
|
||||
let mut logs = tx.open_table(LOGS).expect("actual log table");
|
||||
if gap {
|
||||
logs.remove(3).expect("remove required suffix entry");
|
||||
} else if corruption >= 2 {
|
||||
let mut tail = id(3);
|
||||
if corruption == 2 {
|
||||
tail.index += 1;
|
||||
} else {
|
||||
tail.term += 1;
|
||||
tail.leader_node += 1;
|
||||
}
|
||||
let encoded = encode_id(tail);
|
||||
tx.open_table(META)
|
||||
.expect("actual metadata")
|
||||
.insert("tail", encoded.as_slice())
|
||||
.expect("corrupt durable tail");
|
||||
} else {
|
||||
let mut entry = store.read_entry(2).expect("retained cut").expect("cut entry");
|
||||
entry.id.term += 1;
|
||||
entry.id.leader_node += 1;
|
||||
let encoded = encode_log(&entry).expect("valid conflicting full ID encoding");
|
||||
logs.insert(2, encoded.as_slice()).expect("replace exact cut identity");
|
||||
}
|
||||
drop(logs);
|
||||
tx.commit().expect("persist actual inconsistent recovery input");
|
||||
}
|
||||
assert!(
|
||||
rt.block_on(writer.purge_checkpoint(&receipt)).is_err(),
|
||||
"CHECKPOINT_PURGE_BOUND: invalid recovery cannot authorize cleanup"
|
||||
);
|
||||
if !gap {
|
||||
let store = writer.store.lock().expect("refused purge retains original evidence");
|
||||
assert!(store.read_entry(1).expect("original log retained").is_some(), "CHECKPOINT_PURGE_BOUND");
|
||||
assert!(
|
||||
store.created_apply_result(1).expect("original response retained").is_some(),
|
||||
"CHECKPOINT_PURGE_BOUND"
|
||||
);
|
||||
}
|
||||
drop(writer);
|
||||
drop(rt);
|
||||
assert!(Store::open(dir.path().join("db"), identity()).is_err(), "CHECKPOINT_RECOVERY_BOUND");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_checkpoint_publication_fault_selects_complete_generation() {
|
||||
for (sync, after) in [(1, false), (1, true), (2, false), (2, true)] {
|
||||
let (dir, store, original, first) = fixture();
|
||||
let vote = VoteRecord {
|
||||
term: 9,
|
||||
voted_node: 0,
|
||||
committed: true,
|
||||
};
|
||||
let mut store = store;
|
||||
store.save_vote(vote).expect("local hard state before checkpoint");
|
||||
drop(store);
|
||||
let path = dir.path().join("db");
|
||||
let backend = tests::CrashBackend::new(&path);
|
||||
let store = Store::from_database(backend.database(), identity()).expect("existing simulated durability backend");
|
||||
let writer = NativeLogWriter::new(store);
|
||||
let (entered, release) = backend.pause_sync(sync, after);
|
||||
let worker = writer.clone();
|
||||
let task = std::thread::spawn(move || runtime().block_on(worker.checkpoint_local(CheckpointLimits::default())));
|
||||
entered.wait();
|
||||
backend.cut();
|
||||
release.wait();
|
||||
let _unknown = task.join().expect("interrupted publication worker");
|
||||
drop(writer);
|
||||
let reopened = Store::open(&path, identity()).expect("recover complete generation after simulated sync cut");
|
||||
assert!(matches!(schema(&reopened), 6 | 7), "CHECKPOINT_CRASH_ATOMIC");
|
||||
assert_negative(&reopened, &original, &first);
|
||||
assert_eq!(
|
||||
reopened.read_vote().expect("preserved local vote"),
|
||||
Some(vote),
|
||||
"CHECKPOINT_LOCAL_STATE_BOUND"
|
||||
);
|
||||
assert_eq!(reopened.last_applied().expect("complete applied position"), Some(id(2)));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_checkpoint_recovery_budget_can_be_raised_without_changing_baseline() {
|
||||
use redb::ReadableTableMetadata;
|
||||
let (dir, store, original, first) = fixture();
|
||||
let mut limits = CheckpointLimits::default();
|
||||
let tx = store.database.begin_read().expect("measure real baseline rows");
|
||||
limits.snapshot.records = [
|
||||
application::HEADS,
|
||||
application::VERSIONS,
|
||||
application::EVENTS,
|
||||
application::OPERATIONS,
|
||||
]
|
||||
.into_iter()
|
||||
.map(|definition| {
|
||||
tx.open_table(definition)
|
||||
.expect("actual baseline table")
|
||||
.len()
|
||||
.expect("actual row count")
|
||||
})
|
||||
.sum();
|
||||
drop(tx);
|
||||
let writer = NativeLogWriter::new(store);
|
||||
let receipt = runtime()
|
||||
.block_on(writer.checkpoint_local(limits))
|
||||
.expect("capture exact bounded baseline");
|
||||
{
|
||||
let mut store = writer.store.lock().expect("actual suffix writer");
|
||||
let mut suffix = command(21);
|
||||
suffix.object.key = b"suffix-new-key".to_vec();
|
||||
suffix.prepared.version_id = [22; 16];
|
||||
apply(&mut store, 3, suffix);
|
||||
}
|
||||
drop(writer);
|
||||
let path = dir.path().join("db");
|
||||
assert!(
|
||||
matches!(Store::open(&path, identity()), Err(StoreError::ResourceLimit)),
|
||||
"CHECKPOINT_RESOURCE_BOUND"
|
||||
);
|
||||
let mut raised = limits;
|
||||
raised.snapshot.records += 32;
|
||||
let reopened =
|
||||
Store::open_with_checkpoint_limits(&path, identity(), raised).expect("explicitly raise bounded recovery budget");
|
||||
assert_negative(&reopened, &original, &first);
|
||||
assert_eq!(reopened.last_applied().expect("retained suffix position"), Some(id(3)));
|
||||
let tx = reopened.database.begin_read().expect("unchanged baseline metadata");
|
||||
let meta = tx.open_table(META).expect("actual metadata");
|
||||
let baseline = checkpoint_baseline::read_metadata(&meta)
|
||||
.expect("baseline decode")
|
||||
.expect("baseline selected");
|
||||
assert_eq!(baseline.generation, receipt.generation());
|
||||
assert_eq!(
|
||||
baseline.limits.snapshot.records, limits.snapshot.records,
|
||||
"CHECKPOINT_RESOURCE_BOUND: override never rewrites evidence"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_checkpoint_interrupted_purge_reopens_and_retries_idempotently() {
|
||||
for (sync, after) in [(1, false), (1, true), (2, false), (2, true)] {
|
||||
let (dir, store, original, first) = fixture();
|
||||
let writer = NativeLogWriter::new(store);
|
||||
let receipt = runtime()
|
||||
.block_on(writer.checkpoint_local(CheckpointLimits::default()))
|
||||
.expect("durable baseline before purge");
|
||||
drop(writer);
|
||||
let path = dir.path().join("db");
|
||||
let backend = tests::CrashBackend::new(&path);
|
||||
let store = Store::from_database(backend.database(), identity()).expect("simulated purge durability backend");
|
||||
let writer = NativeLogWriter::new(store);
|
||||
let (entered, release) = backend.pause_sync(sync, after);
|
||||
let worker = writer.clone();
|
||||
let task = std::thread::spawn(move || runtime().block_on(worker.purge_checkpoint(&receipt)));
|
||||
entered.wait();
|
||||
backend.cut();
|
||||
release.wait();
|
||||
let _unknown = task.join().expect("interrupted purge worker");
|
||||
drop(writer);
|
||||
let reopened = Store::open(&path, identity()).expect("baseline survives simulated interrupted purge");
|
||||
assert_negative(&reopened, &original, &first);
|
||||
let retained = (0..=2)
|
||||
.map(|index| reopened.read_entry(index).expect("atomic purge log state").is_some())
|
||||
.collect::<Vec<_>>();
|
||||
assert!(
|
||||
retained.iter().all(|present| *present) || retained.iter().all(|present| !*present),
|
||||
"CHECKPOINT_PURGE_ATOMIC: complete old prefix or complete new purge"
|
||||
);
|
||||
let writer = NativeLogWriter::new(reopened);
|
||||
let rt = runtime();
|
||||
rt.block_on(writer.purge_checkpoint(&receipt)).expect("resume exact purge");
|
||||
rt.block_on(writer.purge_checkpoint(&receipt))
|
||||
.expect("repeat exact purge idempotently");
|
||||
drop(writer);
|
||||
let reopened = Store::open(&path, identity()).expect("reopen resumed purge");
|
||||
assert_negative(&reopened, &original, &first);
|
||||
assert!(reopened.read_entry(2).expect("purged cut absent").is_none(), "CHECKPOINT_PURGE_ATOMIC");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_checkpoint_ack_and_purge_ack_survive_simulated_power_cut() {
|
||||
let (dir, store, original, first) = fixture();
|
||||
drop(store);
|
||||
let path = dir.path().join("db");
|
||||
let backend = tests::CrashBackend::new(&path);
|
||||
let writer =
|
||||
NativeLogWriter::new(Store::from_database(backend.database(), identity()).expect("actual checkpoint fault backend"));
|
||||
let receipt = runtime()
|
||||
.block_on(writer.checkpoint_local(CheckpointLimits::default()))
|
||||
.expect("successful checkpoint ACK");
|
||||
backend.cut();
|
||||
drop(writer);
|
||||
let reopened = Store::open(&path, identity()).expect("ACK checkpoint survives simulated power cut");
|
||||
assert_eq!(schema(&reopened), 7, "CHECKPOINT_ACK_DURABLE");
|
||||
assert_negative(&reopened, &original, &first);
|
||||
let backend = tests::CrashBackend::new(&path);
|
||||
drop(reopened);
|
||||
let writer = NativeLogWriter::new(Store::from_database(backend.database(), identity()).expect("actual purge fault backend"));
|
||||
runtime()
|
||||
.block_on(writer.purge_checkpoint(&receipt))
|
||||
.expect("successful purge ACK");
|
||||
backend.cut();
|
||||
drop(writer);
|
||||
let reopened = Store::open(&path, identity()).expect("ACK purge survives simulated power cut");
|
||||
assert_negative(&reopened, &original, &first);
|
||||
assert!(reopened.read_entry(2).expect("ACK purge cut absent").is_none(), "CHECKPOINT_ACK_DURABLE");
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
//! Owned consistent native application snapshot view.
|
||||
use crate::{Store, StoreError, application, snapshot};
|
||||
use redb::{ReadTransaction, ReadableDatabase, ReadableTable, ReadableTableMetadata};
|
||||
use std::{
|
||||
io::Write,
|
||||
sync::{Arc, Mutex},
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
std::thread_local! {
|
||||
static AFTER_CAPTURE: std::cell::RefCell<Option<Box<dyn FnOnce()>>> = const { std::cell::RefCell::new(None) };
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) struct CaptureHook;
|
||||
#[cfg(test)]
|
||||
impl Drop for CaptureHook {
|
||||
fn drop(&mut self) {
|
||||
AFTER_CAPTURE.with(|hook| {
|
||||
hook.borrow_mut().take();
|
||||
});
|
||||
}
|
||||
}
|
||||
#[cfg(test)]
|
||||
pub(crate) fn after_capture(hook: impl FnOnce() + 'static) -> CaptureHook {
|
||||
AFTER_CAPTURE.with(|slot| {
|
||||
assert!(slot.borrow().is_none(), "one capture hook per test thread");
|
||||
*slot.borrow_mut() = Some(Box::new(hook));
|
||||
});
|
||||
CaptureHook
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum NativeSnapshotError {
|
||||
#[error(transparent)]
|
||||
Store(#[from] StoreError),
|
||||
#[error(transparent)]
|
||||
Snapshot(#[from] snapshot::SnapshotError),
|
||||
#[error("native snapshot Store lock poisoned")]
|
||||
Poisoned,
|
||||
#[error("native snapshot worker failed")]
|
||||
Worker(#[source] tokio::task::JoinError),
|
||||
}
|
||||
|
||||
pub struct NativeSnapshotView {
|
||||
// Drop the transaction before the owner so the Database stays open throughout the view.
|
||||
transaction: ReadTransaction,
|
||||
header: snapshot::Header,
|
||||
counts: [u64; 4],
|
||||
schema: u64,
|
||||
_owner: Arc<Mutex<Store>>,
|
||||
}
|
||||
impl NativeSnapshotView {
|
||||
pub(crate) fn acquire(owner: Arc<Mutex<Store>>) -> Result<Self, NativeSnapshotError> {
|
||||
let transaction = {
|
||||
let store = owner.lock().map_err(|_| NativeSnapshotError::Poisoned)?;
|
||||
if !store.writable {
|
||||
return Err(StoreError::WriteDisabled.into());
|
||||
}
|
||||
store.database.begin_read().map_err(StoreError::from)?
|
||||
};
|
||||
#[cfg(test)]
|
||||
if let Some(hook) = AFTER_CAPTURE.with(|slot| slot.borrow_mut().take()) {
|
||||
hook();
|
||||
}
|
||||
let (applied, membership) = application::native_application_state(&transaction)?;
|
||||
let (header, schema) = {
|
||||
let meta = transaction.open_table(crate::META).map_err(StoreError::from)?;
|
||||
let logs = transaction.open_table(crate::LOGS).map_err(StoreError::from)?;
|
||||
let committed = crate::committed::validate_committed(&meta, &logs)?;
|
||||
if let Some(applied) = applied {
|
||||
let committed = committed.ok_or(StoreError::CorruptRecord)?;
|
||||
if applied.index > committed.index || (applied.index == committed.index && applied != committed) {
|
||||
return Err(StoreError::CorruptRecord.into());
|
||||
}
|
||||
}
|
||||
let identity = meta
|
||||
.get("identity")
|
||||
.map_err(StoreError::from)?
|
||||
.ok_or(StoreError::CorruptRecord)?;
|
||||
let schema = crate::number(
|
||||
meta.get("schema")
|
||||
.map_err(StoreError::from)?
|
||||
.ok_or(StoreError::CorruptRecord)?
|
||||
.value(),
|
||||
)?;
|
||||
(
|
||||
snapshot::Header {
|
||||
cluster_id: crate::decode_identity(identity.value())?.cluster_id,
|
||||
applied,
|
||||
membership,
|
||||
binding: application::read_binding(&meta)?,
|
||||
},
|
||||
schema,
|
||||
)
|
||||
};
|
||||
let mut counts = [0; 4];
|
||||
for (index, definition) in [
|
||||
application::HEADS,
|
||||
application::VERSIONS,
|
||||
application::EVENTS,
|
||||
application::OPERATIONS,
|
||||
]
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
{
|
||||
counts[index] = transaction
|
||||
.open_table(definition)
|
||||
.map_err(StoreError::from)?
|
||||
.len()
|
||||
.map_err(StoreError::from)?;
|
||||
}
|
||||
Ok(Self {
|
||||
transaction,
|
||||
header,
|
||||
counts,
|
||||
schema,
|
||||
_owner: owner,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn stream_capability_artifact<W: Write>(
|
||||
self,
|
||||
writer: W,
|
||||
limits: snapshot::Limits,
|
||||
) -> Result<snapshot::CapabilityArtifact<W>, NativeSnapshotError> {
|
||||
if !matches!(self.schema, 6 | 7) {
|
||||
return Err(StoreError::UnsupportedSchema.into());
|
||||
}
|
||||
let output = snapshot::CapabilityOutput::new(writer, limits.total_bytes);
|
||||
Ok(self.stream(output, limits)?.finish())
|
||||
}
|
||||
|
||||
pub(crate) fn transaction(&self) -> &ReadTransaction {
|
||||
&self.transaction
|
||||
}
|
||||
|
||||
pub(crate) fn header(&self) -> &snapshot::Header {
|
||||
&self.header
|
||||
}
|
||||
|
||||
pub(crate) fn owner(&self) -> &Arc<Mutex<Store>> {
|
||||
&self._owner
|
||||
}
|
||||
|
||||
pub(crate) fn stream_local_capability<W: Write>(
|
||||
&self,
|
||||
writer: W,
|
||||
limits: snapshot::Limits,
|
||||
) -> Result<snapshot::CapabilityArtifact<W>, NativeSnapshotError> {
|
||||
if !matches!(self.schema, 6 | 7) {
|
||||
return Err(StoreError::UnsupportedSchema.into());
|
||||
}
|
||||
let output = snapshot::CapabilityOutput::new(writer, limits.total_bytes);
|
||||
Ok(self.stream_borrowed(output, limits)?.finish())
|
||||
}
|
||||
|
||||
pub fn stream<W: Write>(self, writer: W, limits: snapshot::Limits) -> Result<W, NativeSnapshotError> {
|
||||
self.stream_borrowed(writer, limits)
|
||||
}
|
||||
|
||||
fn stream_borrowed<W: Write>(&self, writer: W, limits: snapshot::Limits) -> Result<W, NativeSnapshotError> {
|
||||
let mut encoder = snapshot::Encoder::new(writer, &self.header, self.counts, limits)?;
|
||||
for (definition, section) in [
|
||||
(application::HEADS, snapshot::Section::Heads),
|
||||
(application::VERSIONS, snapshot::Section::Versions),
|
||||
(application::EVENTS, snapshot::Section::Events),
|
||||
(application::OPERATIONS, snapshot::Section::Operations),
|
||||
] {
|
||||
let table = self.transaction.open_table(definition).map_err(StoreError::from)?;
|
||||
for row in table.iter().map_err(StoreError::from)? {
|
||||
let (key, value) = row.map_err(StoreError::from)?;
|
||||
encoder.row(snapshot::Row {
|
||||
section,
|
||||
key: key.value(),
|
||||
value: value.value(),
|
||||
})?;
|
||||
}
|
||||
}
|
||||
Ok(encoder.finish()?)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
//! Bounded snapshot transport bytes; receive ownership follows the buffer into install.
|
||||
use std::{
|
||||
io::{self, Read, Seek, SeekFrom, Write},
|
||||
pin::Pin,
|
||||
task::{Context, Poll},
|
||||
};
|
||||
use tokio::io::{AsyncRead, AsyncSeek, AsyncWrite, ReadBuf};
|
||||
|
||||
pub struct NativeSnapshotData {
|
||||
bytes: Vec<u8>,
|
||||
position: u64,
|
||||
limit: u64,
|
||||
pub(crate) receive_guard: Option<crate::raft_state_machine::ReceiveGuard>,
|
||||
}
|
||||
|
||||
fn boundary() -> io::Error {
|
||||
io::Error::new(io::ErrorKind::InvalidInput, "native snapshot byte limit")
|
||||
}
|
||||
|
||||
impl NativeSnapshotData {
|
||||
pub fn new(max_bytes: u64) -> io::Result<Self> {
|
||||
if max_bytes == 0 || usize::try_from(max_bytes).is_err() {
|
||||
return Err(boundary());
|
||||
}
|
||||
Ok(Self {
|
||||
bytes: Vec::new(),
|
||||
position: 0,
|
||||
limit: max_bytes,
|
||||
receive_guard: None,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn as_slice(&self) -> &[u8] {
|
||||
&self.bytes
|
||||
}
|
||||
|
||||
pub(crate) fn attach_receive_guard(&mut self, guard: crate::raft_state_machine::ReceiveGuard) {
|
||||
self.receive_guard = Some(guard);
|
||||
}
|
||||
}
|
||||
|
||||
impl Read for NativeSnapshotData {
|
||||
fn read(&mut self, output: &mut [u8]) -> io::Result<usize> {
|
||||
let position = usize::try_from(self.position).map_err(|_| boundary())?;
|
||||
let available = self.bytes.get(position..).unwrap_or_default();
|
||||
let count = output.len().min(available.len());
|
||||
output[..count].copy_from_slice(&available[..count]);
|
||||
self.position = self
|
||||
.position
|
||||
.checked_add(u64::try_from(count).map_err(|_| boundary())?)
|
||||
.ok_or_else(boundary)?;
|
||||
Ok(count)
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for NativeSnapshotData {
|
||||
fn write(&mut self, input: &[u8]) -> io::Result<usize> {
|
||||
if input.is_empty() {
|
||||
return Ok(0);
|
||||
}
|
||||
let count = u64::try_from(input.len()).map_err(|_| boundary())?;
|
||||
let end = self
|
||||
.position
|
||||
.checked_add(count)
|
||||
.filter(|end| *end <= self.limit)
|
||||
.ok_or_else(boundary)?;
|
||||
let position = usize::try_from(self.position).map_err(|_| boundary())?;
|
||||
let length = usize::try_from(end).map_err(|_| boundary())?;
|
||||
if length > self.bytes.len() {
|
||||
self.bytes
|
||||
.try_reserve_exact(length - self.bytes.len())
|
||||
.map_err(|_| io::Error::new(io::ErrorKind::OutOfMemory, "native snapshot allocation"))?;
|
||||
self.bytes.resize(length, 0);
|
||||
}
|
||||
self.bytes[position..length].copy_from_slice(input);
|
||||
self.position = end;
|
||||
Ok(input.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Seek for NativeSnapshotData {
|
||||
fn seek(&mut self, from: SeekFrom) -> io::Result<u64> {
|
||||
let position = match from {
|
||||
SeekFrom::Start(position) => Some(position),
|
||||
SeekFrom::Current(offset) => self.position.checked_add_signed(offset),
|
||||
SeekFrom::End(offset) => u64::try_from(self.bytes.len())
|
||||
.map_err(|_| boundary())?
|
||||
.checked_add_signed(offset),
|
||||
}
|
||||
.filter(|position| *position <= self.limit)
|
||||
.ok_or_else(boundary)?;
|
||||
self.position = position;
|
||||
Ok(position)
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncRead for NativeSnapshotData {
|
||||
fn poll_read(self: Pin<&mut Self>, _cx: &mut Context<'_>, output: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
|
||||
let this = self.get_mut();
|
||||
let position = match usize::try_from(this.position) {
|
||||
Ok(position) => position,
|
||||
Err(_) => return Poll::Ready(Err(boundary())),
|
||||
};
|
||||
let available = this.bytes.get(position..).unwrap_or_default();
|
||||
let count = output.remaining().min(available.len());
|
||||
let end = this
|
||||
.position
|
||||
.checked_add(u64::try_from(count).map_err(|_| boundary())?)
|
||||
.ok_or_else(boundary)?;
|
||||
output.put_slice(&available[..count]);
|
||||
this.position = end;
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncWrite for NativeSnapshotData {
|
||||
fn poll_write(self: Pin<&mut Self>, _cx: &mut Context<'_>, input: &[u8]) -> Poll<io::Result<usize>> {
|
||||
Poll::Ready(Write::write(self.get_mut(), input))
|
||||
}
|
||||
|
||||
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
|
||||
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncSeek for NativeSnapshotData {
|
||||
fn start_seek(self: Pin<&mut Self>, from: SeekFrom) -> io::Result<()> {
|
||||
Seek::seek(self.get_mut(), from).map(|_| ())
|
||||
}
|
||||
|
||||
fn poll_complete(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<u64>> {
|
||||
Poll::Ready(Ok(self.position))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn native_snapshot_data_rejects_growth_before_mutation() {
|
||||
let mut data = NativeSnapshotData::new(4).expect("bounded snapshot");
|
||||
Write::write_all(&mut data, b"ab").expect("initial bytes");
|
||||
assert!(Write::write_all(&mut data, b"cde").is_err());
|
||||
assert_eq!(data.as_slice(), b"ab", "rejected write leaves bytes intact");
|
||||
assert_eq!(Seek::stream_position(&mut data).expect("position"), 2);
|
||||
Seek::seek(&mut data, SeekFrom::Start(4)).expect("seek exactly to limit");
|
||||
Write::write(&mut data, b"").expect("empty write");
|
||||
assert_eq!(data.as_slice(), b"ab", "empty write never allocates a gap");
|
||||
assert!(Write::write(&mut data, b"x").is_err());
|
||||
assert!(Seek::seek(&mut data, SeekFrom::Current(1)).is_err());
|
||||
assert!(Seek::seek(&mut data, SeekFrom::Current(i64::MIN)).is_err());
|
||||
assert_eq!(Seek::stream_position(&mut data).expect("unchanged position"), 4);
|
||||
Seek::seek(&mut data, SeekFrom::Start(3)).expect("bounded sparse position");
|
||||
Write::write_all(&mut data, b"z").expect("write ending exactly at limit");
|
||||
assert_eq!(data.as_slice(), b"ab\0z");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_snapshot_data_checks_length_overflow() {
|
||||
assert!(NativeSnapshotData::new(0).is_err());
|
||||
if usize::BITS == 64 {
|
||||
let mut data = NativeSnapshotData::new(u64::MAX).expect("representable byte limit");
|
||||
Seek::seek(&mut data, SeekFrom::Start(u64::MAX)).expect("seek does not allocate");
|
||||
assert!(Write::write(&mut data, b"x").is_err());
|
||||
assert!(Seek::seek(&mut data, SeekFrom::Current(1)).is_err());
|
||||
assert!(data.as_slice().is_empty(), "overflow never allocates");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn native_snapshot_data_async_io_uses_the_same_bounds() {
|
||||
use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt};
|
||||
let mut data = NativeSnapshotData::new(4).expect("bounded async snapshot");
|
||||
AsyncWriteExt::write_all(&mut data, b"abcd").await.expect("write exact limit");
|
||||
assert!(AsyncWriteExt::write_all(&mut data, b"e").await.is_err());
|
||||
assert!(AsyncSeekExt::seek(&mut data, SeekFrom::Start(5)).await.is_err());
|
||||
AsyncSeekExt::seek(&mut data, SeekFrom::End(-2))
|
||||
.await
|
||||
.expect("rewind within snapshot");
|
||||
let mut output = Vec::new();
|
||||
AsyncReadExt::read_to_end(&mut data, &mut output)
|
||||
.await
|
||||
.expect("read snapshot suffix");
|
||||
assert_eq!(output, b"cd");
|
||||
AsyncWriteExt::shutdown(&mut data).await.expect("transport shutdown");
|
||||
assert_eq!(data.as_slice(), b"abcd");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,217 @@
|
||||
//! Validate isolated received bytes before one durable shared-owner publication.
|
||||
use crate::{
|
||||
CaptureMembership, CheckpointLimits, NativeLogWriter, NativeSnapshotData, StoreError, application, checkpoint_baseline,
|
||||
native_checkpoint, snapshot,
|
||||
};
|
||||
use redb::{ReadableDatabase, ReadableTable};
|
||||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
};
|
||||
|
||||
struct Cancel(Arc<AtomicBool>);
|
||||
impl Drop for Cancel {
|
||||
fn drop(&mut self) {
|
||||
self.0.store(true, Ordering::Release);
|
||||
}
|
||||
}
|
||||
struct Stage {
|
||||
database: redb::Database,
|
||||
_directory: tempfile::TempDir,
|
||||
baseline: checkpoint_baseline::BaselineMetadata,
|
||||
}
|
||||
fn stage(data: &NativeSnapshotData, limits: CheckpointLimits) -> Result<Stage, StoreError> {
|
||||
let bytes = data.as_slice();
|
||||
if u64::try_from(bytes.len()).map_err(|_| StoreError::ResourceLimit)?
|
||||
> limits.snapshot.total_bytes.checked_add(56).ok_or(StoreError::ResourceLimit)?
|
||||
{
|
||||
return Err(StoreError::ResourceLimit);
|
||||
}
|
||||
let manifest = snapshot::CapabilityManifest::decode(bytes.get(..56).ok_or(StoreError::CorruptRecord)?)
|
||||
.map_err(|_| StoreError::CorruptRecord)?;
|
||||
let directory = tempfile::tempdir()?;
|
||||
let database = native_checkpoint::stage_database(directory.path().join("received.redb"), limits.staging_bytes)?;
|
||||
let tx = database.begin_write()?;
|
||||
for definition in [
|
||||
checkpoint_baseline::HEADS,
|
||||
checkpoint_baseline::VERSIONS,
|
||||
checkpoint_baseline::EVENTS,
|
||||
checkpoint_baseline::OPERATIONS,
|
||||
] {
|
||||
tx.open_table(definition)?;
|
||||
}
|
||||
let header = snapshot::decode_capability_artifact(
|
||||
snapshot::CapabilityArtifact {
|
||||
manifest,
|
||||
payload: &bytes[56..],
|
||||
},
|
||||
&mut native_checkpoint::StageSink(&tx),
|
||||
limits.snapshot,
|
||||
)
|
||||
.map_err(|_| StoreError::CorruptRecord)?;
|
||||
let baseline = checkpoint_baseline::BaselineMetadata {
|
||||
generation: 1,
|
||||
header,
|
||||
limits,
|
||||
};
|
||||
let encoded = checkpoint_baseline::encode_metadata(&baseline)?;
|
||||
{
|
||||
let mut meta = tx.open_table(crate::META)?;
|
||||
meta.insert("schema", 7u64.to_be_bytes().as_slice())?;
|
||||
meta.insert(checkpoint_baseline::META_KEY, encoded.as_slice())?;
|
||||
}
|
||||
tx.commit().map_err(StoreError::StagingCommit)?;
|
||||
checkpoint_baseline::validate_baseline_with_budget(&database.begin_read()?, &baseline, limits.staging_bytes)?;
|
||||
Ok(Stage {
|
||||
database,
|
||||
_directory: directory,
|
||||
baseline,
|
||||
})
|
||||
}
|
||||
pub(crate) async fn install(
|
||||
writer: NativeLogWriter,
|
||||
limits: CheckpointLimits,
|
||||
meta: openraft::SnapshotMeta<u64, crate::CaptureNode>,
|
||||
data: Box<NativeSnapshotData>,
|
||||
) -> Result<(), StoreError> {
|
||||
if data
|
||||
.receive_guard
|
||||
.as_ref()
|
||||
.is_none_or(|guard| !guard.belongs_to(&writer.snapshot_receives))
|
||||
{
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let (mut stage, data) = tokio::task::spawn_blocking(move || Ok::<_, StoreError>((stage(&data, limits)?, data)))
|
||||
.await
|
||||
.map_err(|_| StoreError::CorruptRecord)??;
|
||||
if meta.last_log_id.map(crate::LogId::from) != stage.baseline.header.applied {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
if CaptureMembership::try_from(&meta.last_membership)? != stage.baseline.header.membership {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let ordering = writer.ordering.clone().lock_owned().await;
|
||||
let cancel = Cancel(Arc::new(AtomicBool::new(false)));
|
||||
let cancelled = cancel.0.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let _ordering = ordering;
|
||||
let _data = data;
|
||||
if cancelled.load(Ordering::Acquire) {
|
||||
return Err(StoreError::WriteDisabled);
|
||||
}
|
||||
let mut store = writer.store.lock().map_err(|_| StoreError::CorruptRecord)?;
|
||||
let read = store.database.begin_read()?;
|
||||
let old_meta = read.open_table(crate::META)?;
|
||||
let identity = crate::decode_identity(old_meta.get("identity")?.ok_or(StoreError::CorruptRecord)?.value())?;
|
||||
if identity.cluster_id != stage.baseline.header.cluster_id
|
||||
|| application::read_binding(&old_meta)? != stage.baseline.header.binding
|
||||
{
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let old_applied = application::read_applied(&old_meta)?;
|
||||
let old_logs = read.open_table(crate::LOGS)?;
|
||||
let committed = crate::committed::validate_committed(&old_meta, &old_logs)?;
|
||||
let cut = stage.baseline.header.applied;
|
||||
if old_applied.is_some_and(|id| cut.is_none_or(|cut| id.index > cut.index || (id.index == cut.index && id != cut))) {
|
||||
return Err(StoreError::CommittedRegression);
|
||||
}
|
||||
let compatible = match cut {
|
||||
None => true,
|
||||
Some(cut) => {
|
||||
let at_cut = old_logs
|
||||
.get(cut.index)?
|
||||
.map(|v| crate::decode_log(v.value()).map(|v| v.id))
|
||||
.transpose()?
|
||||
.or(checkpoint_baseline::read_baseline_cut(&old_meta)?.filter(|id| id.index == cut.index));
|
||||
if committed
|
||||
.is_some_and(|id| id.index >= cut.index && (at_cut != Some(cut) || (id.index == cut.index && id != cut)))
|
||||
{
|
||||
return Err(StoreError::CommittedRegression);
|
||||
}
|
||||
at_cut == Some(cut)
|
||||
}
|
||||
};
|
||||
let previous = checkpoint_baseline::read_metadata(&old_meta)?;
|
||||
stage.baseline.generation =
|
||||
previous.map_or(Ok(1), |old| old.generation.checked_add(1).ok_or(StoreError::ResourceLimit))?;
|
||||
drop(old_logs);
|
||||
drop(old_meta);
|
||||
drop(read);
|
||||
let source = stage.database.begin_read()?;
|
||||
let tx = store.write_transaction()?;
|
||||
for (baseline, live) in [
|
||||
(checkpoint_baseline::HEADS, application::HEADS),
|
||||
(checkpoint_baseline::VERSIONS, application::VERSIONS),
|
||||
(checkpoint_baseline::EVENTS, application::EVENTS),
|
||||
(checkpoint_baseline::OPERATIONS, application::OPERATIONS),
|
||||
] {
|
||||
let table = source.open_table(baseline)?;
|
||||
for target in [baseline, live] {
|
||||
let mut target = tx.open_table(target)?;
|
||||
target.retain(|_, _| false)?;
|
||||
for row in table.iter()? {
|
||||
let (key, value) = row?;
|
||||
target.insert(key.value(), value.value())?;
|
||||
}
|
||||
}
|
||||
}
|
||||
{
|
||||
let mut index = tx.open_table(crate::source_page::EVENT_SEQUENCE)?;
|
||||
index.retain(|_, _| false)?;
|
||||
}
|
||||
for row in source.open_table(checkpoint_baseline::EVENTS)?.iter()? {
|
||||
let (key, value) = row?;
|
||||
let event = application::decode_event(value.value())?;
|
||||
crate::source_page::insert_event_index(
|
||||
&tx,
|
||||
event.decision_log_id,
|
||||
key.value().try_into().map_err(|_| StoreError::CorruptRecord)?,
|
||||
)?;
|
||||
}
|
||||
let tail = {
|
||||
let mut logs = tx.open_table(crate::LOGS)?;
|
||||
logs.retain(|index, _| compatible && cut.is_none_or(|cut| index > cut.index))?;
|
||||
logs.last()?
|
||||
.map(|(_, v)| crate::decode_log(v.value()).map(|v| v.id))
|
||||
.transpose()?
|
||||
.or(cut)
|
||||
};
|
||||
tx.open_table(application::APPLY_RESULTS)?.retain(|_, _| false)?;
|
||||
{
|
||||
let mut target = tx.open_table(crate::META)?;
|
||||
target.insert("schema", 7u64.to_be_bytes().as_slice())?;
|
||||
target.insert(
|
||||
checkpoint_baseline::META_KEY,
|
||||
checkpoint_baseline::encode_metadata(&stage.baseline)?.as_slice(),
|
||||
)?;
|
||||
let mut applied = vec![0];
|
||||
if let Some(cut) = cut {
|
||||
target.insert(checkpoint_baseline::PURGED_KEY, checkpoint_baseline::encode_purged(cut).as_slice())?;
|
||||
applied = vec![1];
|
||||
applied.extend(crate::encode_id(cut));
|
||||
} else {
|
||||
target.remove(checkpoint_baseline::PURGED_KEY)?;
|
||||
}
|
||||
target.insert("last_applied", applied.as_slice())?;
|
||||
target.insert("native_membership", stage.baseline.header.membership.encode()?.as_slice())?;
|
||||
if let Some(tail) = tail {
|
||||
target.insert("tail", crate::encode_id(tail).as_slice())?;
|
||||
} else {
|
||||
target.remove("tail")?;
|
||||
}
|
||||
let committed = committed.filter(|id| cut.is_none_or(|cut| id.index > cut.index)).or(cut);
|
||||
if let Some(committed) = committed {
|
||||
target.insert("committed", crate::encode_id(committed).as_slice())?;
|
||||
} else {
|
||||
target.remove("committed")?;
|
||||
}
|
||||
crate::raft_state_machine::publish_meta(&mut target, &stage.baseline, meta)?;
|
||||
}
|
||||
if cancelled.load(Ordering::Acquire) {
|
||||
return Err(StoreError::WriteDisabled);
|
||||
}
|
||||
store.commit(tx)
|
||||
})
|
||||
.await
|
||||
.map_err(|_| StoreError::CorruptRecord)?
|
||||
}
|
||||
@@ -0,0 +1,536 @@
|
||||
//! Atomic installation failures over real native stores and disk fault backend.
|
||||
use super::raft_state_machine_tests::{Fixture, entries, entry, id, install, receive, snapshot_bytes};
|
||||
use super::*;
|
||||
use openraft::{
|
||||
RaftLogReader, RaftSnapshotBuilder,
|
||||
storage::{RaftLogStorage, RaftLogStorageExt, RaftStateMachine},
|
||||
};
|
||||
|
||||
async fn media_target() -> (Fixture, tests::CrashBackend) {
|
||||
let mut f = Fixture::new(1);
|
||||
f.apply(entries(1)).await;
|
||||
f.logs
|
||||
.save_vote(&openraft::Vote::new_committed(9, 1))
|
||||
.await
|
||||
.expect("preserved target vote");
|
||||
let Fixture {
|
||||
directory,
|
||||
writer,
|
||||
logs,
|
||||
sm,
|
||||
identity,
|
||||
} = f;
|
||||
drop(sm);
|
||||
drop(logs);
|
||||
drop(writer);
|
||||
let backend = tests::CrashBackend::new(&directory.path().join("db"));
|
||||
let writer = NativeLogWriter::new(Store::from_database(backend.database(), identity).expect("actual redb fault target"));
|
||||
let logs = NativeRaftLogStore::new(writer.clone());
|
||||
let sm = NativeRaftStateMachine::new(writer.clone(), CheckpointLimits::default()).expect("fault target state machine");
|
||||
(
|
||||
Fixture {
|
||||
directory,
|
||||
writer,
|
||||
logs,
|
||||
sm,
|
||||
identity,
|
||||
},
|
||||
backend,
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn native_snapshot_install_corruption_preserves_state() {
|
||||
let mut source = Fixture::new(0);
|
||||
source.apply(entries(2)).await;
|
||||
let mut snapshot = source.build().await;
|
||||
let bytes = snapshot_bytes(&mut snapshot).await;
|
||||
let mut target = Fixture::new(1);
|
||||
target.apply(entries(1)).await;
|
||||
let before = target.page().await;
|
||||
let state = target.sm.applied_state().await.expect("old applied state");
|
||||
let mut corrupted = bytes.clone();
|
||||
*corrupted.last_mut().expect("nonempty snapshot") ^= 1;
|
||||
for payload in [corrupted, bytes[..bytes.len() - 1].to_vec()] {
|
||||
let stable = std::fs::read(target.directory.path().join("db")).expect("old stable bytes");
|
||||
let data = receive(&mut target.sm, &payload).await;
|
||||
assert!(
|
||||
target.sm.install_snapshot(&snapshot.meta, data).await.is_err(),
|
||||
"SNAPSHOT_INSTALL_CORRUPT_FRAME_REJECTED"
|
||||
);
|
||||
assert_eq!(target.sm.applied_state().await.expect("unchanged applied state"), state);
|
||||
assert_eq!(target.page().await, before, "SNAPSHOT_INSTALL_CORRUPTION_NO_PARTIAL_TABLES");
|
||||
assert_eq!(std::fs::read(target.directory.path().join("db")).expect("unchanged stable bytes"), stable);
|
||||
assert!(
|
||||
target
|
||||
.sm
|
||||
.get_current_snapshot()
|
||||
.await
|
||||
.expect("unpublished corrupt snapshot")
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
install(&mut target.sm, &mut snapshot).await;
|
||||
assert_eq!(target.page().await, source.page().await, "SNAPSHOT_INSTALL_RETRY_AFTER_CORRUPT_RECEIVE");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn native_snapshot_install_stale_cut_rejected() {
|
||||
let mut source = Fixture::new(0);
|
||||
source.apply(entries(1)).await;
|
||||
let mut snapshot = source.build().await;
|
||||
let bytes = snapshot_bytes(&mut snapshot).await;
|
||||
let mut target = Fixture::new(1);
|
||||
target.apply(entries(3)).await;
|
||||
let current = target.build().await.meta;
|
||||
let events = target.page().await;
|
||||
let data = receive(&mut target.sm, &bytes).await;
|
||||
assert!(
|
||||
target.sm.install_snapshot(&snapshot.meta, data).await.is_err(),
|
||||
"SNAPSHOT_INSTALL_NEVER_ROLLS_BACK_APPLIED_STATE"
|
||||
);
|
||||
assert_eq!(target.sm.applied_state().await.expect("non-regressed applied").0, Some(id(3).into()));
|
||||
assert_eq!(
|
||||
target
|
||||
.sm
|
||||
.get_current_snapshot()
|
||||
.await
|
||||
.expect("old current snapshot")
|
||||
.expect("present")
|
||||
.meta,
|
||||
current
|
||||
);
|
||||
assert_eq!(target.page().await, events);
|
||||
let mut source = Fixture::new(0);
|
||||
source.apply(entries(2)).await;
|
||||
let mut snapshot = source.build().await;
|
||||
let bytes = snapshot_bytes(&mut snapshot).await;
|
||||
let mut target = Fixture::new(1);
|
||||
let mut batch = entries(3);
|
||||
for entry in &mut batch[2..] {
|
||||
entry.log_id = LogId {
|
||||
term: 9,
|
||||
..id(entry.log_id.index)
|
||||
}
|
||||
.into();
|
||||
}
|
||||
target
|
||||
.logs
|
||||
.blocking_append(batch)
|
||||
.await
|
||||
.expect("conflicting committed suffix logs");
|
||||
target.sm.apply(entries(1)).await.expect("existing applied prefix");
|
||||
target
|
||||
.logs
|
||||
.save_committed(Some(LogId { term: 9, ..id(3) }.into()))
|
||||
.await
|
||||
.expect("protect conflicting committed suffix");
|
||||
let before = target
|
||||
.logs
|
||||
.get_log_reader()
|
||||
.await
|
||||
.try_get_log_entries(..)
|
||||
.await
|
||||
.expect("protected logs");
|
||||
let data = receive(&mut target.sm, &bytes).await;
|
||||
assert!(
|
||||
target.sm.install_snapshot(&snapshot.meta, data).await.is_err(),
|
||||
"SNAPSHOT_INSTALL_NEVER_DISCARDS_COMMITTED_CONFLICT"
|
||||
);
|
||||
assert_eq!(
|
||||
target
|
||||
.logs
|
||||
.get_log_reader()
|
||||
.await
|
||||
.try_get_log_entries(..)
|
||||
.await
|
||||
.expect("unchanged protected logs"),
|
||||
before
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn native_snapshot_install_cancelled_owner() {
|
||||
let mut source = Fixture::new(0);
|
||||
source.apply(entries(2)).await;
|
||||
let mut snapshot = source.build().await;
|
||||
let bytes = snapshot_bytes(&mut snapshot).await;
|
||||
let (mut target, backend) = media_target().await;
|
||||
let data = receive(&mut target.sm, &bytes).await;
|
||||
let (entered, resume) = backend.pause_sync(1, false);
|
||||
let mut installing = target.sm.clone();
|
||||
let meta = snapshot.meta.clone();
|
||||
let task = tokio::spawn(async move { installing.install_snapshot(&meta, data).await });
|
||||
entered.wait();
|
||||
assert!(!task.is_finished(), "SNAPSHOT_INSTALL_NOT_ACKED_BEFORE_SYNC");
|
||||
task.abort();
|
||||
assert!(task.await.expect_err("cancelled install waiter").is_cancelled());
|
||||
assert!(
|
||||
target.writer.ordering.try_lock().is_err(),
|
||||
"SNAPSHOT_INSTALL_CANCELLED_WAITER_RETAINS_OWNER"
|
||||
);
|
||||
resume.wait();
|
||||
// Acquiring the production owner waits for the abandoned blocking commit.
|
||||
drop(target.writer.ordering.lock().await);
|
||||
assert_eq!(
|
||||
target.sm.applied_state().await.expect("completed background install").0,
|
||||
Some(id(2).into()),
|
||||
"SNAPSHOT_INSTALL_COMMIT_NOT_INTERRUPTED_BY_WAITER_CANCEL"
|
||||
);
|
||||
let vote = target.logs.read_vote().await.expect("preserved vote");
|
||||
assert_eq!(vote, Some(openraft::Vote::new_committed(9, 1)));
|
||||
backend.cut();
|
||||
let Fixture {
|
||||
directory,
|
||||
writer,
|
||||
logs,
|
||||
sm,
|
||||
identity,
|
||||
} = target;
|
||||
drop(sm);
|
||||
drop(logs);
|
||||
drop(writer);
|
||||
let writer = NativeLogWriter::new(Store::open(directory.path().join("db"), identity).expect("reopen cancelled installation"));
|
||||
let mut sm =
|
||||
NativeRaftStateMachine::new(writer.clone(), CheckpointLimits::default()).expect("reopen installed state machine");
|
||||
assert_eq!(
|
||||
sm.get_current_snapshot()
|
||||
.await
|
||||
.expect("completed durable snapshot")
|
||||
.expect("present")
|
||||
.meta,
|
||||
snapshot.meta
|
||||
);
|
||||
assert_eq!(
|
||||
writer.created_events_page(None, 256).await.expect("installed events"),
|
||||
source.page().await
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn native_snapshot_install_power_cut_atomic() {
|
||||
let mut source = Fixture::new(0);
|
||||
source.apply(entries(2)).await;
|
||||
let mut snapshot = source.build().await;
|
||||
let bytes = snapshot_bytes(&mut snapshot).await;
|
||||
let new_events = source.page().await;
|
||||
for after_sync in [false, true] {
|
||||
let (mut target, backend) = media_target().await;
|
||||
let old_events = target.page().await;
|
||||
let data = receive(&mut target.sm, &bytes).await;
|
||||
let (entered, resume) = backend.pause_sync(1, after_sync);
|
||||
let meta = snapshot.meta.clone();
|
||||
let mut installing = target.sm.clone();
|
||||
let task = tokio::spawn(async move { installing.install_snapshot(&meta, data).await });
|
||||
entered.wait();
|
||||
backend.cut();
|
||||
resume.wait();
|
||||
assert!(
|
||||
task.await.expect("faulted install task").is_err(),
|
||||
"SNAPSHOT_INSTALL_POWER_CUT_NO_FALSE_ACK"
|
||||
);
|
||||
assert!(
|
||||
target.sm.get_current_snapshot().await.is_err(),
|
||||
"SNAPSHOT_INSTALL_FENCED_HANDLE_CANNOT_EXPORT"
|
||||
);
|
||||
let Fixture {
|
||||
directory,
|
||||
writer,
|
||||
logs,
|
||||
sm,
|
||||
identity,
|
||||
} = target;
|
||||
drop(sm);
|
||||
drop(logs);
|
||||
drop(writer);
|
||||
let writer =
|
||||
NativeLogWriter::new(Store::open(directory.path().join("db"), identity).expect("atomic installation recovery"));
|
||||
let mut logs = NativeRaftLogStore::new(writer.clone());
|
||||
let mut sm = NativeRaftStateMachine::new(writer.clone(), CheckpointLimits::default()).expect("recovered SM");
|
||||
let events = writer
|
||||
.created_events_page(None, 256)
|
||||
.await
|
||||
.expect("complete recovered tables");
|
||||
let cut = sm.applied_state().await.expect("recovered applied marker").0;
|
||||
if events == old_events {
|
||||
assert_eq!(cut, Some(id(1).into()), "SNAPSHOT_INSTALL_OLD_TABLES_OLD_MARKER");
|
||||
assert!(sm.get_current_snapshot().await.expect("old snapshot selection").is_none());
|
||||
} else {
|
||||
assert_eq!(events, new_events, "SNAPSHOT_INSTALL_ALL_TABLES_OLD_OR_NEW");
|
||||
assert_eq!(cut, Some(id(2).into()), "SNAPSHOT_INSTALL_NEW_TABLES_NEW_MARKER");
|
||||
let mut current = sm
|
||||
.get_current_snapshot()
|
||||
.await
|
||||
.expect("new snapshot selection")
|
||||
.expect("new snapshot published");
|
||||
assert_eq!(current.meta, snapshot.meta);
|
||||
assert_eq!(snapshot_bytes(&mut current).await, bytes);
|
||||
}
|
||||
assert_eq!(writer.durable_identity().await.expect("recovered local identity"), identity);
|
||||
assert_eq!(
|
||||
logs.read_vote().await.expect("recovered local vote"),
|
||||
Some(openraft::Vote::new_committed(9, 1))
|
||||
);
|
||||
let state = logs.get_log_state().await.expect("coherent recovered log state");
|
||||
assert!(
|
||||
state
|
||||
.last_log_id
|
||||
.is_some_and(|last| cut.is_some_and(|applied| last.index >= applied.index)),
|
||||
"SNAPSHOT_INSTALL_LOG_STATE_COVERS_APPLIED_CUT"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn native_snapshot_install_ack_survives_power_cut() {
|
||||
let mut source = Fixture::new(0);
|
||||
source.apply(entries(2)).await;
|
||||
let mut snapshot = source.build().await;
|
||||
let bytes = snapshot_bytes(&mut snapshot).await;
|
||||
let (mut target, backend) = media_target().await;
|
||||
install(&mut target.sm, &mut snapshot).await;
|
||||
backend.cut();
|
||||
let Fixture {
|
||||
directory,
|
||||
writer,
|
||||
logs,
|
||||
sm,
|
||||
identity,
|
||||
} = target;
|
||||
drop(sm);
|
||||
drop(logs);
|
||||
drop(writer);
|
||||
let writer = NativeLogWriter::new(
|
||||
Store::open(directory.path().join("db"), identity).expect("acknowledged installation survives power cut"),
|
||||
);
|
||||
let mut logs = NativeRaftLogStore::new(writer.clone());
|
||||
let mut sm = NativeRaftStateMachine::new(writer.clone(), CheckpointLimits::default()).expect("reopened durable SM");
|
||||
let mut current = sm
|
||||
.get_current_snapshot()
|
||||
.await
|
||||
.expect("durable current snapshot")
|
||||
.expect("present");
|
||||
assert_eq!(current.meta, snapshot.meta, "SNAPSHOT_INSTALL_ACK_COMPLETE_META_DURABLE");
|
||||
assert_eq!(snapshot_bytes(&mut current).await, bytes);
|
||||
assert_eq!(
|
||||
writer.created_events_page(None, 256).await.expect("durable installed events"),
|
||||
source.page().await
|
||||
);
|
||||
logs.blocking_append(vec![entry(3)]).await.expect("append after ACK recovery");
|
||||
sm.apply(vec![entry(3)]).await.expect("apply after ACK recovery");
|
||||
assert_eq!(sm.applied_state().await.expect("continued state").0, Some(id(3).into()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn native_snapshot_install_purge_without_pending_rejected() {
|
||||
let mut target = Fixture::new(1);
|
||||
target.apply(entries(1)).await;
|
||||
let before = target.page().await;
|
||||
let bytes = std::fs::read(target.directory.path().join("db")).expect("old unauthorized purge bytes");
|
||||
let result = tokio::time::timeout(std::time::Duration::from_secs(2), target.logs.purge(id(3).into()))
|
||||
.await
|
||||
.expect("no pending receiver means purge cannot wait for hypothetical installation");
|
||||
assert!(result.is_err(), "SNAPSHOT_PURGE_WITHOUT_PENDING_FAILS_IMMEDIATELY");
|
||||
assert_eq!(target.page().await, before);
|
||||
assert_eq!(
|
||||
std::fs::read(target.directory.path().join("db")).expect("rejected purge preserves bytes"),
|
||||
bytes
|
||||
);
|
||||
}
|
||||
|
||||
async fn wait_for_pending_purge(writer: &NativeLogWriter) {
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
while writer.snapshot_receives.waiters.load(std::sync::atomic::Ordering::Acquire) == 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("actual trait purge reached pending-receive wait");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn native_snapshot_install_pending_purge_waits_for_install() {
|
||||
let mut source = Fixture::new(0);
|
||||
source.apply(entries(3)).await;
|
||||
let mut snapshot = source.build().await;
|
||||
let bytes = snapshot_bytes(&mut snapshot).await;
|
||||
let mut target = Fixture::new(1);
|
||||
target.apply(entries(1)).await;
|
||||
let data = receive(&mut target.sm, &bytes).await;
|
||||
let mut purging = target.logs.clone();
|
||||
let purge = tokio::spawn(async move { purging.purge(id(3).into()).await });
|
||||
wait_for_pending_purge(&target.writer).await;
|
||||
assert!(!purge.is_finished(), "SNAPSHOT_PURGE_WAITS_FOR_INCOMING_CUT_BEYOND_LOCAL_TAIL");
|
||||
target
|
||||
.sm
|
||||
.install_snapshot(&snapshot.meta, data)
|
||||
.await
|
||||
.expect("actual install while OpenRaft-style purge waits");
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), purge)
|
||||
.await
|
||||
.expect("pending purge wakes after install")
|
||||
.expect("purge task")
|
||||
.expect("installed checkpoint now authorizes exact purge");
|
||||
assert_eq!(
|
||||
target
|
||||
.logs
|
||||
.get_log_state()
|
||||
.await
|
||||
.expect("installed log state")
|
||||
.last_purged_log_id,
|
||||
Some(id(3).into()),
|
||||
"SNAPSHOT_PENDING_PURGE_USES_DURABLE_INSTALLED_CUT"
|
||||
);
|
||||
assert_eq!(target.page().await, source.page().await);
|
||||
assert_eq!(
|
||||
target
|
||||
.writer
|
||||
.snapshot_receives
|
||||
.waiters
|
||||
.load(std::sync::atomic::Ordering::Acquire),
|
||||
0,
|
||||
"SNAPSHOT_PURGE_WAIT_OWNER_RELEASED"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn native_snapshot_install_cancelled_receive_wakes_purge() {
|
||||
let mut target = Fixture::new(1);
|
||||
target.apply(entries(1)).await;
|
||||
let page = target.page().await;
|
||||
let before = std::fs::read(target.directory.path().join("db")).expect("stable target before cancelled receive");
|
||||
let data = target.sm.begin_receiving_snapshot().await.expect("actual pending receiver");
|
||||
let mut purging = target.logs.clone();
|
||||
let purge = tokio::spawn(async move { purging.purge(id(3).into()).await });
|
||||
wait_for_pending_purge(&target.writer).await;
|
||||
assert!(!purge.is_finished(), "SNAPSHOT_CANCELLED_RECEIVE_TEST_HAS_REAL_WAITING_PURGE");
|
||||
drop(data);
|
||||
assert!(
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), purge)
|
||||
.await
|
||||
.expect("cancelled receive wakes pending purge")
|
||||
.expect("purge task")
|
||||
.is_err(),
|
||||
"SNAPSHOT_CANCELLED_RECEIVE_CANNOT_AUTHORIZE_PURGE"
|
||||
);
|
||||
assert_eq!(target.page().await, page);
|
||||
assert_eq!(
|
||||
std::fs::read(target.directory.path().join("db")).expect("retained cancelled-receive state"),
|
||||
before
|
||||
);
|
||||
assert_eq!(
|
||||
target
|
||||
.writer
|
||||
.snapshot_receives
|
||||
.waiters
|
||||
.load(std::sync::atomic::Ordering::Acquire),
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn native_snapshot_install_failed_receive_wakes_purge() {
|
||||
let mut source = Fixture::new(0);
|
||||
source.apply(entries(3)).await;
|
||||
let mut snapshot = source.build().await;
|
||||
let bytes = snapshot_bytes(&mut snapshot).await;
|
||||
let mut target = Fixture::new(1);
|
||||
target.apply(entries(1)).await;
|
||||
let page = target.page().await;
|
||||
let before = std::fs::read(target.directory.path().join("db")).expect("stable target before failed install");
|
||||
let data = receive(&mut target.sm, &bytes).await;
|
||||
let mut purging = target.logs.clone();
|
||||
let purge = tokio::spawn(async move { purging.purge(id(3).into()).await });
|
||||
wait_for_pending_purge(&target.writer).await;
|
||||
assert!(!purge.is_finished(), "SNAPSHOT_FAILED_INSTALL_TEST_HAS_REAL_WAITING_PURGE");
|
||||
let mut wrong = snapshot.meta.clone();
|
||||
wrong.last_log_id = Some(LogId { term: 9, ..id(3) }.into());
|
||||
assert!(
|
||||
target.sm.install_snapshot(&wrong, data).await.is_err(),
|
||||
"SNAPSHOT_INVALID_META_INSTALL_REJECTED"
|
||||
);
|
||||
assert!(
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), purge)
|
||||
.await
|
||||
.expect("failed install wakes pending purge")
|
||||
.expect("purge task")
|
||||
.is_err(),
|
||||
"SNAPSHOT_FAILED_INSTALL_CANNOT_AUTHORIZE_PURGE"
|
||||
);
|
||||
assert_eq!(target.page().await, page);
|
||||
assert_eq!(
|
||||
std::fs::read(target.directory.path().join("db")).expect("retained failed-install state"),
|
||||
before
|
||||
);
|
||||
assert_eq!(
|
||||
target
|
||||
.writer
|
||||
.snapshot_receives
|
||||
.waiters
|
||||
.load(std::sync::atomic::Ordering::Acquire),
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn native_snapshot_install_completed_between_purge_query_and_wait() {
|
||||
let mut source = Fixture::new(0);
|
||||
source.apply(entries(2)).await;
|
||||
let mut snapshot = source.build().await;
|
||||
let data_bytes = snapshot_bytes(&mut snapshot).await;
|
||||
let mut target = Fixture::new(1);
|
||||
let data = receive(&mut target.sm, &data_bytes).await;
|
||||
let entered = std::sync::Arc::new(tokio::sync::Notify::new());
|
||||
let resume = std::sync::Arc::new(tokio::sync::Notify::new());
|
||||
*target.writer.snapshot_receives.purge_recheck.lock().expect("test pause") = Some((entered.clone(), resume.clone()));
|
||||
let mut logs = target.logs.clone();
|
||||
let purge = tokio::spawn(async move { logs.purge(id(2).into()).await });
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), entered.notified())
|
||||
.await
|
||||
.expect("purge failed coverage under owner before continuation");
|
||||
target
|
||||
.sm
|
||||
.install_snapshot(&snapshot.meta, data)
|
||||
.await
|
||||
.expect("install completes before purge continuation");
|
||||
resume.notify_one();
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), purge)
|
||||
.await
|
||||
.expect("completed install wake retained")
|
||||
.expect("purge task")
|
||||
.expect("old StaleCheckpoint rechecked after install");
|
||||
assert_eq!(
|
||||
target.page().await,
|
||||
source.page().await,
|
||||
"SNAPSHOT_COMPLETED_INSTALL_RECHECKS_STALE_PURGE"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn native_snapshot_install_empty_baseline_pending_purge() {
|
||||
let mut source = Fixture::new(0);
|
||||
source.apply(entries(2)).await;
|
||||
let mut snapshot = source.build().await;
|
||||
let bytes = snapshot_bytes(&mut snapshot).await;
|
||||
let mut target = Fixture::new(1);
|
||||
target.sm.build_snapshot().await.expect("real local empty baseline");
|
||||
let data = receive(&mut target.sm, &bytes).await;
|
||||
let mut logs = target.logs.clone();
|
||||
let purge = tokio::spawn(async move { logs.purge(id(2).into()).await });
|
||||
wait_for_pending_purge(&target.writer).await;
|
||||
target
|
||||
.sm
|
||||
.install_snapshot(&snapshot.meta, data)
|
||||
.await
|
||||
.expect("install supersedes empty baseline");
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), purge)
|
||||
.await
|
||||
.expect("empty coverage wake")
|
||||
.expect("purge task")
|
||||
.expect("new covered purge");
|
||||
assert_eq!(
|
||||
target.page().await,
|
||||
source.page().await,
|
||||
"SNAPSHOT_EMPTY_BASELINE_IS_UNCOVERED_NOT_CORRUPT"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,371 @@
|
||||
//! Validated capture peer metadata and its independent CNOD v1 codec.
|
||||
use serde::{Deserialize, Deserializer, Serialize, Serializer, ser::SerializeStruct};
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct CaptureNode {
|
||||
rpc_uri: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum CaptureNodeError {
|
||||
#[error("invalid RPC URI")]
|
||||
InvalidRpcUri,
|
||||
#[error("invalid RPC URI")]
|
||||
InvalidRpcUriParse(#[source] url::ParseError),
|
||||
#[error("RPC URI too long")]
|
||||
RpcUriTooLong,
|
||||
#[error("invalid node magic")]
|
||||
InvalidNodeMagic,
|
||||
#[error("unsupported node version")]
|
||||
UnsupportedNodeVersion,
|
||||
#[error("invalid node length")]
|
||||
InvalidNodeLength,
|
||||
#[error("truncated node record")]
|
||||
TruncatedNodeRecord,
|
||||
#[error("invalid node UTF-8")]
|
||||
InvalidNodeUtf8(#[source] std::str::Utf8Error),
|
||||
#[error("trailing node bytes")]
|
||||
TrailingNodeBytes,
|
||||
}
|
||||
impl CaptureNodeError {
|
||||
pub fn code(&self) -> &'static str {
|
||||
match self {
|
||||
Self::InvalidRpcUri | Self::InvalidRpcUriParse(_) => "invalid_rpc_uri",
|
||||
Self::RpcUriTooLong => "rpc_uri_too_long",
|
||||
Self::InvalidNodeMagic => "invalid_node_magic",
|
||||
Self::UnsupportedNodeVersion => "unsupported_node_version",
|
||||
Self::InvalidNodeLength => "invalid_node_length",
|
||||
Self::TruncatedNodeRecord => "truncated_node_record",
|
||||
Self::InvalidNodeUtf8(_) => "invalid_node_utf8",
|
||||
Self::TrailingNodeBytes => "trailing_node_bytes",
|
||||
}
|
||||
}
|
||||
}
|
||||
impl CaptureNode {
|
||||
pub fn new(rpc_uri: String) -> Result<Self, CaptureNodeError> {
|
||||
let node = Self { rpc_uri };
|
||||
node.validate()?;
|
||||
Ok(node)
|
||||
}
|
||||
|
||||
pub fn rpc_uri(&self) -> &str {
|
||||
&self.rpc_uri
|
||||
}
|
||||
|
||||
pub fn validate(&self) -> Result<(), CaptureNodeError> {
|
||||
let uri = &self.rpc_uri;
|
||||
if uri.len() > 4096 {
|
||||
return Err(CaptureNodeError::RpcUriTooLong);
|
||||
}
|
||||
if uri.is_empty() || uri.chars().any(|c| c.is_whitespace() || c.is_control() || c == '\\') {
|
||||
return Err(CaptureNodeError::InvalidRpcUri);
|
||||
}
|
||||
let rest = uri
|
||||
.strip_prefix("http://")
|
||||
.or_else(|| uri.strip_prefix("https://"))
|
||||
.ok_or(CaptureNodeError::InvalidRpcUri)?;
|
||||
if uri.contains('?') || uri.contains('#') {
|
||||
return Err(CaptureNodeError::InvalidRpcUri);
|
||||
}
|
||||
let (authority, path) = rest.split_once('/').map_or((rest, ""), |(a, p)| (a, p));
|
||||
// An empty suffix after the first slash is the only allowed path.
|
||||
if !path.is_empty() || authority.contains('@') {
|
||||
return Err(CaptureNodeError::InvalidRpcUri);
|
||||
}
|
||||
let (host, port) = if authority.starts_with('[') {
|
||||
let end = authority.find(']').ok_or(CaptureNodeError::InvalidRpcUri)?;
|
||||
let port = authority
|
||||
.get(end + 1..)
|
||||
.and_then(|suffix| suffix.strip_prefix(':'))
|
||||
.ok_or(CaptureNodeError::InvalidRpcUri)?;
|
||||
(&authority[..=end], port)
|
||||
} else {
|
||||
authority.split_once(':').ok_or(CaptureNodeError::InvalidRpcUri)?
|
||||
};
|
||||
if host.is_empty() || port.is_empty() || !port.bytes().all(|b| b.is_ascii_digit()) {
|
||||
return Err(CaptureNodeError::InvalidRpcUri);
|
||||
}
|
||||
let port = port.parse::<u16>().map_err(|_| CaptureNodeError::InvalidRpcUri)?;
|
||||
if port == 0 {
|
||||
return Err(CaptureNodeError::InvalidRpcUri);
|
||||
}
|
||||
let parsed_url = url::Url::parse(uri).map_err(CaptureNodeError::InvalidRpcUriParse)?;
|
||||
if parsed_url.host().is_none() {
|
||||
return Err(CaptureNodeError::InvalidRpcUri);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn encode(&self) -> Result<Vec<u8>, CaptureNodeError> {
|
||||
self.validate()?;
|
||||
let length = u32::try_from(self.rpc_uri.len()).map_err(|_| CaptureNodeError::InvalidNodeLength)?;
|
||||
let mut bytes = b"CNOD\x01".to_vec();
|
||||
bytes.extend_from_slice(&length.to_be_bytes());
|
||||
bytes.extend_from_slice(self.rpc_uri.as_bytes());
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
pub fn decode(bytes: &[u8]) -> Result<Self, CaptureNodeError> {
|
||||
if bytes.len() < 9 {
|
||||
return Err(CaptureNodeError::TruncatedNodeRecord);
|
||||
}
|
||||
if &bytes[..4] != b"CNOD" {
|
||||
return Err(CaptureNodeError::InvalidNodeMagic);
|
||||
}
|
||||
if bytes[4] != 1 {
|
||||
return Err(CaptureNodeError::UnsupportedNodeVersion);
|
||||
}
|
||||
let length = u32::from_be_bytes(bytes[5..9].try_into().map_err(|_| CaptureNodeError::InvalidNodeLength)?);
|
||||
if !(1..=4096).contains(&length) {
|
||||
return Err(CaptureNodeError::InvalidNodeLength);
|
||||
}
|
||||
let length = usize::try_from(length).map_err(|_| CaptureNodeError::InvalidNodeLength)?;
|
||||
let end = 9_usize.checked_add(length).ok_or(CaptureNodeError::InvalidNodeLength)?;
|
||||
if bytes.len() < end {
|
||||
return Err(CaptureNodeError::TruncatedNodeRecord);
|
||||
}
|
||||
if bytes.len() != end {
|
||||
return Err(CaptureNodeError::TrailingNodeBytes);
|
||||
}
|
||||
let uri = std::str::from_utf8(&bytes[9..end]).map_err(CaptureNodeError::InvalidNodeUtf8)?;
|
||||
Self::new(uri.to_owned())
|
||||
}
|
||||
}
|
||||
impl Serialize for CaptureNode {
|
||||
fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
|
||||
self.validate().map_err(serde::ser::Error::custom)?;
|
||||
let mut state = serializer.serialize_struct("CaptureNode", 1)?;
|
||||
state.serialize_field("rpc_uri", &self.rpc_uri)?;
|
||||
state.end()
|
||||
}
|
||||
}
|
||||
impl<'de> Deserialize<'de> for CaptureNode {
|
||||
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct Fields {
|
||||
rpc_uri: String,
|
||||
}
|
||||
let fields = Fields::deserialize(deserializer).map_err(|_| serde::de::Error::custom("invalid capture node fields"))?;
|
||||
Self::new(fields.rpc_uri).map_err(serde::de::Error::custom)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::error::Error;
|
||||
|
||||
fn record(payload: &[u8]) -> Vec<u8> {
|
||||
let mut bytes = b"CNOD\x01".to_vec();
|
||||
bytes.extend_from_slice(&u32::try_from(payload.len()).expect("test payload length").to_be_bytes());
|
||||
bytes.extend_from_slice(payload);
|
||||
bytes
|
||||
}
|
||||
fn reject_uri(uri: &str) {
|
||||
let result = CaptureNode::new(uri.to_owned());
|
||||
assert!(result.is_err(), "URI should reject: {uri:?}");
|
||||
assert_eq!(result.expect_err("rejected URI").code(), "invalid_rpc_uri");
|
||||
}
|
||||
fn reject_record(bytes: &[u8], code: &str) {
|
||||
let result = CaptureNode::decode(bytes);
|
||||
assert!(result.is_err(), "record should reject with {code}");
|
||||
assert_eq!(result.expect_err("rejected record").code(), code);
|
||||
}
|
||||
#[test]
|
||||
fn actual_openraft_node_bound() {
|
||||
fn require_node<T: openraft::Node>() {}
|
||||
require_node::<CaptureNode>();
|
||||
assert_eq!(CaptureNode::default().rpc_uri(), "");
|
||||
}
|
||||
#[test]
|
||||
fn default_and_empty_rejected_at_every_boundary() {
|
||||
let node = CaptureNode::default();
|
||||
assert_eq!(node.rpc_uri(), "");
|
||||
reject_uri("");
|
||||
assert_eq!(node.validate().expect_err("empty validation").code(), "invalid_rpc_uri");
|
||||
assert!(serde_json::to_string(&node).is_err());
|
||||
assert!(serde_json::from_str::<CaptureNode>(r#"{"rpc_uri":""}"#).is_err());
|
||||
assert_eq!(node.encode().expect_err("empty encoding").code(), "invalid_rpc_uri");
|
||||
reject_record(&record(b""), "invalid_node_length");
|
||||
}
|
||||
#[test]
|
||||
fn accepted_uris_preserve_exact_bytes() {
|
||||
for uri in [
|
||||
"http://node:80",
|
||||
"https://node:443",
|
||||
"http://node:1",
|
||||
"https://node:65535/",
|
||||
"http://127.0.0.1:9000",
|
||||
"https://[::1]:443/",
|
||||
"http://[2001:db8::1]:1",
|
||||
"http://NoDe:00080/",
|
||||
] {
|
||||
let result = CaptureNode::new(uri.to_owned());
|
||||
assert!(result.is_ok(), "valid URI construction: {uri}");
|
||||
let node = result.expect("valid URI");
|
||||
assert_eq!(node.rpc_uri(), uri, "preserve explicit default port and raw URI");
|
||||
assert_eq!(node.encode().expect("encode URI"), record(uri.as_bytes()));
|
||||
assert_eq!(CaptureNode::decode(&record(uri.as_bytes())).expect("decode URI"), node);
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn golden_cnod_bytes() {
|
||||
let node = CaptureNode {
|
||||
rpc_uri: "http://node:80".to_owned(),
|
||||
};
|
||||
let golden = b"CNOD\x01\x00\x00\x00\x0ehttp://node:80";
|
||||
assert_eq!(node.encode().ok().as_deref(), Some(golden.as_slice()), "golden CNOD bytes");
|
||||
}
|
||||
#[test]
|
||||
fn valid_cnod_decode() {
|
||||
let golden = b"CNOD\x01\x00\x00\x00\x0ehttp://node:80";
|
||||
let decoded = CaptureNode::decode(golden);
|
||||
assert!(decoded.is_ok(), "golden decode succeeds");
|
||||
assert_eq!(decoded.expect("golden decoded node").rpc_uri(), "http://node:80");
|
||||
}
|
||||
#[test]
|
||||
fn serde_shape_roundtrip_and_strict_fields() {
|
||||
let node = CaptureNode::new("https://node:443/".to_owned()).expect("serde node");
|
||||
let json = r#"{"rpc_uri":"https://node:443/"}"#;
|
||||
assert_eq!(serde_json::to_string(&node).expect("serialize node"), json);
|
||||
assert_eq!(serde_json::from_str::<CaptureNode>(json).expect("deserialize node"), node);
|
||||
for json in [
|
||||
r#"{}"#,
|
||||
r#"{"rpc_uri":1}"#,
|
||||
r#"{"rpc_uri":null}"#,
|
||||
r#"{"rpc_uri":"http://node:1","extra":0}"#,
|
||||
r#"{"rpc_uri":"http://node:1","rpc_uri":"http://node:2"}"#,
|
||||
] {
|
||||
assert!(serde_json::from_str::<CaptureNode>(json).is_err(), "strict fields: {json}");
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn credentials_and_raw_uri_shapes_reject() {
|
||||
for uri in [
|
||||
"http://user@node:1",
|
||||
"http://user:password@node:1",
|
||||
"http://@node:1",
|
||||
"HTTP://node:1",
|
||||
"Https://node:1",
|
||||
"http://node",
|
||||
"http://node:",
|
||||
"http://node:0",
|
||||
"http://node:-1",
|
||||
"http://node:+1",
|
||||
"http://node:65536",
|
||||
"http://node:999999999999999999999",
|
||||
"http://[::1:1",
|
||||
"http://[::1]extra:1",
|
||||
"http://[invalid]:1",
|
||||
"http://::1:1",
|
||||
"http://:1",
|
||||
"/disk/path",
|
||||
"file:///disk/path",
|
||||
"http://node:1/./",
|
||||
"http://node:1/../",
|
||||
"http://node:1/%2e/",
|
||||
"http://node:1/%2E%2e/",
|
||||
"http://node:1/path",
|
||||
"http://node:1\\",
|
||||
" http://node:1",
|
||||
"http://node:1\n",
|
||||
"http://no\tde:1",
|
||||
"http://node:1\u{00a0}",
|
||||
"http://node:1\u{0085}",
|
||||
"http://node:1\0",
|
||||
] {
|
||||
reject_uri(uri);
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn query_and_fragment_markers_reject() {
|
||||
for uri in [
|
||||
"http://node:1?",
|
||||
"http://node:1?x=y",
|
||||
"http://node:1/?",
|
||||
"http://node:1#",
|
||||
"http://node:1#fragment",
|
||||
"http://node:1/#",
|
||||
] {
|
||||
reject_uri(uri);
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn utf8_byte_length_boundaries() {
|
||||
for length in [4095, 4096] {
|
||||
let prefix = "http://node:";
|
||||
let uri = format!("{prefix}{}1", "0".repeat(length - prefix.len() - 1));
|
||||
assert_eq!(uri.len(), length);
|
||||
assert!(url::Url::parse(&uri).is_ok(), "boundary fixture accepted by actual URL parser");
|
||||
let node = CaptureNode::new(uri.clone()).expect("boundary node");
|
||||
assert_eq!(node.rpc_uri(), uri);
|
||||
assert_eq!(
|
||||
CaptureNode::decode(&node.encode().expect("boundary encoding")).expect("boundary decoding"),
|
||||
node
|
||||
);
|
||||
}
|
||||
let uri = format!("http://node:{}1", "0".repeat(4097 - "http://node:".len() - 1));
|
||||
assert_eq!(CaptureNode::new(uri).expect_err("oversize URI").code(), "rpc_uri_too_long");
|
||||
let unicode = format!("http://{}:1", "é".repeat(2044));
|
||||
assert!(unicode.chars().count() < 4096);
|
||||
assert!(unicode.len() > 4096);
|
||||
assert_eq!(CaptureNode::new(unicode).expect_err("UTF-8 byte limit").code(), "rpc_uri_too_long");
|
||||
}
|
||||
#[test]
|
||||
fn cnod_headers_and_lengths_reject_exact_codes() {
|
||||
let valid = record(b"http://node:80");
|
||||
for end in 0..9 {
|
||||
reject_record(&valid[..end], "truncated_node_record");
|
||||
}
|
||||
let mut bad = valid.clone();
|
||||
bad[0] = b'X';
|
||||
reject_record(&bad, "invalid_node_magic");
|
||||
bad = valid.clone();
|
||||
bad[4] = 2;
|
||||
reject_record(&bad, "unsupported_node_version");
|
||||
for length in [0_u32, 4097, u32::MAX] {
|
||||
bad = b"CNOD\x01".to_vec();
|
||||
bad.extend_from_slice(&length.to_be_bytes());
|
||||
reject_record(&bad, "invalid_node_length");
|
||||
}
|
||||
for end in 9..valid.len() {
|
||||
reject_record(&valid[..end], "truncated_node_record");
|
||||
}
|
||||
reject_record(&record(&[0xff]), "invalid_node_utf8");
|
||||
reject_record(&record(&[0xc3, 0x28]), "invalid_node_utf8");
|
||||
reject_record(&record(b"invalid URI"), "invalid_rpc_uri");
|
||||
}
|
||||
#[test]
|
||||
fn trailing_bytes_reject() {
|
||||
for tail in [&b"x"[..], &b"xyz"[..]] {
|
||||
let mut bytes = record(b"http://node:80");
|
||||
bytes.extend_from_slice(tail);
|
||||
reject_record(&bytes, "trailing_node_bytes");
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn errors_and_sources_redact_input() {
|
||||
let sentinel = "http://SECRET_USER:SECRET_PASSWORD@node:1";
|
||||
let error = CaptureNode::new(sentinel.to_owned()).expect_err("credentials reject");
|
||||
let mut current: &dyn Error = &error;
|
||||
loop {
|
||||
let rendered = format!("{current} {current:?}");
|
||||
assert!(!rendered.contains("SECRET"));
|
||||
match current.source() {
|
||||
Some(source) => current = source,
|
||||
None => break,
|
||||
}
|
||||
}
|
||||
let json = format!(r#"{{"rpc_uri":"{sentinel}"}}"#);
|
||||
let error = serde_json::from_str::<CaptureNode>(&json).expect_err("serde credentials reject");
|
||||
assert!(!format!("{error} {error:?}").contains("SECRET"));
|
||||
let json = format!(r#"{{"{sentinel}":0}}"#);
|
||||
let error = serde_json::from_str::<CaptureNode>(&json).expect_err("unknown fields reject");
|
||||
assert!(!format!("{error} {error:?}").contains("SECRET"));
|
||||
let error = CaptureNode::new("http://[bad]:1".to_owned()).expect_err("parser rejection");
|
||||
assert!(error.source().is_some(), "retain URL parser source");
|
||||
let error = CaptureNode::decode(&record(&[0xff])).expect_err("UTF-8 rejection");
|
||||
assert!(error.source().is_some(), "retain UTF-8 source");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
//! OpenRaft log storage backed by the existing native writer and local durable baseline.
|
||||
use crate::{CaptureRaftEntry, CaptureTypeConfig, NativeLogReader, NativeLogWriter, NativeWriteError, StoreError};
|
||||
use openraft::{
|
||||
LogId, OptionalSend, RaftLogReader, StorageError, StorageIOError, Vote,
|
||||
storage::{LogFlushed, RaftLogStorage},
|
||||
};
|
||||
use redb::{ReadableDatabase, ReadableTable};
|
||||
|
||||
/// Clones share one native Store, write ordering and checkpoint authority.
|
||||
#[derive(Clone)]
|
||||
pub struct NativeRaftLogStore {
|
||||
writer: NativeLogWriter,
|
||||
// Last: owner drain is signaled only after this clone releases its writer.
|
||||
pub(crate) owner_lease: Option<std::sync::Arc<crate::raft_owner::OwnerLease>>,
|
||||
}
|
||||
impl NativeRaftLogStore {
|
||||
pub fn new(writer: NativeLogWriter) -> Self {
|
||||
Self {
|
||||
writer,
|
||||
owner_lease: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
fn error(error: NativeWriteError, operation: fn(openraft::AnyError) -> StorageIOError<u64>) -> StorageError<u64> {
|
||||
StorageError::IO {
|
||||
source: operation(openraft::AnyError::new(&error)),
|
||||
}
|
||||
}
|
||||
impl RaftLogReader<CaptureTypeConfig> for NativeRaftLogStore {
|
||||
async fn try_get_log_entries<RB: std::ops::RangeBounds<u64> + Clone + std::fmt::Debug + OptionalSend>(
|
||||
&mut self,
|
||||
range: RB,
|
||||
) -> Result<Vec<CaptureRaftEntry>, StorageError<u64>> {
|
||||
let mut reader = self.writer.reader();
|
||||
reader.owner_lease = self.owner_lease.clone();
|
||||
reader.try_get_log_entries(range).await
|
||||
}
|
||||
}
|
||||
impl RaftLogStorage<CaptureTypeConfig> for NativeRaftLogStore {
|
||||
type LogReader = NativeLogReader;
|
||||
async fn get_log_state(&mut self) -> Result<openraft::LogState<CaptureTypeConfig>, StorageError<u64>> {
|
||||
self.writer
|
||||
.write(|store| {
|
||||
if !store.writable {
|
||||
return Err(StoreError::WriteDisabled);
|
||||
}
|
||||
let read = store.database.begin_read()?;
|
||||
let meta = read.open_table(crate::META)?;
|
||||
let purged = crate::checkpoint_baseline::read_purged(&meta)?;
|
||||
let logs = read.open_table(crate::LOGS)?;
|
||||
let last = logs
|
||||
.last()?
|
||||
.map(|(index, value)| {
|
||||
let id = crate::decode_log(value.value())?.id;
|
||||
if id.index != index.value() {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
Ok(id)
|
||||
})
|
||||
.transpose()?
|
||||
.or(purged);
|
||||
let tail = meta.get("tail")?.map(|value| crate::decode_id(value.value())).transpose()?;
|
||||
if tail != last {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
Ok(openraft::LogState {
|
||||
last_purged_log_id: purged.map(Into::into),
|
||||
last_log_id: last.map(Into::into),
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(|e| error(e, StorageIOError::read_logs))
|
||||
}
|
||||
async fn get_log_reader(&mut self) -> Self::LogReader {
|
||||
let mut reader = self.writer.reader();
|
||||
reader.owner_lease = self.owner_lease.clone();
|
||||
reader
|
||||
}
|
||||
async fn save_vote(&mut self, vote: &Vote<u64>) -> Result<(), StorageError<u64>> {
|
||||
self.writer
|
||||
.save_vote(*vote)
|
||||
.await
|
||||
.map_err(|e| error(e, StorageIOError::write_vote))
|
||||
}
|
||||
async fn read_vote(&mut self) -> Result<Option<Vote<u64>>, StorageError<u64>> {
|
||||
self.writer.read_vote().await.map_err(|e| error(e, StorageIOError::read_vote))
|
||||
}
|
||||
async fn save_committed(&mut self, committed: Option<LogId<u64>>) -> Result<(), StorageError<u64>> {
|
||||
self.writer
|
||||
.write(move |store| match committed {
|
||||
Some(id) => store.save_committed(id.into()),
|
||||
None if store.read_committed()?.is_none() => Ok(()),
|
||||
None => Err(StoreError::CommittedRegression),
|
||||
})
|
||||
.await
|
||||
.map_err(|e| error(e, StorageIOError::write_logs))
|
||||
}
|
||||
async fn read_committed(&mut self) -> Result<Option<LogId<u64>>, StorageError<u64>> {
|
||||
self.writer
|
||||
.write(|store| store.read_committed().map(|id| id.map(Into::into)))
|
||||
.await
|
||||
.map_err(|e| error(e, StorageIOError::read_logs))
|
||||
}
|
||||
async fn append<I>(&mut self, entries: I, callback: LogFlushed<CaptureTypeConfig>) -> Result<(), StorageError<u64>>
|
||||
where
|
||||
I: IntoIterator<Item = CaptureRaftEntry> + OptionalSend,
|
||||
I::IntoIter: OptionalSend,
|
||||
{
|
||||
let entries = entries.into_iter().collect::<Vec<_>>();
|
||||
// The original blocking worker owns ordering and callback through durable commit,
|
||||
// even if OpenRaft cancels its awaiting append future.
|
||||
self.writer
|
||||
.write(move |store| {
|
||||
let result = store.append_native(&entries);
|
||||
callback.log_io_completed(
|
||||
result
|
||||
.as_ref()
|
||||
.map(|_| ())
|
||||
.map_err(|error| std::io::Error::other(error.code())),
|
||||
);
|
||||
result
|
||||
})
|
||||
.await
|
||||
.map_err(|e| error(e, StorageIOError::write_logs))
|
||||
}
|
||||
async fn truncate(&mut self, log_id: LogId<u64>) -> Result<(), StorageError<u64>> {
|
||||
self.writer
|
||||
.truncate_suffix(log_id.index)
|
||||
.await
|
||||
.map_err(|e| error(e, StorageIOError::write_logs))
|
||||
}
|
||||
async fn purge(&mut self, log_id: LogId<u64>) -> Result<(), StorageError<u64>> {
|
||||
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(30);
|
||||
loop {
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return Err(error(NativeWriteError::Store(StoreError::StaleCheckpoint), StorageIOError::write_logs));
|
||||
}
|
||||
let notified = self.writer.snapshot_receives.changed.notified();
|
||||
tokio::pin!(notified);
|
||||
notified.as_mut().enable();
|
||||
let receives = self.writer.snapshot_receives.clone();
|
||||
// Observe coverage and receive ownership while install is excluded by ordering.
|
||||
let (result, pending) = self
|
||||
.writer
|
||||
.write(move |store| Ok((store.purge_native_prefix(log_id.into(), None), receives.pending())))
|
||||
.await
|
||||
.map_err(|e| error(e, StorageIOError::write_logs))?;
|
||||
if matches!(&result, Err(StoreError::StaleCheckpoint)) {
|
||||
#[cfg(test)]
|
||||
{
|
||||
let hook = self
|
||||
.writer
|
||||
.snapshot_receives
|
||||
.purge_recheck
|
||||
.lock()
|
||||
.expect("test hook lock")
|
||||
.take();
|
||||
if let Some((entered, resume)) = hook {
|
||||
entered.notify_one();
|
||||
resume.notified().await;
|
||||
}
|
||||
}
|
||||
if pending {
|
||||
#[cfg(test)]
|
||||
let _waiter = crate::raft_state_machine::Waiter::new(self.writer.snapshot_receives.clone());
|
||||
if tokio::time::timeout_at(deadline, notified).await.is_err() {
|
||||
return Err(error(NativeWriteError::Store(StoreError::StaleCheckpoint), StorageIOError::write_logs));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
return result.map_err(|e| error(e.into(), StorageIOError::write_logs));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,439 @@
|
||||
//! Exercises actual OpenRaft storage traits, not a substitute consensus engine.
|
||||
use super::*;
|
||||
use openraft::{
|
||||
RaftLogReader,
|
||||
storage::{RaftLogStorage, RaftLogStorageExt},
|
||||
};
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
fn identity() -> NodeIdentity {
|
||||
NodeIdentity {
|
||||
cluster_id: [7; 16],
|
||||
node_id: 0,
|
||||
}
|
||||
}
|
||||
fn binding() -> CaptureBinding {
|
||||
CaptureBinding {
|
||||
revision: 9,
|
||||
binding_id: [8; 16],
|
||||
target_id: [9; 16],
|
||||
}
|
||||
}
|
||||
fn id(index: u64) -> LogId {
|
||||
LogId {
|
||||
term: 8,
|
||||
leader_node: 0,
|
||||
index,
|
||||
}
|
||||
}
|
||||
fn entry(index: u64) -> CaptureRaftEntry {
|
||||
let payload = if index == 0 {
|
||||
let membership = CaptureMembership::new(
|
||||
Some(id(0)),
|
||||
vec![BTreeSet::from([0])],
|
||||
BTreeMap::from([(0, CaptureNode::new("http://127.0.0.1:7000".into()).expect("test member"))]),
|
||||
)
|
||||
.expect("native membership");
|
||||
let membership = openraft::StoredMembership::try_from(&membership).expect("stored native membership");
|
||||
openraft::EntryPayload::Membership(membership.membership().clone())
|
||||
} else {
|
||||
let byte = u8::try_from(index).expect("small test event index");
|
||||
let command = DecideCreated {
|
||||
operation_id: [byte; 16],
|
||||
object: ObjectIdentity {
|
||||
bucket_incarnation: [2; 16],
|
||||
key: vec![byte],
|
||||
},
|
||||
expected_head: None,
|
||||
prepared: PreparedIdentity {
|
||||
version_id: [byte; 16],
|
||||
preparation_id: [4; 16],
|
||||
content_digest: [5; 32],
|
||||
content_length: 123,
|
||||
},
|
||||
binding_revision: binding().revision,
|
||||
};
|
||||
openraft::EntryPayload::Normal(CaptureCommand::try_created(command).expect("native test command"))
|
||||
};
|
||||
CaptureRaftEntry {
|
||||
log_id: id(index).into(),
|
||||
payload,
|
||||
}
|
||||
}
|
||||
fn entries(end: u64) -> Vec<CaptureRaftEntry> {
|
||||
(0..=end).map(entry).collect()
|
||||
}
|
||||
fn fixture() -> (tempfile::TempDir, NativeLogWriter, NativeRaftLogStore) {
|
||||
let directory = tempfile::tempdir().expect("actual native database directory");
|
||||
let mut store = Store::initialize(directory.path().join("db"), identity()).expect("actual redb initialization");
|
||||
store.pin_capture_binding(binding()).expect("capture binding");
|
||||
store.enable_native_empty().expect("native mode");
|
||||
store.promote_created_provenance().expect("native provenance");
|
||||
let writer = NativeLogWriter::new(store);
|
||||
let adapter = NativeRaftLogStore::new(writer.clone());
|
||||
(directory, writer, adapter)
|
||||
}
|
||||
fn media_fixture() -> (tempfile::TempDir, tests::CrashBackend, NativeLogWriter, NativeRaftLogStore) {
|
||||
let (directory, writer, adapter) = fixture();
|
||||
drop(adapter);
|
||||
drop(writer);
|
||||
let backend = tests::CrashBackend::new(&directory.path().join("db"));
|
||||
let store = Store::from_database(backend.database(), identity()).expect("existing real disk fault backend");
|
||||
let writer = NativeLogWriter::new(store);
|
||||
let adapter = NativeRaftLogStore::new(writer.clone());
|
||||
(directory, backend, writer, adapter)
|
||||
}
|
||||
async fn logs(adapter: &mut NativeRaftLogStore) -> Vec<CaptureRaftEntry> {
|
||||
adapter
|
||||
.get_log_reader()
|
||||
.await
|
||||
.try_get_log_entries(..)
|
||||
.await
|
||||
.expect("actual trait reader")
|
||||
}
|
||||
async fn checkpoint(writer: &NativeLogWriter, adapter: &mut NativeRaftLogStore) -> CheckpointReceipt {
|
||||
let batch = entries(2);
|
||||
adapter
|
||||
.blocking_append(batch.clone())
|
||||
.await
|
||||
.expect("trait append before checkpoint");
|
||||
// Prepares genuine applied/checkpoint evidence; this does not claim Raft consensus.
|
||||
writer
|
||||
.apply_committed(batch)
|
||||
.await
|
||||
.expect("trusted test owner applies preparation entries");
|
||||
writer
|
||||
.checkpoint_local(CheckpointLimits::default())
|
||||
.await
|
||||
.expect("real durable baseline")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raft_log_storage_flush_restart() {
|
||||
let (directory, backend, writer, mut adapter) = media_fixture();
|
||||
let batch = entries(2);
|
||||
// blocking_append constructs the real library-private LogFlushed callback.
|
||||
adapter
|
||||
.blocking_append(batch.clone())
|
||||
.await
|
||||
.expect("real durable flush callback");
|
||||
let state = adapter.get_log_state().await.expect("trait log state");
|
||||
assert_eq!(state.last_log_id, Some(id(2).into()), "RAFT_LOG_TAIL_NOT_APPLIED_STATE");
|
||||
assert_eq!(state.last_purged_log_id, None);
|
||||
assert_eq!(writer.applied_state().await.expect("application state").0, None);
|
||||
assert_eq!(logs(&mut adapter).await, batch);
|
||||
backend.cut();
|
||||
drop(adapter);
|
||||
drop(writer);
|
||||
let reopened = Store::open(directory.path().join("db"), identity()).expect("reopen after successful callback and power cut");
|
||||
let mut adapter = NativeRaftLogStore::new(NativeLogWriter::new(reopened));
|
||||
assert_eq!(logs(&mut adapter).await, batch, "RAFT_LOG_FLUSH_CALLBACK_DURABLE");
|
||||
assert_eq!(
|
||||
adapter.get_log_state().await.expect("reopened trait state").last_log_id,
|
||||
Some(id(2).into())
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raft_log_storage_vote_restart() {
|
||||
let (directory, backend, writer, mut adapter) = media_fixture();
|
||||
let vote = openraft::Vote::new_committed(9, 0);
|
||||
adapter.save_vote(&vote).await.expect("trait durable vote");
|
||||
assert_eq!(adapter.read_vote().await.expect("trait vote read"), Some(vote));
|
||||
adapter.blocking_append(entries(2)).await.expect("trait append");
|
||||
adapter
|
||||
.save_committed(Some(id(1).into()))
|
||||
.await
|
||||
.expect("trait durable commit marker");
|
||||
assert!(adapter.save_committed(None).await.is_err(), "RAFT_LOG_COMMITTED_NONE_CANNOT_RESET");
|
||||
assert_eq!(adapter.read_committed().await.expect("exact marker"), Some(id(1).into()));
|
||||
assert!(
|
||||
adapter.save_committed(Some(id(0).into())).await.is_err(),
|
||||
"RAFT_LOG_COMMITTED_NO_REGRESSION"
|
||||
);
|
||||
let mut wrong = id(2);
|
||||
wrong.term += 1;
|
||||
assert!(
|
||||
adapter.save_committed(Some(wrong.into())).await.is_err(),
|
||||
"RAFT_LOG_COMMITTED_EXACT_IDENTITY"
|
||||
);
|
||||
backend.cut();
|
||||
drop(adapter);
|
||||
drop(writer);
|
||||
let mut reopened = NativeRaftLogStore::new(NativeLogWriter::new(
|
||||
Store::open(directory.path().join("db"), identity()).expect("durable vote reopen"),
|
||||
));
|
||||
assert_eq!(
|
||||
reopened.read_vote().await.expect("reopened vote"),
|
||||
Some(vote),
|
||||
"RAFT_LOG_VOTE_ACK_DURABLE"
|
||||
);
|
||||
assert_eq!(reopened.read_committed().await.expect("reopened committed marker"), Some(id(1).into()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raft_log_storage_partial_purge_restart() {
|
||||
let (directory, writer, mut adapter) = fixture();
|
||||
let receipt = checkpoint(&writer, &mut adapter).await;
|
||||
adapter
|
||||
.blocking_append(vec![entry(3), entry(4)])
|
||||
.await
|
||||
.expect("uncovered suffix append");
|
||||
let events = writer
|
||||
.created_events_page(None, 256)
|
||||
.await
|
||||
.expect("pre-purge authoritative events");
|
||||
adapter
|
||||
.purge(id(1).into())
|
||||
.await
|
||||
.expect("checkpoint authorizes strict prefix");
|
||||
assert_eq!(
|
||||
logs(&mut adapter).await,
|
||||
vec![entry(2), entry(3), entry(4)],
|
||||
"RAFT_LOG_PARTIAL_PURGE_RETAINS_SUFFIX"
|
||||
);
|
||||
let state = adapter.get_log_state().await.expect("partially purged state");
|
||||
assert_eq!(state.last_purged_log_id, Some(id(1).into()));
|
||||
assert_eq!(state.last_log_id, Some(id(4).into()));
|
||||
adapter.purge(id(1).into()).await.expect("repeat exact purge is idempotent");
|
||||
assert!(adapter.purge(id(0).into()).await.is_err(), "RAFT_LOG_PURGE_NO_REGRESSION");
|
||||
{
|
||||
let store = writer.store.lock().expect("inspect task-owned store");
|
||||
let tx = store.database.begin_read().expect("real persisted prefix inspection");
|
||||
let results = tx.open_table(application::APPLY_RESULTS).expect("apply receipts");
|
||||
assert!(results.get(0).expect("membership receipt").is_none());
|
||||
assert!(results.get(1).expect("first receipt").is_none());
|
||||
assert!(
|
||||
results.get(2).expect("retained receipt").is_some(),
|
||||
"RAFT_LOG_PARTIAL_PURGE_RECEIPT_SUFFIX"
|
||||
);
|
||||
}
|
||||
drop(adapter);
|
||||
drop(writer);
|
||||
let writer = NativeLogWriter::new(Store::open(directory.path().join("db"), identity()).expect("restart partial purge"));
|
||||
let mut adapter = NativeRaftLogStore::new(writer.clone());
|
||||
assert_eq!(
|
||||
logs(&mut adapter).await,
|
||||
vec![entry(2), entry(3), entry(4)],
|
||||
"RAFT_LOG_PARTIAL_PURGE_REOPEN"
|
||||
);
|
||||
adapter
|
||||
.blocking_append(vec![entry(5)])
|
||||
.await
|
||||
.expect("continue append after partial purge");
|
||||
writer
|
||||
.apply_committed(vec![entry(3), entry(4), entry(5)])
|
||||
.await
|
||||
.expect("continue application from true applied state");
|
||||
assert_eq!(
|
||||
writer
|
||||
.created_events_page(None, 256)
|
||||
.await
|
||||
.expect("retained plus new events")
|
||||
.events
|
||||
.len(),
|
||||
5,
|
||||
"RAFT_LOG_PURGE_NEVER_DELETES_EVENTS"
|
||||
);
|
||||
writer
|
||||
.purge_checkpoint(&receipt)
|
||||
.await
|
||||
.expect("existing exact receipt API remains compatible after partial purge");
|
||||
assert_eq!(
|
||||
adapter
|
||||
.get_log_state()
|
||||
.await
|
||||
.expect("exact compatibility state")
|
||||
.last_purged_log_id,
|
||||
Some(id(2).into())
|
||||
);
|
||||
assert_eq!(logs(&mut adapter).await, vec![entry(3), entry(4), entry(5)]);
|
||||
assert_eq!(
|
||||
writer
|
||||
.created_events_page(None, 2)
|
||||
.await
|
||||
.expect("original events retained")
|
||||
.events,
|
||||
events.events
|
||||
);
|
||||
drop(adapter);
|
||||
drop(writer);
|
||||
let mut reopened = NativeRaftLogStore::new(NativeLogWriter::new(
|
||||
Store::open(directory.path().join("db"), identity()).expect("reopen continued apply and exact purge"),
|
||||
));
|
||||
assert_eq!(logs(&mut reopened).await, vec![entry(3), entry(4), entry(5)]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raft_log_storage_purge_rejects_uncovered() {
|
||||
let (directory, writer, mut adapter) = fixture();
|
||||
adapter
|
||||
.blocking_append(entries(2))
|
||||
.await
|
||||
.expect("trait append without snapshot authority");
|
||||
writer
|
||||
.apply_committed(entries(2))
|
||||
.await
|
||||
.expect("applied preparation without checkpoint");
|
||||
let before = std::fs::read(directory.path().join("db")).expect("stable pre-rejection bytes");
|
||||
assert!(
|
||||
adapter.purge(id(1).into()).await.is_err(),
|
||||
"PURGE_REQUIRES_CHECKPOINT_COVERAGE: application progress is not a durable checkpoint"
|
||||
);
|
||||
assert_eq!(
|
||||
std::fs::read(directory.path().join("db")).expect("rejected purge bytes"),
|
||||
before,
|
||||
"RAFT_LOG_UNAUTHORIZED_PURGE_NO_MUTATION"
|
||||
);
|
||||
writer
|
||||
.checkpoint_local(CheckpointLimits::default())
|
||||
.await
|
||||
.expect("checkpoint at applied cut 2");
|
||||
adapter
|
||||
.blocking_append(vec![entry(3)])
|
||||
.await
|
||||
.expect("log beyond checkpoint cut");
|
||||
let before = std::fs::read(directory.path().join("db")).expect("post-checkpoint bytes");
|
||||
assert!(
|
||||
adapter.purge(id(3).into()).await.is_err(),
|
||||
"PURGE_REQUIRES_CHECKPOINT_COVERAGE: retained target exceeds durable cut"
|
||||
);
|
||||
assert_eq!(
|
||||
std::fs::read(directory.path().join("db")).expect("uncovered target rejection bytes"),
|
||||
before,
|
||||
"RAFT_LOG_UNCOVERED_PURGE_NO_MUTATION"
|
||||
);
|
||||
assert_eq!(logs(&mut adapter).await, entries(3));
|
||||
assert_eq!(adapter.get_log_state().await.expect("unchanged marker").last_purged_log_id, None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raft_log_storage_purge_identity() {
|
||||
let (directory, writer, mut adapter) = fixture();
|
||||
checkpoint(&writer, &mut adapter).await;
|
||||
for target in [
|
||||
LogId { term: 9, ..id(1) },
|
||||
LogId { leader_node: 1, ..id(1) },
|
||||
LogId { term: 9, ..id(2) },
|
||||
] {
|
||||
let before = std::fs::read(directory.path().join("db")).expect("before wrong complete LogId");
|
||||
assert!(adapter.purge(target.into()).await.is_err(), "RAFT_LOG_PURGE_EXACT_LOG_ID");
|
||||
assert_eq!(std::fs::read(directory.path().join("db")).expect("after identity rejection"), before);
|
||||
}
|
||||
adapter.purge(id(1).into()).await.expect("valid identity");
|
||||
let mut wrong = id(1);
|
||||
wrong.term += 1;
|
||||
assert!(adapter.purge(wrong.into()).await.is_err(), "RAFT_LOG_PURGED_REPEAT_IDENTITY_BOUND");
|
||||
assert_eq!(
|
||||
adapter
|
||||
.get_log_state()
|
||||
.await
|
||||
.expect("unchanged identity marker")
|
||||
.last_purged_log_id,
|
||||
Some(id(1).into())
|
||||
);
|
||||
adapter.purge(id(2).into()).await.expect("final exact covered target");
|
||||
let state = adapter.get_log_state().await.expect("no remaining logs");
|
||||
assert_eq!(state.last_purged_log_id, Some(id(2).into()));
|
||||
assert_eq!(state.last_log_id, Some(id(2).into()), "RAFT_LOG_EMPTY_TAIL_USES_PURGED_LOG_ID");
|
||||
assert!(logs(&mut adapter).await.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn raft_log_storage_cancelled_append() {
|
||||
let (directory, backend, writer, adapter) = media_fixture();
|
||||
let (entered, resume) = backend.pause_sync(1, false);
|
||||
let mut first = adapter.clone();
|
||||
let task = tokio::spawn(async move { first.blocking_append(entries(2)).await });
|
||||
entered.wait(); // Actual redb commit has begun but its first sync is paused.
|
||||
assert!(!task.is_finished(), "RAFT_LOG_FLUSH_NOT_BEFORE_DURABILITY");
|
||||
task.abort();
|
||||
assert!(task.await.expect_err("cancelled blocking append waiter").is_cancelled());
|
||||
assert!(writer.ordering.try_lock().is_err(), "RAFT_LOG_CANCELLED_APPEND_RETAINS_ORDERING_OWNER");
|
||||
let mut following = adapter.clone();
|
||||
let next = tokio::spawn(async move { following.blocking_append(vec![entry(3)]).await });
|
||||
assert!(writer.ordering.try_lock().is_err(), "RAFT_LOG_NEXT_APPEND_CANNOT_BYPASS_PENDING_SYNC");
|
||||
resume.wait();
|
||||
next.await.expect("next trait append task").expect("ordered follow-up append");
|
||||
let mut adapter = adapter;
|
||||
assert_eq!(
|
||||
logs(&mut adapter).await,
|
||||
entries(3),
|
||||
"RAFT_LOG_CANCELLED_WAITER_BACKGROUND_COMMIT_COMPLETE"
|
||||
);
|
||||
backend.cut();
|
||||
drop(adapter);
|
||||
drop(writer);
|
||||
let mut reopened = NativeRaftLogStore::new(NativeLogWriter::new(
|
||||
Store::open(directory.path().join("db"), identity()).expect("reopen cancelled waiter background commit"),
|
||||
));
|
||||
assert_eq!(logs(&mut reopened).await, entries(3));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn raft_log_storage_failed_append() {
|
||||
let (directory, backend, writer, adapter) = media_fixture();
|
||||
let (entered, resume) = backend.pause_sync(1, false);
|
||||
let mut appending = adapter.clone();
|
||||
let task = tokio::spawn(async move { appending.blocking_append(entries(2)).await });
|
||||
entered.wait();
|
||||
assert!(!task.is_finished(), "RAFT_LOG_FAILED_FLUSH_NOT_EARLY_SUCCESS");
|
||||
backend.cut();
|
||||
resume.wait();
|
||||
assert!(
|
||||
task.await.expect("failed actual trait append task").is_err(),
|
||||
"RAFT_LOG_FAILED_APPEND_NEVER_REPORTS_CALLBACK_SUCCESS"
|
||||
);
|
||||
let mut adapter = adapter;
|
||||
assert!(
|
||||
adapter.blocking_append(entries(2)).await.is_err(),
|
||||
"RAFT_LOG_INDETERMINATE_STORE_REJECTS_LATER_APPEND"
|
||||
);
|
||||
assert!(
|
||||
adapter.save_vote(&openraft::Vote::new_committed(9, 0)).await.is_err(),
|
||||
"RAFT_LOG_INDETERMINATE_STORE_REJECTS_LATER_VOTE"
|
||||
);
|
||||
drop(adapter);
|
||||
drop(writer);
|
||||
let mut reopened = NativeRaftLogStore::new(NativeLogWriter::new(
|
||||
Store::open(directory.path().join("db"), identity()).expect("reopen failed-before-first-sync append"),
|
||||
));
|
||||
assert!(logs(&mut reopened).await.is_empty(), "RAFT_LOG_FAILURE_NO_PARTIAL_BATCH");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raft_log_storage_truncate_uncommitted_suffix() {
|
||||
let (directory, writer, mut adapter) = fixture();
|
||||
adapter.blocking_append(entries(4)).await.expect("trait append full log");
|
||||
adapter
|
||||
.save_committed(Some(id(2).into()))
|
||||
.await
|
||||
.expect("protect committed prefix");
|
||||
assert!(
|
||||
adapter.truncate(id(2).into()).await.is_err(),
|
||||
"RAFT_LOG_TRUNCATE_PROTECTS_COMMITTED_PREFIX"
|
||||
);
|
||||
adapter
|
||||
.truncate(id(3).into())
|
||||
.await
|
||||
.expect("truncate includes uncommitted target");
|
||||
assert_eq!(logs(&mut adapter).await, entries(2), "RAFT_LOG_TRUNCATE_INCLUSIVE");
|
||||
let mut replacement = entry(3);
|
||||
replacement.log_id = LogId { term: 9, ..id(3) }.into();
|
||||
adapter
|
||||
.blocking_append(vec![replacement.clone()])
|
||||
.await
|
||||
.expect("trait append replacement suffix");
|
||||
assert_eq!(
|
||||
adapter.get_log_state().await.expect("replacement tail").last_log_id,
|
||||
Some(replacement.log_id)
|
||||
);
|
||||
drop(adapter);
|
||||
drop(writer);
|
||||
let mut reopened = NativeRaftLogStore::new(NativeLogWriter::new(
|
||||
Store::open(directory.path().join("db"), identity()).expect("reopen truncate and replacement"),
|
||||
));
|
||||
let mut expected = entries(2);
|
||||
expected.push(replacement);
|
||||
assert_eq!(logs(&mut reopened).await, expected);
|
||||
}
|
||||
@@ -0,0 +1,251 @@
|
||||
//! Explicit lifecycle for real OpenRaft over one durable native capture owner.
|
||||
use crate::{
|
||||
CaptureCommand, CaptureNode, CaptureNodeError, CaptureTypeConfig, CheckpointLimits, NativeLogWriter, NativeRaftLogStore,
|
||||
NativeRaftStateMachine, NativeSnapshotError, NativeWriteError, StoreError,
|
||||
};
|
||||
use openraft::{
|
||||
Config, ConfigError, Raft, RaftMetrics, RaftNetworkFactory,
|
||||
error::{ClientWriteError, Fatal, InitializeError, RaftError},
|
||||
raft::ClientWriteResponse,
|
||||
};
|
||||
use redb::ReadableDatabase;
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
},
|
||||
};
|
||||
use tokio::sync::watch;
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum NativeRaftOwnerError {
|
||||
#[error(transparent)]
|
||||
Config(#[from] Box<ConfigError>),
|
||||
#[error(transparent)]
|
||||
Store(#[from] NativeWriteError),
|
||||
#[error(transparent)]
|
||||
Snapshot(#[from] NativeSnapshotError),
|
||||
#[error("runtime node ID {runtime} differs from durable node ID {durable}")]
|
||||
IdentityMismatch { runtime: u64, durable: u64 },
|
||||
#[error("bootstrap must contain the local node")]
|
||||
MissingLocalMember,
|
||||
#[error(transparent)]
|
||||
Node(#[from] CaptureNodeError),
|
||||
#[error("native Raft owner is stopping or stopped")]
|
||||
Stopped,
|
||||
#[error(transparent)]
|
||||
Fatal(Box<Fatal<u64>>),
|
||||
#[error(transparent)]
|
||||
Initialize(Box<RaftError<u64, InitializeError<u64, CaptureNode>>>),
|
||||
#[error(transparent)]
|
||||
Write(Box<RaftError<u64, ClientWriteError<u64, CaptureNode>>>),
|
||||
#[error("native Raft shutdown task failed")]
|
||||
Shutdown(#[source] tokio::task::JoinError),
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
#[error(transparent)]
|
||||
Reader(#[from] crate::source_reader::ReaderError),
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct DrainState {
|
||||
complete: AtomicBool,
|
||||
changed: tokio::sync::Notify,
|
||||
#[cfg(test)]
|
||||
waiters: Arc<std::sync::atomic::AtomicUsize>,
|
||||
}
|
||||
impl DrainState {
|
||||
async fn wait(&self) {
|
||||
#[cfg(test)]
|
||||
let _waiter = DrainWaiter::new(self.waiters.clone());
|
||||
loop {
|
||||
let notified = self.changed.notified();
|
||||
tokio::pin!(notified);
|
||||
notified.as_mut().enable();
|
||||
if self.complete.load(Ordering::Acquire) {
|
||||
return;
|
||||
}
|
||||
notified.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
#[cfg(test)]
|
||||
struct DrainWaiter(Arc<std::sync::atomic::AtomicUsize>);
|
||||
#[cfg(test)]
|
||||
impl DrainWaiter {
|
||||
fn new(waiters: Arc<std::sync::atomic::AtomicUsize>) -> Self {
|
||||
waiters.fetch_add(1, Ordering::AcqRel);
|
||||
Self(waiters)
|
||||
}
|
||||
}
|
||||
#[cfg(test)]
|
||||
impl Drop for DrainWaiter {
|
||||
fn drop(&mut self) {
|
||||
self.0.fetch_sub(1, Ordering::AcqRel);
|
||||
}
|
||||
}
|
||||
|
||||
// The real worker, snapshot builders and replication readers share this one lease.
|
||||
// Its final drop runs after their NativeLogWriter fields have been released.
|
||||
pub(crate) struct OwnerLease {
|
||||
drain: Arc<DrainState>,
|
||||
}
|
||||
impl Drop for OwnerLease {
|
||||
fn drop(&mut self) {
|
||||
self.drain.complete.store(true, Ordering::Release);
|
||||
self.drain.changed.notify_waiters();
|
||||
}
|
||||
}
|
||||
|
||||
/// A lifecycle owner, not a transport. All storage adapters share its original writer.
|
||||
pub struct NativeRaftOwner {
|
||||
raft: Raft<CaptureTypeConfig>,
|
||||
writer: NativeLogWriter,
|
||||
identity: crate::NodeIdentity,
|
||||
accepting: AtomicBool,
|
||||
drain: Arc<DrainState>,
|
||||
#[cfg(test)]
|
||||
pub(crate) snapshot_build_pause: Arc<crate::raft_state_machine::TestPause>,
|
||||
}
|
||||
impl NativeRaftOwner {
|
||||
pub async fn start<N>(
|
||||
writer: NativeLogWriter,
|
||||
node_id: u64,
|
||||
config: Arc<Config>,
|
||||
limits: CheckpointLimits,
|
||||
network: N,
|
||||
) -> Result<Self, NativeRaftOwnerError>
|
||||
where
|
||||
N: RaftNetworkFactory<CaptureTypeConfig>,
|
||||
{
|
||||
// All operator-input checks precede Raft::new and its background workers.
|
||||
let config = Arc::new(
|
||||
(*config)
|
||||
.clone()
|
||||
.validate()
|
||||
.map_err(|error| NativeRaftOwnerError::Config(Box::new(error)))?,
|
||||
);
|
||||
let mut state_machine = NativeRaftStateMachine::new(writer.clone(), limits)?;
|
||||
let identity = writer
|
||||
.write(|store| {
|
||||
if !store.writable {
|
||||
return Err(StoreError::WriteDisabled);
|
||||
}
|
||||
let transaction = store.database.begin_read()?;
|
||||
let meta = transaction.open_table(crate::META)?;
|
||||
crate::application::native_application_state(&transaction)?;
|
||||
crate::application::read_binding(&meta)?;
|
||||
crate::decode_identity(meta.get("identity")?.ok_or(StoreError::CorruptRecord)?.value())
|
||||
})
|
||||
.await?;
|
||||
if identity.node_id != node_id {
|
||||
return Err(NativeRaftOwnerError::IdentityMismatch {
|
||||
runtime: node_id,
|
||||
durable: identity.node_id,
|
||||
});
|
||||
}
|
||||
let drain = Arc::new(DrainState::default());
|
||||
let lease = Arc::new(OwnerLease { drain: drain.clone() });
|
||||
state_machine.owner_lease = Some(lease.clone());
|
||||
let mut logs = NativeRaftLogStore::new(writer.clone());
|
||||
logs.owner_lease = Some(lease);
|
||||
#[cfg(test)]
|
||||
let snapshot_build_pause = state_machine.build_before_ordering.clone();
|
||||
let raft = Raft::new(node_id, config, network, logs, state_machine)
|
||||
.await
|
||||
.map_err(|error| NativeRaftOwnerError::Fatal(Box::new(error)))?;
|
||||
Ok(Self {
|
||||
raft,
|
||||
writer,
|
||||
identity,
|
||||
accepting: AtomicBool::new(true),
|
||||
drain,
|
||||
#[cfg(test)]
|
||||
snapshot_build_pause,
|
||||
})
|
||||
}
|
||||
|
||||
/// Identity read from durable META and checked before starting this owner.
|
||||
pub fn identity(&self) -> crate::NodeIdentity {
|
||||
self.identity
|
||||
}
|
||||
|
||||
fn require_running(&self) -> Result<(), NativeRaftOwnerError> {
|
||||
if self.accepting.load(Ordering::Acquire) {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(NativeRaftOwnerError::Stopped)
|
||||
}
|
||||
}
|
||||
|
||||
/// Only explicit first-cluster formation calls initialize; start never does.
|
||||
pub async fn bootstrap(&self, members: BTreeMap<u64, CaptureNode>) -> Result<(), NativeRaftOwnerError> {
|
||||
self.require_running()?;
|
||||
if !members.contains_key(&self.identity.node_id) {
|
||||
return Err(NativeRaftOwnerError::MissingLocalMember);
|
||||
}
|
||||
for node in members.values() {
|
||||
node.validate()?;
|
||||
}
|
||||
self.raft
|
||||
.initialize(members)
|
||||
.await
|
||||
.map_err(|error| NativeRaftOwnerError::Initialize(Box::new(error)))
|
||||
}
|
||||
|
||||
/// The returned LogId and application response come from actual consensus apply.
|
||||
pub async fn client_write(
|
||||
&self,
|
||||
command: CaptureCommand,
|
||||
) -> Result<ClientWriteResponse<CaptureTypeConfig>, NativeRaftOwnerError> {
|
||||
self.require_running()?;
|
||||
self.raft
|
||||
.client_write(command)
|
||||
.await
|
||||
.map_err(|error| NativeRaftOwnerError::Write(Box::new(error)))
|
||||
}
|
||||
|
||||
/// Re-read an exact successful first decision from this owner's durable view.
|
||||
/// Use the Created result's decision LogId, not an idempotent retry entry's outer LogId.
|
||||
pub async fn authorize_created(
|
||||
&self,
|
||||
expected: crate::DecideCreated,
|
||||
expected_decision: crate::LogId,
|
||||
) -> Result<crate::CommittedCreatedAuthorization, NativeRaftOwnerError> {
|
||||
self.require_running()?;
|
||||
Ok(self.writer.authorize_created(expected, expected_decision).await?)
|
||||
}
|
||||
|
||||
pub fn metrics(&self) -> watch::Receiver<RaftMetrics<u64, CaptureNode>> {
|
||||
self.raft.metrics()
|
||||
}
|
||||
|
||||
/// Transport registration only; raw clones remain trusted caller-owned handles.
|
||||
pub fn raft(&self) -> &Raft<CaptureTypeConfig> {
|
||||
&self.raft
|
||||
}
|
||||
|
||||
/// The caller owns serve/shutdown and must drain it before releasing the Store.
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
pub fn bind_reader(
|
||||
&self,
|
||||
path: impl AsRef<std::path::Path>,
|
||||
) -> Result<(crate::source_reader::SourceReaderServer, crate::source_reader::Shutdown), NativeRaftOwnerError> {
|
||||
self.require_running()?;
|
||||
Ok(crate::source_reader::SourceReaderServer::bind(path, self.writer.clone())?)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn shutdown_waiters(&self) -> Arc<std::sync::atomic::AtomicUsize> {
|
||||
self.drain.waiters.clone()
|
||||
}
|
||||
|
||||
/// Stop admission before signaling Raft; await its real core/tick shutdown.
|
||||
pub async fn shutdown(&self) -> Result<(), NativeRaftOwnerError> {
|
||||
self.accepting.store(false, Ordering::Release);
|
||||
self.raft.shutdown().await.map_err(NativeRaftOwnerError::Shutdown)?;
|
||||
// OpenRaft joins core/tick but detaches the SM worker and snapshot builders.
|
||||
self.drain.wait().await;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,885 @@
|
||||
//! Real Raft engines and real redb; direct RPC routing exists only inside these tests.
|
||||
use super::*;
|
||||
use openraft::{
|
||||
Raft, RaftNetwork, RaftNetworkFactory, ServerState,
|
||||
error::{InstallSnapshotError, RPCError, RaftError, RemoteError, Unreachable},
|
||||
network::{Backoff, RPCOption},
|
||||
raft::{
|
||||
AppendEntriesRequest, AppendEntriesResponse, InstallSnapshotRequest, InstallSnapshotResponse, VoteRequest, VoteResponse,
|
||||
},
|
||||
};
|
||||
use std::{
|
||||
collections::{BTreeMap, BTreeSet},
|
||||
sync::{
|
||||
Arc, Mutex,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
type Engine = Raft<CaptureTypeConfig>;
|
||||
type RpcError = RPCError<u64, CaptureNode, RaftError<u64>>;
|
||||
type SnapshotRpcError = RPCError<u64, CaptureNode, RaftError<u64, InstallSnapshotError>>;
|
||||
type PausedLinks = Mutex<BTreeMap<(u64, u64), (Arc<tokio::sync::Notify>, Arc<tokio::sync::Notify>)>>;
|
||||
#[derive(Default)]
|
||||
struct Routes {
|
||||
nodes: Mutex<BTreeMap<u64, Engine>>,
|
||||
blocked: Mutex<BTreeSet<(u64, u64)>>,
|
||||
snapshot_installs: AtomicUsize,
|
||||
pause_append: PausedLinks,
|
||||
}
|
||||
impl Routes {
|
||||
fn register(&self, id: u64, raft: Engine) {
|
||||
self.nodes.lock().expect("routes").insert(id, raft);
|
||||
}
|
||||
fn remove(&self, id: u64) {
|
||||
self.nodes.lock().expect("routes").remove(&id);
|
||||
}
|
||||
fn isolate(&self, id: u64, members: &[u64]) {
|
||||
let mut blocked = self.blocked.lock().expect("partition routes");
|
||||
for peer in members {
|
||||
if *peer != id {
|
||||
blocked.insert((id, *peer));
|
||||
blocked.insert((*peer, id));
|
||||
}
|
||||
}
|
||||
}
|
||||
fn heal(&self) {
|
||||
self.blocked.lock().expect("restore routes").clear();
|
||||
}
|
||||
fn endpoint(&self, source: u64, target: u64) -> Result<Engine, Unreachable> {
|
||||
if self.blocked.lock().expect("partition check").contains(&(source, target)) {
|
||||
return Err(Unreachable::new(&std::io::Error::other("test partition")));
|
||||
}
|
||||
self.nodes
|
||||
.lock()
|
||||
.expect("endpoint registry")
|
||||
.get(&target)
|
||||
.cloned()
|
||||
.ok_or_else(|| Unreachable::new(&std::io::Error::other("test endpoint unavailable")))
|
||||
}
|
||||
}
|
||||
#[derive(Clone)]
|
||||
struct DirectFactory {
|
||||
source: u64,
|
||||
routes: Arc<Routes>,
|
||||
}
|
||||
struct DirectClient {
|
||||
source: u64,
|
||||
target: u64,
|
||||
node: CaptureNode,
|
||||
routes: Arc<Routes>,
|
||||
}
|
||||
impl RaftNetworkFactory<CaptureTypeConfig> for DirectFactory {
|
||||
type Network = DirectClient;
|
||||
async fn new_client(&mut self, target: u64, node: &CaptureNode) -> DirectClient {
|
||||
DirectClient {
|
||||
source: self.source,
|
||||
target,
|
||||
node: node.clone(),
|
||||
routes: self.routes.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
impl RaftNetwork<CaptureTypeConfig> for DirectClient {
|
||||
async fn append_entries(
|
||||
&mut self,
|
||||
request: AppendEntriesRequest<CaptureTypeConfig>,
|
||||
_option: RPCOption,
|
||||
) -> Result<AppendEntriesResponse<u64>, RpcError> {
|
||||
let pause = self
|
||||
.routes
|
||||
.pause_append
|
||||
.lock()
|
||||
.expect("actual RPC pause hook")
|
||||
.remove(&(self.source, self.target));
|
||||
if let Some((entered, resume)) = pause {
|
||||
entered.notify_one();
|
||||
resume.notified().await;
|
||||
}
|
||||
let raft = self.routes.endpoint(self.source, self.target)?;
|
||||
raft.append_entries(request)
|
||||
.await
|
||||
.map_err(|error| RemoteError::new_with_node(self.target, self.node.clone(), error).into())
|
||||
}
|
||||
async fn vote(&mut self, request: VoteRequest<u64>, _option: RPCOption) -> Result<VoteResponse<u64>, RpcError> {
|
||||
let raft = self.routes.endpoint(self.source, self.target)?;
|
||||
raft.vote(request)
|
||||
.await
|
||||
.map_err(|error| RemoteError::new_with_node(self.target, self.node.clone(), error).into())
|
||||
}
|
||||
async fn install_snapshot(
|
||||
&mut self,
|
||||
request: InstallSnapshotRequest<CaptureTypeConfig>,
|
||||
_option: RPCOption,
|
||||
) -> Result<InstallSnapshotResponse<u64>, SnapshotRpcError> {
|
||||
let raft = self.routes.endpoint(self.source, self.target)?;
|
||||
let done = request.done;
|
||||
let result = raft
|
||||
.install_snapshot(request)
|
||||
.await
|
||||
.map_err(|error| RemoteError::new_with_node(self.target, self.node.clone(), error).into());
|
||||
if done && result.is_ok() {
|
||||
self.routes.snapshot_installs.fetch_add(1, Ordering::AcqRel);
|
||||
}
|
||||
result
|
||||
}
|
||||
fn backoff(&self) -> Backoff {
|
||||
Backoff::new(std::iter::repeat(Duration::from_millis(25)))
|
||||
}
|
||||
}
|
||||
fn identity(node_id: u64) -> NodeIdentity {
|
||||
NodeIdentity {
|
||||
cluster_id: [7; 16],
|
||||
node_id,
|
||||
}
|
||||
}
|
||||
fn binding() -> CaptureBinding {
|
||||
CaptureBinding {
|
||||
revision: 9,
|
||||
binding_id: [8; 16],
|
||||
target_id: [9; 16],
|
||||
}
|
||||
}
|
||||
fn command(value: u8) -> CaptureCommand {
|
||||
CaptureCommand::try_created(DecideCreated {
|
||||
operation_id: [value; 16],
|
||||
object: ObjectIdentity {
|
||||
bucket_incarnation: [2; 16],
|
||||
key: vec![value],
|
||||
},
|
||||
expected_head: None,
|
||||
prepared: PreparedIdentity {
|
||||
version_id: [value; 16],
|
||||
preparation_id: [4; 16],
|
||||
content_digest: [5; 32],
|
||||
content_length: 123,
|
||||
},
|
||||
binding_revision: 9,
|
||||
})
|
||||
.expect("native client command")
|
||||
}
|
||||
fn config(snapshot_every: u64) -> Arc<openraft::Config> {
|
||||
Arc::new(
|
||||
openraft::Config {
|
||||
cluster_name: "native-owner-tests".into(),
|
||||
heartbeat_interval: 100,
|
||||
election_timeout_min: 500,
|
||||
election_timeout_max: 1000,
|
||||
snapshot_policy: openraft::SnapshotPolicy::LogsSinceLast(snapshot_every),
|
||||
max_in_snapshot_log_to_keep: 0,
|
||||
purge_batch_size: 1,
|
||||
..openraft::Config::default()
|
||||
}
|
||||
.validate()
|
||||
.expect("valid real Raft test configuration"),
|
||||
)
|
||||
}
|
||||
fn members(ids: &[u64]) -> BTreeMap<u64, CaptureNode> {
|
||||
ids.iter()
|
||||
.map(|id| {
|
||||
(
|
||||
*id,
|
||||
CaptureNode::new(format!("http://127.0.0.1:{}", 7000 + id)).expect("bounded test endpoint"),
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
struct Node {
|
||||
directory: tempfile::TempDir,
|
||||
writer: NativeLogWriter,
|
||||
owner: Option<NativeRaftOwner>,
|
||||
id: u64,
|
||||
}
|
||||
impl Node {
|
||||
async fn start(id: u64, routes: Arc<Routes>, config: Arc<openraft::Config>) -> Self {
|
||||
let directory = tempfile::tempdir().expect("real Raft owner directory");
|
||||
let mut store = Store::initialize(directory.path().join("db"), identity(id)).expect("real owner redb");
|
||||
store.pin_capture_binding(binding()).expect("captured binding");
|
||||
store.enable_native_empty().expect("native storage mode");
|
||||
store.promote_created_provenance().expect("native provenance");
|
||||
let writer = NativeLogWriter::new(store);
|
||||
let owner = NativeRaftOwner::start(
|
||||
writer.clone(),
|
||||
id,
|
||||
config,
|
||||
CheckpointLimits::default(),
|
||||
DirectFactory {
|
||||
source: id,
|
||||
routes: routes.clone(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("actual owner start");
|
||||
routes.register(id, owner.raft().clone());
|
||||
Self {
|
||||
directory,
|
||||
writer,
|
||||
owner: Some(owner),
|
||||
id,
|
||||
}
|
||||
}
|
||||
fn owner(&self) -> &NativeRaftOwner {
|
||||
self.owner.as_ref().expect("live owner")
|
||||
}
|
||||
async fn page(&self) -> EventPage {
|
||||
self.writer
|
||||
.created_events_page(None, 256)
|
||||
.await
|
||||
.expect("actual committed events")
|
||||
}
|
||||
async fn stop(&mut self, routes: &Routes) {
|
||||
self.owner
|
||||
.take()
|
||||
.expect("running owner")
|
||||
.shutdown()
|
||||
.await
|
||||
.expect("real Raft shutdown");
|
||||
routes.remove(self.id);
|
||||
}
|
||||
}
|
||||
struct Cluster {
|
||||
nodes: Vec<Node>,
|
||||
routes: Arc<Routes>,
|
||||
}
|
||||
impl Cluster {
|
||||
async fn start(ids: &[u64], snapshot_every: u64) -> Self {
|
||||
let routes = Arc::new(Routes::default());
|
||||
let mut nodes = Vec::new();
|
||||
for id in ids {
|
||||
nodes.push(Node::start(*id, routes.clone(), config(snapshot_every)).await);
|
||||
}
|
||||
nodes[0]
|
||||
.owner()
|
||||
.bootstrap(members(ids))
|
||||
.await
|
||||
.expect("explicit initial cluster bootstrap");
|
||||
let cluster = Self { nodes, routes };
|
||||
cluster.leader(ids).await;
|
||||
cluster
|
||||
}
|
||||
fn node(&self, id: u64) -> &Node {
|
||||
self.nodes.iter().find(|node| node.id == id).expect("cluster node")
|
||||
}
|
||||
async fn leader(&self, eligible: &[u64]) -> u64 {
|
||||
tokio::time::timeout(Duration::from_secs(15), async {
|
||||
loop {
|
||||
for id in eligible {
|
||||
let node = self.node(*id);
|
||||
let Some(owner) = &node.owner else {
|
||||
continue;
|
||||
};
|
||||
let metrics = owner.metrics().borrow().clone();
|
||||
assert!(metrics.running_state.is_ok(), "real Raft owner fatal state: {:?}", metrics.running_state);
|
||||
if metrics.state == ServerState::Leader && metrics.current_leader == Some(*id) {
|
||||
return *id;
|
||||
}
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("real leader election readiness")
|
||||
}
|
||||
async fn applied(&self, ids: &[u64], index: u64) {
|
||||
for id in ids {
|
||||
self.node(*id)
|
||||
.owner()
|
||||
.raft()
|
||||
.wait(Some(Duration::from_secs(15)))
|
||||
.applied_index_at_least(Some(index), "real replicated apply readiness")
|
||||
.await
|
||||
.expect("committed log applied on real node");
|
||||
}
|
||||
}
|
||||
async fn stop(mut self) {
|
||||
for node in &mut self.nodes {
|
||||
if node.owner.is_some() {
|
||||
node.stop(&self.routes).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raft_owner_single_node_bootstrap() {
|
||||
let cluster = Cluster::start(&[0], 100).await;
|
||||
let response = cluster
|
||||
.node(0)
|
||||
.owner()
|
||||
.client_write(command(1))
|
||||
.await
|
||||
.expect("real client_write");
|
||||
assert!(
|
||||
matches!(response.data.created_result(),Some(CreatedResult::Created { event_id,.. }) if *event_id==[1;16]),
|
||||
"OWNER_TRUE_RAFT_CREATED_RESPONSE"
|
||||
);
|
||||
let page = cluster.node(0).page().await;
|
||||
assert_eq!(page.events.len(), 1);
|
||||
assert_eq!(
|
||||
page.events[0].captured.decision_log_id,
|
||||
LogId::from(response.log_id),
|
||||
"OWNER_COMMITTED_EVENT_LOG_ID"
|
||||
);
|
||||
assert!(
|
||||
cluster.node(0).owner().bootstrap(members(&[0])).await.is_err(),
|
||||
"OWNER_BOOTSTRAP_NOT_REINITIALIZATION"
|
||||
);
|
||||
cluster.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raft_owner_single_node_restart() {
|
||||
let routes = Arc::new(Routes::default());
|
||||
let mut node = Node::start(0, routes.clone(), config(100)).await;
|
||||
node.owner().bootstrap(members(&[0])).await.expect("first bootstrap only");
|
||||
node.owner()
|
||||
.raft()
|
||||
.wait(Some(Duration::from_secs(15)))
|
||||
.state(ServerState::Leader, "single leader")
|
||||
.await
|
||||
.expect("leader ready");
|
||||
node.owner().client_write(command(1)).await.expect("first genuine write");
|
||||
let before = node.page().await;
|
||||
node.stop(&routes).await;
|
||||
let Node {
|
||||
directory,
|
||||
writer,
|
||||
owner: _,
|
||||
id,
|
||||
} = node;
|
||||
let vote = writer.read_vote().await.expect("durable vote before reopen");
|
||||
drop(writer);
|
||||
let store = Store::open(directory.path().join("db"), identity(id)).expect("restart same durable identity");
|
||||
assert_eq!(store.read_vote().expect("persisted vote"), vote.map(Into::into));
|
||||
let writer = NativeLogWriter::new(store);
|
||||
let owner = NativeRaftOwner::start(
|
||||
writer.clone(),
|
||||
id,
|
||||
config(100),
|
||||
CheckpointLimits::default(),
|
||||
DirectFactory {
|
||||
source: id,
|
||||
routes: routes.clone(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("restart existing Raft");
|
||||
routes.register(id, owner.raft().clone());
|
||||
assert!(
|
||||
owner.raft().is_initialized().await.expect("restored cluster membership"),
|
||||
"OWNER_RESTART_DOES_NOT_REQUIRE_BOOTSTRAP"
|
||||
);
|
||||
owner
|
||||
.raft()
|
||||
.wait(Some(Duration::from_secs(15)))
|
||||
.state(ServerState::Leader, "restored leader")
|
||||
.await
|
||||
.expect("restarted leader ready");
|
||||
owner
|
||||
.client_write(command(2))
|
||||
.await
|
||||
.expect("continue after restart without initialize");
|
||||
let after = writer.created_events_page(None, 256).await.expect("restart continued events");
|
||||
assert_eq!(after.events.len(), 2);
|
||||
assert_eq!(after.events[0], before.events[0], "OWNER_RESTART_PRESERVES_FIRST_DECISION");
|
||||
owner.shutdown().await.expect("restart owner shutdown");
|
||||
routes.remove(id);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raft_owner_idempotent_retry() {
|
||||
let cluster = Cluster::start(&[0], 100).await;
|
||||
let owner = cluster.node(0).owner();
|
||||
let first = owner.client_write(command(1)).await.expect("first write");
|
||||
let replay = owner.client_write(command(1)).await.expect("same operation retry");
|
||||
assert_eq!(
|
||||
replay.data.created_result(),
|
||||
first.data.created_result(),
|
||||
"OWNER_RETRY_PRESERVES_FIRST_DECISION_RESULT"
|
||||
);
|
||||
assert!(replay.log_id.index > first.log_id.index, "real retry can append another Raft log");
|
||||
assert_eq!(cluster.node(0).page().await.events.len(), 1, "OWNER_RETRY_NO_DUPLICATE_EVENT");
|
||||
cluster.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raft_owner_identity_binding() {
|
||||
let routes = Arc::new(Routes::default());
|
||||
let mut node = Node::start(0, routes.clone(), config(100)).await;
|
||||
node.owner().bootstrap(members(&[0])).await.expect("real durable membership");
|
||||
node.owner()
|
||||
.raft()
|
||||
.wait(Some(Duration::from_secs(15)))
|
||||
.state(ServerState::Leader, "single leader")
|
||||
.await
|
||||
.expect("leader");
|
||||
node.owner().client_write(command(1)).await.expect("nonempty durable store");
|
||||
node.stop(&routes).await;
|
||||
let before = std::fs::read(node.directory.path().join("db")).expect("stable identity-bound store");
|
||||
let result = NativeRaftOwner::start(
|
||||
node.writer.clone(),
|
||||
1,
|
||||
config(100),
|
||||
CheckpointLimits::default(),
|
||||
DirectFactory {
|
||||
source: 1,
|
||||
routes: routes.clone(),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
matches!(result, Err(NativeRaftOwnerError::IdentityMismatch { runtime: 1, durable: 0 })),
|
||||
"OWNER_REQUIRES_DURABLE_NODE_ID: runtime node cannot reinterpret stored membership/identity"
|
||||
);
|
||||
assert_eq!(
|
||||
std::fs::read(node.directory.path().join("db")).expect("unchanged identity-bound store"),
|
||||
before
|
||||
);
|
||||
let invalid_config = Arc::new(openraft::Config {
|
||||
election_timeout_min: 1000,
|
||||
election_timeout_max: 500,
|
||||
..openraft::Config::default()
|
||||
});
|
||||
let invalid = NativeRaftOwner::start(
|
||||
node.writer.clone(),
|
||||
0,
|
||||
invalid_config,
|
||||
CheckpointLimits::default(),
|
||||
DirectFactory {
|
||||
source: 0,
|
||||
routes: routes.clone(),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
matches!(invalid, Err(NativeRaftOwnerError::Config(_))),
|
||||
"OWNER_REJECTS_INVALID_CONFIG_BEFORE_START"
|
||||
);
|
||||
let mut limits = CheckpointLimits::default();
|
||||
limits.snapshot.total_bytes = 0;
|
||||
let invalid = NativeRaftOwner::start(node.writer.clone(), 0, config(100), limits, DirectFactory { source: 0, routes }).await;
|
||||
assert!(
|
||||
matches!(invalid, Err(NativeRaftOwnerError::Snapshot(_))),
|
||||
"OWNER_REJECTS_INVALID_LIMITS_BEFORE_START"
|
||||
);
|
||||
assert_eq!(
|
||||
std::fs::read(node.directory.path().join("db")).expect("invalid start leaves stable bytes"),
|
||||
before
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
#[tokio::test]
|
||||
async fn raft_owner_shared_reader() {
|
||||
use source_reader::SourceReaderClient;
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let cluster = Cluster::start(&[0], 100).await;
|
||||
let node = cluster.node(0);
|
||||
std::fs::set_permissions(node.directory.path(), std::fs::Permissions::from_mode(0o700)).expect("private reader parent");
|
||||
let socket = node.directory.path().join("reader.sock");
|
||||
let (server, shutdown) = node
|
||||
.owner()
|
||||
.bind_reader(&socket)
|
||||
.expect("owner shares live writer with reader");
|
||||
let task = tokio::spawn(server.serve());
|
||||
let client = SourceReaderClient::new(&socket).expect("live reader");
|
||||
assert!(
|
||||
client
|
||||
.page(identity(0).cluster_id, None, 1)
|
||||
.await
|
||||
.expect("initial reader page")
|
||||
.page
|
||||
.events
|
||||
.is_empty()
|
||||
);
|
||||
let response = node
|
||||
.owner()
|
||||
.client_write(command(1))
|
||||
.await
|
||||
.expect("real write while reader runs");
|
||||
let page = client
|
||||
.page(identity(0).cluster_id, None, 1)
|
||||
.await
|
||||
.expect("live committed event");
|
||||
assert_eq!(page.identity, identity(0));
|
||||
assert_eq!(
|
||||
page.page.events[0].captured.decision_log_id,
|
||||
LogId::from(response.log_id),
|
||||
"OWNER_READER_USES_REAL_COMMITTED_WRITER"
|
||||
);
|
||||
shutdown.shutdown();
|
||||
task.await.expect("reader serve task").expect("reader drains");
|
||||
cluster.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raft_owner_shutdown() {
|
||||
let cluster = Cluster::start(&[0], 100).await;
|
||||
let owner = cluster.node(0).owner();
|
||||
owner.client_write(command(1)).await.expect("accepted before shutdown");
|
||||
owner.shutdown().await.expect("actual owner shutdown");
|
||||
assert!(owner.client_write(command(2)).await.is_err(), "OWNER_SHUTDOWN_REJECTS_NEW_WRITES");
|
||||
assert_eq!(cluster.node(0).page().await.events.len(), 1);
|
||||
cluster.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn raft_owner_three_node_majority() {
|
||||
let cluster = Cluster::start(&[0, 1, 2], 100).await;
|
||||
let leader = cluster.leader(&[0, 1, 2]).await;
|
||||
let first = cluster
|
||||
.node(leader)
|
||||
.owner()
|
||||
.client_write(command(1))
|
||||
.await
|
||||
.expect("real quorum write");
|
||||
cluster.applied(&[0, 1, 2], first.log_id.index).await;
|
||||
let follower = [0, 1, 2].into_iter().find(|id| *id != leader).expect("nonleader");
|
||||
let rejected = cluster.node(follower).owner().client_write(command(254)).await;
|
||||
match rejected {
|
||||
Err(NativeRaftOwnerError::Write(error)) => {
|
||||
assert!(
|
||||
matches!(error.api_error(), Some(openraft::error::ClientWriteError::ForwardToLeader(_))),
|
||||
"OWNER_PRESERVES_TYPED_FORWARD_TO_LEADER"
|
||||
);
|
||||
}
|
||||
other => panic!("OWNER_NONLEADER_WRITE_MUST_NOT_SUCCEED: {other:?}"),
|
||||
}
|
||||
cluster.routes.isolate(leader, &[0, 1, 2]);
|
||||
let uncertain =
|
||||
tokio::time::timeout(Duration::from_millis(1500), cluster.node(leader).owner().client_write(command(2))).await;
|
||||
assert!(!matches!(uncertain, Ok(Ok(_))), "OWNER_MAJORITY_REQUIRED_FOR_SUCCESS");
|
||||
let others: Vec<_> = [0, 1, 2].into_iter().filter(|id| *id != leader).collect();
|
||||
let next = cluster.leader(&others).await;
|
||||
let response = cluster
|
||||
.node(next)
|
||||
.owner()
|
||||
.client_write(command(2))
|
||||
.await
|
||||
.expect("majority-owned retry of exact same operation");
|
||||
cluster.routes.heal();
|
||||
cluster.applied(&[0, 1, 2], response.log_id.index).await;
|
||||
for node in &cluster.nodes {
|
||||
assert_eq!(node.page().await.events.len(), 2, "OWNER_QUORUM_RETRY_NO_DUPLICATE_EVENT");
|
||||
}
|
||||
cluster.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn raft_owner_leader_failover() {
|
||||
let mut cluster = Cluster::start(&[0, 1, 2], 100).await;
|
||||
let leader = cluster.leader(&[0, 1, 2]).await;
|
||||
let first = cluster
|
||||
.node(leader)
|
||||
.owner()
|
||||
.client_write(command(1))
|
||||
.await
|
||||
.expect("first confirmed write");
|
||||
cluster.applied(&[0, 1, 2], first.log_id.index).await;
|
||||
let before = cluster.node(leader).page().await.events[0].clone();
|
||||
cluster
|
||||
.nodes
|
||||
.iter_mut()
|
||||
.find(|node| node.id == leader)
|
||||
.expect("old leader")
|
||||
.stop(&cluster.routes)
|
||||
.await;
|
||||
let survivors: Vec<_> = [0, 1, 2].into_iter().filter(|id| *id != leader).collect();
|
||||
let next = cluster.leader(&survivors).await;
|
||||
let response = cluster
|
||||
.node(next)
|
||||
.owner()
|
||||
.client_write(command(2))
|
||||
.await
|
||||
.expect("new leader actual quorum write");
|
||||
cluster.applied(&survivors, response.log_id.index).await;
|
||||
for id in survivors {
|
||||
let page = cluster.node(id).page().await;
|
||||
assert_eq!(page.events.len(), 2);
|
||||
assert_eq!(page.events[0], before, "OWNER_FAILOVER_PRESERVES_CONFIRMED_DECISION");
|
||||
}
|
||||
cluster.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn raft_owner_snapshot_catchup() {
|
||||
let cluster = Cluster::start(&[0, 1, 2], 1000).await;
|
||||
let leader = cluster.leader(&[0, 1, 2]).await;
|
||||
let follower = [0, 1, 2].into_iter().find(|id| *id != leader).expect("lagging follower");
|
||||
let first = cluster
|
||||
.node(leader)
|
||||
.owner()
|
||||
.client_write(command(1))
|
||||
.await
|
||||
.expect("initial confirmed event");
|
||||
cluster.applied(&[0, 1, 2], first.log_id.index).await;
|
||||
// Keep the catch-up test focused on replication rather than an isolated follower's election.
|
||||
cluster.node(follower).owner().raft().runtime_config().elect(false);
|
||||
cluster.routes.isolate(follower, &[0, 1, 2]);
|
||||
let mut last = first;
|
||||
for value in 2..=12 {
|
||||
last = cluster
|
||||
.node(leader)
|
||||
.owner()
|
||||
.client_write(command(value))
|
||||
.await
|
||||
.expect("majority writes while follower offline");
|
||||
}
|
||||
cluster
|
||||
.node(leader)
|
||||
.owner()
|
||||
.raft()
|
||||
.trigger()
|
||||
.snapshot()
|
||||
.await
|
||||
.expect("true Raft snapshot trigger");
|
||||
cluster
|
||||
.node(leader)
|
||||
.owner()
|
||||
.raft()
|
||||
.wait(Some(Duration::from_secs(15)))
|
||||
.metrics(
|
||||
|metrics| metrics.purged.is_some_and(|cut| cut.index >= last.log_id.index),
|
||||
"snapshot durably purged lagging log prefix",
|
||||
)
|
||||
.await
|
||||
.expect("real snapshot and purge");
|
||||
let before = cluster.routes.snapshot_installs.load(Ordering::Acquire);
|
||||
cluster.routes.heal();
|
||||
cluster.applied(&[0, 1, 2], last.log_id.index).await;
|
||||
cluster
|
||||
.node(follower)
|
||||
.owner()
|
||||
.raft()
|
||||
.wait(Some(Duration::from_secs(15)))
|
||||
.metrics(
|
||||
|metrics| metrics.snapshot.is_some_and(|cut| cut.index >= last.log_id.index),
|
||||
"follower installed actual transferred snapshot",
|
||||
)
|
||||
.await
|
||||
.expect("real snapshot catch-up");
|
||||
tokio::time::timeout(Duration::from_secs(15), async {
|
||||
while cluster.routes.snapshot_installs.load(Ordering::Acquire) <= before {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("snapshot readiness includes actual successful done-RPC return");
|
||||
assert!(
|
||||
cluster.routes.snapshot_installs.load(Ordering::Acquire) > before,
|
||||
"OWNER_CATCHUP_USED_ACTUAL_SNAPSHOT_RPC"
|
||||
);
|
||||
let expected = cluster.node(leader).page().await;
|
||||
for node in &cluster.nodes {
|
||||
assert_eq!(node.page().await, expected, "OWNER_SNAPSHOT_CATCHUP_FULL_SOURCE_EVENTS");
|
||||
}
|
||||
cluster.node(follower).owner().raft().runtime_config().elect(true);
|
||||
let current = cluster.leader(&[0, 1, 2]).await;
|
||||
let next = cluster
|
||||
.node(current)
|
||||
.owner()
|
||||
.client_write(command(13))
|
||||
.await
|
||||
.expect("continue after real snapshot install");
|
||||
cluster.applied(&[0, 1, 2], next.log_id.index).await;
|
||||
for node in &cluster.nodes {
|
||||
assert_eq!(node.page().await.events.len(), 13);
|
||||
}
|
||||
cluster.stop().await;
|
||||
}
|
||||
|
||||
async fn wait_for_shutdown_drain(waiters: &AtomicUsize) {
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while waiters.load(Ordering::Acquire) == 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("OWNER_SHUTDOWN_DRAINS_INFLIGHT_WORK: owner reached real worker drain wait");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raft_owner_shutdown_inflight_snapshot() {
|
||||
let routes = Arc::new(Routes::default());
|
||||
let mut node = Node::start(0, routes.clone(), config(1000)).await;
|
||||
node.owner()
|
||||
.bootstrap(members(&[0]))
|
||||
.await
|
||||
.expect("real single-node bootstrap");
|
||||
node.owner()
|
||||
.raft()
|
||||
.wait(Some(Duration::from_secs(15)))
|
||||
.state(ServerState::Leader, "leader before snapshot")
|
||||
.await
|
||||
.expect("leader readiness");
|
||||
node.owner()
|
||||
.client_write(command(1))
|
||||
.await
|
||||
.expect("real committed snapshot input");
|
||||
let entered = Arc::new(tokio::sync::Notify::new());
|
||||
let resume = Arc::new(tokio::sync::Notify::new());
|
||||
*node.owner().snapshot_build_pause.lock().expect("real builder hook") = Some((entered.clone(), resume.clone()));
|
||||
node.owner()
|
||||
.raft()
|
||||
.trigger()
|
||||
.snapshot()
|
||||
.await
|
||||
.expect("real Raft snapshot request");
|
||||
tokio::time::timeout(Duration::from_secs(5), entered.notified())
|
||||
.await
|
||||
.expect("real detached snapshot builder paused");
|
||||
let owner = node.owner.take().expect("running owner");
|
||||
let waiters = owner.shutdown_waiters();
|
||||
let observer = owner.raft().clone();
|
||||
let shutdown = tokio::spawn(async move {
|
||||
let result = owner.shutdown().await;
|
||||
(owner, result)
|
||||
});
|
||||
wait_for_shutdown_drain(&waiters).await;
|
||||
assert!(
|
||||
observer.metrics().borrow().running_state.is_err(),
|
||||
"OWNER_CORE_STOPPED_WHILE_SNAPSHOT_DRAIN_PENDING"
|
||||
);
|
||||
assert!(!shutdown.is_finished(), "OWNER_SHUTDOWN_DRAINS_INFLIGHT_SNAPSHOT");
|
||||
resume.notify_one();
|
||||
let (owner, result) = tokio::time::timeout(Duration::from_secs(10), shutdown)
|
||||
.await
|
||||
.expect("released snapshot lets shutdown finish")
|
||||
.expect("shutdown task");
|
||||
result.expect("snapshot work drained");
|
||||
routes.remove(0);
|
||||
drop(observer);
|
||||
drop(owner);
|
||||
drop(waiters);
|
||||
let Node {
|
||||
directory,
|
||||
writer,
|
||||
owner: _,
|
||||
id,
|
||||
} = node;
|
||||
drop(writer);
|
||||
let store = Store::open(directory.path().join("db"), identity(id)).expect("OWNER_SHUTDOWN_SNAPSHOT_RELEASES_LIVE_DATABASE");
|
||||
assert_eq!(
|
||||
store
|
||||
.created_events_page(None, 256)
|
||||
.expect("retained committed source events")
|
||||
.events
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn raft_owner_shutdown_inflight_replication() {
|
||||
// OpenRaft times append_entries by heartbeat_interval. Keep the real deadline
|
||||
// longer than readiness checks so only explicit RPC release ends this pause.
|
||||
let slow = Arc::new(
|
||||
openraft::Config {
|
||||
heartbeat_interval: 10_000,
|
||||
election_timeout_min: 20_000,
|
||||
election_timeout_max: 30_000,
|
||||
snapshot_policy: openraft::SnapshotPolicy::LogsSinceLast(1000),
|
||||
..openraft::Config::default()
|
||||
}
|
||||
.validate()
|
||||
.expect("long real RPC deadline"),
|
||||
);
|
||||
let routes = Arc::new(Routes::default());
|
||||
let mut nodes = Vec::new();
|
||||
for id in [0, 1, 2] {
|
||||
nodes.push(Node::start(id, routes.clone(), slow.clone()).await);
|
||||
}
|
||||
nodes[0]
|
||||
.owner()
|
||||
.bootstrap(members(&[0, 1, 2]))
|
||||
.await
|
||||
.expect("real three-node bootstrap");
|
||||
nodes[0]
|
||||
.owner()
|
||||
.raft()
|
||||
.trigger()
|
||||
.elect()
|
||||
.await
|
||||
.expect("explicit test election readiness");
|
||||
let mut cluster = Cluster { nodes, routes };
|
||||
let leader = cluster.leader(&[0, 1, 2]).await;
|
||||
let response = cluster
|
||||
.node(leader)
|
||||
.owner()
|
||||
.client_write(command(1))
|
||||
.await
|
||||
.expect("real replicated write before pause");
|
||||
cluster.applied(&[0, 1, 2], response.log_id.index).await;
|
||||
let follower = [0, 1, 2].into_iter().find(|id| *id != leader).expect("replication target");
|
||||
let entered = Arc::new(tokio::sync::Notify::new());
|
||||
let resume = Arc::new(tokio::sync::Notify::new());
|
||||
cluster
|
||||
.routes
|
||||
.pause_append
|
||||
.lock()
|
||||
.expect("actual request pause")
|
||||
.insert((leader, follower), (entered.clone(), resume.clone()));
|
||||
cluster
|
||||
.node(leader)
|
||||
.owner()
|
||||
.raft()
|
||||
.trigger()
|
||||
.heartbeat()
|
||||
.await
|
||||
.expect("real replication RPC");
|
||||
tokio::time::timeout(Duration::from_secs(5), entered.notified())
|
||||
.await
|
||||
.expect("actual AppendEntries in flight");
|
||||
let node = cluster.nodes.iter_mut().find(|node| node.id == leader).expect("leader node");
|
||||
let owner = node.owner.take().expect("leader owner");
|
||||
let waiters = owner.shutdown_waiters();
|
||||
let observer = owner.raft().clone();
|
||||
let shutdown = tokio::spawn(async move {
|
||||
let result = owner.shutdown().await;
|
||||
(owner, result)
|
||||
});
|
||||
wait_for_shutdown_drain(&waiters).await;
|
||||
assert!(
|
||||
observer.metrics().borrow().running_state.is_err(),
|
||||
"OWNER_CORE_STOPPED_WHILE_REPLICATION_DRAIN_PENDING"
|
||||
);
|
||||
assert!(!shutdown.is_finished(), "OWNER_SHUTDOWN_DRAINS_INFLIGHT_REPLICATION");
|
||||
resume.notify_one();
|
||||
let (owner, result) = tokio::time::timeout(Duration::from_secs(10), shutdown)
|
||||
.await
|
||||
.expect("released real RPC lets shutdown finish")
|
||||
.expect("replication shutdown task");
|
||||
result.expect("actual replication reader drained");
|
||||
cluster.routes.remove(leader);
|
||||
drop(observer);
|
||||
drop(owner);
|
||||
drop(waiters);
|
||||
let position = cluster
|
||||
.nodes
|
||||
.iter()
|
||||
.position(|node| node.id == leader)
|
||||
.expect("stopped leader");
|
||||
let Node {
|
||||
directory,
|
||||
writer,
|
||||
owner: _,
|
||||
id,
|
||||
} = cluster.nodes.remove(position);
|
||||
drop(writer);
|
||||
let store =
|
||||
Store::open(directory.path().join("db"), identity(id)).expect("OWNER_SHUTDOWN_REPLICATION_RELEASES_LIVE_DATABASE");
|
||||
assert_eq!(
|
||||
store
|
||||
.created_events_page(None, 256)
|
||||
.expect("real quorum event remains")
|
||||
.events
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
drop(store);
|
||||
cluster.stop().await;
|
||||
}
|
||||
|
||||
#[path = "committed_created_authorization_tests.rs"]
|
||||
mod authorization;
|
||||
@@ -0,0 +1,327 @@
|
||||
//! Durable OpenRaft application state and selected baseline snapshots.
|
||||
use crate::{
|
||||
CaptureRaftEntry, CaptureResponse, CaptureTypeConfig, CheckpointLimits, NativeLogWriter, NativeSnapshotData,
|
||||
NativeSnapshotError, StoreError, checkpoint_baseline, snapshot,
|
||||
};
|
||||
use openraft::{
|
||||
OptionalSend, RaftSnapshotBuilder, Snapshot, SnapshotMeta, StorageError, StorageIOError, StoredMembership,
|
||||
storage::RaftStateMachine,
|
||||
};
|
||||
use redb::{ReadableDatabase, ReadableTable, ReadableTableMetadata};
|
||||
use std::{
|
||||
io::{Seek, Write},
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
},
|
||||
};
|
||||
|
||||
pub(crate) const SNAPSHOT_META: &str = "native_snapshot_meta_v1";
|
||||
type Meta = SnapshotMeta<u64, crate::CaptureNode>;
|
||||
#[derive(serde::Serialize, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct SelectedMeta {
|
||||
generation: u64,
|
||||
meta: Meta,
|
||||
}
|
||||
|
||||
pub(crate) fn publish_meta(
|
||||
meta: &mut redb::Table<&str, &[u8]>,
|
||||
baseline: &checkpoint_baseline::BaselineMetadata,
|
||||
value: Meta,
|
||||
) -> Result<(), StoreError> {
|
||||
let bytes = serde_json::to_vec(&SelectedMeta {
|
||||
generation: baseline.generation,
|
||||
meta: value,
|
||||
})
|
||||
.map_err(|_| StoreError::CorruptRecord)?;
|
||||
if bytes.len() > 1024 * 1024 {
|
||||
return Err(StoreError::ResourceLimit);
|
||||
}
|
||||
meta.insert(SNAPSHOT_META, bytes.as_slice())?;
|
||||
meta.insert("native_snapshot_meta_required", &[1u8][..])?;
|
||||
Ok(())
|
||||
}
|
||||
pub(crate) fn local_meta(
|
||||
baseline: &checkpoint_baseline::BaselineMetadata,
|
||||
identity: crate::NodeIdentity,
|
||||
) -> Result<Meta, StoreError> {
|
||||
Ok(Meta {
|
||||
last_log_id: baseline.header.applied.map(Into::into),
|
||||
last_membership: StoredMembership::try_from(&baseline.header.membership)?,
|
||||
snapshot_id: format!("native-local-{:x?}-{}-{}", identity.cluster_id, identity.node_id, baseline.generation),
|
||||
})
|
||||
}
|
||||
pub(crate) fn publish_local_meta(
|
||||
meta: &mut redb::Table<&str, &[u8]>,
|
||||
baseline: &checkpoint_baseline::BaselineMetadata,
|
||||
) -> Result<(), StoreError> {
|
||||
let identity = crate::decode_identity(meta.get("identity")?.ok_or(StoreError::CorruptRecord)?.value())?;
|
||||
publish_meta(meta, baseline, local_meta(baseline, identity)?)
|
||||
}
|
||||
pub(crate) fn selected_meta(
|
||||
meta: &impl ReadableTable<&'static str, &'static [u8]>,
|
||||
baseline: &checkpoint_baseline::BaselineMetadata,
|
||||
) -> Result<Meta, StoreError> {
|
||||
let Some(bytes) = meta.get(SNAPSHOT_META)? else {
|
||||
if meta.get("native_snapshot_meta_required")?.is_some() {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
return local_meta(
|
||||
baseline,
|
||||
crate::decode_identity(meta.get("identity")?.ok_or(StoreError::CorruptRecord)?.value())?,
|
||||
);
|
||||
};
|
||||
if bytes.value().len() > 1024 * 1024 {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let selected: SelectedMeta = serde_json::from_slice(bytes.value()).map_err(|_| StoreError::CorruptRecord)?;
|
||||
if selected.generation != baseline.generation
|
||||
|| selected.meta.last_log_id.map(crate::LogId::from) != baseline.header.applied
|
||||
|| crate::CaptureMembership::try_from(&selected.meta.last_membership)? != baseline.header.membership
|
||||
{
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
Ok(selected.meta)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) type TestPause = std::sync::Mutex<Option<(Arc<tokio::sync::Notify>, Arc<tokio::sync::Notify>)>>;
|
||||
|
||||
#[derive(Default)]
|
||||
pub(crate) struct ReceiveState {
|
||||
count: AtomicUsize,
|
||||
#[cfg(test)]
|
||||
pub(crate) purge_recheck: TestPause,
|
||||
pub(crate) changed: tokio::sync::Notify,
|
||||
#[cfg(test)]
|
||||
pub(crate) waiters: AtomicUsize,
|
||||
}
|
||||
pub(crate) struct ReceiveGuard {
|
||||
state: Arc<ReceiveState>,
|
||||
}
|
||||
impl ReceiveGuard {
|
||||
pub(crate) fn belongs_to(&self, state: &Arc<ReceiveState>) -> bool {
|
||||
Arc::ptr_eq(&self.state, state)
|
||||
}
|
||||
fn new(state: Arc<ReceiveState>) -> Self {
|
||||
state.count.fetch_add(1, Ordering::AcqRel);
|
||||
Self { state }
|
||||
}
|
||||
}
|
||||
impl Drop for ReceiveGuard {
|
||||
fn drop(&mut self) {
|
||||
self.state.count.fetch_sub(1, Ordering::AcqRel);
|
||||
self.state.changed.notify_waiters();
|
||||
}
|
||||
}
|
||||
impl ReceiveState {
|
||||
pub(crate) fn pending(&self) -> bool {
|
||||
self.count.load(Ordering::Acquire) > 0
|
||||
}
|
||||
}
|
||||
|
||||
/// Both adapters must use clones of the same writer. Limits apply while receiving.
|
||||
#[derive(Clone)]
|
||||
pub struct NativeRaftStateMachine {
|
||||
pub(crate) writer: NativeLogWriter,
|
||||
pub(crate) limits: CheckpointLimits,
|
||||
#[cfg(test)]
|
||||
pub(crate) build_before_ordering: Arc<TestPause>,
|
||||
// Declared last: every clone releases its writer before the final owner lease.
|
||||
pub(crate) owner_lease: Option<Arc<crate::raft_owner::OwnerLease>>,
|
||||
}
|
||||
impl NativeRaftStateMachine {
|
||||
pub fn new(writer: NativeLogWriter, limits: CheckpointLimits) -> Result<Self, NativeSnapshotError> {
|
||||
limits.validate()?;
|
||||
limits.snapshot.total_bytes.checked_add(56).ok_or(StoreError::ResourceLimit)?;
|
||||
Ok(Self {
|
||||
writer,
|
||||
limits,
|
||||
#[cfg(test)]
|
||||
build_before_ordering: Arc::new(TestPause::default()),
|
||||
owner_lease: None,
|
||||
})
|
||||
}
|
||||
fn maximum(&self) -> u64 {
|
||||
self.limits.snapshot.total_bytes + 56
|
||||
}
|
||||
async fn current(&self) -> Result<Option<Snapshot<CaptureTypeConfig>>, NativeSnapshotError> {
|
||||
let owner = self.writer.store.clone();
|
||||
let limits = self.limits;
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let store = owner.lock().map_err(|_| StoreError::CorruptRecord)?;
|
||||
if !store.writable {
|
||||
return Err(StoreError::WriteDisabled);
|
||||
}
|
||||
let tx = store.database.begin_read()?;
|
||||
drop(store);
|
||||
let table = tx.open_table(crate::META)?;
|
||||
let Some(baseline) = checkpoint_baseline::read_metadata(&table)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
let meta = selected_meta(&table, &baseline)?;
|
||||
let definitions = [
|
||||
checkpoint_baseline::HEADS,
|
||||
checkpoint_baseline::VERSIONS,
|
||||
checkpoint_baseline::EVENTS,
|
||||
checkpoint_baseline::OPERATIONS,
|
||||
];
|
||||
let mut counts = [0; 4];
|
||||
for (i, definition) in definitions.iter().enumerate() {
|
||||
counts[i] = tx.open_table(*definition)?.len()?;
|
||||
}
|
||||
let out = snapshot::CapabilityOutput::new(Vec::new(), limits.snapshot.total_bytes);
|
||||
let mut encoder =
|
||||
snapshot::Encoder::new(out, &baseline.header, counts, limits.snapshot).map_err(|_| StoreError::CorruptRecord)?;
|
||||
for (definition, section) in definitions.into_iter().zip([
|
||||
snapshot::Section::Heads,
|
||||
snapshot::Section::Versions,
|
||||
snapshot::Section::Events,
|
||||
snapshot::Section::Operations,
|
||||
]) {
|
||||
for row in tx.open_table(definition)?.iter()? {
|
||||
let (key, value) = row?;
|
||||
encoder
|
||||
.row(snapshot::Row {
|
||||
section,
|
||||
key: key.value(),
|
||||
value: value.value(),
|
||||
})
|
||||
.map_err(|_| StoreError::CorruptRecord)?;
|
||||
}
|
||||
}
|
||||
let artifact = encoder.finish().map_err(|_| StoreError::CorruptRecord)?.finish();
|
||||
let mut data = NativeSnapshotData::new(limits.snapshot.total_bytes + 56).map_err(|_| StoreError::ResourceLimit)?;
|
||||
data.write_all(&artifact.manifest.bytes)
|
||||
.map_err(|_| StoreError::ResourceLimit)?;
|
||||
data.write_all(&artifact.payload).map_err(|_| StoreError::ResourceLimit)?;
|
||||
data.rewind().map_err(|_| StoreError::CorruptRecord)?;
|
||||
Ok::<_, StoreError>(Some(Snapshot {
|
||||
meta,
|
||||
snapshot: Box::new(data),
|
||||
}))
|
||||
})
|
||||
.await
|
||||
.map_err(NativeSnapshotError::Worker)?
|
||||
.map_err(NativeSnapshotError::Store)
|
||||
}
|
||||
}
|
||||
fn failure(error: impl std::error::Error + 'static) -> StorageError<u64> {
|
||||
StorageIOError::read_state_machine(openraft::AnyError::new(&error)).into()
|
||||
}
|
||||
impl RaftStateMachine<CaptureTypeConfig> for NativeRaftStateMachine {
|
||||
type SnapshotBuilder = Self;
|
||||
async fn applied_state(
|
||||
&mut self,
|
||||
) -> Result<(Option<openraft::LogId<u64>>, StoredMembership<u64, crate::CaptureNode>), StorageError<u64>> {
|
||||
let (id, membership) = self.writer.applied_state().await.map_err(failure)?;
|
||||
Ok((id.map(Into::into), StoredMembership::try_from(&membership).map_err(failure)?))
|
||||
}
|
||||
async fn apply<I>(&mut self, entries: I) -> Result<Vec<CaptureResponse>, StorageError<u64>>
|
||||
where
|
||||
I: IntoIterator<Item = CaptureRaftEntry> + OptionalSend,
|
||||
I::IntoIter: OptionalSend,
|
||||
{
|
||||
self.writer
|
||||
.apply_committed(entries.into_iter().collect())
|
||||
.await
|
||||
.map_err(failure)
|
||||
}
|
||||
async fn get_snapshot_builder(&mut self) -> Self {
|
||||
self.clone()
|
||||
}
|
||||
async fn begin_receiving_snapshot(&mut self) -> Result<Box<NativeSnapshotData>, StorageError<u64>> {
|
||||
let mut data = NativeSnapshotData::new(self.maximum()).map_err(failure)?;
|
||||
data.attach_receive_guard(ReceiveGuard::new(self.writer.snapshot_receives.clone()));
|
||||
Ok(Box::new(data))
|
||||
}
|
||||
async fn install_snapshot(&mut self, meta: &Meta, data: Box<NativeSnapshotData>) -> Result<(), StorageError<u64>> {
|
||||
crate::native_snapshot_install::install(self.writer.clone(), self.limits, meta.clone(), data)
|
||||
.await
|
||||
.map_err(failure)
|
||||
}
|
||||
async fn get_current_snapshot(&mut self) -> Result<Option<Snapshot<CaptureTypeConfig>>, StorageError<u64>> {
|
||||
self.current().await.map_err(failure)
|
||||
}
|
||||
}
|
||||
impl RaftSnapshotBuilder<CaptureTypeConfig> for NativeRaftStateMachine {
|
||||
async fn build_snapshot(&mut self) -> Result<Snapshot<CaptureTypeConfig>, StorageError<u64>> {
|
||||
#[cfg(test)]
|
||||
{
|
||||
let hook = self.build_before_ordering.lock().expect("build test hook").take();
|
||||
if let Some((entered, resume)) = hook {
|
||||
entered.notify_one();
|
||||
resume.notified().await;
|
||||
}
|
||||
}
|
||||
let limits = self.limits;
|
||||
let published_empty = self
|
||||
.writer
|
||||
.write(move |store| {
|
||||
let tx = store.write_transaction()?;
|
||||
let mut meta = tx.open_table(crate::META)?;
|
||||
// Decide the empty/nonempty path under the original writer ordering.
|
||||
if crate::application::read_applied(&meta)?.is_some() {
|
||||
return Ok(false);
|
||||
}
|
||||
let identity = crate::decode_identity(meta.get("identity")?.ok_or(StoreError::CorruptRecord)?.value())?;
|
||||
let membership =
|
||||
crate::CaptureMembership::decode(meta.get("native_membership")?.ok_or(StoreError::CorruptRecord)?.value())?;
|
||||
let previous = checkpoint_baseline::read_metadata(&meta)?;
|
||||
let generation = previous.map_or(Ok(1), |v| v.generation.checked_add(1).ok_or(StoreError::ResourceLimit))?;
|
||||
let baseline = checkpoint_baseline::BaselineMetadata {
|
||||
generation,
|
||||
header: snapshot::Header {
|
||||
cluster_id: identity.cluster_id,
|
||||
applied: None,
|
||||
membership,
|
||||
binding: crate::application::read_binding(&meta)?,
|
||||
},
|
||||
limits,
|
||||
};
|
||||
for (live, base) in [
|
||||
(crate::application::HEADS, checkpoint_baseline::HEADS),
|
||||
(crate::application::VERSIONS, checkpoint_baseline::VERSIONS),
|
||||
(crate::application::EVENTS, checkpoint_baseline::EVENTS),
|
||||
(crate::application::OPERATIONS, checkpoint_baseline::OPERATIONS),
|
||||
] {
|
||||
if tx.open_table(live)?.len()? != 0 {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
tx.open_table(base)?.retain(|_, _| false)?;
|
||||
}
|
||||
meta.insert("schema", 7u64.to_be_bytes().as_slice())?;
|
||||
meta.insert(checkpoint_baseline::META_KEY, checkpoint_baseline::encode_metadata(&baseline)?.as_slice())?;
|
||||
publish_local_meta(&mut meta, &baseline)?;
|
||||
drop(meta);
|
||||
store.commit(tx)?;
|
||||
Ok(true)
|
||||
})
|
||||
.await
|
||||
.map_err(failure)?;
|
||||
if !published_empty {
|
||||
self.writer.checkpoint_local(self.limits).await.map_err(failure)?;
|
||||
}
|
||||
self.current()
|
||||
.await
|
||||
.map_err(failure)?
|
||||
.ok_or_else(|| failure(StoreError::CorruptRecord))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) struct Waiter(Arc<ReceiveState>);
|
||||
#[cfg(test)]
|
||||
impl Waiter {
|
||||
pub(crate) fn new(state: Arc<ReceiveState>) -> Self {
|
||||
state.waiters.fetch_add(1, Ordering::AcqRel);
|
||||
Self(state)
|
||||
}
|
||||
}
|
||||
#[cfg(test)]
|
||||
impl Drop for Waiter {
|
||||
fn drop(&mut self) {
|
||||
self.0.waiters.fetch_sub(1, Ordering::AcqRel);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,583 @@
|
||||
//! Actual OpenRaft storage traits over real redb; no simulated state machine.
|
||||
use super::*;
|
||||
use openraft::{
|
||||
RaftLogReader, RaftSnapshotBuilder,
|
||||
storage::{RaftLogStorage, RaftLogStorageExt, RaftStateMachine},
|
||||
};
|
||||
use std::{
|
||||
collections::{BTreeMap, BTreeSet},
|
||||
io::SeekFrom,
|
||||
};
|
||||
use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt};
|
||||
|
||||
pub(super) type Snapshot = openraft::Snapshot<CaptureTypeConfig>;
|
||||
pub(super) fn identity(node_id: u64) -> NodeIdentity {
|
||||
NodeIdentity {
|
||||
cluster_id: [7; 16],
|
||||
node_id,
|
||||
}
|
||||
}
|
||||
pub(super) fn binding() -> CaptureBinding {
|
||||
CaptureBinding {
|
||||
revision: 9,
|
||||
binding_id: [8; 16],
|
||||
target_id: [9; 16],
|
||||
}
|
||||
}
|
||||
pub(super) fn id(index: u64) -> LogId {
|
||||
LogId {
|
||||
term: 8,
|
||||
leader_node: 0,
|
||||
index,
|
||||
}
|
||||
}
|
||||
pub(super) fn entry(index: u64) -> CaptureRaftEntry {
|
||||
let payload = if index == 0 {
|
||||
let membership = CaptureMembership::new(
|
||||
Some(id(0)),
|
||||
vec![BTreeSet::from([0])],
|
||||
BTreeMap::from([(0, CaptureNode::new("http://127.0.0.1:7000".into()).expect("member"))]),
|
||||
)
|
||||
.expect("native membership");
|
||||
let stored = openraft::StoredMembership::try_from(&membership).expect("stored membership");
|
||||
openraft::EntryPayload::Membership(stored.membership().clone())
|
||||
} else {
|
||||
let byte = u8::try_from(index).expect("bounded test event");
|
||||
openraft::EntryPayload::Normal(
|
||||
CaptureCommand::try_created(DecideCreated {
|
||||
operation_id: [byte; 16],
|
||||
object: ObjectIdentity {
|
||||
bucket_incarnation: [2; 16],
|
||||
key: vec![byte],
|
||||
},
|
||||
expected_head: None,
|
||||
prepared: PreparedIdentity {
|
||||
version_id: [byte; 16],
|
||||
preparation_id: [4; 16],
|
||||
content_digest: [5; 32],
|
||||
content_length: 123,
|
||||
},
|
||||
binding_revision: binding().revision,
|
||||
})
|
||||
.expect("native command"),
|
||||
)
|
||||
};
|
||||
CaptureRaftEntry {
|
||||
log_id: id(index).into(),
|
||||
payload,
|
||||
}
|
||||
}
|
||||
pub(super) fn entries(end: u64) -> Vec<CaptureRaftEntry> {
|
||||
(0..=end).map(entry).collect()
|
||||
}
|
||||
pub(super) struct Fixture {
|
||||
pub directory: tempfile::TempDir,
|
||||
pub writer: NativeLogWriter,
|
||||
pub logs: NativeRaftLogStore,
|
||||
pub sm: NativeRaftStateMachine,
|
||||
pub identity: NodeIdentity,
|
||||
}
|
||||
impl Fixture {
|
||||
pub fn new(node_id: u64) -> Self {
|
||||
Self::with_identity_binding(identity(node_id), binding())
|
||||
}
|
||||
pub fn with_identity_binding(identity: NodeIdentity, binding: CaptureBinding) -> Self {
|
||||
let directory = tempfile::tempdir().expect("real state machine directory");
|
||||
let mut store = Store::initialize(directory.path().join("db"), identity).expect("real redb initialization");
|
||||
store.pin_capture_binding(binding).expect("native binding");
|
||||
store.enable_native_empty().expect("native mode");
|
||||
store.promote_created_provenance().expect("provenance");
|
||||
let writer = NativeLogWriter::new(store);
|
||||
let logs = NativeRaftLogStore::new(writer.clone());
|
||||
let sm = NativeRaftStateMachine::new(writer.clone(), CheckpointLimits::default()).expect("real state machine adapter");
|
||||
Self {
|
||||
directory,
|
||||
writer,
|
||||
logs,
|
||||
sm,
|
||||
identity,
|
||||
}
|
||||
}
|
||||
pub async fn apply(&mut self, batch: Vec<CaptureRaftEntry>) -> Vec<CaptureResponse> {
|
||||
self.logs
|
||||
.blocking_append(batch.clone())
|
||||
.await
|
||||
.expect("actual trait append and library flush callback");
|
||||
self.sm.apply(batch).await.expect("actual state machine trait apply")
|
||||
}
|
||||
pub async fn build(&mut self) -> Snapshot {
|
||||
self.sm
|
||||
.get_snapshot_builder()
|
||||
.await
|
||||
.build_snapshot()
|
||||
.await
|
||||
.expect("actual durable snapshot builder")
|
||||
}
|
||||
pub async fn page(&self) -> EventPage {
|
||||
self.writer
|
||||
.created_events_page(None, 256)
|
||||
.await
|
||||
.expect("authoritative native events")
|
||||
}
|
||||
}
|
||||
pub(super) async fn snapshot_bytes(snapshot: &mut Snapshot) -> Vec<u8> {
|
||||
snapshot.snapshot.seek(SeekFrom::Start(0)).await.expect("snapshot rewind");
|
||||
let mut bytes = Vec::new();
|
||||
snapshot
|
||||
.snapshot
|
||||
.read_to_end(&mut bytes)
|
||||
.await
|
||||
.expect("actual snapshot bytes");
|
||||
bytes
|
||||
}
|
||||
pub(super) async fn receive(
|
||||
sm: &mut NativeRaftStateMachine,
|
||||
bytes: &[u8],
|
||||
) -> Box<<CaptureTypeConfig as openraft::RaftTypeConfig>::SnapshotData> {
|
||||
let mut data = sm.begin_receiving_snapshot().await.expect("actual bounded receive handle");
|
||||
data.write_all(bytes).await.expect("receive real snapshot bytes");
|
||||
data.seek(SeekFrom::Start(0)).await.expect("received snapshot rewind");
|
||||
data
|
||||
}
|
||||
pub(super) async fn install(sm: &mut NativeRaftStateMachine, snapshot: &mut Snapshot) {
|
||||
let bytes = snapshot_bytes(snapshot).await;
|
||||
let data = receive(sm, &bytes).await;
|
||||
sm.install_snapshot(&snapshot.meta, data)
|
||||
.await
|
||||
.expect("actual state machine trait install");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raft_state_machine_apply_restart() {
|
||||
let mut f = Fixture::new(0);
|
||||
assert_eq!(f.sm.applied_state().await.expect("empty applied state").0, None);
|
||||
assert!(f.sm.get_current_snapshot().await.expect("no published snapshot").is_none());
|
||||
let batch = entries(2);
|
||||
let responses = f.apply(batch.clone()).await;
|
||||
assert_eq!(responses.len(), 3, "RAFT_SM_REAL_RESPONSE_COUNT");
|
||||
assert_eq!(responses[0].kind(), CaptureResponseKind::Membership);
|
||||
assert!(matches!(responses[1].created_result(), Some(CreatedResult::Created { .. })));
|
||||
let state = f.sm.applied_state().await.expect("durable applied state");
|
||||
assert_eq!(state.0, Some(id(2).into()));
|
||||
assert_eq!(state.1.log_id(), &Some(id(0).into()));
|
||||
let before = f.page().await;
|
||||
let Fixture {
|
||||
directory,
|
||||
writer,
|
||||
logs,
|
||||
sm,
|
||||
identity,
|
||||
} = f;
|
||||
drop(sm);
|
||||
drop(logs);
|
||||
drop(writer);
|
||||
let writer = NativeLogWriter::new(Store::open(directory.path().join("db"), identity).expect("restart real store"));
|
||||
let mut sm = NativeRaftStateMachine::new(writer.clone(), CheckpointLimits::default()).expect("reopened state machine");
|
||||
assert_eq!(
|
||||
sm.applied_state().await.expect("reopened applied state"),
|
||||
state,
|
||||
"RAFT_SM_APPLY_ACK_DURABLE"
|
||||
);
|
||||
assert_eq!(writer.created_events_page(None, 256).await.expect("reopened events"), before);
|
||||
assert_eq!(
|
||||
sm.apply(batch).await.expect("exact replay responses"),
|
||||
responses,
|
||||
"RAFT_SM_REPLAY_SAME_RESPONSES"
|
||||
);
|
||||
assert_eq!(
|
||||
writer
|
||||
.created_events_page(None, 256)
|
||||
.await
|
||||
.expect("no duplicate replay events"),
|
||||
before
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raft_state_machine_snapshot_build_get_restart() {
|
||||
let mut f = Fixture::new(0);
|
||||
f.apply(entries(2)).await;
|
||||
let mut built = f.build().await;
|
||||
let meta = built.meta.clone();
|
||||
let bytes = snapshot_bytes(&mut built).await;
|
||||
assert_eq!(meta.last_log_id, Some(id(2).into()), "RAFT_SM_SNAPSHOT_META_CUT");
|
||||
assert_eq!(meta.last_membership, f.sm.applied_state().await.expect("membership").1);
|
||||
f.apply(vec![entry(3)]).await;
|
||||
let mut current =
|
||||
f.sm.get_current_snapshot()
|
||||
.await
|
||||
.expect("persisted snapshot")
|
||||
.expect("snapshot exists");
|
||||
assert_eq!(current.meta, meta, "RAFT_SM_LIVE_APPLY_DOES_NOT_REPIN_SNAPSHOT");
|
||||
assert_eq!(snapshot_bytes(&mut current).await, bytes, "RAFT_SM_SNAPSHOT_FROZEN_BYTES");
|
||||
let Fixture {
|
||||
directory,
|
||||
writer,
|
||||
logs,
|
||||
sm,
|
||||
identity,
|
||||
} = f;
|
||||
drop(current);
|
||||
drop(built);
|
||||
drop(sm);
|
||||
drop(logs);
|
||||
drop(writer);
|
||||
let writer = NativeLogWriter::new(Store::open(directory.path().join("db"), identity).expect("restart snapshot owner"));
|
||||
let mut sm = NativeRaftStateMachine::new(writer, CheckpointLimits::default()).expect("reopen adapter");
|
||||
let mut current = sm
|
||||
.get_current_snapshot()
|
||||
.await
|
||||
.expect("restarted current snapshot")
|
||||
.expect("durable snapshot exists");
|
||||
assert_eq!(current.meta, meta, "RAFT_SM_COMPLETE_META_RECONSTRUCTED");
|
||||
assert_eq!(snapshot_bytes(&mut current).await, bytes, "RAFT_SM_RESTART_SNAPSHOT_BYTES");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raft_state_machine_receive_bounds() {
|
||||
let f = Fixture::new(0);
|
||||
let mut limits = CheckpointLimits::default();
|
||||
limits.snapshot.total_bytes = 1024;
|
||||
let mut sm = NativeRaftStateMachine::new(f.writer.clone(), limits).expect("bounded state machine");
|
||||
let mut data = sm.begin_receiving_snapshot().await.expect("bounded receive");
|
||||
data.write_all(&[1; 16]).await.expect("small write");
|
||||
assert!(data.seek(SeekFrom::Start(u64::MAX)).await.is_err(), "RAFT_SM_RECEIVE_SEEK_BOUND");
|
||||
data.seek(SeekFrom::Start(0)).await.expect("restore seek");
|
||||
let mut bytes = [0; 16];
|
||||
data.read_exact(&mut bytes).await.expect("small received bytes");
|
||||
assert_eq!(bytes, [1; 16], "RAFT_SM_BAD_SEEK_PRESERVES_BYTES");
|
||||
assert!(data.write_all(&[2; 2048]).await.is_err(), "RAFT_SM_RECEIVE_WRITE_BOUND_BEFORE_INSTALL");
|
||||
assert_eq!(sm.applied_state().await.expect("receive is not install").0, None);
|
||||
assert!(sm.get_current_snapshot().await.expect("no received publication").is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raft_state_machine_install_rejects_meta_identity() {
|
||||
let mut source = Fixture::new(0);
|
||||
source.apply(entries(2)).await;
|
||||
let mut snapshot = source.build().await;
|
||||
let bytes = snapshot_bytes(&mut snapshot).await;
|
||||
let mut target = Fixture::new(1);
|
||||
target.apply(entries(1)).await;
|
||||
let old_page = target.page().await;
|
||||
let stable = std::fs::read(target.directory.path().join("db")).expect("nonempty target bytes");
|
||||
let mut wrong = snapshot.meta.clone();
|
||||
wrong.last_log_id = Some(LogId { term: 9, ..id(2) }.into());
|
||||
let data = receive(&mut target.sm, &bytes).await;
|
||||
assert!(
|
||||
target.sm.install_snapshot(&wrong, data).await.is_err(),
|
||||
"SNAPSHOT_META_BOUND_TO_FRAME: full LogId term cannot differ from snapshot header"
|
||||
);
|
||||
assert_eq!(target.sm.applied_state().await.expect("rejection state").0, Some(id(1).into()));
|
||||
assert_eq!(target.page().await, old_page, "RAFT_SM_META_REJECTION_RETAINS_EVENTS");
|
||||
assert_eq!(
|
||||
std::fs::read(target.directory.path().join("db")).expect("unchanged nonempty target"),
|
||||
stable,
|
||||
"RAFT_SM_META_REJECTION_NO_MUTATION"
|
||||
);
|
||||
assert!(target.sm.get_current_snapshot().await.expect("rejection snapshot").is_none());
|
||||
let mut wrong = snapshot.meta.clone();
|
||||
wrong.last_membership = openraft::StoredMembership::default();
|
||||
let data = receive(&mut target.sm, &bytes).await;
|
||||
assert!(
|
||||
target.sm.install_snapshot(&wrong, data).await.is_err(),
|
||||
"RAFT_SM_MEMBERSHIP_BOUND_TO_FRAME"
|
||||
);
|
||||
let mut foreign = Fixture::with_identity_binding(
|
||||
NodeIdentity {
|
||||
cluster_id: [6; 16],
|
||||
..identity(1)
|
||||
},
|
||||
binding(),
|
||||
);
|
||||
let data = receive(&mut foreign.sm, &bytes).await;
|
||||
assert!(
|
||||
foreign.sm.install_snapshot(&snapshot.meta, data).await.is_err(),
|
||||
"RAFT_SM_CLUSTER_BOUND_TO_DURABLE_OWNER"
|
||||
);
|
||||
let mut foreign = Fixture::with_identity_binding(
|
||||
identity(1),
|
||||
CaptureBinding {
|
||||
target_id: [5; 16],
|
||||
..binding()
|
||||
},
|
||||
);
|
||||
let data = receive(&mut foreign.sm, &bytes).await;
|
||||
assert!(
|
||||
foreign.sm.install_snapshot(&snapshot.meta, data).await.is_err(),
|
||||
"RAFT_SM_CAPTURE_BINDING_BOUND"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raft_state_machine_install_preserves_local_identity_vote() {
|
||||
let mut source = Fixture::new(0);
|
||||
source.apply(entries(2)).await;
|
||||
let mut snapshot = source.build().await;
|
||||
snapshot.meta.snapshot_id = "foreign-snapshot-id-preserved-verbatim".into();
|
||||
let mut target = Fixture::new(1);
|
||||
let vote = openraft::Vote::new_committed(10, 1);
|
||||
target.logs.save_vote(&vote).await.expect("local election vote");
|
||||
install(&mut target.sm, &mut snapshot).await;
|
||||
assert_eq!(
|
||||
target.writer.durable_identity().await.expect("local identity"),
|
||||
identity(1),
|
||||
"RAFT_SM_INSTALL_PRESERVES_LOCAL_NODE"
|
||||
);
|
||||
assert_eq!(
|
||||
target.logs.read_vote().await.expect("local vote"),
|
||||
Some(vote),
|
||||
"RAFT_SM_INSTALL_PRESERVES_LOCAL_VOTE"
|
||||
);
|
||||
let mut current = target
|
||||
.sm
|
||||
.get_current_snapshot()
|
||||
.await
|
||||
.expect("installed snapshot")
|
||||
.expect("present");
|
||||
assert_eq!(current.meta, snapshot.meta, "RAFT_SM_INSTALL_PRESERVES_COMPLETE_INCOMING_META");
|
||||
assert_eq!(snapshot_bytes(&mut current).await, snapshot_bytes(&mut snapshot).await);
|
||||
let Fixture {
|
||||
directory,
|
||||
writer,
|
||||
logs,
|
||||
sm,
|
||||
identity,
|
||||
} = target;
|
||||
drop(current);
|
||||
drop(sm);
|
||||
drop(logs);
|
||||
drop(writer);
|
||||
let writer = NativeLogWriter::new(Store::open(directory.path().join("db"), identity).expect("restart installed target"));
|
||||
let mut logs = NativeRaftLogStore::new(writer.clone());
|
||||
let mut sm = NativeRaftStateMachine::new(writer.clone(), CheckpointLimits::default()).expect("reopened installed SM");
|
||||
assert_eq!(writer.durable_identity().await.expect("restarted local identity"), identity);
|
||||
assert_eq!(logs.read_vote().await.expect("restarted vote"), Some(vote));
|
||||
assert_eq!(
|
||||
sm.get_current_snapshot()
|
||||
.await
|
||||
.expect("restarted snapshot")
|
||||
.expect("present")
|
||||
.meta,
|
||||
snapshot.meta,
|
||||
"RAFT_SM_INCOMING_SNAPSHOT_ID_DURABLE"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raft_state_machine_install_suffix() {
|
||||
let mut source = Fixture::new(0);
|
||||
source.apply(entries(2)).await;
|
||||
let mut snapshot = source.build().await;
|
||||
let mut target = Fixture::new(1);
|
||||
target
|
||||
.logs
|
||||
.blocking_append(entries(4))
|
||||
.await
|
||||
.expect("retained local log suffix");
|
||||
target.sm.apply(entries(1)).await.expect("old local applied prefix");
|
||||
target
|
||||
.logs
|
||||
.save_committed(Some(id(4).into()))
|
||||
.await
|
||||
.expect("known committed compatible suffix");
|
||||
install(&mut target.sm, &mut snapshot).await;
|
||||
assert_eq!(
|
||||
target
|
||||
.logs
|
||||
.get_log_reader()
|
||||
.await
|
||||
.try_get_log_entries(3..)
|
||||
.await
|
||||
.expect("compatible suffix"),
|
||||
vec![entry(3), entry(4)],
|
||||
"RAFT_SM_INSTALL_PRESERVES_COMPATIBLE_COMMITTED_SUFFIX"
|
||||
);
|
||||
assert_eq!(
|
||||
target.logs.read_committed().await.expect("preserved committed marker"),
|
||||
Some(id(4).into())
|
||||
);
|
||||
target
|
||||
.sm
|
||||
.apply(vec![entry(3), entry(4)])
|
||||
.await
|
||||
.expect("apply retained committed suffix");
|
||||
assert_eq!(target.page().await.events.len(), 4);
|
||||
let mut conflict = Fixture::new(2);
|
||||
let mut batch = entries(4);
|
||||
for entry in &mut batch[2..] {
|
||||
entry.log_id = LogId {
|
||||
term: 9,
|
||||
..id(entry.log_id.index)
|
||||
}
|
||||
.into();
|
||||
}
|
||||
conflict
|
||||
.logs
|
||||
.blocking_append(batch)
|
||||
.await
|
||||
.expect("conflicting uncommitted suffix");
|
||||
conflict.sm.apply(entries(1)).await.expect("protected applied prefix");
|
||||
install(&mut conflict.sm, &mut snapshot).await;
|
||||
assert!(
|
||||
conflict
|
||||
.logs
|
||||
.get_log_reader()
|
||||
.await
|
||||
.try_get_log_entries(3..)
|
||||
.await
|
||||
.expect("discarded conflict")
|
||||
.is_empty(),
|
||||
"RAFT_SM_INSTALL_DROPS_ONLY_UNCOMMITTED_CONFLICT"
|
||||
);
|
||||
conflict.apply(vec![entry(3)]).await;
|
||||
assert_eq!(conflict.sm.applied_state().await.expect("new suffix applied").0, Some(id(3).into()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raft_state_machine_install_restart_continue() {
|
||||
let mut source = Fixture::new(0);
|
||||
source.apply(entries(2)).await;
|
||||
let expected = source.page().await;
|
||||
let mut snapshot = source.build().await;
|
||||
let mut target = Fixture::new(1);
|
||||
install(&mut target.sm, &mut snapshot).await;
|
||||
assert_eq!(target.page().await, expected, "RAFT_SM_INSTALL_REBUILDS_SOURCE_EVENTS");
|
||||
let Fixture {
|
||||
directory,
|
||||
writer,
|
||||
logs,
|
||||
sm,
|
||||
identity,
|
||||
} = target;
|
||||
drop(sm);
|
||||
drop(logs);
|
||||
drop(writer);
|
||||
let writer = NativeLogWriter::new(Store::open(directory.path().join("db"), identity).expect("restart installed redb"));
|
||||
let mut logs = NativeRaftLogStore::new(writer.clone());
|
||||
let mut sm = NativeRaftStateMachine::new(writer.clone(), CheckpointLimits::default()).expect("restart installed adapter");
|
||||
assert_eq!(sm.applied_state().await.expect("snapshot applied state").0, Some(id(2).into()));
|
||||
logs.blocking_append(vec![entry(3)])
|
||||
.await
|
||||
.expect("append after install restart");
|
||||
sm.apply(vec![entry(3)]).await.expect("apply after install restart");
|
||||
let all = writer.created_events_page(None, 256).await.expect("retained plus new event");
|
||||
assert_eq!(all.events.len(), 3);
|
||||
assert_eq!(all.events[..2], expected.events);
|
||||
let next = writer
|
||||
.created_events_page(expected.next, 1)
|
||||
.await
|
||||
.expect("snapshot cursor resumes");
|
||||
assert_eq!(next.events[0].event_id, [3; 16], "RAFT_SM_INSTALL_CURSOR_CONTINUITY");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raft_state_machine_empty_snapshot_restart_continue() {
|
||||
let mut source = Fixture::new(0);
|
||||
let mut target = Fixture::new(42);
|
||||
let mut snapshot = source.sm.build_snapshot().await.expect("durable empty baseline");
|
||||
assert_eq!(snapshot.meta.last_log_id, None);
|
||||
let empty_meta = snapshot.meta.clone();
|
||||
let received = receive(&mut target.sm, &snapshot_bytes(&mut snapshot).await).await;
|
||||
target
|
||||
.sm
|
||||
.install_snapshot(&empty_meta, received)
|
||||
.await
|
||||
.expect("install empty frame");
|
||||
let mut target = target.reopen();
|
||||
let current = target
|
||||
.sm
|
||||
.get_current_snapshot()
|
||||
.await
|
||||
.expect("empty after reopen")
|
||||
.expect("durable selection");
|
||||
assert_eq!(current.meta, empty_meta);
|
||||
target
|
||||
.logs
|
||||
.blocking_append(entries(1))
|
||||
.await
|
||||
.expect("real append after empty install");
|
||||
target.sm.apply(entries(1)).await.expect("real apply after empty install");
|
||||
let mut target = target.reopen();
|
||||
assert_eq!(
|
||||
target.sm.applied_state().await.expect("replay empty baseline + suffix").0,
|
||||
Some(id(1).into())
|
||||
);
|
||||
assert_eq!(
|
||||
target
|
||||
.writer
|
||||
.created_events_page(None, 16)
|
||||
.await
|
||||
.expect("event after empty baseline")
|
||||
.events
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
let refreshed = target
|
||||
.sm
|
||||
.build_snapshot()
|
||||
.await
|
||||
.expect("nonempty checkpoint supersedes empty");
|
||||
assert_eq!(refreshed.meta.last_log_id, Some(id(1).into()));
|
||||
}
|
||||
|
||||
impl Fixture {
|
||||
fn reopen(self) -> Self {
|
||||
let Self {
|
||||
directory,
|
||||
writer,
|
||||
logs,
|
||||
sm,
|
||||
identity,
|
||||
} = self;
|
||||
drop(sm);
|
||||
drop(logs);
|
||||
drop(writer);
|
||||
let writer =
|
||||
NativeLogWriter::new(Store::open(directory.path().join("db"), identity).expect("real empty baseline restart"));
|
||||
let logs = NativeRaftLogStore::new(writer.clone());
|
||||
let sm =
|
||||
NativeRaftStateMachine::new(writer.clone(), CheckpointLimits::default()).expect("reopened empty baseline adapter");
|
||||
Self {
|
||||
directory,
|
||||
writer,
|
||||
logs,
|
||||
sm,
|
||||
identity,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raft_state_machine_build_competes_with_initial_apply() {
|
||||
let mut target = Fixture::new(0);
|
||||
target
|
||||
.logs
|
||||
.blocking_append(entries(1))
|
||||
.await
|
||||
.expect("real pending initial batch");
|
||||
let entered = std::sync::Arc::new(tokio::sync::Notify::new());
|
||||
let resume = std::sync::Arc::new(tokio::sync::Notify::new());
|
||||
*target.sm.build_before_ordering.lock().expect("build pause") = Some((entered.clone(), resume.clone()));
|
||||
let mut builder = target.sm.get_snapshot_builder().await;
|
||||
let task = tokio::spawn(async move { builder.build_snapshot().await });
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), entered.notified())
|
||||
.await
|
||||
.expect("builder began before initial apply");
|
||||
target
|
||||
.sm
|
||||
.apply(entries(1))
|
||||
.await
|
||||
.expect("actual initial apply while builder pending");
|
||||
resume.notify_one();
|
||||
let snapshot = tokio::time::timeout(std::time::Duration::from_secs(5), task)
|
||||
.await
|
||||
.expect("build continues")
|
||||
.expect("builder task")
|
||||
.expect("initial apply does not make snapshot build a fatal storage error");
|
||||
assert_eq!(
|
||||
snapshot.meta.last_log_id,
|
||||
Some(id(1).into()),
|
||||
"SNAPSHOT_BUILD_INITIAL_APPLY_RACE_USES_NONEMPTY_CHECKPOINT"
|
||||
);
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,559 @@
|
||||
//! Disposable event ordering index. EVENTS remains the sole event authority.
|
||||
use crate::{
|
||||
CaptureBinding, CreatedEvent, CreatedResult, LogId, NativeLogWriter, NativeWriteError, Store, StoreError, application,
|
||||
decode_id, encode_id,
|
||||
};
|
||||
use redb::{ReadableDatabase, ReadableTable, ReadableTableMetadata, TableDefinition, WriteTransaction};
|
||||
|
||||
pub const MAX_EVENT_PAGE: usize = 256;
|
||||
pub(crate) const EVENT_SEQUENCE: TableDefinition<u64, &[u8]> = TableDefinition::new("event_sequence_cache_v1");
|
||||
|
||||
/// Persist only after every event preceding this cursor has a durable consumer ACK.
|
||||
/// Deserialization does not grant authority: each use checks the stored event.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct EventCursor {
|
||||
cluster: [u8; 16],
|
||||
term: u64,
|
||||
leader: u64,
|
||||
index: u64,
|
||||
event: [u8; 16],
|
||||
binding_id: [u8; 16],
|
||||
target_id: [u8; 16],
|
||||
revision: u64,
|
||||
}
|
||||
impl EventCursor {
|
||||
pub(crate) fn new(cluster: [u8; 16], event: [u8; 16], value: &CreatedEvent) -> Self {
|
||||
Self {
|
||||
cluster,
|
||||
term: value.decision_log_id.term,
|
||||
leader: value.decision_log_id.leader_node,
|
||||
index: value.decision_log_id.index,
|
||||
event,
|
||||
binding_id: value.binding.binding_id,
|
||||
target_id: value.binding.target_id,
|
||||
revision: value.binding.revision,
|
||||
}
|
||||
}
|
||||
fn matches(&self, cluster: [u8; 16], event: [u8; 16], value: &CreatedEvent) -> bool {
|
||||
self == &Self::new(cluster, event, value)
|
||||
}
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct SourceEvent {
|
||||
pub event_id: [u8; 16],
|
||||
pub captured: CreatedEvent,
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct EventPage {
|
||||
pub events: Vec<SourceEvent>,
|
||||
pub next: Option<EventCursor>,
|
||||
}
|
||||
fn encoded_index(id: LogId, event: [u8; 16]) -> Vec<u8> {
|
||||
let mut bytes = encode_id(id).to_vec();
|
||||
bytes.extend(event);
|
||||
bytes
|
||||
}
|
||||
fn decode_index(bytes: &[u8]) -> Result<(LogId, [u8; 16]), StoreError> {
|
||||
if bytes.len() != 40 {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let id = decode_id(&bytes[..24])?;
|
||||
let event: [u8; 16] = bytes[24..].try_into().map_err(|_| StoreError::CorruptRecord)?;
|
||||
if event == [0; 16] {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
Ok((id, event))
|
||||
}
|
||||
pub(crate) fn insert_event_index(tx: &WriteTransaction, id: LogId, event: [u8; 16]) -> Result<(), StoreError> {
|
||||
let bytes = encoded_index(id, event);
|
||||
let mut index = tx.open_table(EVENT_SEQUENCE)?;
|
||||
if index.get(id.index)?.is_some() {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
index.insert(id.index, bytes.as_slice())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Check the event semantics required to derive ordering from checkpoint authority.
|
||||
/// This deliberately does not grant snapshot installation or source authority.
|
||||
fn validated_event(
|
||||
key: &[u8],
|
||||
bytes: &[u8],
|
||||
operations: &impl ReadableTable<&'static [u8], &'static [u8]>,
|
||||
versions: &impl ReadableTable<&'static [u8], &'static [u8]>,
|
||||
schema: u64,
|
||||
binding: &CaptureBinding,
|
||||
applied: Option<LogId>,
|
||||
) -> Result<([u8; 16], CreatedEvent), StoreError> {
|
||||
let event_id: [u8; 16] = key.try_into().map_err(|_| StoreError::CorruptRecord)?;
|
||||
let event = application::decode_event(bytes)?;
|
||||
let cut = applied.ok_or(StoreError::CorruptRecord)?;
|
||||
let operation = operations.get(key)?.ok_or(StoreError::CorruptRecord)?;
|
||||
let first_decision = if matches!(schema, 5 | 6 | 7) {
|
||||
Some(application::decode_provenance(operation.value())?.first_decision_log_id)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let operation = application::decode_stored_operation(operation.value(), schema)?;
|
||||
let command = crate::DecideCreated::decode(&operation.canonical_command)?;
|
||||
let version_key = application::version_key(&event.object, event.prepared.version_id)?;
|
||||
let version = versions.get(version_key.as_slice())?.ok_or(StoreError::CorruptRecord)?;
|
||||
let version = application::decode_version(version.value())?;
|
||||
if first_decision.is_some_and(|first| first != event.decision_log_id)
|
||||
|| event_id == [0; 16]
|
||||
|| event.decision_log_id.index > cut.index
|
||||
|| (event.decision_log_id.index == cut.index && event.decision_log_id != cut)
|
||||
|| event.binding != *binding
|
||||
|| command.operation_id != event_id
|
||||
|| command.object != event.object
|
||||
|| command.prepared != event.prepared
|
||||
|| command.binding_revision != event.binding.revision
|
||||
|| operation.result
|
||||
!= (CreatedResult::Created {
|
||||
version_id: event.prepared.version_id,
|
||||
event_id,
|
||||
decision_log_id: event.decision_log_id,
|
||||
})
|
||||
|| version
|
||||
!= (crate::CreatedVersion {
|
||||
prepared: event.prepared.clone(),
|
||||
decision_log_id: event.decision_log_id,
|
||||
binding_revision: event.binding.revision,
|
||||
})
|
||||
{
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
Ok((event_id, event))
|
||||
}
|
||||
|
||||
/// Rebuild from the same authority rows emitted by the existing four-section checkpoint.
|
||||
/// Callers must independently establish authority and validate all other state before promotion.
|
||||
pub(crate) fn rebuild_from_authority(
|
||||
tx: &WriteTransaction,
|
||||
schema: u64,
|
||||
binding: &CaptureBinding,
|
||||
applied: Option<LogId>,
|
||||
) -> Result<(), StoreError> {
|
||||
let events = tx.open_table(application::EVENTS)?;
|
||||
let operations = tx.open_table(application::OPERATIONS)?;
|
||||
let versions = tx.open_table(application::VERSIONS)?;
|
||||
{
|
||||
let mut index = tx.open_table(EVENT_SEQUENCE)?;
|
||||
index.retain(|_, _| false)?;
|
||||
}
|
||||
for row in events.iter()? {
|
||||
let (key, bytes) = row?;
|
||||
let (event, value) = validated_event(key.value(), bytes.value(), &operations, &versions, schema, binding, applied)?;
|
||||
insert_event_index(tx, value.decision_log_id, event)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
impl Store {
|
||||
pub(crate) fn ensure_event_index(&mut self) -> Result<(), StoreError> {
|
||||
let read = self.database.begin_read()?;
|
||||
let meta = read.open_table(crate::META)?;
|
||||
let schema = crate::number(meta.get("schema")?.ok_or(StoreError::CorruptRecord)?.value())?;
|
||||
if !matches!(schema, 3..=7) {
|
||||
return Ok(());
|
||||
}
|
||||
let binding = application::read_binding(&meta)?;
|
||||
let applied = application::read_applied(&meta)?;
|
||||
let events = read.open_table(application::EVENTS)?;
|
||||
let complete = match read.open_table(EVENT_SEQUENCE) {
|
||||
Ok(index) => {
|
||||
let mut complete = index.len()? == events.len()?;
|
||||
if complete {
|
||||
for row in events.iter()? {
|
||||
let (key, bytes) = row?;
|
||||
let value = application::decode_event(bytes.value())?;
|
||||
let expected =
|
||||
encoded_index(value.decision_log_id, key.value().try_into().map_err(|_| StoreError::CorruptRecord)?);
|
||||
if index
|
||||
.get(value.decision_log_id.index)?
|
||||
.is_none_or(|stored| stored.value() != expected.as_slice())
|
||||
{
|
||||
complete = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
complete
|
||||
}
|
||||
Err(redb::TableError::TableDoesNotExist(_)) => false,
|
||||
Err(error) => return Err(error.into()),
|
||||
};
|
||||
drop(events);
|
||||
drop(meta);
|
||||
drop(read);
|
||||
if !complete {
|
||||
let tx = self.write_transaction()?;
|
||||
rebuild_from_authority(&tx, schema, &binding, applied)?;
|
||||
self.commit(tx)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
/// Read a bounded page without changing any delivery ACK or cursor state.
|
||||
pub fn created_events_page(&self, cursor: Option<&EventCursor>, limit: usize) -> Result<EventPage, StoreError> {
|
||||
if !self.writable {
|
||||
return Err(StoreError::WriteDisabled);
|
||||
}
|
||||
if limit == 0 || limit > MAX_EVENT_PAGE {
|
||||
return Err(StoreError::InvalidEventPage);
|
||||
}
|
||||
let tx = self.database.begin_read()?;
|
||||
let meta = tx.open_table(crate::META)?;
|
||||
let schema = crate::number(meta.get("schema")?.ok_or(StoreError::CorruptRecord)?.value())?;
|
||||
if !matches!(schema, 3..=7) {
|
||||
return Err(StoreError::UnsupportedSchema);
|
||||
}
|
||||
let identity = crate::decode_identity(meta.get("identity")?.ok_or(StoreError::CorruptRecord)?.value())?;
|
||||
let events = tx.open_table(application::EVENTS)?;
|
||||
let index = tx.open_table(EVENT_SEQUENCE)?;
|
||||
if index.len()? != events.len()? {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let read_event = |id: u64, bytes: &[u8]| -> Result<SourceEvent, StoreError> {
|
||||
let (log, event_id) = decode_index(bytes)?;
|
||||
let value = events.get(event_id.as_slice())?.ok_or(StoreError::CorruptRecord)?;
|
||||
let captured = application::decode_event(value.value())?;
|
||||
if log.index != id || captured.decision_log_id != log {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
Ok(SourceEvent { event_id, captured })
|
||||
};
|
||||
if let Some(cursor) = cursor {
|
||||
let stored = index.get(cursor.index)?.ok_or(StoreError::IdentityMismatch)?;
|
||||
let event = read_event(cursor.index, stored.value())?;
|
||||
if !cursor.matches(identity.cluster_id, event.event_id, &event.captured) {
|
||||
return Err(StoreError::IdentityMismatch);
|
||||
}
|
||||
}
|
||||
let start = cursor.map_or(0, |c| c.index);
|
||||
let range = if cursor.is_some() {
|
||||
(std::ops::Bound::Excluded(start), std::ops::Bound::Unbounded)
|
||||
} else {
|
||||
(std::ops::Bound::Included(start), std::ops::Bound::Unbounded)
|
||||
};
|
||||
let mut page = EventPage {
|
||||
events: Vec::with_capacity(limit),
|
||||
next: cursor.cloned(),
|
||||
};
|
||||
for row in index.range(range)?.take(limit) {
|
||||
let (id, bytes) = row?;
|
||||
let event = read_event(id.value(), bytes.value())?;
|
||||
page.next = Some(EventCursor::new(identity.cluster_id, event.event_id, &event.captured));
|
||||
page.events.push(event);
|
||||
}
|
||||
Ok(page)
|
||||
}
|
||||
}
|
||||
impl NativeLogWriter {
|
||||
pub async fn created_events_page(&self, cursor: Option<EventCursor>, limit: usize) -> Result<EventPage, NativeWriteError> {
|
||||
let owner = self.store.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
owner
|
||||
.lock()
|
||||
.map_err(|_| NativeWriteError::Poisoned)?
|
||||
.created_events_page(cursor.as_ref(), limit)
|
||||
.map_err(NativeWriteError::Store)
|
||||
})
|
||||
.await
|
||||
.map_err(NativeWriteError::Worker)?
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::{CaptureCommand, CaptureRaftEntry, DecideCreated, NodeIdentity, ObjectIdentity, PreparedIdentity, snapshot};
|
||||
use std::io;
|
||||
fn identity() -> NodeIdentity {
|
||||
NodeIdentity {
|
||||
cluster_id: [7; 16],
|
||||
node_id: 0,
|
||||
}
|
||||
}
|
||||
fn binding() -> CaptureBinding {
|
||||
CaptureBinding {
|
||||
revision: 9,
|
||||
binding_id: [8; 16],
|
||||
target_id: [9; 16],
|
||||
}
|
||||
}
|
||||
fn id(index: u64) -> LogId {
|
||||
LogId {
|
||||
term: 8,
|
||||
leader_node: 0,
|
||||
index,
|
||||
}
|
||||
}
|
||||
fn command(event: u8) -> DecideCreated {
|
||||
DecideCreated {
|
||||
operation_id: [event; 16],
|
||||
object: ObjectIdentity {
|
||||
bucket_incarnation: [2; 16],
|
||||
key: vec![event],
|
||||
},
|
||||
expected_head: None,
|
||||
prepared: PreparedIdentity {
|
||||
version_id: [event; 16],
|
||||
preparation_id: [4; 16],
|
||||
content_digest: [5; 32],
|
||||
content_length: 123,
|
||||
},
|
||||
binding_revision: 9,
|
||||
}
|
||||
}
|
||||
fn apply(store: &mut Store, index: u64, command: DecideCreated) -> CreatedResult {
|
||||
store
|
||||
.append_native(&[CaptureRaftEntry {
|
||||
log_id: id(index).into(),
|
||||
payload: openraft::EntryPayload::Normal(CaptureCommand::try_created(command).expect("command")),
|
||||
}])
|
||||
.expect("append native");
|
||||
store.save_committed(id(index)).expect("commit native");
|
||||
store
|
||||
.apply_next_native(id(index))
|
||||
.expect("apply native")
|
||||
.created_result()
|
||||
.expect("created response")
|
||||
.clone()
|
||||
}
|
||||
fn fixture() -> (tempfile::TempDir, Store) {
|
||||
let dir = tempfile::tempdir().expect("directory");
|
||||
let mut store = Store::initialize(dir.path().join("source.redb"), identity()).expect("initialize");
|
||||
store.pin_capture_binding(binding()).expect("pin");
|
||||
store.enable_native_empty().expect("native");
|
||||
apply(&mut store, 0, command(9));
|
||||
apply(&mut store, 1, command(7));
|
||||
(dir, store)
|
||||
}
|
||||
#[test]
|
||||
fn committed_events_page_in_decision_order() {
|
||||
let (_dir, store) = fixture();
|
||||
let first = store.created_events_page(None, 1).expect("first");
|
||||
assert_eq!(
|
||||
first.events.iter().map(|e| e.event_id).collect::<Vec<_>>(),
|
||||
vec![[9; 16]],
|
||||
"SOURCE_PAGE_ORDER_BOUND"
|
||||
);
|
||||
let next = store.created_events_page(first.next.as_ref(), 1).expect("second");
|
||||
assert_eq!(next.events[0].event_id, [7; 16]);
|
||||
assert_eq!(next.events[0].captured.binding, binding());
|
||||
assert!(
|
||||
store
|
||||
.created_events_page(next.next.as_ref(), 1)
|
||||
.expect("end")
|
||||
.events
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn replay_and_rejections_do_not_duplicate_events() {
|
||||
let (_dir, mut store) = fixture();
|
||||
apply(&mut store, 2, command(9));
|
||||
let mut rejected = command(3);
|
||||
rejected.object = command(9).object;
|
||||
assert!(matches!(apply(&mut store, 3, rejected), CreatedResult::HeadMismatch { .. }));
|
||||
let page = store.created_events_page(None, 256).expect("page");
|
||||
assert_eq!(page.events.len(), 2, "SOURCE_PAGE_REPLAY_BOUND");
|
||||
}
|
||||
#[test]
|
||||
fn reopen_rebuilds_missing_or_stale_index() {
|
||||
let (dir, mut store) = fixture();
|
||||
apply(&mut store, 2, command(1));
|
||||
let expected = store.created_events_page(None, 256).expect("before");
|
||||
let tx = store.write_transaction().expect("remove cache");
|
||||
tx.delete_table(EVENT_SEQUENCE).expect("delete cache");
|
||||
store.commit(tx).expect("cache deletion");
|
||||
drop(store);
|
||||
let mut store = Store::open(dir.path().join("source.redb"), identity()).expect("rebuild missing");
|
||||
assert_eq!(store.created_events_page(None, 256).expect("rebuilt"), expected);
|
||||
let tx = store.write_transaction().expect("stale cache");
|
||||
{
|
||||
let mut index = tx.open_table(EVENT_SEQUENCE).expect("cache");
|
||||
index.remove(1).expect("omit middle item");
|
||||
index.insert(200, b"bad".as_slice()).expect("same-count stale item");
|
||||
}
|
||||
store.commit(tx).expect("cache damage");
|
||||
drop(store);
|
||||
let store = Store::open(dir.path().join("source.redb"), identity()).expect("rebuild stale");
|
||||
assert_eq!(store.created_events_page(None, 256).expect("rebuilt stale"), expected);
|
||||
}
|
||||
#[test]
|
||||
fn cursor_binds_cluster_term_leader_and_event() {
|
||||
let (_dir, store) = fixture();
|
||||
let cursor = store.created_events_page(None, 1).expect("page").next.expect("cursor");
|
||||
let mut bad = Vec::new();
|
||||
let mut c = cursor.clone();
|
||||
c.cluster = [6; 16];
|
||||
bad.push(c);
|
||||
let mut c = cursor.clone();
|
||||
c.term += 1;
|
||||
bad.push(c);
|
||||
let mut c = cursor.clone();
|
||||
c.leader += 1;
|
||||
bad.push(c);
|
||||
let mut c = cursor.clone();
|
||||
c.event = [7; 16];
|
||||
bad.push(c);
|
||||
let mut c = cursor.clone();
|
||||
c.revision += 1;
|
||||
bad.push(c);
|
||||
let mut c = cursor.clone();
|
||||
c.binding_id = [6; 16];
|
||||
bad.push(c);
|
||||
let mut c = cursor.clone();
|
||||
c.target_id = [6; 16];
|
||||
bad.push(c);
|
||||
for cursor in bad {
|
||||
assert_eq!(
|
||||
store.created_events_page(Some(&cursor), 1).err().map(|e| e.code()),
|
||||
Some("identity_mismatch"),
|
||||
"SOURCE_PAGE_CURSOR_BOUND"
|
||||
);
|
||||
}
|
||||
let encoded = serde_json::to_vec(&cursor).expect("cursor bytes");
|
||||
let decoded: EventCursor = serde_json::from_slice(&encoded).expect("cursor decode");
|
||||
assert_eq!(store.created_events_page(Some(&decoded), 1).expect("resume").events[0].event_id, [7; 16]);
|
||||
}
|
||||
#[test]
|
||||
fn page_is_bounded_and_new_lower_uuid_is_not_lost() {
|
||||
let (_dir, mut store) = fixture();
|
||||
let page = store.created_events_page(None, 1).expect("one");
|
||||
assert_eq!(page.events.len(), 1);
|
||||
apply(&mut store, 2, command(1));
|
||||
let next = store.created_events_page(page.next.as_ref(), 256).expect("later");
|
||||
assert_eq!(next.events.iter().map(|e| e.event_id).collect::<Vec<_>>(), vec![[7; 16], [1; 16]]);
|
||||
for limit in [0, MAX_EVENT_PAGE + 1, usize::MAX] {
|
||||
assert_eq!(store.created_events_page(None, limit).err().map(|e| e.code()), Some("invalid_event_page"));
|
||||
}
|
||||
store.writable = false;
|
||||
assert_eq!(store.created_events_page(None, 1).err().map(|e| e.code()), Some("write_disabled"));
|
||||
store.writable = true;
|
||||
let mut max = page.next.expect("cursor");
|
||||
max.index = u64::MAX;
|
||||
assert_eq!(
|
||||
store.created_events_page(Some(&max), 1).err().map(|e| e.code()),
|
||||
Some("identity_mismatch")
|
||||
);
|
||||
}
|
||||
struct Staging<'a>(&'a WriteTransaction);
|
||||
impl snapshot::StagingSink for Staging<'_> {
|
||||
fn row(&mut self, section: snapshot::Section, key: &[u8], value: &[u8]) -> io::Result<()> {
|
||||
let table = match section {
|
||||
snapshot::Section::Heads => application::HEADS,
|
||||
snapshot::Section::Versions => application::VERSIONS,
|
||||
snapshot::Section::Events => application::EVENTS,
|
||||
snapshot::Section::Operations => application::OPERATIONS,
|
||||
};
|
||||
self.0
|
||||
.open_table(table)
|
||||
.map_err(io::Error::other)?
|
||||
.insert(key, value)
|
||||
.map_err(io::Error::other)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
fn limits() -> snapshot::Limits {
|
||||
snapshot::Limits {
|
||||
total_bytes: 1_048_576,
|
||||
membership_bytes: 4096,
|
||||
key_bytes: 4096,
|
||||
value_bytes: 8192,
|
||||
records: 1024,
|
||||
}
|
||||
}
|
||||
async fn artifact(store: Store) -> (snapshot::CapabilityArtifact<Vec<u8>>, EventPage) {
|
||||
let writer = NativeLogWriter::new(store);
|
||||
let before = writer.created_events_page(None, 256).await.expect("async page");
|
||||
let artifact = writer
|
||||
.snapshot_view()
|
||||
.await
|
||||
.expect("snapshot view")
|
||||
.stream_capability_artifact(Vec::new(), limits())
|
||||
.expect("capability export");
|
||||
(artifact, before)
|
||||
}
|
||||
fn stage(artifact: snapshot::CapabilityArtifact<Vec<u8>>, damage: bool) -> Result<(tempfile::TempDir, Store), StoreError> {
|
||||
let dir = tempfile::tempdir().expect("stage directory");
|
||||
let database = redb::Database::create(dir.path().join("stage.redb")).expect("isolated database");
|
||||
let tx = database.begin_write()?;
|
||||
let header = snapshot::decode_capability_artifact(
|
||||
snapshot::CapabilityArtifact {
|
||||
manifest: artifact.manifest,
|
||||
payload: std::io::Cursor::new(artifact.payload),
|
||||
},
|
||||
&mut Staging(&tx),
|
||||
limits(),
|
||||
)
|
||||
.expect("real capability decoder");
|
||||
if damage {
|
||||
let mut events = tx.open_table(application::EVENTS)?;
|
||||
let value = events.get([9; 16].as_slice())?.expect("event").value().to_vec();
|
||||
events.insert([6; 16].as_slice(), value.as_slice())?;
|
||||
events.remove([9; 16].as_slice())?;
|
||||
}
|
||||
rebuild_from_authority(&tx, 6, &header.binding, header.applied)?;
|
||||
{
|
||||
let mut meta = tx.open_table(crate::META)?;
|
||||
meta.insert("schema", 6u64.to_be_bytes().as_slice())?;
|
||||
meta.insert(
|
||||
"identity",
|
||||
crate::encode_identity(NodeIdentity {
|
||||
cluster_id: header.cluster_id,
|
||||
..identity()
|
||||
})
|
||||
.as_slice(),
|
||||
)?;
|
||||
meta.insert("capture_binding", crate::created::binding_bytes(&header.binding)?.as_slice())?;
|
||||
let mut applied = vec![1];
|
||||
applied.extend(encode_id(header.applied.expect("cut")));
|
||||
meta.insert("last_applied", applied.as_slice())?;
|
||||
}
|
||||
tx.commit().expect("stage commit");
|
||||
// A test-only staging handle, not a production installer or authority grant.
|
||||
Ok((
|
||||
dir,
|
||||
Store {
|
||||
database,
|
||||
writable: true,
|
||||
},
|
||||
))
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn snapshot_authority_rebuild_matches_source() {
|
||||
let (_dir, mut store) = fixture();
|
||||
apply(&mut store, 2, command(9));
|
||||
let mut rejected = command(3);
|
||||
rejected.object = command(9).object;
|
||||
apply(&mut store, 3, rejected);
|
||||
store.promote_created_provenance().expect("schema6");
|
||||
let (artifact, before) = artifact(store).await;
|
||||
let (_stage, restored) = stage(artifact, false).expect("staged authority index");
|
||||
assert_eq!(
|
||||
restored.created_events_page(None, 256).expect("restored page"),
|
||||
before,
|
||||
"SOURCE_PAGE_SNAPSHOT_BOUND"
|
||||
);
|
||||
let first = restored.created_events_page(None, 1).expect("first");
|
||||
assert_eq!(
|
||||
restored.created_events_page(first.next.as_ref(), 1).expect("resume").events[0],
|
||||
before.events[1]
|
||||
);
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn snapshot_event_operation_mismatch_rejected() {
|
||||
let (_dir, mut store) = fixture();
|
||||
store.promote_created_provenance().expect("schema6");
|
||||
let (artifact, _) = artifact(store).await;
|
||||
assert_eq!(
|
||||
stage(artifact, true).err().map(|e| e.code()),
|
||||
Some("corrupt_record"),
|
||||
"SOURCE_PAGE_SNAPSHOT_BOUND"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,451 @@
|
||||
//! Same-UID local reads from an owner's live native store. No ACK or cursor writes.
|
||||
use crate::{EventCursor, EventPage, NativeLogWriter, NativeWriteError, NodeIdentity, StoreError};
|
||||
use redb::ReadableDatabase;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{
|
||||
fs, io,
|
||||
os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt},
|
||||
path::{Path, PathBuf},
|
||||
time::Duration,
|
||||
};
|
||||
use tokio::{
|
||||
io::{AsyncReadExt, AsyncWriteExt},
|
||||
net::{UnixListener, UnixStream},
|
||||
sync::{OwnedSemaphorePermit, watch},
|
||||
task::JoinSet,
|
||||
};
|
||||
|
||||
pub const WIRE_VERSION: u16 = 1;
|
||||
pub const MAX_REQUEST_BYTES: usize = 4096;
|
||||
pub const MAX_RESPONSE_BYTES: usize = 8 * 1024 * 1024;
|
||||
pub const MAX_CONNECTIONS: usize = 4;
|
||||
pub const IO_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ReadPage {
|
||||
pub version: u16,
|
||||
pub expected_cluster: [u8; 16],
|
||||
pub cursor: Option<EventCursor>,
|
||||
pub limit: usize,
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ReadPageResponse {
|
||||
pub version: u16,
|
||||
pub identity: NodeIdentity,
|
||||
pub page: EventPage,
|
||||
}
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub enum Rejection {
|
||||
InvalidRequest,
|
||||
WrongCluster,
|
||||
InvalidCursor,
|
||||
StoreUnavailable,
|
||||
ResponseLimit,
|
||||
}
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
#[serde(tag = "status", deny_unknown_fields)]
|
||||
pub enum ReadPageReply {
|
||||
Page { response: ReadPageResponse },
|
||||
Rejected { version: u16, error: Rejection },
|
||||
}
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum ReaderError {
|
||||
#[error("SOURCE_READER_IO")]
|
||||
Io(#[from] io::Error),
|
||||
#[error("SOURCE_READER_PERMISSION")]
|
||||
Permission,
|
||||
#[error("SOURCE_READER_PATH_OCCUPIED")]
|
||||
Occupied,
|
||||
#[error("SOURCE_READER_PATH_CHANGED")]
|
||||
PathChanged,
|
||||
#[error("SOURCE_READER_TIMEOUT")]
|
||||
Timeout,
|
||||
#[error("SOURCE_READER_PROTOCOL")]
|
||||
Protocol,
|
||||
#[error("SOURCE_READER_REJECTED: {0:?}")]
|
||||
Rejected(Rejection),
|
||||
#[error("SOURCE_READER_WORKER")]
|
||||
Worker(#[source] tokio::task::JoinError),
|
||||
}
|
||||
|
||||
impl crate::Store {
|
||||
fn durable_source_identity(&self) -> Result<NodeIdentity, StoreError> {
|
||||
let tx = self.database.begin_read()?;
|
||||
let meta = tx.open_table(crate::META)?;
|
||||
let identity = meta.get("identity")?.ok_or(StoreError::CorruptRecord)?;
|
||||
crate::decode_identity(identity.value())
|
||||
}
|
||||
}
|
||||
impl NativeLogWriter {
|
||||
/// Read the validated durable identity, never an owner-supplied cluster label.
|
||||
pub async fn durable_identity(&self) -> Result<NodeIdentity, NativeWriteError> {
|
||||
let owner = self.store.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
owner
|
||||
.lock()
|
||||
.map_err(|_| NativeWriteError::Poisoned)?
|
||||
.durable_source_identity()
|
||||
.map_err(NativeWriteError::Store)
|
||||
})
|
||||
.await
|
||||
.map_err(NativeWriteError::Worker)?
|
||||
}
|
||||
}
|
||||
|
||||
fn uid() -> u32 {
|
||||
rustix::process::geteuid().as_raw()
|
||||
}
|
||||
fn fingerprint(meta: &fs::Metadata) -> (u64, u64) {
|
||||
(meta.dev(), meta.ino())
|
||||
}
|
||||
fn private_parent(path: &Path) -> Result<(PathBuf, (u64, u64)), ReaderError> {
|
||||
if !path.is_absolute() || path.file_name().is_none() {
|
||||
return Err(ReaderError::Permission);
|
||||
}
|
||||
let parent = path.parent().ok_or(ReaderError::Permission)?;
|
||||
let meta = fs::symlink_metadata(parent)?;
|
||||
if !meta.is_dir() || meta.uid() != uid() || meta.mode() & 0o077 != 0 {
|
||||
return Err(ReaderError::Permission);
|
||||
}
|
||||
let resolved = fs::canonicalize(parent)?;
|
||||
let resolved_meta = fs::symlink_metadata(&resolved)?;
|
||||
if fingerprint(&meta) != fingerprint(&resolved_meta) {
|
||||
return Err(ReaderError::PathChanged);
|
||||
}
|
||||
Ok((resolved.join(path.file_name().ok_or(ReaderError::Permission)?), fingerprint(&meta)))
|
||||
}
|
||||
fn owned_socket(path: &Path) -> Result<fs::Metadata, ReaderError> {
|
||||
let meta = fs::symlink_metadata(path)?;
|
||||
if !meta.file_type().is_socket() || meta.uid() != uid() {
|
||||
return Err(ReaderError::Permission);
|
||||
}
|
||||
Ok(meta)
|
||||
}
|
||||
fn socket_metadata(path: &Path) -> Result<(u64, u64), ReaderError> {
|
||||
let meta = owned_socket(path)?;
|
||||
if meta.mode() & 0o077 != 0 {
|
||||
return Err(ReaderError::Permission);
|
||||
}
|
||||
Ok(fingerprint(&meta))
|
||||
}
|
||||
fn peer(stream: &UnixStream) -> Result<(), ReaderError> {
|
||||
if stream.peer_cred()?.uid() != uid() {
|
||||
return Err(ReaderError::Permission);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
struct SocketPath {
|
||||
path: PathBuf,
|
||||
inode: Option<(u64, u64)>,
|
||||
parent: (u64, u64),
|
||||
}
|
||||
impl SocketPath {
|
||||
fn remove(&mut self) -> Result<(), ReaderError> {
|
||||
// One cleanup attempt only: never let Drop re-evaluate a later namespace.
|
||||
let Some(inode) = self.inode.take() else {
|
||||
return Ok(());
|
||||
};
|
||||
let (_, parent) = private_parent(&self.path)?;
|
||||
if parent != self.parent || fingerprint(&owned_socket(&self.path)?) != inode {
|
||||
return Err(ReaderError::PathChanged);
|
||||
}
|
||||
fs::remove_file(&self.path)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
impl Drop for SocketPath {
|
||||
fn drop(&mut self) {
|
||||
// Explicit shutdown reports cleanup failure; cancellation preserves detected replacements.
|
||||
let _ = self.remove();
|
||||
}
|
||||
}
|
||||
|
||||
/// Signal shutdown, then await the owner's `serve` task to drain workers and clean up.
|
||||
#[derive(Clone)]
|
||||
pub struct Shutdown(watch::Sender<bool>);
|
||||
impl Shutdown {
|
||||
pub fn shutdown(&self) {
|
||||
let _ = self.0.send(true);
|
||||
}
|
||||
}
|
||||
pub struct SourceReaderServer {
|
||||
listener: UnixListener,
|
||||
writer: NativeLogWriter,
|
||||
socket: SocketPath,
|
||||
shutdown: watch::Receiver<bool>,
|
||||
}
|
||||
impl SourceReaderServer {
|
||||
pub fn bind(path: impl AsRef<Path>, writer: NativeLogWriter) -> Result<(Self, Shutdown), ReaderError> {
|
||||
let (path, parent) = private_parent(path.as_ref())?;
|
||||
match fs::symlink_metadata(&path) {
|
||||
Ok(_) => return Err(ReaderError::Occupied),
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => {}
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
// The private parent prevents a foreign UID from observing the initial socket mode.
|
||||
let listener = UnixListener::bind(&path)?;
|
||||
let inode = fingerprint(&fs::symlink_metadata(&path)?);
|
||||
let socket = SocketPath {
|
||||
path,
|
||||
inode: Some(inode),
|
||||
parent,
|
||||
};
|
||||
fs::set_permissions(&socket.path, fs::Permissions::from_mode(0o600))?;
|
||||
if private_parent(&socket.path)?.1 != parent || socket_metadata(&socket.path)? != inode {
|
||||
return Err(ReaderError::PathChanged);
|
||||
}
|
||||
let (sender, shutdown) = watch::channel(false);
|
||||
Ok((
|
||||
Self {
|
||||
listener,
|
||||
writer,
|
||||
socket,
|
||||
shutdown,
|
||||
},
|
||||
Shutdown(sender),
|
||||
))
|
||||
}
|
||||
pub async fn serve(mut self) -> Result<(), ReaderError> {
|
||||
let mut tasks = JoinSet::new();
|
||||
let outcome = async {
|
||||
loop {
|
||||
if *self.shutdown.borrow() {
|
||||
break;
|
||||
}
|
||||
// Admission is shared by every clone of this writer, including re-bound services.
|
||||
let admission = self.writer.source_reader_admission.clone();
|
||||
let permit = tokio::select! {
|
||||
result = admission.acquire_owned() => result.map_err(|_| ReaderError::Protocol)?,
|
||||
_ = self.shutdown.changed() => break,
|
||||
};
|
||||
let stream = tokio::select! {
|
||||
result = self.listener.accept() => result?.0,
|
||||
_ = self.shutdown.changed() => break,
|
||||
};
|
||||
if peer(&stream).is_err() {
|
||||
continue;
|
||||
}
|
||||
let writer = self.writer.clone();
|
||||
tasks.spawn(async move {
|
||||
let _ = connection(stream, writer, permit).await;
|
||||
});
|
||||
while let Some(result) = tasks.try_join_next() {
|
||||
result.map_err(ReaderError::Worker)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
.await;
|
||||
// Ordinary shutdown drains connections and blocking workers, never aborts storage work.
|
||||
while let Some(result) = tasks.join_next().await {
|
||||
result.map_err(ReaderError::Worker)?;
|
||||
}
|
||||
drop(self.listener);
|
||||
self.socket.remove()?;
|
||||
outcome
|
||||
}
|
||||
}
|
||||
|
||||
fn reject(error: Rejection) -> ReadPageReply {
|
||||
ReadPageReply::Rejected {
|
||||
version: WIRE_VERSION,
|
||||
error,
|
||||
}
|
||||
}
|
||||
fn read_page(writer: &NativeLogWriter, request: ReadPage) -> ReadPageReply {
|
||||
if request.version != WIRE_VERSION
|
||||
|| request.expected_cluster == [0; 16]
|
||||
|| request.limit == 0
|
||||
|| request.limit > crate::MAX_EVENT_PAGE
|
||||
{
|
||||
return reject(Rejection::InvalidRequest);
|
||||
}
|
||||
// Lock order: connection admission, then Store. Native reads take no writer-ordering lock.
|
||||
let read = || -> Result<ReadPageReply, NativeWriteError> {
|
||||
let store = writer.store.lock().map_err(|_| NativeWriteError::Poisoned)?;
|
||||
let identity = store.durable_source_identity()?;
|
||||
if identity.cluster_id != request.expected_cluster {
|
||||
return Ok(reject(Rejection::WrongCluster));
|
||||
}
|
||||
let page = match store.created_events_page(request.cursor.as_ref(), request.limit) {
|
||||
Ok(page) => page,
|
||||
Err(StoreError::IdentityMismatch) => return Ok(reject(Rejection::InvalidCursor)),
|
||||
Err(error) => return Err(error.into()),
|
||||
};
|
||||
Ok(ReadPageReply::Page {
|
||||
response: ReadPageResponse {
|
||||
version: WIRE_VERSION,
|
||||
identity,
|
||||
page,
|
||||
},
|
||||
})
|
||||
};
|
||||
read().unwrap_or_else(|_| reject(Rejection::StoreUnavailable))
|
||||
}
|
||||
|
||||
struct BoundedBytes {
|
||||
bytes: Vec<u8>,
|
||||
limit: usize,
|
||||
}
|
||||
impl io::Write for BoundedBytes {
|
||||
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
|
||||
if bytes.len() > self.limit.saturating_sub(self.bytes.len()) {
|
||||
return Err(io::Error::other("source reader frame limit"));
|
||||
}
|
||||
self.bytes.extend_from_slice(bytes);
|
||||
Ok(bytes.len())
|
||||
}
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
fn encode(value: &impl Serialize, limit: usize) -> Result<Vec<u8>, ReaderError> {
|
||||
let mut buffer = BoundedBytes {
|
||||
bytes: Vec::new(),
|
||||
limit,
|
||||
};
|
||||
serde_json::to_writer(&mut buffer, value).map_err(|_| ReaderError::Protocol)?;
|
||||
Ok(buffer.bytes)
|
||||
}
|
||||
async fn frame_read(stream: &mut UnixStream, limit: usize) -> Result<Vec<u8>, ReaderError> {
|
||||
let len = usize::try_from(stream.read_u32().await?).map_err(|_| ReaderError::Protocol)?;
|
||||
if len == 0 || len > limit {
|
||||
return Err(ReaderError::Protocol);
|
||||
}
|
||||
let mut bytes = vec![0; len];
|
||||
stream.read_exact(&mut bytes).await?;
|
||||
Ok(bytes)
|
||||
}
|
||||
async fn frame_write(stream: &mut UnixStream, bytes: &[u8]) -> Result<(), ReaderError> {
|
||||
stream
|
||||
.write_u32(u32::try_from(bytes.len()).map_err(|_| ReaderError::Protocol)?)
|
||||
.await?;
|
||||
stream.write_all(bytes).await?;
|
||||
Ok(())
|
||||
}
|
||||
async fn connection(mut stream: UnixStream, writer: NativeLogWriter, permit: OwnedSemaphorePermit) -> Result<(), ReaderError> {
|
||||
let bytes = tokio::time::timeout(IO_TIMEOUT, frame_read(&mut stream, MAX_REQUEST_BYTES))
|
||||
.await
|
||||
.map_err(|_| ReaderError::Timeout)??;
|
||||
let request: ReadPage = match serde_json::from_slice(&bytes) {
|
||||
Ok(request) => request,
|
||||
Err(_) => {
|
||||
let bytes = encode(&reject(Rejection::InvalidRequest), MAX_RESPONSE_BYTES)?;
|
||||
return tokio::time::timeout(IO_TIMEOUT, frame_write(&mut stream, &bytes))
|
||||
.await
|
||||
.map_err(|_| ReaderError::Timeout)?;
|
||||
}
|
||||
};
|
||||
// The blocking worker owns admission until read + bounded serialization finish, even
|
||||
// if a deadline, shutdown or a dropped serve task cancels its awaiting connection.
|
||||
let (bytes, _permit) = tokio::task::spawn_blocking(move || {
|
||||
let reply = read_page(&writer, request);
|
||||
let bytes = encode(&reply, MAX_RESPONSE_BYTES).or_else(|_| encode(&reject(Rejection::ResponseLimit), MAX_RESPONSE_BYTES));
|
||||
(bytes, permit)
|
||||
})
|
||||
.await
|
||||
.map_err(ReaderError::Worker)?;
|
||||
tokio::time::timeout(IO_TIMEOUT, frame_write(&mut stream, &bytes?))
|
||||
.await
|
||||
.map_err(|_| ReaderError::Timeout)?
|
||||
}
|
||||
|
||||
/// Connect only to a private socket owned by the effective UID, verified at both ends.
|
||||
#[derive(Clone)]
|
||||
pub struct SourceReaderClient {
|
||||
path: PathBuf,
|
||||
}
|
||||
impl SourceReaderClient {
|
||||
pub fn new(path: impl AsRef<Path>) -> Result<Self, ReaderError> {
|
||||
let (path, _) = private_parent(path.as_ref())?;
|
||||
socket_metadata(&path)?;
|
||||
Ok(Self { path })
|
||||
}
|
||||
pub async fn page(
|
||||
&self,
|
||||
expected_cluster: [u8; 16],
|
||||
cursor: Option<EventCursor>,
|
||||
limit: usize,
|
||||
) -> Result<ReadPageResponse, ReaderError> {
|
||||
let request = ReadPage {
|
||||
version: WIRE_VERSION,
|
||||
expected_cluster,
|
||||
cursor,
|
||||
limit,
|
||||
};
|
||||
tokio::time::timeout(IO_TIMEOUT, self.page_inner(request))
|
||||
.await
|
||||
.map_err(|_| ReaderError::Timeout)?
|
||||
}
|
||||
async fn page_inner(&self, request: ReadPage) -> Result<ReadPageResponse, ReaderError> {
|
||||
let (_, parent) = private_parent(&self.path)?;
|
||||
let inode = socket_metadata(&self.path)?;
|
||||
let mut stream = UnixStream::connect(&self.path).await?;
|
||||
peer(&stream)?;
|
||||
if private_parent(&self.path)?.1 != parent || socket_metadata(&self.path)? != inode {
|
||||
return Err(ReaderError::PathChanged);
|
||||
}
|
||||
frame_write(&mut stream, &encode(&request, MAX_REQUEST_BYTES)?).await?;
|
||||
let bytes = frame_read(&mut stream, MAX_RESPONSE_BYTES).await?;
|
||||
match serde_json::from_slice::<ReadPageReply>(&bytes).map_err(|_| ReaderError::Protocol)? {
|
||||
ReadPageReply::Page { response } => {
|
||||
if response.version != WIRE_VERSION
|
||||
|| response.identity.cluster_id != request.expected_cluster
|
||||
|| response.identity.cluster_id == [0; 16]
|
||||
|| response.page.events.len() > request.limit
|
||||
|| request.limit == 0
|
||||
|| request.limit > crate::MAX_EVENT_PAGE
|
||||
{
|
||||
return Err(ReaderError::Protocol);
|
||||
}
|
||||
let next = response
|
||||
.page
|
||||
.events
|
||||
.last()
|
||||
.map(|event| EventCursor::new(response.identity.cluster_id, event.event_id, &event.captured))
|
||||
.or(request.cursor);
|
||||
if response.page.next != next {
|
||||
return Err(ReaderError::Protocol);
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
ReadPageReply::Rejected { version, error } if version == WIRE_VERSION => Err(ReaderError::Rejected(error)),
|
||||
_ => Err(ReaderError::Protocol),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod cleanup_tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn source_reader_cleanup_is_disarmed_before_rebind() {
|
||||
let directory = tempfile::tempdir().expect("private socket owner");
|
||||
fs::set_permissions(directory.path(), fs::Permissions::from_mode(0o700)).expect("private directory mode");
|
||||
let (path, parent) = private_parent(&directory.path().join("reader.sock")).expect("private path");
|
||||
let listener = std::os::unix::net::UnixListener::bind(&path).expect("first real socket");
|
||||
fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).expect("first socket mode");
|
||||
let inode = socket_metadata(&path).expect("first socket identity");
|
||||
let mut guard = SocketPath {
|
||||
path: path.clone(),
|
||||
inode: Some(inode),
|
||||
parent,
|
||||
};
|
||||
drop(listener);
|
||||
guard.remove().expect("explicit owner cleanup");
|
||||
assert!(guard.inode.is_none(), "SOURCE_READER_CLEANUP_DISARMED");
|
||||
let listener = std::os::unix::net::UnixListener::bind(&path).expect("immediate new owner bind");
|
||||
fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).expect("new socket mode");
|
||||
let replacement = socket_metadata(&path).expect("new owner socket");
|
||||
drop(guard);
|
||||
assert_eq!(
|
||||
socket_metadata(&path).expect("new socket preserved"),
|
||||
replacement,
|
||||
"SOURCE_READER_OLD_GUARD_PRESERVES_REBIND"
|
||||
);
|
||||
let _client = std::os::unix::net::UnixStream::connect(&path).expect("new service reachable");
|
||||
let _connection = listener.accept().expect("new owner accepts");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,502 @@
|
||||
//! Real NativeLogWriter and socket owner is this test process, not a production S3 owner.
|
||||
use crate::{source_reader::*, *};
|
||||
use std::{
|
||||
collections::{BTreeMap, BTreeSet},
|
||||
os::unix::fs::{MetadataExt, PermissionsExt},
|
||||
path::PathBuf,
|
||||
};
|
||||
use tokio::{
|
||||
io::{AsyncReadExt, AsyncWriteExt},
|
||||
net::UnixStream,
|
||||
};
|
||||
fn identity() -> NodeIdentity {
|
||||
NodeIdentity {
|
||||
cluster_id: [7; 16],
|
||||
node_id: 0,
|
||||
}
|
||||
}
|
||||
fn binding() -> CaptureBinding {
|
||||
CaptureBinding {
|
||||
revision: 9,
|
||||
binding_id: [8; 16],
|
||||
target_id: [9; 16],
|
||||
}
|
||||
}
|
||||
fn id(index: u64) -> LogId {
|
||||
LogId {
|
||||
term: 8,
|
||||
leader_node: 0,
|
||||
index,
|
||||
}
|
||||
}
|
||||
fn command(event: u8) -> DecideCreated {
|
||||
DecideCreated {
|
||||
operation_id: [event; 16],
|
||||
object: ObjectIdentity {
|
||||
bucket_incarnation: [2; 16],
|
||||
key: vec![event],
|
||||
},
|
||||
expected_head: None,
|
||||
prepared: PreparedIdentity {
|
||||
version_id: [event; 16],
|
||||
preparation_id: [4; 16],
|
||||
content_digest: [5; 32],
|
||||
content_length: 123,
|
||||
},
|
||||
binding_revision: 9,
|
||||
}
|
||||
}
|
||||
fn fixture() -> (tempfile::TempDir, NativeLogWriter) {
|
||||
let dir = tempfile::tempdir().expect("real source reader directory");
|
||||
std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o700)).expect("private owner directory");
|
||||
let mut store = Store::initialize(dir.path().join("db"), identity()).expect("initialize real redb");
|
||||
store.pin_capture_binding(binding()).expect("binding");
|
||||
store.enable_native_empty().expect("native schema");
|
||||
store.promote_created_provenance().expect("provenance");
|
||||
let membership = CaptureMembership::new(
|
||||
Some(id(0)),
|
||||
vec![BTreeSet::from([0])],
|
||||
BTreeMap::from([(0, CaptureNode::new("http://127.0.0.1:7000".into()).expect("member"))]),
|
||||
)
|
||||
.expect("membership");
|
||||
let membership = openraft::StoredMembership::try_from(&membership).expect("stored membership");
|
||||
store
|
||||
.append_native(&[CaptureRaftEntry {
|
||||
log_id: id(0).into(),
|
||||
payload: openraft::EntryPayload::Membership(membership.membership().clone()),
|
||||
}])
|
||||
.expect("membership log");
|
||||
store.save_committed(id(0)).expect("membership commit");
|
||||
store.apply_next_native(id(0)).expect("membership apply");
|
||||
(dir, NativeLogWriter::new(store))
|
||||
}
|
||||
async fn apply(writer: &NativeLogWriter, index: u64, event: u8) {
|
||||
let entry = CaptureRaftEntry {
|
||||
log_id: id(index).into(),
|
||||
payload: openraft::EntryPayload::Normal(CaptureCommand::try_created(command(event)).expect("canonical native command")),
|
||||
};
|
||||
writer
|
||||
.append_native(vec![entry.clone()])
|
||||
.await
|
||||
.expect("live durable log append");
|
||||
let response = writer.apply_committed(vec![entry]).await.expect("live committed apply");
|
||||
assert!(
|
||||
matches!(response[0].created_result(), Some(CreatedResult::Created { .. })),
|
||||
"SOURCE_READER_REAL_APPLY"
|
||||
);
|
||||
}
|
||||
fn start(
|
||||
dir: &tempfile::TempDir,
|
||||
writer: &NativeLogWriter,
|
||||
) -> (PathBuf, SourceReaderClient, Shutdown, tokio::task::JoinHandle<Result<(), ReaderError>>) {
|
||||
let path = dir.path().join("reader.sock");
|
||||
let (server, shutdown) = SourceReaderServer::bind(&path, writer.clone()).expect("owner socket bind");
|
||||
let client = SourceReaderClient::new(&path).expect("trusted local client");
|
||||
let task = tokio::spawn(server.serve());
|
||||
(path, client, shutdown, task)
|
||||
}
|
||||
async fn stop(path: PathBuf, shutdown: Shutdown, task: tokio::task::JoinHandle<Result<(), ReaderError>>) {
|
||||
shutdown.shutdown();
|
||||
task.await.expect("serve task").expect("clean shutdown");
|
||||
assert!(!path.exists(), "SOURCE_READER_SOCKET_CLEANUP");
|
||||
}
|
||||
async fn raw(path: &std::path::Path, value: serde_json::Value) -> ReadPageReply {
|
||||
let mut stream = UnixStream::connect(path).await.expect("real socket connect");
|
||||
let bytes = serde_json::to_vec(&value).expect("wire request");
|
||||
stream
|
||||
.write_u32(bytes.len().try_into().expect("request length"))
|
||||
.await
|
||||
.expect("frame prefix");
|
||||
stream.write_all(&bytes).await.expect("frame bytes");
|
||||
let len = stream.read_u32().await.expect("reply prefix");
|
||||
let mut bytes = vec![0; usize::try_from(len).expect("reply length")];
|
||||
stream.read_exact(&mut bytes).await.expect("reply bytes");
|
||||
serde_json::from_slice(&bytes).expect("strict native reply")
|
||||
}
|
||||
fn request() -> serde_json::Value {
|
||||
serde_json::to_value(ReadPage {
|
||||
version: WIRE_VERSION,
|
||||
expected_cluster: identity().cluster_id,
|
||||
cursor: None,
|
||||
limit: 2,
|
||||
})
|
||||
.expect("native request")
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn source_reader_live_apply() {
|
||||
let (dir, writer) = fixture();
|
||||
let (path, client, shutdown, task) = start(&dir, &writer);
|
||||
assert_eq!(writer.durable_identity().await.expect("durable META identity"), identity());
|
||||
assert!(
|
||||
client
|
||||
.page(identity().cluster_id, None, 2)
|
||||
.await
|
||||
.expect("empty live page")
|
||||
.page
|
||||
.events
|
||||
.is_empty()
|
||||
);
|
||||
apply(&writer, 1, 9).await;
|
||||
let response = client
|
||||
.page(identity().cluster_id, None, 2)
|
||||
.await
|
||||
.expect("live committed page");
|
||||
assert_eq!(response.identity, identity(), "SOURCE_READER_DURABLE_IDENTITY");
|
||||
assert_eq!(response.page.events[0].event_id, [9; 16], "SOURCE_READER_LIVE_EVENT");
|
||||
assert_eq!(response.page.events[0].captured.binding, binding());
|
||||
{
|
||||
let mut corrupt = writer.store.lock().expect("test owner");
|
||||
let tx = corrupt.write_transaction().expect("corrupt META transaction");
|
||||
tx.open_table(META)
|
||||
.expect("META")
|
||||
.insert("identity", &[0; 24][..])
|
||||
.expect("corrupt identity");
|
||||
corrupt.commit(tx).expect("persist corruption");
|
||||
}
|
||||
assert!(writer.durable_identity().await.is_err(), "SOURCE_READER_META_CORRUPTION_REJECTED");
|
||||
assert!(
|
||||
matches!(
|
||||
client.page(identity().cluster_id, None, 2).await,
|
||||
Err(ReaderError::Rejected(Rejection::StoreUnavailable))
|
||||
),
|
||||
"SOURCE_READER_NO_CALLER_IDENTITY_SUBSTITUTION"
|
||||
);
|
||||
stop(path, shutdown, task).await;
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn source_reader_restart_cursor() {
|
||||
let (dir, writer) = fixture();
|
||||
apply(&writer, 1, 9).await;
|
||||
apply(&writer, 2, 7).await;
|
||||
let (path, client, shutdown, task) = start(&dir, &writer);
|
||||
let first = client.page(identity().cluster_id, None, 1).await.expect("first");
|
||||
stop(path, shutdown, task).await;
|
||||
drop(writer);
|
||||
let writer = NativeLogWriter::new(Store::open(dir.path().join("db"), identity()).expect("restart same durable store"));
|
||||
let (path, client, shutdown, task) = start(&dir, &writer);
|
||||
let next = client
|
||||
.page(identity().cluster_id, first.page.next, 1)
|
||||
.await
|
||||
.expect("resume durable cursor");
|
||||
assert_eq!(next.page.events[0].event_id, [7; 16], "SOURCE_READER_RESTART_CONTINUES");
|
||||
stop(path, shutdown, task).await;
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn source_reader_page_boundaries() {
|
||||
let (dir, writer) = fixture();
|
||||
for (index, event) in [(1, 9), (2, 7), (3, 1)] {
|
||||
apply(&writer, index, event).await;
|
||||
}
|
||||
let (path, client, shutdown, task) = start(&dir, &writer);
|
||||
let first = client.page(identity().cluster_id, None, 2).await.expect("first two");
|
||||
assert_eq!(
|
||||
first.page.events.iter().map(|e| e.event_id).collect::<Vec<_>>(),
|
||||
vec![[9; 16], [7; 16]],
|
||||
"SOURCE_READER_DECISION_ORDER"
|
||||
);
|
||||
let next = client
|
||||
.page(identity().cluster_id, first.page.next, 2)
|
||||
.await
|
||||
.expect("remaining");
|
||||
assert_eq!(next.page.events.len(), 1);
|
||||
let end = client
|
||||
.page(identity().cluster_id, next.page.next.clone(), 2)
|
||||
.await
|
||||
.expect("end");
|
||||
assert!(end.page.events.is_empty());
|
||||
assert_eq!(end.page.next, next.page.next, "SOURCE_READER_CURSOR_RETAINED_AT_END");
|
||||
for limit in [0, MAX_EVENT_PAGE + 1] {
|
||||
assert!(
|
||||
matches!(
|
||||
client.page(identity().cluster_id, None, limit).await,
|
||||
Err(ReaderError::Rejected(Rejection::InvalidRequest))
|
||||
),
|
||||
"SOURCE_READER_LIMIT_REJECTED"
|
||||
);
|
||||
}
|
||||
stop(path, shutdown, task).await;
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn source_reader_wrong_identity() {
|
||||
let (dir, writer) = fixture();
|
||||
apply(&writer, 1, 9).await;
|
||||
let (path, client, shutdown, task) = start(&dir, &writer);
|
||||
let mut wrong = request();
|
||||
wrong["expected_cluster"] = serde_json::to_value([6; 16]).expect("wrong cluster");
|
||||
assert!(
|
||||
matches!(
|
||||
raw(&path, wrong).await,
|
||||
ReadPageReply::Rejected {
|
||||
error: Rejection::WrongCluster,
|
||||
..
|
||||
}
|
||||
),
|
||||
"SOURCE_READER_CLUSTER: reject before reading or returning foreign events"
|
||||
);
|
||||
let first = client.page(identity().cluster_id, None, 1).await.expect("real cursor");
|
||||
let mut wrong = serde_json::to_value(first.page.next.expect("cursor")).expect("cursor wire");
|
||||
wrong["event"] = serde_json::to_value([1; 16]).expect("wrong event");
|
||||
let wrong: EventCursor = serde_json::from_value(wrong).expect("syntactically valid foreign cursor");
|
||||
assert!(
|
||||
matches!(
|
||||
client.page(identity().cluster_id, Some(wrong), 1).await,
|
||||
Err(ReaderError::Rejected(Rejection::InvalidCursor))
|
||||
),
|
||||
"SOURCE_READER_CURSOR_BOUND"
|
||||
);
|
||||
for (field, value) in [
|
||||
("version", serde_json::json!(2)),
|
||||
("unknown", serde_json::json!(true)),
|
||||
("cursor", serde_json::json!({"unknown":true})),
|
||||
] {
|
||||
let mut wrong = request();
|
||||
wrong[field] = value;
|
||||
assert!(
|
||||
matches!(
|
||||
raw(&path, wrong).await,
|
||||
ReadPageReply::Rejected {
|
||||
error: Rejection::InvalidRequest,
|
||||
..
|
||||
}
|
||||
),
|
||||
"SOURCE_READER_STRICT_REQUEST"
|
||||
);
|
||||
}
|
||||
stop(path, shutdown, task).await;
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn source_reader_socket_ownership() {
|
||||
let (dir, writer) = fixture();
|
||||
let path = dir.path().join("reader.sock");
|
||||
std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o750)).expect("insecure directory");
|
||||
assert!(
|
||||
matches!(SourceReaderServer::bind(&path, writer.clone()), Err(ReaderError::Permission)),
|
||||
"SOURCE_READER_PRIVATE_DIRECTORY"
|
||||
);
|
||||
std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o700)).expect("restore private directory");
|
||||
std::fs::write(&path, b"preserve me").expect("occupied file");
|
||||
assert!(
|
||||
matches!(SourceReaderServer::bind(&path, writer.clone()), Err(ReaderError::Occupied)),
|
||||
"SOURCE_READER_OCCUPIED_PATH"
|
||||
);
|
||||
assert_eq!(std::fs::read(&path).expect("unchanged file"), b"preserve me");
|
||||
std::fs::remove_file(&path).expect("remove own fixture");
|
||||
std::os::unix::fs::symlink(dir.path().join("missing"), &path).expect("unknown symlink");
|
||||
assert!(matches!(SourceReaderServer::bind(&path, writer.clone()), Err(ReaderError::Occupied)));
|
||||
assert!(SourceReaderClient::new(&path).is_err());
|
||||
std::fs::remove_file(&path).expect("remove own symlink");
|
||||
let long = dir.path().join("x".repeat(300));
|
||||
assert!(SourceReaderServer::bind(&long, writer.clone()).is_err(), "SOURCE_READER_LONG_SOCKET_PATH");
|
||||
let parent_link = dir.path().join("parent-link");
|
||||
std::os::unix::fs::symlink(dir.path(), &parent_link).expect("parent symlink");
|
||||
assert!(
|
||||
matches!(
|
||||
SourceReaderServer::bind(parent_link.join("socket"), writer.clone()),
|
||||
Err(ReaderError::Permission)
|
||||
),
|
||||
"SOURCE_READER_PARENT_SYMLINK_DENIED"
|
||||
);
|
||||
let (path, client, shutdown, task) = start(&dir, &writer);
|
||||
let meta = std::fs::symlink_metadata(&path).expect("socket metadata");
|
||||
assert_eq!(meta.mode() & 0o777, 0o600);
|
||||
assert_eq!(meta.uid(), rustix::process::geteuid().as_raw());
|
||||
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o660)).expect("insecure socket");
|
||||
assert!(
|
||||
matches!(client.page(identity().cluster_id, None, 1).await, Err(ReaderError::Permission)),
|
||||
"SOURCE_READER_CLIENT_VALIDATES_SOCKET"
|
||||
);
|
||||
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)).expect("restore socket");
|
||||
std::fs::remove_file(&path).expect("test owner replaces socket");
|
||||
std::fs::write(&path, b"replacement").expect("replacement file");
|
||||
shutdown.shutdown();
|
||||
assert!(task.await.expect("serve exit").is_err(), "SOURCE_READER_REPLACEMENT_CLEANUP_REJECTED");
|
||||
assert_eq!(std::fs::read(&path).expect("replacement preserved"), b"replacement");
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn source_reader_transport_bounds() {
|
||||
let (dir, writer) = fixture();
|
||||
let (path, client, shutdown, task) = start(&dir, &writer);
|
||||
let mut stream = UnixStream::connect(&path).await.expect("oversized input peer");
|
||||
stream
|
||||
.write_u32(u32::try_from(MAX_REQUEST_BYTES + 1).expect("length"))
|
||||
.await
|
||||
.expect("overlong prefix");
|
||||
let mut byte = [0];
|
||||
assert_eq!(
|
||||
stream.read(&mut byte).await.expect("bounded rejection closes"),
|
||||
0,
|
||||
"SOURCE_READER_INPUT_BOUND"
|
||||
);
|
||||
let mut idle = UnixStream::connect(&path).await.expect("idle connection");
|
||||
let start = std::time::Instant::now();
|
||||
assert_eq!(
|
||||
idle.read(&mut byte).await.expect("deadline closes connection"),
|
||||
0,
|
||||
"SOURCE_READER_READ_DEADLINE"
|
||||
);
|
||||
assert!(start.elapsed() >= IO_TIMEOUT, "SOURCE_READER_REAL_READ_DEADLINE");
|
||||
assert!(
|
||||
client.page(identity().cluster_id, None, 1).await.is_ok(),
|
||||
"SOURCE_READER_TIMEOUT_RELEASES_CONNECTION"
|
||||
);
|
||||
stop(path.clone(), shutdown, task).await;
|
||||
// A same-UID peer still cannot supply unbounded responses to the client.
|
||||
let listener = tokio::net::UnixListener::bind(&path).expect("controlled response peer");
|
||||
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)).expect("private response socket");
|
||||
let client = SourceReaderClient::new(&path).expect("response client");
|
||||
let peer = tokio::spawn(async move {
|
||||
let (mut stream, _) = listener.accept().await.expect("client connects");
|
||||
stream
|
||||
.write_u32(u32::try_from(MAX_RESPONSE_BYTES + 1).expect("oversized response"))
|
||||
.await
|
||||
.expect("response prefix");
|
||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||
});
|
||||
assert!(
|
||||
matches!(client.page(identity().cluster_id, None, 1).await, Err(ReaderError::Protocol)),
|
||||
"SOURCE_READER_RESPONSE_BOUND"
|
||||
);
|
||||
peer.await.expect("peer task");
|
||||
}
|
||||
fn hold_store(writer: &NativeLogWriter) -> (std::sync::mpsc::Sender<()>, std::thread::JoinHandle<()>) {
|
||||
let owner = writer.clone();
|
||||
let (locked_tx, locked_rx) = std::sync::mpsc::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel();
|
||||
let thread = std::thread::spawn(move || {
|
||||
let _store = owner.store.lock().expect("actual Store held by test owner");
|
||||
locked_tx.send(()).expect("lock visible");
|
||||
let _ = release_rx.recv();
|
||||
});
|
||||
locked_rx.recv().expect("store locked");
|
||||
(release_tx, thread)
|
||||
}
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn source_reader_shutdown() {
|
||||
let (dir, writer) = fixture();
|
||||
apply(&writer, 1, 9).await;
|
||||
let (path, _client, shutdown, task) = start(&dir, &writer);
|
||||
let (release_tx, blocker) = hold_store(&writer);
|
||||
let mut streams = Vec::new();
|
||||
for _ in 0..MAX_CONNECTIONS {
|
||||
let mut stream = UnixStream::connect(&path).await.expect("active read socket");
|
||||
let bytes = serde_json::to_vec(&request()).expect("request");
|
||||
stream
|
||||
.write_u32(bytes.len().try_into().expect("length"))
|
||||
.await
|
||||
.expect("prefix");
|
||||
stream.write_all(&bytes).await.expect("request bytes");
|
||||
streams.push(stream);
|
||||
}
|
||||
tokio::time::timeout(std::time::Duration::from_secs(2), async {
|
||||
while writer.source_reader_admission.available_permits() != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("all admissions active");
|
||||
drop(streams);
|
||||
tokio::time::sleep(IO_TIMEOUT + std::time::Duration::from_millis(100)).await;
|
||||
assert_eq!(
|
||||
writer.source_reader_admission.available_permits(),
|
||||
0,
|
||||
"SOURCE_READER_CLIENT_TIMEOUT_KEEPS_WORKER_ADMISSION"
|
||||
);
|
||||
shutdown.shutdown();
|
||||
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||
assert!(!task.is_finished(), "SOURCE_READER_SHUTDOWN_DRAINS_READERS");
|
||||
assert_eq!(
|
||||
writer.source_reader_admission.available_permits(),
|
||||
0,
|
||||
"SOURCE_READER_CANCELLED_CLIENTS_KEEP_ADMISSION"
|
||||
);
|
||||
release_tx.send(()).expect("release real blocking reads");
|
||||
blocker.join().expect("test lock owner exits");
|
||||
task.await.expect("serve completes").expect("shutdown clean");
|
||||
assert!(!path.exists());
|
||||
assert_eq!(
|
||||
writer.source_reader_admission.available_permits(),
|
||||
MAX_CONNECTIONS,
|
||||
"SOURCE_READER_ADMISSION_RELEASED_AFTER_WORKERS"
|
||||
);
|
||||
let (release, blocker) = hold_store(&writer);
|
||||
let (path, _, _shutdown, task) = start(&dir, &writer);
|
||||
let bytes = serde_json::to_vec(&request()).expect("native request");
|
||||
let mut sockets = Vec::new();
|
||||
for _ in 0..MAX_CONNECTIONS {
|
||||
let mut stream = UnixStream::connect(&path).await.expect("cancelled service peer");
|
||||
stream
|
||||
.write_u32(bytes.len().try_into().expect("length"))
|
||||
.await
|
||||
.expect("prefix");
|
||||
stream.write_all(&bytes).await.expect("request");
|
||||
sockets.push(stream);
|
||||
}
|
||||
tokio::time::timeout(std::time::Duration::from_secs(2), async {
|
||||
while writer.source_reader_admission.available_permits() != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("cancelled service admits bounded workers");
|
||||
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||
task.abort();
|
||||
assert!(task.await.expect_err("serve cancellation").is_cancelled());
|
||||
drop(sockets);
|
||||
assert_eq!(
|
||||
writer.source_reader_admission.available_permits(),
|
||||
0,
|
||||
"SOURCE_READER_CANCELLED_SERVE_RETAINS_WORKER_ADMISSION"
|
||||
);
|
||||
let (path, client, shutdown, task) = start(&dir, &writer);
|
||||
let pending = tokio::spawn(async move { client.page(identity().cluster_id, None, 1).await });
|
||||
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||
assert!(!pending.is_finished(), "SOURCE_READER_REBIND_CANNOT_BYPASS_ADMISSION");
|
||||
assert_eq!(writer.source_reader_admission.available_permits(), 0);
|
||||
release.send(()).expect("release cancelled worker store lock");
|
||||
blocker.join().expect("lock owner completes");
|
||||
assert_eq!(
|
||||
pending
|
||||
.await
|
||||
.expect("new service client")
|
||||
.expect("new service waits for old workers")
|
||||
.page
|
||||
.events
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
stop(path, shutdown, task).await;
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn source_reader_checkpoint_replay() {
|
||||
let (dir, writer) = fixture();
|
||||
apply(&writer, 1, 9).await;
|
||||
apply(&writer, 2, 7).await;
|
||||
let (path, client, shutdown, task) = start(&dir, &writer);
|
||||
let first = client
|
||||
.page(identity().cluster_id, None, 1)
|
||||
.await
|
||||
.expect("pre-checkpoint cursor");
|
||||
let checkpoint = writer
|
||||
.checkpoint_local(CheckpointLimits::default())
|
||||
.await
|
||||
.expect("real checkpoint");
|
||||
assert_eq!(checkpoint.cut(), id(2));
|
||||
let next = client
|
||||
.page(identity().cluster_id, first.page.next.clone(), 1)
|
||||
.await
|
||||
.expect("checkpoint retained events");
|
||||
assert_eq!(next.page.events[0].event_id, [7; 16], "SOURCE_READER_CHECKPOINT_REPLAY");
|
||||
stop(path, shutdown, task).await;
|
||||
drop(writer);
|
||||
let writer = NativeLogWriter::new(Store::open(dir.path().join("db"), identity()).expect("reopen checkpointed store"));
|
||||
let (path, client, shutdown, task) = start(&dir, &writer);
|
||||
assert_eq!(
|
||||
client
|
||||
.page(identity().cluster_id, first.page.next, 1)
|
||||
.await
|
||||
.expect("replay after restart"),
|
||||
next,
|
||||
"SOURCE_READER_CHECKPOINT_RESTART"
|
||||
);
|
||||
stop(path, shutdown, task).await;
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
use crate::{LOGS, META, Store, StoreError, decode_id, decode_log, encode_id};
|
||||
use redb::ReadableTable;
|
||||
|
||||
impl Store {
|
||||
/// Atomically remove every log entry at or after `from`.
|
||||
pub fn truncate_suffix(&mut self, from: u64) -> Result<(), StoreError> {
|
||||
let tx = self.write_transaction()?;
|
||||
{
|
||||
let mut meta = tx.open_table(META)?;
|
||||
let logs = tx.open_table(LOGS)?;
|
||||
if crate::committed::validate_committed(&meta, &logs)?.is_some_and(|id| from <= id.index) {
|
||||
return Err(StoreError::CommittedLogProtected);
|
||||
}
|
||||
drop(logs);
|
||||
let tail = meta.get("tail")?.map(|v| decode_id(v.value())).transpose()?;
|
||||
if tail.is_none_or(|id| from > id.index) {
|
||||
return Ok(());
|
||||
}
|
||||
let mut logs = tx.open_table(LOGS)?;
|
||||
let new_tail = if from == 0 {
|
||||
None
|
||||
} else {
|
||||
let index = from - 1;
|
||||
let id = match logs.get(index)? {
|
||||
Some(record) => decode_log(record.value())?.id,
|
||||
None => crate::checkpoint_baseline::read_purged(&meta)?
|
||||
.filter(|id| id.index == index)
|
||||
.ok_or(StoreError::InvalidLogSequence)?,
|
||||
};
|
||||
if id.index != index {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
Some(id)
|
||||
};
|
||||
logs.retain(|index, _| index < from)?;
|
||||
match new_tail {
|
||||
Some(id) => {
|
||||
meta.insert("tail", encode_id(id).as_slice())?;
|
||||
}
|
||||
None => {
|
||||
meta.remove("tail")?;
|
||||
}
|
||||
}
|
||||
}
|
||||
self.commit(tx)
|
||||
}
|
||||
}
|
||||
@@ -1,512 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! What one stack answered, and how two answers are compared.
|
||||
//!
|
||||
//! An [`Observation`] keeps only what a client acts on: the HTTP status, the S3 error code, the
|
||||
//! allowlisted response headers and a few typed fields decoded from the response. The error
|
||||
//! message is kept for diagnostics and never compared: message wording is judged by the wire
|
||||
//! corpus, not here. The comparator is pure so it can be tested without a server.
|
||||
|
||||
use serde::Serialize;
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
/// Response headers compared exactly (after [`normalized_header_value`]).
|
||||
const HEADER_ALLOWLIST: [&str; 7] = [
|
||||
"content-type",
|
||||
"x-amz-version-id",
|
||||
"etag",
|
||||
"x-amz-delete-marker",
|
||||
"content-range",
|
||||
"accept-ranges",
|
||||
"x-amz-request-charged",
|
||||
];
|
||||
|
||||
/// Every response header with this prefix is compared as well.
|
||||
const HEADER_ALLOWLIST_PREFIX: &str = "x-amz-server-side-encryption";
|
||||
|
||||
/// Placeholder for a field one side did not send.
|
||||
const ABSENT: &str = "<absent>";
|
||||
|
||||
/// The lowercase header name if the comparator looks at this header.
|
||||
pub(crate) fn compared_header(name: &str) -> Option<String> {
|
||||
let name = name.to_ascii_lowercase();
|
||||
(HEADER_ALLOWLIST.contains(&name.as_str()) || name.starts_with(HEADER_ALLOWLIST_PREFIX)).then_some(name)
|
||||
}
|
||||
|
||||
/// A version ID is minted per process, so its value never matches across two servers; only
|
||||
/// whether it is the literal `null` (an unversioned write) carries meaning.
|
||||
fn normalized_header_value(name: &str, value: &str) -> String {
|
||||
if name == "x-amz-version-id" && value != "null" {
|
||||
"<version-id>".to_owned()
|
||||
} else {
|
||||
value.to_owned()
|
||||
}
|
||||
}
|
||||
|
||||
/// One request's outcome as a client sees it.
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub(crate) struct Observation {
|
||||
/// The S3 operation, named as in the `s3s::S3` trait (`GetObject`).
|
||||
pub op: &'static str,
|
||||
/// Which request of the script this was (`normal`, `negative`, or a setup label).
|
||||
pub case: &'static str,
|
||||
/// HTTP status; 0 when no response arrived.
|
||||
pub status: u16,
|
||||
/// The S3 error code, or the SDK's error kind when there was no service error.
|
||||
pub error_code: Option<String>,
|
||||
/// Allowlisted headers only, lowercase names, normalized values.
|
||||
pub headers: BTreeMap<String, String>,
|
||||
/// Typed values decoded from the response body, never free text.
|
||||
pub fields: BTreeMap<String, String>,
|
||||
/// Diagnostic only; never compared.
|
||||
#[serde(skip)]
|
||||
pub message: Option<String>,
|
||||
}
|
||||
|
||||
impl Observation {
|
||||
/// Builds an observation, keeping only the allowlisted headers.
|
||||
pub(crate) fn new<'h>(
|
||||
op: &'static str,
|
||||
case: &'static str,
|
||||
status: u16,
|
||||
error_code: Option<String>,
|
||||
headers: impl IntoIterator<Item = (&'h str, &'h str)>,
|
||||
) -> Self {
|
||||
let headers = headers
|
||||
.into_iter()
|
||||
.filter_map(|(name, value)| compared_header(name).map(|name| (name, value)))
|
||||
.map(|(name, value)| {
|
||||
let value = normalized_header_value(&name, value);
|
||||
(name, value)
|
||||
})
|
||||
.collect();
|
||||
Self {
|
||||
op,
|
||||
case,
|
||||
status,
|
||||
error_code,
|
||||
headers,
|
||||
fields: BTreeMap::new(),
|
||||
message: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn with_field(mut self, name: &str, value: impl Into<String>) -> Self {
|
||||
self.fields.insert(name.to_owned(), value.into());
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn is_success(&self) -> bool {
|
||||
(200..300).contains(&self.status)
|
||||
}
|
||||
}
|
||||
|
||||
/// One field on which the two stacks answered differently.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
|
||||
pub(crate) struct Difference {
|
||||
pub op: String,
|
||||
pub case: String,
|
||||
pub field: String,
|
||||
pub a: String,
|
||||
pub b: String,
|
||||
}
|
||||
|
||||
/// Every difference between two transcripts of the same script, matched by `(op, case)`.
|
||||
pub(crate) fn compare(a: &[Observation], b: &[Observation]) -> Vec<Difference> {
|
||||
let mut differences = Vec::new();
|
||||
let (index_a, duplicates_a) = index(a);
|
||||
let (index_b, duplicates_b) = index(b);
|
||||
for (op, case) in duplicates_a.union(&duplicates_b) {
|
||||
let count = |side: &[Observation]| {
|
||||
side.iter()
|
||||
.filter(|observation| (observation.op, observation.case) == (*op, *case))
|
||||
.count()
|
||||
.to_string()
|
||||
};
|
||||
differences.push(difference(op, case, "duplicate_case", count(a), count(b)));
|
||||
}
|
||||
let keys: BTreeSet<_> = index_a.keys().chain(index_b.keys()).copied().collect();
|
||||
for key @ (op, case) in keys {
|
||||
match (index_a.get(&key), index_b.get(&key)) {
|
||||
(Some(a), Some(b)) => compare_one(a, b, &mut differences),
|
||||
(a, b) => {
|
||||
let presence = |side: Option<&&Observation>| if side.is_some() { "present" } else { ABSENT }.to_owned();
|
||||
differences.push(difference(op, case, "observation", presence(a), presence(b)));
|
||||
}
|
||||
}
|
||||
}
|
||||
differences
|
||||
}
|
||||
|
||||
type Key = (&'static str, &'static str);
|
||||
|
||||
fn index(side: &[Observation]) -> (BTreeMap<Key, &Observation>, BTreeSet<Key>) {
|
||||
let mut index = BTreeMap::new();
|
||||
let mut duplicates = BTreeSet::new();
|
||||
for observation in side {
|
||||
let key = (observation.op, observation.case);
|
||||
if index.insert(key, observation).is_some() {
|
||||
duplicates.insert(key);
|
||||
}
|
||||
}
|
||||
(index, duplicates)
|
||||
}
|
||||
|
||||
fn difference(op: &str, case: &str, field: &str, a: String, b: String) -> Difference {
|
||||
Difference {
|
||||
op: op.to_owned(),
|
||||
case: case.to_owned(),
|
||||
field: field.to_owned(),
|
||||
a,
|
||||
b,
|
||||
}
|
||||
}
|
||||
|
||||
fn compare_one(a: &Observation, b: &Observation, differences: &mut Vec<Difference>) {
|
||||
let mut field = |name: String, value_a: Option<&str>, value_b: Option<&str>| {
|
||||
if value_a != value_b {
|
||||
let show = |value: Option<&str>| value.unwrap_or(ABSENT).to_owned();
|
||||
differences.push(difference(a.op, a.case, &name, show(value_a), show(value_b)));
|
||||
}
|
||||
};
|
||||
field(
|
||||
"status".to_owned(),
|
||||
Some(a.status.to_string().as_str()),
|
||||
Some(b.status.to_string().as_str()),
|
||||
);
|
||||
field("error_code".to_owned(), a.error_code.as_deref(), b.error_code.as_deref());
|
||||
let headers: BTreeSet<_> = a.headers.keys().chain(b.headers.keys()).collect();
|
||||
for name in headers {
|
||||
field(
|
||||
format!("header.{name}"),
|
||||
a.headers.get(name).map(String::as_str),
|
||||
b.headers.get(name).map(String::as_str),
|
||||
);
|
||||
}
|
||||
let fields: BTreeSet<_> = a.fields.keys().chain(b.fields.keys()).collect();
|
||||
for name in fields {
|
||||
field(
|
||||
format!("body.{name}"),
|
||||
a.fields.get(name).map(String::as_str),
|
||||
b.fields.get(name).map(String::as_str),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Operations observed on both sides.
|
||||
pub(crate) fn compared_ops(a: &[Observation], b: &[Observation]) -> BTreeSet<&'static str> {
|
||||
let ops_b: BTreeSet<_> = b.iter().map(|observation| observation.op).collect();
|
||||
a.iter()
|
||||
.map(|observation| observation.op)
|
||||
.filter(|op| ops_b.contains(op))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// `(op, case)` pairs observed on both sides.
|
||||
pub(crate) fn compared_cases(a: &[Observation], b: &[Observation]) -> usize {
|
||||
let cases_b: BTreeSet<_> = b.iter().map(|observation| (observation.op, observation.case)).collect();
|
||||
a.iter()
|
||||
.map(|observation| (observation.op, observation.case))
|
||||
.collect::<BTreeSet<_>>()
|
||||
.intersection(&cases_b)
|
||||
.count()
|
||||
}
|
||||
|
||||
/// The JSON report written for every run.
|
||||
#[derive(Debug, Serialize)]
|
||||
pub(crate) struct Report {
|
||||
pub schema: u32,
|
||||
pub run: String,
|
||||
pub stack_a: String,
|
||||
pub stack_b: String,
|
||||
pub region_a: Option<String>,
|
||||
pub region_b: Option<String>,
|
||||
pub compared_ops: usize,
|
||||
pub compared_cases: usize,
|
||||
pub differences: Vec<Difference>,
|
||||
}
|
||||
|
||||
impl Report {
|
||||
/// Writes the report as `<directory>/<run>.json` and returns its path.
|
||||
pub(crate) fn write_to(&self, directory: &Path) -> std::io::Result<PathBuf> {
|
||||
std::fs::create_dir_all(directory)?;
|
||||
let path = directory.join(format!("{}.json", self.run));
|
||||
let json = serde_json::to_vec_pretty(self).map_err(std::io::Error::other)?;
|
||||
std::fs::write(&path, json)?;
|
||||
Ok(path)
|
||||
}
|
||||
}
|
||||
|
||||
/// One line per difference, for a failing assertion.
|
||||
pub(crate) fn render(differences: &[Difference]) -> String {
|
||||
differences
|
||||
.iter()
|
||||
.map(|d| format!("({}, {}, {}, a={:?}, b={:?})", d.op, d.case, d.field, d.a, d.b))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn ok(op: &'static str) -> Observation {
|
||||
Observation::new(op, "normal", 200, None, [("content-type", "application/xml")])
|
||||
}
|
||||
|
||||
fn fields(differences: &[Difference]) -> Vec<&str> {
|
||||
differences.iter().map(|d| d.field.as_str()).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identical_transcripts_have_no_difference() {
|
||||
let a = vec![
|
||||
ok("HeadBucket"),
|
||||
ok("GetBucketLocation").with_field("location_constraint", "us-east-1"),
|
||||
];
|
||||
let b = a.clone();
|
||||
assert_eq!(compare(&a, &b), Vec::new());
|
||||
assert_eq!(compared_ops(&a, &b).len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_status_difference_is_reported() {
|
||||
let a = vec![ok("PutBucketAcl")];
|
||||
let b = vec![Observation::new(
|
||||
"PutBucketAcl",
|
||||
"normal",
|
||||
403,
|
||||
None,
|
||||
[("content-type", "application/xml")],
|
||||
)];
|
||||
assert_eq!(
|
||||
compare(&a, &b),
|
||||
vec![Difference {
|
||||
op: "PutBucketAcl".into(),
|
||||
case: "normal".into(),
|
||||
field: "status".into(),
|
||||
a: "200".into(),
|
||||
b: "403".into(),
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_error_code_difference_is_reported() {
|
||||
let a = vec![Observation::new("GetObject", "negative", 404, Some("NoSuchKey".into()), [])];
|
||||
let b = vec![Observation::new(
|
||||
"GetObject",
|
||||
"negative",
|
||||
404,
|
||||
Some("NoSuchBucket".into()),
|
||||
[],
|
||||
)];
|
||||
let differences = compare(&a, &b);
|
||||
assert_eq!(fields(&differences), ["error_code"]);
|
||||
assert_eq!((differences[0].a.as_str(), differences[0].b.as_str()), ("NoSuchKey", "NoSuchBucket"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_error_code_on_one_side_only_is_reported() {
|
||||
let a = vec![Observation::new("HeadBucket", "negative", 404, Some("NotFound".into()), [])];
|
||||
let b = vec![Observation::new("HeadBucket", "negative", 404, None, [])];
|
||||
let differences = compare(&a, &b);
|
||||
assert_eq!(fields(&differences), ["error_code"]);
|
||||
assert_eq!(differences[0].b, ABSENT);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_allowlisted_header_difference_is_reported() {
|
||||
let a = vec![Observation::new("HeadObject", "normal", 200, None, [("ETag", "\"aa\"")])];
|
||||
let b = vec![Observation::new("HeadObject", "normal", 200, None, [("etag", "\"bb\"")])];
|
||||
assert_eq!(fields(&compare(&a, &b)), ["header.etag"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_header_sent_by_one_side_only_is_reported() {
|
||||
let a = vec![Observation::new(
|
||||
"DeleteObject",
|
||||
"normal",
|
||||
204,
|
||||
None,
|
||||
[("x-amz-delete-marker", "true")],
|
||||
)];
|
||||
let b = vec![Observation::new("DeleteObject", "normal", 204, None, [])];
|
||||
let differences = compare(&a, &b);
|
||||
assert_eq!(fields(&differences), ["header.x-amz-delete-marker"]);
|
||||
assert_eq!((differences[0].a.as_str(), differences[0].b.as_str()), ("true", ABSENT));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_server_side_encryption_header_is_compared() {
|
||||
let header = "x-amz-server-side-encryption-customer-key-md5";
|
||||
let a = vec![Observation::new("PutObject", "sse-c", 200, None, [(header, "one")])];
|
||||
let b = vec![Observation::new("PutObject", "sse-c", 200, None, [(header, "two")])];
|
||||
assert_eq!(fields(&compare(&a, &b)), [format!("header.{header}")]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn headers_outside_the_allowlist_are_not_compared() {
|
||||
let a = vec![Observation::new(
|
||||
"ListBuckets",
|
||||
"normal",
|
||||
200,
|
||||
None,
|
||||
[
|
||||
("date", "Mon"),
|
||||
("x-amz-request-id", "1"),
|
||||
("server", "a"),
|
||||
("content-length", "10"),
|
||||
],
|
||||
)];
|
||||
let b = vec![Observation::new(
|
||||
"ListBuckets",
|
||||
"normal",
|
||||
200,
|
||||
None,
|
||||
[
|
||||
("date", "Tue"),
|
||||
("x-amz-request-id", "2"),
|
||||
("server", "b"),
|
||||
("content-length", "11"),
|
||||
],
|
||||
)];
|
||||
assert_eq!(compare(&a, &b), Vec::new());
|
||||
assert!(a[0].headers.is_empty(), "non-allowlisted headers were kept: {:?}", a[0].headers);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn distinct_version_ids_are_not_a_difference() {
|
||||
let a = vec![Observation::new(
|
||||
"PutObject",
|
||||
"normal",
|
||||
200,
|
||||
None,
|
||||
[("x-amz-version-id", "0b1c")],
|
||||
)];
|
||||
let b = vec![Observation::new(
|
||||
"PutObject",
|
||||
"normal",
|
||||
200,
|
||||
None,
|
||||
[("x-amz-version-id", "9f8e")],
|
||||
)];
|
||||
assert_eq!(compare(&a, &b), Vec::new());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_null_version_id_against_a_real_one_is_reported() {
|
||||
let a = vec![Observation::new(
|
||||
"PutObject",
|
||||
"normal",
|
||||
200,
|
||||
None,
|
||||
[("x-amz-version-id", "null")],
|
||||
)];
|
||||
let b = vec![Observation::new(
|
||||
"PutObject",
|
||||
"normal",
|
||||
200,
|
||||
None,
|
||||
[("x-amz-version-id", "9f8e")],
|
||||
)];
|
||||
assert_eq!(fields(&compare(&a, &b)), ["header.x-amz-version-id"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_body_field_difference_is_reported() {
|
||||
let a = vec![ok("GetBucketLocation").with_field("location_constraint", "us-east-1")];
|
||||
let b = vec![ok("GetBucketLocation").with_field("location_constraint", "eu-west-1")];
|
||||
let differences = compare(&a, &b);
|
||||
assert_eq!(fields(&differences), ["body.location_constraint"]);
|
||||
assert_eq!((differences[0].a.as_str(), differences[0].b.as_str()), ("us-east-1", "eu-west-1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_body_field_on_one_side_only_is_reported() {
|
||||
let a = vec![ok("ListObjectsV2").with_field("keys", "[]")];
|
||||
let b = vec![ok("ListObjectsV2")];
|
||||
assert_eq!(fields(&compare(&a, &b)), ["body.keys"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn error_messages_are_never_compared() {
|
||||
let mut a = Observation::new("GetObject", "negative", 404, Some("NoSuchKey".into()), []);
|
||||
let mut b = a.clone();
|
||||
a.message = Some("The specified key does not exist.".into());
|
||||
b.message = Some("Object not found".into());
|
||||
assert_eq!(compare(&[a], &[b]), Vec::new());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_case_observed_on_one_side_only_is_reported() {
|
||||
let a = vec![ok("HeadBucket"), ok("ListBuckets")];
|
||||
let b = vec![ok("HeadBucket")];
|
||||
let differences = compare(&a, &b);
|
||||
assert_eq!(fields(&differences), ["observation"]);
|
||||
assert_eq!(differences[0].op, "ListBuckets");
|
||||
assert_eq!(compared_ops(&a, &b).into_iter().collect::<Vec<_>>(), ["HeadBucket"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_case_recorded_twice_is_reported() {
|
||||
let a = vec![ok("HeadBucket"), ok("HeadBucket")];
|
||||
let b = vec![ok("HeadBucket")];
|
||||
assert_eq!(fields(&compare(&a, &b)), ["duplicate_case"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compared_ops_counts_operations_not_cases() {
|
||||
let negative = Observation::new("HeadBucket", "negative", 404, Some("NotFound".into()), []);
|
||||
let a = vec![ok("HeadBucket"), negative, ok("ListBuckets")];
|
||||
let b = a.clone();
|
||||
assert_eq!(compared_ops(&a, &b).len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_report_lists_each_difference_as_op_field_a_b() {
|
||||
let directory = std::env::temp_dir().join(format!("dual-stack-report-{}", uuid::Uuid::new_v4()));
|
||||
let report = Report {
|
||||
schema: 1,
|
||||
run: "unit".into(),
|
||||
stack_a: "legacy".into(),
|
||||
stack_b: "legacy".into(),
|
||||
region_a: None,
|
||||
region_b: Some("eu-west-1".into()),
|
||||
compared_ops: 1,
|
||||
compared_cases: 1,
|
||||
differences: compare(
|
||||
&[ok("GetBucketLocation").with_field("location_constraint", "us-east-1")],
|
||||
&[ok("GetBucketLocation").with_field("location_constraint", "eu-west-1")],
|
||||
),
|
||||
};
|
||||
let path = report.write_to(&directory).expect("write the report");
|
||||
let json: serde_json::Value = serde_json::from_slice(&std::fs::read(&path).expect("read the report")).expect("parse");
|
||||
let _ = std::fs::remove_dir_all(&directory);
|
||||
assert_eq!(path.file_name().and_then(|name| name.to_str()), Some("unit.json"));
|
||||
assert_eq!(json["compared_ops"], 1);
|
||||
assert_eq!(
|
||||
json["differences"],
|
||||
serde_json::json!([{
|
||||
"op": "GetBucketLocation",
|
||||
"case": "normal",
|
||||
"field": "body.location_constraint",
|
||||
"a": "us-east-1",
|
||||
"b": "eu-west-1",
|
||||
}])
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,197 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Dual-stack differential: one `rustfs` binary, two processes, one SDK script (rustfs/backlog#2740).
|
||||
//!
|
||||
//! Stack A always runs `RUSTFS_S3_STACK=legacy`. Stack B runs the value of `RUSTFS_E2E_STACK_B`
|
||||
//! (default `legacy` until the gateway stack is assembled), and `RUSTFS_E2E_STACK_B_REGION`, when
|
||||
//! set, starts B in another region so that a difference can be injected by hand. Both servers
|
||||
//! answer the script in `scenarios`, which covers every operation of `impl S3 for FS`; `compare`
|
||||
//! reports each `(op, field, a, b)` on which the answers differ, and every run writes its report
|
||||
//! to `<target>/dual-stack/<run>.json`. Two processes, not two embedded servers: bucket metadata
|
||||
//! is process-global, so two stacks in one process would read each other's buckets.
|
||||
|
||||
mod compare;
|
||||
mod scenarios;
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, workspace_root};
|
||||
use compare::{Difference, Observation, Report, compare, compared_cases, compared_ops, render};
|
||||
use std::path::PathBuf;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
type BoxError = Box<dyn std::error::Error + Send + Sync>;
|
||||
|
||||
const STACK_A: &str = "legacy";
|
||||
const ENV_STACK_B: &str = "RUSTFS_E2E_STACK_B";
|
||||
const ENV_STACK_B_REGION: &str = "RUSTFS_E2E_STACK_B_REGION";
|
||||
/// Becomes `gateway` once rustfs/backlog#2734 T2.5 assembles the gateway stack.
|
||||
const DEFAULT_STACK_B: &str = "legacy";
|
||||
/// The region the negative control gives stack A explicitly, so an inherited `RUSTFS_REGION`
|
||||
/// cannot erase the injected difference.
|
||||
const CONTROL_REGION_A: &str = "us-east-1";
|
||||
const CONTROL_REGION_B: &str = "eu-west-1";
|
||||
/// `normal` cases the legacy stack must answer with a 2xx. The comparison alone cannot tell a
|
||||
/// script that exercises real behavior from one in which both stacks fail the same way. Five of
|
||||
/// the 73 fail on legacy by design of this script: PutBucketReplication (no registered remote
|
||||
/// target), GetBucketReplication (so no configuration), PutBucketNotificationConfiguration (no
|
||||
/// notification target configured), RestoreObject (the object is not archived) and
|
||||
/// GetObjectTorrent (legacy answers `NoSuchKey`).
|
||||
const MIN_NORMAL_SUCCESSES: usize = 68;
|
||||
|
||||
struct PairConfig {
|
||||
stack_b: String,
|
||||
region_a: Option<String>,
|
||||
region_b: Option<String>,
|
||||
}
|
||||
|
||||
impl PairConfig {
|
||||
fn from_env() -> Self {
|
||||
Self {
|
||||
stack_b: std::env::var(ENV_STACK_B).unwrap_or_else(|_| DEFAULT_STACK_B.to_owned()),
|
||||
region_a: None,
|
||||
region_b: std::env::var(ENV_STACK_B_REGION).ok(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Starts one server on `stack`, overriding its region only when asked.
|
||||
async fn start_stack(stack: &str, region: Option<&str>) -> Result<RustFSTestEnvironment, BoxError> {
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
let mut vars = vec![("RUSTFS_S3_STACK", stack)];
|
||||
if let Some(region) = region {
|
||||
vars.push(("RUSTFS_REGION", region));
|
||||
}
|
||||
env.start_rustfs_server_without_cleanup_with_env(&vars).await?;
|
||||
Ok(env)
|
||||
}
|
||||
|
||||
/// Starts stack A and stack B side by side, each with its own port and data directory.
|
||||
async fn spawn_pair(config: &PairConfig) -> Result<(RustFSTestEnvironment, RustFSTestEnvironment), BoxError> {
|
||||
tokio::try_join!(
|
||||
start_stack(STACK_A, config.region_a.as_deref()),
|
||||
start_stack(&config.stack_b, config.region_b.as_deref())
|
||||
)
|
||||
}
|
||||
|
||||
fn report_directory() -> PathBuf {
|
||||
let target = std::env::var_os("CARGO_TARGET_DIR").map_or_else(|| PathBuf::from("target"), PathBuf::from);
|
||||
let target = if target.is_absolute() {
|
||||
target
|
||||
} else {
|
||||
workspace_root().join(target)
|
||||
};
|
||||
target.join("dual-stack")
|
||||
}
|
||||
|
||||
struct Run {
|
||||
report: Report,
|
||||
path: PathBuf,
|
||||
a: Vec<Observation>,
|
||||
}
|
||||
|
||||
/// Runs the script against both stacks concurrently and writes the report.
|
||||
async fn run_differential(label: &str, config: &PairConfig) -> Run {
|
||||
let (server_a, server_b) = spawn_pair(config).await.expect("start stack A and stack B");
|
||||
let (a, b) = tokio::join!(Box::pin(scenarios::run(&server_a)), Box::pin(scenarios::run(&server_b)));
|
||||
let millis = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map_or(0, |elapsed| elapsed.as_millis());
|
||||
let report = Report {
|
||||
schema: 1,
|
||||
run: format!("{label}-{millis}-{}", std::process::id()),
|
||||
stack_a: STACK_A.to_owned(),
|
||||
stack_b: config.stack_b.clone(),
|
||||
region_a: config.region_a.clone(),
|
||||
region_b: config.region_b.clone(),
|
||||
compared_ops: compared_ops(&a, &b).len(),
|
||||
compared_cases: compared_cases(&a, &b),
|
||||
differences: compare(&a, &b),
|
||||
};
|
||||
let path = report.write_to(&report_directory()).expect("write the dual-stack report");
|
||||
Run { report, path, a }
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn both_stacks_answer_every_operation_identically() {
|
||||
init_logging();
|
||||
let config = PairConfig::from_env();
|
||||
let run = run_differential("self-comparison", &config).await;
|
||||
let report = &run.report;
|
||||
|
||||
let mut observed: Vec<_> = run.a.iter().map(|observation| observation.op).collect();
|
||||
observed.sort_unstable();
|
||||
observed.dedup();
|
||||
assert_eq!(
|
||||
observed,
|
||||
scenarios::OPERATIONS,
|
||||
"the script must drive exactly the operations of impl S3 for FS"
|
||||
);
|
||||
assert_eq!(report.compared_ops, scenarios::OPERATIONS.len(), "report: {}", run.path.display());
|
||||
assert!(
|
||||
report.differences.is_empty(),
|
||||
"stack {} and stack {} differ on {} of {} cases (report: {}):\n{}",
|
||||
report.stack_a,
|
||||
report.stack_b,
|
||||
report.differences.len(),
|
||||
report.compared_cases,
|
||||
run.path.display(),
|
||||
render(&report.differences)
|
||||
);
|
||||
|
||||
let not_4xx: Vec<_> = run
|
||||
.a
|
||||
.iter()
|
||||
.filter(|observation| observation.case == "negative" && !(400..500).contains(&observation.status))
|
||||
.collect();
|
||||
assert!(not_4xx.is_empty(), "negative cases must fail with a 4xx on {STACK_A}: {not_4xx:#?}");
|
||||
let successes = run
|
||||
.a
|
||||
.iter()
|
||||
.filter(|observation| observation.case == "normal" && observation.is_success())
|
||||
.count();
|
||||
assert!(
|
||||
successes >= MIN_NORMAL_SUCCESSES,
|
||||
"only {successes} normal cases succeeded on {STACK_A}: {:#?}",
|
||||
run.a
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn differential_detects_an_injected_difference() {
|
||||
init_logging();
|
||||
let config = PairConfig {
|
||||
stack_b: STACK_A.to_owned(),
|
||||
region_a: Some(CONTROL_REGION_A.to_owned()),
|
||||
region_b: Some(CONTROL_REGION_B.to_owned()),
|
||||
};
|
||||
let run = run_differential("injected-region", &config).await;
|
||||
let expected = Difference {
|
||||
op: "GetBucketLocation".to_owned(),
|
||||
case: "normal".to_owned(),
|
||||
field: "body.location_constraint".to_owned(),
|
||||
a: CONTROL_REGION_A.to_owned(),
|
||||
b: CONTROL_REGION_B.to_owned(),
|
||||
};
|
||||
assert!(
|
||||
run.report.differences.contains(&expected),
|
||||
"B started in {CONTROL_REGION_B} but GetBucketLocation was not reported (report: {}):\n{}",
|
||||
run.path.display(),
|
||||
render(&run.report.differences)
|
||||
);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -31,12 +31,6 @@ pub mod chaos;
|
||||
#[cfg(test)]
|
||||
pub mod fake_s3_target;
|
||||
|
||||
// Dual-stack differential (rustfs/backlog#2740): one binary, two processes on
|
||||
// different RUSTFS_S3_STACK values, the same SDK script for every S3 operation,
|
||||
// compared on status, error code, allowlisted headers and typed body fields.
|
||||
#[cfg(test)]
|
||||
mod dual_stack_differential;
|
||||
|
||||
// On-demand migration (backlog#2147): shared two-server environment, admin
|
||||
// wrappers, and the harness self-test (backlog#2151). Behavior scenarios are
|
||||
// added by later ODM tasks.
|
||||
|
||||
@@ -122,6 +122,8 @@ test-util = []
|
||||
e2e-test-hooks = []
|
||||
|
||||
[dependencies]
|
||||
rustfs-capture-authority = { workspace = true }
|
||||
openraft = { workspace = true, features = ["storage-v2"] }
|
||||
hotpath.workspace = true
|
||||
rustfs-filemeta.workspace = true
|
||||
rustfs-utils = { workspace = true, features = ["full"] }
|
||||
|
||||
@@ -371,6 +371,43 @@ pub mod config {
|
||||
with_config_object_read_lock, with_config_object_write_lock, with_server_config_read_lock,
|
||||
with_server_config_write_lock,
|
||||
};
|
||||
|
||||
/// Observe persisted config bytes and their reader-associated metadata within a byte limit.
|
||||
pub async fn read_config_observation(
|
||||
api: std::sync::Arc<crate::store::ECStore>,
|
||||
file: &str,
|
||||
max_bytes: usize,
|
||||
) -> crate::error::Result<(Vec<u8>, crate::object_api::ObjectInfo)> {
|
||||
crate::config::com::read_config_observation(api, file, max_bytes).await
|
||||
}
|
||||
|
||||
/// Observe bounded config bytes while the caller retains the object's write owner.
|
||||
pub async fn read_config_observation_no_lock(
|
||||
api: std::sync::Arc<crate::store::ECStore>,
|
||||
file: &str,
|
||||
max_bytes: usize,
|
||||
guard: &crate::object_api::WriteCommitGuard,
|
||||
) -> crate::error::Result<(Vec<u8>, crate::object_api::ObjectInfo)> {
|
||||
crate::config::com::read_config_observation_no_lock(api, file, max_bytes, guard).await
|
||||
}
|
||||
|
||||
/// Read config bytes and metadata without treating an empty payload as absent.
|
||||
pub async fn read_config_preserve_empty_with_metadata_opts(
|
||||
api: std::sync::Arc<crate::store::ECStore>,
|
||||
file: &str,
|
||||
opts: &crate::object_api::ObjectOptions,
|
||||
) -> crate::error::Result<(Vec<u8>, crate::object_api::ObjectInfo)> {
|
||||
crate::config::com::read_config_preserve_empty_with_metadata_opts(api, file, opts).await
|
||||
}
|
||||
|
||||
pub async fn read_config_limited_preserve_empty_with_metadata_opts(
|
||||
api: std::sync::Arc<crate::store::ECStore>,
|
||||
file: &str,
|
||||
opts: &crate::object_api::ObjectOptions,
|
||||
max_bytes: usize,
|
||||
) -> crate::error::Result<(Vec<u8>, crate::object_api::ObjectInfo)> {
|
||||
crate::config::com::read_config_limited_preserve_empty_with_metadata_opts(api, file, opts, max_bytes).await
|
||||
}
|
||||
}
|
||||
|
||||
pub mod storageclass {
|
||||
@@ -560,6 +597,7 @@ pub mod rio {
|
||||
}
|
||||
|
||||
pub mod rpc {
|
||||
pub use crate::cluster::rpc::capture_raft::{CaptureRaftError, CaptureRaftLimits, CaptureRaftNetworkFactory, CaptureRaftRpc};
|
||||
pub use crate::cluster::rpc::{
|
||||
AuthenticatedChannel, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, MAX_NETWORK_PROBE_BYTES, MAX_NETWORK_PROBE_DURATION,
|
||||
NetworkPeerProbeClient, NetworkPeerProbeError, NetworkPeerProbeMeasurement, NetworkPeerTarget, PEER_RESTDRY_RUN,
|
||||
@@ -606,7 +644,13 @@ pub mod store_list {
|
||||
|
||||
pub mod storage {
|
||||
pub use crate::core::pools::{HealLifecycleExpiryContext, POOL_META_NAME};
|
||||
pub use crate::disk::local::native_prepare::{
|
||||
NativePreparationDescription, NativePreparationProbe, NativePreparationReceipt,
|
||||
};
|
||||
pub use crate::store::HealWalkVersion;
|
||||
pub use crate::store::native_prepare::{
|
||||
NativePreparationFinalizeRequest, NativePreparationRecovery, NativePreparationTarget,
|
||||
};
|
||||
pub use crate::store::{
|
||||
BootstrapLocalTarget, ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk,
|
||||
all_local_disk_path, find_local_disk_by_ref, init_local_disks, init_local_disks_with_instance_ctx, init_lock_clients,
|
||||
|
||||
@@ -2939,6 +2939,20 @@ fn resolve_tier_free_version_recovery_enabled(value: Result<String, env::VarErro
|
||||
}
|
||||
|
||||
fn spawn_tier_free_version_recovery_once(api: Arc<ECStore>, started: &OnceLock<()>) -> Option<JoinHandle<()>> {
|
||||
spawn_tier_free_version_recovery_once_with_options(
|
||||
api,
|
||||
started,
|
||||
TIER_FREE_VERSION_RECOVERY_LEADER_LOCK.to_string(),
|
||||
tier_free_version_recovery_leader_retry_delay,
|
||||
)
|
||||
}
|
||||
|
||||
fn spawn_tier_free_version_recovery_once_with_options(
|
||||
api: Arc<ECStore>,
|
||||
started: &OnceLock<()>,
|
||||
leader_lock_name: String,
|
||||
retry_delay: fn() -> StdDuration,
|
||||
) -> Option<JoinHandle<()>> {
|
||||
if !tier_free_version_recovery_enabled() {
|
||||
warn!(
|
||||
event = EVENT_LIFECYCLE_WORKER_STATE,
|
||||
@@ -2956,15 +2970,15 @@ fn spawn_tier_free_version_recovery_once(api: Arc<ECStore>, started: &OnceLock<(
|
||||
Some(tokio::spawn(async move {
|
||||
let cancel_token = api.ctx.background_cancel_token().unwrap_or_default();
|
||||
let acquire_api = Arc::clone(&api);
|
||||
let acquire_lock_name = Arc::<str>::from(leader_lock_name);
|
||||
run_tier_free_version_recovery_coordinator(
|
||||
cancel_token,
|
||||
tier_free_version_recovery_leader_retry_delay,
|
||||
retry_delay,
|
||||
move || {
|
||||
let api = Arc::clone(&acquire_api);
|
||||
let lock_name = Arc::clone(&acquire_lock_name);
|
||||
async move {
|
||||
let lock = api
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, TIER_FREE_VERSION_RECOVERY_LEADER_LOCK)
|
||||
.await?;
|
||||
let lock = api.new_ns_lock(RUSTFS_META_BUCKET, &lock_name).await?;
|
||||
match lock.get_write_lock_quiet(TIER_FREE_VERSION_RECOVERY_LOCK_TIMEOUT).await {
|
||||
Ok(guard) => Ok(Some(guard)),
|
||||
Err(LockError::Timeout { .. }) => Ok(None),
|
||||
@@ -14193,14 +14207,14 @@ mod tests {
|
||||
.await;
|
||||
}
|
||||
|
||||
fn fast_tier_free_version_recovery_leader_retry_delay() -> StdDuration {
|
||||
StdDuration::from_millis(10)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn tier_free_version_recovery_production_entrypoint_enqueues_seeded_item() {
|
||||
let (disk_paths, ecstore) =
|
||||
temp_env::async_with_vars([(super::ENV_TIER_FREE_VERSION_RECOVERY_ENABLED, Some("false"))], async {
|
||||
setup_test_env().await
|
||||
})
|
||||
.await;
|
||||
let (disk_paths, ecstore) = setup_test_env().await;
|
||||
let runtime_state = install_unconsumed_runtime_expiry_worker(&ecstore, 1).await;
|
||||
let recovery_rx = {
|
||||
let state = runtime_state.read().await;
|
||||
@@ -14212,8 +14226,13 @@ mod tests {
|
||||
seed_recoverable_free_version(&disk_paths, &bucket, object, None, None).await;
|
||||
|
||||
let started = OnceLock::new();
|
||||
let recovery = super::spawn_tier_free_version_recovery_once(Arc::clone(&ecstore), &started)
|
||||
.expect("production recovery entrypoint should start once");
|
||||
let recovery = super::spawn_tier_free_version_recovery_once_with_options(
|
||||
Arc::clone(&ecstore),
|
||||
&started,
|
||||
format!("tier-free-version-recovery/test/{}", Uuid::new_v4()),
|
||||
fast_tier_free_version_recovery_leader_retry_delay,
|
||||
)
|
||||
.expect("production recovery entrypoint should start once");
|
||||
let task = tokio::time::timeout(StdDuration::from_secs(30), async { recovery_rx.lock().await.recv().await })
|
||||
.await
|
||||
.expect("production recovery entrypoint should enqueue the seeded item")
|
||||
|
||||
@@ -439,8 +439,6 @@ pub struct BucketMetadata {
|
||||
pub bucket_incarnation_id: Uuid,
|
||||
pub(crate) bucket_incarnation_sidecar: bool,
|
||||
pub(crate) bucket_creation_committed: bool,
|
||||
// A legacy default is not proof of an interrupted creation.
|
||||
pub(crate) bucket_creation_commit_record_present: bool,
|
||||
pub policy_config_json: Vec<u8>,
|
||||
pub notification_config_xml: Vec<u8>,
|
||||
pub lifecycle_config_xml: Vec<u8>,
|
||||
@@ -516,7 +514,6 @@ impl Default for BucketMetadata {
|
||||
bucket_incarnation_id: Uuid::nil(),
|
||||
bucket_incarnation_sidecar: false,
|
||||
bucket_creation_committed: false,
|
||||
bucket_creation_commit_record_present: false,
|
||||
policy_config_json: Default::default(),
|
||||
notification_config_xml: Default::default(),
|
||||
lifecycle_config_xml: Default::default(),
|
||||
@@ -755,10 +752,7 @@ impl BucketMetadata {
|
||||
"TableBucketConfigUpdatedAt" => self.table_bucket_config_updated_at = read_msgp_time_value(rd)?,
|
||||
"DurabilityConfigUpdatedAt" => self.durability_config_updated_at = read_msgp_time_value(rd)?,
|
||||
"OnDemandMigrationConfigUpdatedAt" => self.on_demand_migration_config_updated_at = read_msgp_time_value(rd)?,
|
||||
"BucketCreationCommitted" => {
|
||||
self.bucket_creation_committed = read_msgp_bool(rd)?;
|
||||
self.bucket_creation_commit_record_present = true;
|
||||
}
|
||||
"BucketCreationCommitted" => self.bucket_creation_committed = read_msgp_bool(rd)?,
|
||||
other => {
|
||||
tracing::debug!(field = %other, "BucketMetadata decode_from: skipping unknown field");
|
||||
skip_msgp_value(rd)?;
|
||||
@@ -771,10 +765,8 @@ impl BucketMetadata {
|
||||
|
||||
/// Encode to msgp bytes. Field order follows MinIO BucketMetadata for compatibility.
|
||||
pub fn encode_to<W: Write>(&self, wr: &mut W) -> Result<()> {
|
||||
// Preserve the absence of a legacy commit record across config rewrites.
|
||||
let commit_record_present = self.bucket_creation_commit_record_present || self.bucket_creation_committed;
|
||||
// Map size: MinIO fields (25) + RustFS extensions (21) + optional commit record.
|
||||
let map_len: u32 = 46 + u32::from(commit_record_present);
|
||||
// Map size: MinIO fields (25) + RustFS extensions (22)
|
||||
let map_len: u32 = 47;
|
||||
rmp::encode::write_map_len(wr, map_len)?;
|
||||
|
||||
// MinIO field order (same as Go struct)
|
||||
@@ -855,10 +847,8 @@ impl BucketMetadata {
|
||||
write_msgp_time(wr, self.durability_config_updated_at)?;
|
||||
rmp::encode::write_str(wr, "OnDemandMigrationConfigUpdatedAt")?;
|
||||
write_msgp_time(wr, self.on_demand_migration_config_updated_at)?;
|
||||
if commit_record_present {
|
||||
rmp::encode::write_str(wr, "BucketCreationCommitted")?;
|
||||
rmp::encode::write_bool(wr, self.bucket_creation_committed)?;
|
||||
}
|
||||
rmp::encode::write_str(wr, "BucketCreationCommitted")?;
|
||||
rmp::encode::write_bool(wr, self.bucket_creation_committed)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1701,13 +1691,6 @@ mod test {
|
||||
let mut bm = BucketMetadata::unmarshal(&body[4..]).expect("unmarshal recovered blob");
|
||||
assert_eq!(bm.name, "interop");
|
||||
assert!(!bm.bucket_creation_committed, "legacy metadata must default to uncommitted");
|
||||
assert!(!bm.bucket_creation_commit_record_present, "legacy metadata has no creation-commit record");
|
||||
let rewritten = BucketMetadata::unmarshal(&bm.marshal_msg().expect("rewrite legacy metadata"))
|
||||
.expect("read rewritten legacy metadata");
|
||||
assert!(
|
||||
!rewritten.bucket_creation_commit_record_present,
|
||||
"rewriting legacy metadata must not fabricate a creation intent"
|
||||
);
|
||||
bm.parse_all_configs().expect("parse recovered configs");
|
||||
assert!(bm.lifecycle_config.is_some());
|
||||
}
|
||||
@@ -1725,7 +1708,6 @@ mod test {
|
||||
|
||||
assert_eq!(bm.name, new.name);
|
||||
assert!(new.bucket_creation_committed);
|
||||
assert!(new.bucket_creation_commit_record_present);
|
||||
assert!(!bm.bucket_incarnation_id.is_nil());
|
||||
assert_eq!(bm.bucket_incarnation_id, new.bucket_incarnation_id);
|
||||
}
|
||||
|
||||
@@ -1292,14 +1292,34 @@ pub(crate) async fn get_object_lock_config_state_in(
|
||||
pub(crate) async fn get_object_lock_config_and_incarnation_from_disk_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
) -> Result<(ObjectLockConfigState, Uuid, OffsetDateTime)> {
|
||||
get_object_lock_config_and_incarnation_from_disk_with_guard_in(ctx, bucket, None).await
|
||||
}
|
||||
|
||||
/// Bucket deletion already owns the namespace/lifecycle fences. Reuse its
|
||||
/// live guard rather than recursively acquiring the bucket namespace lock.
|
||||
/// The caller must also hold the metadata transaction read lock.
|
||||
pub(crate) async fn get_object_lock_config_and_incarnation_from_disk_with_guard_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
guard: Option<&rustfs_lock::NamespaceLockGuard>,
|
||||
) -> Result<(ObjectLockConfigState, Uuid, OffsetDateTime)> {
|
||||
let bucket_meta_sys_lock = bucket_metadata_sys_of(ctx)?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await.clone();
|
||||
|
||||
match bucket_meta_sys
|
||||
.read_authoritative_metadata_from_disk_under_transaction_lock(bucket)
|
||||
.await?
|
||||
{
|
||||
let authority = match guard {
|
||||
Some(guard) => {
|
||||
bucket_meta_sys
|
||||
.read_authoritative_metadata_from_disk_under_guard(bucket, guard)
|
||||
.await?
|
||||
}
|
||||
None => {
|
||||
bucket_meta_sys
|
||||
.read_authoritative_metadata_from_disk_under_transaction_lock(bucket)
|
||||
.await?
|
||||
}
|
||||
};
|
||||
match authority {
|
||||
BucketMetadataAuthority::Authoritative(metadata)
|
||||
if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() =>
|
||||
{
|
||||
@@ -1553,27 +1573,9 @@ struct MetadataPublishGuard {
|
||||
_guard: tokio::sync::OwnedMutexGuard<MetadataPublishLockState>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct PendingBucketCreationProof {
|
||||
pub(crate) generation: Uuid,
|
||||
pub(crate) intent: Vec<u8>,
|
||||
pub(crate) versioning_requested: bool,
|
||||
pub(crate) witnesses: Vec<crate::disk::os::NativeBucketCreationWitness>,
|
||||
}
|
||||
|
||||
impl PendingBucketCreationProof {
|
||||
pub(crate) fn matches_metadata(&self, metadata: &BucketMetadata) -> Result<bool> {
|
||||
Ok(!metadata.bucket_creation_committed
|
||||
&& metadata.bucket_incarnation_sidecar
|
||||
&& metadata.bucket_incarnation_id == self.generation
|
||||
&& metadata.marshal_msg()? == self.intent)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BucketMetadataSys {
|
||||
metadata_map: Arc<RwLock<HashMap<String, Arc<BucketMetadata>>>>,
|
||||
pending_bucket_creations: Arc<RwLock<HashMap<String, PendingBucketCreationProof>>>,
|
||||
/// Serializes metadata-map commits and their derived cache updates for one
|
||||
/// bucket. Namespace locks, when present, are acquired before this lock.
|
||||
metadata_publish_locks: Arc<MetadataPublishLockRegistry>,
|
||||
@@ -1617,7 +1619,6 @@ impl BucketMetadataSys {
|
||||
pub fn new(api: Arc<ECStore>) -> Self {
|
||||
Self {
|
||||
metadata_map: Arc::new(RwLock::new(HashMap::new())),
|
||||
pending_bucket_creations: Arc::new(RwLock::new(HashMap::new())),
|
||||
metadata_publish_locks: Arc::new(MetadataPublishLockRegistry {
|
||||
locks: StdMutex::new(HashMap::new()),
|
||||
}),
|
||||
@@ -1648,81 +1649,6 @@ impl BucketMetadataSys {
|
||||
.expect("bucket metadata object store should still be live")
|
||||
}
|
||||
|
||||
pub(crate) async fn pending_bucket_creation(&self, bucket: &str) -> Option<PendingBucketCreationProof> {
|
||||
self.pending_bucket_creations.read().await.get(bucket).cloned()
|
||||
}
|
||||
|
||||
pub(crate) async fn grant_pending_bucket_creation(&self, bucket: &str, proof: PendingBucketCreationProof) {
|
||||
self.pending_bucket_creations.write().await.insert(bucket.to_owned(), proof);
|
||||
}
|
||||
|
||||
pub(crate) async fn revoke_pending_bucket_creation(&self, bucket: &str) {
|
||||
self.pending_bucket_creations.write().await.remove(bucket);
|
||||
}
|
||||
|
||||
pub(crate) async fn consume_pending_bucket_creation(
|
||||
&self,
|
||||
bucket: &str,
|
||||
metadata: &BucketMetadata,
|
||||
witnesses: &[crate::disk::os::NativeBucketCreationWitness],
|
||||
) -> Result<bool> {
|
||||
let mut pending = self.pending_bucket_creations.write().await;
|
||||
let Some(proof) = pending.get(bucket) else {
|
||||
return Ok(false);
|
||||
};
|
||||
if !proof.matches_metadata(metadata)?
|
||||
|| proof.witnesses.len() != witnesses.len()
|
||||
|| !proof
|
||||
.witnesses
|
||||
.iter()
|
||||
.zip(witnesses)
|
||||
.all(|(old, current)| old.matches(current))
|
||||
|| !crate::disk::os::consume_native_creation_witnesses(witnesses)
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
pending.remove(bucket);
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn refresh_bucket_for_test(&self, bucket: &str) -> Result<()> {
|
||||
let expected = self.metadata_map.read().await.get(bucket).cloned();
|
||||
let lock = self.object_store().new_ns_lock(bucket, bucket).await?;
|
||||
let guard = lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
|
||||
self.load_bucket_under_namespace(bucket, MetadataLoadMode::Refresh, expected.as_ref(), &guard)
|
||||
.await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn publish_persisted_metadata_for_test(&self, bucket: &str) -> Result<()> {
|
||||
let expected = self.metadata_map.read().await.get(bucket).cloned();
|
||||
let lock = self.object_store().new_ns_lock(bucket, bucket).await?;
|
||||
let guard = lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
|
||||
let (metadata, persisted) = await_bucket_namespace_operation(
|
||||
Some(&guard),
|
||||
bucket,
|
||||
"test cached metadata load",
|
||||
load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true),
|
||||
)
|
||||
.await?;
|
||||
self.publish_if_unchanged(bucket, expected.as_ref(), metadata, persisted, &guard)
|
||||
.await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn migrate_bucket_creation_commit_for_test(&self, bucket: &str) -> Result<()> {
|
||||
self.migrate_bucket_creation_commit(bucket).await.map(|_| ())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn legacy_migration_reports_missing_bucket_for_test(&self, bucket: &str) -> Result<bool> {
|
||||
Ok(matches!(
|
||||
self.migrate_legacy_metadata(bucket).await?,
|
||||
BucketMetadataAuthority::MissingBucket
|
||||
))
|
||||
}
|
||||
|
||||
fn object_store_if_live(&self) -> Option<Arc<ECStore>> {
|
||||
self.api.upgrade()
|
||||
}
|
||||
@@ -1913,7 +1839,6 @@ impl BucketMetadataSys {
|
||||
expected: Option<&Arc<BucketMetadata>>,
|
||||
namespace_guard: &rustfs_lock::NamespaceLockGuard,
|
||||
) -> Result<()> {
|
||||
self.revoke_pending_bucket_creation(bucket).await;
|
||||
if !await_bucket_namespace_operation(
|
||||
Some(namespace_guard),
|
||||
bucket,
|
||||
@@ -1997,7 +1922,6 @@ impl BucketMetadataSys {
|
||||
persisted: bool,
|
||||
namespace_guard: &rustfs_lock::NamespaceLockGuard,
|
||||
) -> Result<()> {
|
||||
self.revoke_pending_bucket_creation(bucket).await;
|
||||
if !persisted {
|
||||
let _publish_guard = self
|
||||
.lock_metadata_publish(bucket, namespace_guard, "refreshed bucket metadata absence publish")
|
||||
@@ -2059,7 +1983,6 @@ impl BucketMetadataSys {
|
||||
}
|
||||
|
||||
pub async fn set(&self, bucket: String, bm: Arc<BucketMetadata>) {
|
||||
self.revoke_pending_bucket_creation(&bucket).await;
|
||||
if !is_meta_bucketname(&bucket) {
|
||||
let publish_lock = self.metadata_publish_lock(&bucket);
|
||||
let _publish_guard = publish_lock.lock().await;
|
||||
@@ -2081,7 +2004,6 @@ impl BucketMetadataSys {
|
||||
if is_meta_bucketname(bucket) {
|
||||
return false;
|
||||
}
|
||||
self.revoke_pending_bucket_creation(bucket).await;
|
||||
let publish_lock = self.metadata_publish_lock(bucket);
|
||||
let _publish_guard = publish_lock.lock().await;
|
||||
let mut map = self.metadata_map.write().await;
|
||||
@@ -2099,7 +2021,6 @@ impl BucketMetadataSys {
|
||||
}
|
||||
|
||||
async fn _reset(&mut self) {
|
||||
self.pending_bucket_creations.write().await.clear();
|
||||
let mut map = self.metadata_map.write().await;
|
||||
map.clear();
|
||||
drop(map);
|
||||
@@ -2238,7 +2159,6 @@ impl BucketMetadataSys {
|
||||
/// server's metadata never leaks into the ambient (first) instance.
|
||||
pub(crate) async fn persist_and_set(&self, bm: BucketMetadata) -> Result<()> {
|
||||
let mut bm = bm;
|
||||
self.revoke_pending_bucket_creation(&bm.name).await;
|
||||
bm.save_with_store(self.object_store()).await?;
|
||||
|
||||
self.set(bm.name.clone(), Arc::new(bm)).await;
|
||||
@@ -2247,7 +2167,6 @@ impl BucketMetadataSys {
|
||||
}
|
||||
|
||||
async fn persist_new_and_set(&self, mut bm: BucketMetadata) -> Result<()> {
|
||||
self.revoke_pending_bucket_creation(&bm.name).await;
|
||||
bm.bucket_creation_committed = true;
|
||||
bm.save_with_store_committed(self.object_store()).await?;
|
||||
save_bucket_incarnation(self.object_store(), &bm.name, bm.bucket_incarnation_id).await?;
|
||||
@@ -2257,9 +2176,7 @@ impl BucketMetadataSys {
|
||||
}
|
||||
|
||||
async fn persist_new_bucket_metadata_intent(&self, mut bm: BucketMetadata) -> Result<()> {
|
||||
self.revoke_pending_bucket_creation(&bm.name).await;
|
||||
bm.bucket_creation_committed = false;
|
||||
bm.bucket_creation_commit_record_present = true;
|
||||
bm.save_with_store(self.object_store()).await?;
|
||||
save_bucket_incarnation(self.object_store(), &bm.name, bm.bucket_incarnation_id).await?;
|
||||
bm.bucket_incarnation_sidecar = true;
|
||||
@@ -2269,7 +2186,6 @@ impl BucketMetadataSys {
|
||||
}
|
||||
|
||||
async fn commit_bucket_metadata(&self, mut bm: BucketMetadata) -> Result<()> {
|
||||
self.revoke_pending_bucket_creation(&bm.name).await;
|
||||
bm.parse_all_configs()?;
|
||||
bm.bucket_incarnation_sidecar = true;
|
||||
// The caller holds the full bucket creation fence (lifecycle, metadata
|
||||
@@ -2318,7 +2234,6 @@ impl BucketMetadataSys {
|
||||
bucket: &str,
|
||||
namespace_guard: &rustfs_lock::NamespaceLockGuard,
|
||||
) -> Result<()> {
|
||||
self.revoke_pending_bucket_creation(bucket).await;
|
||||
let expected = self.metadata_map.read().await.get(bucket).cloned();
|
||||
if !self
|
||||
.bucket_exists(bucket, namespace_guard, "peer bucket metadata existence check")
|
||||
@@ -2375,7 +2290,6 @@ impl BucketMetadataSys {
|
||||
#[cfg(test)]
|
||||
self.lazy_disk_loads.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
|
||||
self.revoke_pending_bucket_creation(bucket).await;
|
||||
let lock = self.object_store().new_ns_lock(bucket, bucket).await?;
|
||||
let guard = lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
|
||||
#[cfg(test)]
|
||||
@@ -2684,7 +2598,6 @@ impl BucketMetadataSys {
|
||||
/// revalidated at write quorum, so a stale snapshot can neither revert an
|
||||
/// acknowledged configuration update nor outlive a delete/recreate.
|
||||
async fn migrate_bucket_creation_commit(&self, bucket: &str) -> Result<BucketMetadataAuthority> {
|
||||
self.revoke_pending_bucket_creation(bucket).await;
|
||||
let transaction_lock = self
|
||||
.object_store()
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
|
||||
@@ -2762,7 +2675,6 @@ impl BucketMetadataSys {
|
||||
&self,
|
||||
bucket: &str,
|
||||
) -> Result<BucketMetadataAuthority> {
|
||||
self.revoke_pending_bucket_creation(bucket).await;
|
||||
#[cfg(test)]
|
||||
if self.object_lock_disk_read_errors.write().await.remove(bucket) {
|
||||
return Err(Error::other(format!("injected Object Lock metadata disk read failure: {bucket}")));
|
||||
@@ -3134,7 +3046,6 @@ mod tests {
|
||||
};
|
||||
use crate::bucket::target::{BucketTarget, BucketTargetType, Credentials};
|
||||
use crate::config::com::read_config;
|
||||
use crate::disk::{DiskAPI as _, RUSTFS_META_TMP_BUCKET};
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions};
|
||||
use byteorder::{ByteOrder as _, LittleEndian};
|
||||
use serial_test::serial;
|
||||
@@ -3189,180 +3100,6 @@ mod tests {
|
||||
),
|
||||
];
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(storage_class_env)]
|
||||
async fn persistence_entry_failures_revoke_pending_creation_before_cache_publish() {
|
||||
let (dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let sys = bucket_metadata_sys_of(&store.ctx).expect("instance metadata system");
|
||||
let sys = sys.read().await.clone();
|
||||
let options = MakeBucketOptions {
|
||||
lock_enabled: true,
|
||||
..Default::default()
|
||||
};
|
||||
for entry in ["save", "persist", "persist-new", "commit"] {
|
||||
let bucket = format!("revoke-{entry}-{}", Uuid::new_v4());
|
||||
store.fail_next_bucket_creation_after_intent_for_test(&bucket);
|
||||
store
|
||||
.make_bucket(&bucket, &options)
|
||||
.await
|
||||
.expect_err("stop the real producer before physical publication");
|
||||
sys.pending_bucket_creation(&bucket)
|
||||
.await
|
||||
.expect("the producer must leave a real live creation permission");
|
||||
let (intent, persisted) = get_config_from_disk_with_presence_in(&store.ctx, &bucket)
|
||||
.await
|
||||
.expect("read the original producer intent");
|
||||
assert!(persisted, "{entry}: the complete original intent must be persisted");
|
||||
let original_intent = intent.marshal_msg().expect("encode the original intent");
|
||||
|
||||
// Keep the creation fence order used by the real producer.
|
||||
let lifecycle_guard = store
|
||||
.acquire_bucket_lifecycle_write_lock(&bucket)
|
||||
.await
|
||||
.expect("hold the bucket lifecycle fence");
|
||||
let transaction_guard = acquire_bucket_metadata_transaction_lock_in(&store.ctx, &bucket)
|
||||
.await
|
||||
.expect("hold the metadata transaction fence");
|
||||
let namespace_lock = store
|
||||
.new_ns_lock(&bucket, &bucket)
|
||||
.await
|
||||
.expect("create the exact bucket namespace lock");
|
||||
let namespace_guard = namespace_lock
|
||||
.get_write_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await
|
||||
.expect("hold the exact bucket namespace fence");
|
||||
|
||||
// A file at each native staging root causes actual IO to fail before
|
||||
// metadata can reach the cache-publishing set call. Preserve the
|
||||
// original staging directories so no fixture data is discarded.
|
||||
for dir in &dirs {
|
||||
let staging = dir.path().join(RUSTFS_META_TMP_BUCKET);
|
||||
let saved = dir.path().join(".rustfs.sys/revocation-saved-tmp");
|
||||
std::fs::rename(&staging, &saved).expect("preserve the native staging directory");
|
||||
std::fs::write(&staging, b"block metadata staging with a non-directory")
|
||||
.expect("install an actual non-directory IO obstruction");
|
||||
}
|
||||
let result = match entry {
|
||||
"save" => sys.save(intent).await,
|
||||
"persist" => sys.persist_and_set(intent).await,
|
||||
"persist-new" => sys.persist_new_and_set(intent).await,
|
||||
"commit" => sys.commit_bucket_metadata(intent).await,
|
||||
_ => unreachable!(),
|
||||
};
|
||||
for dir in &dirs {
|
||||
let staging = dir.path().join(RUSTFS_META_TMP_BUCKET);
|
||||
let saved = dir.path().join(".rustfs.sys/revocation-saved-tmp");
|
||||
std::fs::remove_file(&staging).expect("remove only the fixture's staging obstruction");
|
||||
std::fs::rename(&saved, &staging).expect("restore the complete original staging directory");
|
||||
}
|
||||
drop((namespace_guard, transaction_guard, lifecycle_guard));
|
||||
result.expect_err("native persistence must fail before its downstream cache set");
|
||||
assert!(
|
||||
sys.pending_bucket_creation(&bucket).await.is_none(),
|
||||
"{entry}: the entry revoker must retire live permission even when persistence fails"
|
||||
);
|
||||
let (after_failure, present) = get_config_from_disk_with_presence_in(&store.ctx, &bucket)
|
||||
.await
|
||||
.expect("read the intent after the failed persistence entry");
|
||||
assert!(present, "{entry}: a failed persistence entry must retain the original metadata");
|
||||
assert_eq!(
|
||||
after_failure
|
||||
.marshal_msg()
|
||||
.expect("encode the original intent after persistence failure"),
|
||||
original_intent,
|
||||
"{entry}: native IO failure must not change the original generation or complete intent"
|
||||
);
|
||||
let error = store
|
||||
.make_bucket(&bucket, &options)
|
||||
.await
|
||||
.expect_err("a failed ordinary persistence entry cannot preserve creation permission");
|
||||
assert!(error.to_string().contains("administrator recovery"), "{entry}: {error}");
|
||||
let (after_retry, _) = get_config_from_disk_with_presence_in(&store.ctx, &bucket)
|
||||
.await
|
||||
.expect("read the original intent after the rejected retry");
|
||||
assert_eq!(
|
||||
after_retry.marshal_msg().expect("encode the intent after the rejected retry"),
|
||||
original_intent,
|
||||
"{entry}: the rejected retry must preserve the complete original intent"
|
||||
);
|
||||
for disk in store.pools[0].disk_set[0].disks.read().await.iter() {
|
||||
assert!(
|
||||
matches!(
|
||||
disk.as_ref().expect("native disk").stat_volume(&bucket).await,
|
||||
Err(crate::disk::error::DiskError::VolumeNotFound)
|
||||
),
|
||||
"{entry}: failed persistence and the rejected retry must not publish a physical volume"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(storage_class_env)]
|
||||
async fn metadata_reset_revokes_real_pending_creation_without_changing_persisted_intent() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("revoke-reset-{}", Uuid::new_v4());
|
||||
let options = MakeBucketOptions {
|
||||
lock_enabled: true,
|
||||
..Default::default()
|
||||
};
|
||||
store.fail_next_bucket_creation_after_intent_for_test(&bucket);
|
||||
store
|
||||
.make_bucket(&bucket, &options)
|
||||
.await
|
||||
.expect_err("stop the real producer before physical publication");
|
||||
let sys = bucket_metadata_sys_of(&store.ctx).expect("instance metadata system");
|
||||
let mut sys = sys.read().await.clone();
|
||||
sys.pending_bucket_creation(&bucket)
|
||||
.await
|
||||
.expect("the real producer must leave live creation permission before reset");
|
||||
let (intent, persisted) = get_config_from_disk_with_presence_in(&store.ctx, &bucket)
|
||||
.await
|
||||
.expect("read the original intent before cache reset");
|
||||
assert!(persisted, "cache reset starts from a complete real persisted intent");
|
||||
let original_intent = intent.marshal_msg().expect("encode the original reset intent");
|
||||
sys._reset().await;
|
||||
assert!(
|
||||
sys.pending_bucket_creation(&bucket).await.is_none(),
|
||||
"reset must revoke the producer-issued permission shared with the instance metadata system"
|
||||
);
|
||||
let (after_reset, present) = get_config_from_disk_with_presence_in(&store.ctx, &bucket)
|
||||
.await
|
||||
.expect("read the original intent after cache reset");
|
||||
assert!(present, "cache reset must retain the persisted intent");
|
||||
assert_eq!(
|
||||
after_reset.marshal_msg().expect("encode the intent after cache reset"),
|
||||
original_intent,
|
||||
"cache reset must leave the original generation and complete persisted intent unchanged"
|
||||
);
|
||||
let error = store
|
||||
.make_bucket(&bucket, &options)
|
||||
.await
|
||||
.expect_err("reset cannot regrant the producer's old creation permission");
|
||||
assert!(error.to_string().contains("administrator recovery"), "{error}");
|
||||
let (after_retry, _) = get_config_from_disk_with_presence_in(&store.ctx, &bucket)
|
||||
.await
|
||||
.expect("read the intent after the reset-induced retry rejection");
|
||||
assert_eq!(
|
||||
after_retry
|
||||
.marshal_msg()
|
||||
.expect("encode the intent after the rejected reset retry"),
|
||||
original_intent,
|
||||
"the reset-induced rejection must preserve the original complete intent"
|
||||
);
|
||||
for disk in store.pools[0].disk_set[0].disks.read().await.iter() {
|
||||
assert!(
|
||||
matches!(
|
||||
disk.as_ref().expect("native disk").stat_volume(&bucket).await,
|
||||
Err(crate::disk::error::DiskError::VolumeNotFound)
|
||||
),
|
||||
"cache reset and the rejected retry must not publish a physical volume"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn g_d3_003_new_writer_replication_loads_without_fail_closed_state() {
|
||||
let (dirs, store) = isolated_store_over_temp_disks().await;
|
||||
|
||||
@@ -0,0 +1,806 @@
|
||||
//! Bounded authenticated Raft RPC, mountable without a second storage owner.
|
||||
use super::{client, http_auth};
|
||||
use openraft::{
|
||||
DefensiveError, ErrorSubject, ErrorVerb, RaftNetwork, RaftNetworkFactory, StorageError, StorageIOError, Violation,
|
||||
error::{Fatal, InstallSnapshotError, RPCError, RaftError, RemoteError, Unreachable},
|
||||
network::RPCOption,
|
||||
raft::{
|
||||
AppendEntriesRequest, AppendEntriesResponse, InstallSnapshotRequest, InstallSnapshotResponse, VoteRequest, VoteResponse,
|
||||
},
|
||||
};
|
||||
use rustfs_capture_authority::{CaptureMembership, CaptureNode, CaptureTypeConfig, NativeRaftOwner, NodeIdentity};
|
||||
use rustfs_protos::proto_gen::node_service::{
|
||||
CaptureRaftRequest, CaptureRaftResponse,
|
||||
capture_raft_service_server::{CaptureRaftService, CaptureRaftServiceServer},
|
||||
};
|
||||
use serde::{Deserialize, Serialize, de::DeserializeOwned};
|
||||
use serde_json::value::RawValue;
|
||||
use std::{collections::BTreeMap, io::Write, sync::Arc, time::Duration};
|
||||
use tonic::{Request, Response, Status};
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct CaptureRaftLimits {
|
||||
pub max_request_bytes: usize,
|
||||
pub max_response_bytes: usize,
|
||||
pub max_entries: usize,
|
||||
pub max_snapshot_chunk_bytes: usize,
|
||||
pub max_inflight: usize,
|
||||
pub connect_timeout: Duration,
|
||||
pub request_timeout: Duration,
|
||||
}
|
||||
impl Default for CaptureRaftLimits {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_request_bytes: 16 * 1024 * 1024,
|
||||
max_response_bytes: 1024 * 1024,
|
||||
max_entries: 300,
|
||||
max_snapshot_chunk_bytes: 3 * 1024 * 1024,
|
||||
max_inflight: 16,
|
||||
connect_timeout: Duration::from_secs(3),
|
||||
request_timeout: Duration::from_secs(5),
|
||||
}
|
||||
}
|
||||
}
|
||||
impl CaptureRaftLimits {
|
||||
fn validate(&self) -> Result<(), CaptureRaftError> {
|
||||
let chunk = self
|
||||
.max_snapshot_chunk_bytes
|
||||
.checked_mul(4)
|
||||
.and_then(|n| n.checked_add(16384))
|
||||
.ok_or(CaptureRaftError::Limits)?;
|
||||
if self.max_request_bytes < chunk
|
||||
|| self.max_request_bytes > 64 * 1024 * 1024
|
||||
|| self.max_response_bytes < 4096
|
||||
|| self.max_response_bytes > 4 * 1024 * 1024
|
||||
|| self.max_entries == 0
|
||||
|| self.max_entries > 1024
|
||||
|| self.max_snapshot_chunk_bytes == 0
|
||||
|| self.max_inflight == 0
|
||||
|| self.max_inflight > 128
|
||||
|| self.connect_timeout.is_zero()
|
||||
|| self.request_timeout.is_zero()
|
||||
|| self.request_timeout > Duration::from_secs(60)
|
||||
|| self.connect_timeout > Duration::from_secs(60)
|
||||
{
|
||||
return Err(CaptureRaftError::Limits);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum CaptureRaftError {
|
||||
#[error("capture Raft message limit exceeded")]
|
||||
Limits,
|
||||
#[error("capture Raft request requires a smaller append batch")]
|
||||
RequestTooLarge,
|
||||
#[error("invalid capture Raft message")]
|
||||
Protocol,
|
||||
#[error("capture Raft identity mismatch")]
|
||||
Identity,
|
||||
#[error("capture Raft authentication failed")]
|
||||
Authentication,
|
||||
#[error("capture Raft channel unavailable")]
|
||||
ChannelUnavailable,
|
||||
#[error("capture Raft deadline exceeded")]
|
||||
Deadline,
|
||||
#[error(transparent)]
|
||||
Io(#[from] std::io::Error),
|
||||
#[error(transparent)]
|
||||
Json(#[from] serde_json::Error),
|
||||
#[error(transparent)]
|
||||
Node(#[from] rustfs_capture_authority::CaptureNodeError),
|
||||
#[error(transparent)]
|
||||
Status(Box<Status>),
|
||||
}
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub(crate) enum Method {
|
||||
AppendEntries,
|
||||
Vote,
|
||||
InstallSnapshot,
|
||||
}
|
||||
impl Method {
|
||||
pub(crate) fn path(self) -> &'static str {
|
||||
match self {
|
||||
Self::AppendEntries => "/node_service.CaptureRaftService/AppendEntries",
|
||||
Self::Vote => "/node_service.CaptureRaftService/Vote",
|
||||
Self::InstallSnapshot => "/node_service.CaptureRaftService/InstallSnapshot",
|
||||
}
|
||||
}
|
||||
}
|
||||
#[derive(Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub(crate) struct WireRequest {
|
||||
pub version: u32,
|
||||
pub cluster: [u8; 16],
|
||||
pub source: u64,
|
||||
pub target: u64,
|
||||
pub method: Method,
|
||||
pub nonce: [u8; 16],
|
||||
pub payload: Box<RawValue>,
|
||||
}
|
||||
struct Bounded {
|
||||
bytes: Vec<u8>,
|
||||
max: usize,
|
||||
}
|
||||
impl Write for Bounded {
|
||||
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
|
||||
let end = self
|
||||
.bytes
|
||||
.len()
|
||||
.checked_add(bytes.len())
|
||||
.ok_or_else(|| std::io::Error::other("capture RPC limit"))?;
|
||||
if end > self.max {
|
||||
return Err(std::io::Error::other("capture RPC limit"));
|
||||
}
|
||||
self.bytes.try_reserve(bytes.len()).map_err(std::io::Error::other)?;
|
||||
self.bytes.extend_from_slice(bytes);
|
||||
Ok(bytes.len())
|
||||
}
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
fn encoded<T: Serialize>(value: &T, max: usize) -> Result<Vec<u8>, CaptureRaftError> {
|
||||
let mut out = Bounded { bytes: Vec::new(), max };
|
||||
serde_json::to_writer(&mut out, value).map_err(|error| {
|
||||
if error.is_io() {
|
||||
CaptureRaftError::Limits
|
||||
} else {
|
||||
CaptureRaftError::Json(error)
|
||||
}
|
||||
})?;
|
||||
Ok(out.bytes)
|
||||
}
|
||||
fn raw<T: Serialize>(value: &T, max: usize) -> Result<Box<RawValue>, CaptureRaftError> {
|
||||
Ok(RawValue::from_string(
|
||||
String::from_utf8(encoded(value, max)?).map_err(|_| CaptureRaftError::Protocol)?,
|
||||
)?)
|
||||
}
|
||||
pub(crate) fn encode_request<P: Serialize>(
|
||||
local: NodeIdentity,
|
||||
target: u64,
|
||||
method: Method,
|
||||
payload: &P,
|
||||
limits: CaptureRaftLimits,
|
||||
) -> Result<CaptureRaftRequest, CaptureRaftError> {
|
||||
let wire = WireRequest {
|
||||
version: 1,
|
||||
cluster: local.cluster_id,
|
||||
source: local.node_id,
|
||||
target,
|
||||
method,
|
||||
nonce: *uuid::Uuid::new_v4().as_bytes(),
|
||||
payload: raw(payload, limits.max_request_bytes)?,
|
||||
};
|
||||
Ok(CaptureRaftRequest {
|
||||
body: encoded(&wire, limits.max_request_bytes)?.into(),
|
||||
})
|
||||
}
|
||||
pub(crate) fn canonical_request(body: &[u8]) -> Vec<u8> {
|
||||
let mut out = b"rustfs.capture-raft.request.v1\0".to_vec();
|
||||
out.extend_from_slice(body);
|
||||
out
|
||||
}
|
||||
fn decode_wire(body: &[u8], max: usize) -> Result<WireRequest, CaptureRaftError> {
|
||||
if body.len() > max {
|
||||
return Err(CaptureRaftError::Limits);
|
||||
}
|
||||
let wire: WireRequest = serde_json::from_slice(body)?;
|
||||
if wire.version != 1 || wire.cluster == [0; 16] || wire.nonce == [0; 16] || encoded(&wire, max)? != body {
|
||||
return Err(CaptureRaftError::Protocol);
|
||||
}
|
||||
Ok(wire)
|
||||
}
|
||||
fn decode_payload<P: Serialize + DeserializeOwned>(wire: &WireRequest, max: usize) -> Result<P, CaptureRaftError> {
|
||||
let value: P = serde_json::from_str(wire.payload.get())?;
|
||||
if encoded(&value, max)?.as_slice() != wire.payload.get().as_bytes() {
|
||||
return Err(CaptureRaftError::Protocol);
|
||||
}
|
||||
Ok(value)
|
||||
}
|
||||
#[derive(Serialize, Deserialize)]
|
||||
#[serde(tag = "kind", deny_unknown_fields)]
|
||||
pub(crate) enum RemoteFailure {
|
||||
Stopped,
|
||||
Panicked,
|
||||
StorageIo {
|
||||
operation: Option<IoOperation>,
|
||||
},
|
||||
StorageDefensive {
|
||||
subject: ErrorSubject<u64>,
|
||||
violation: Violation<u64>,
|
||||
},
|
||||
SnapshotMismatch {
|
||||
error: InstallSnapshotError,
|
||||
},
|
||||
}
|
||||
#[derive(Serialize, Deserialize)]
|
||||
pub(crate) struct IoOperation {
|
||||
subject: ErrorSubject<u64>,
|
||||
verb: ErrorVerb,
|
||||
}
|
||||
fn fatal(error: Fatal<u64>) -> RemoteFailure {
|
||||
match error {
|
||||
Fatal::Stopped => RemoteFailure::Stopped,
|
||||
Fatal::Panicked => RemoteFailure::Panicked,
|
||||
Fatal::StorageError(StorageError::IO { source }) => RemoteFailure::StorageIo {
|
||||
operation: encoded(&source, 16384)
|
||||
.ok()
|
||||
.and_then(|bytes| serde_json::from_slice(&bytes).ok()),
|
||||
},
|
||||
Fatal::StorageError(StorageError::Defensive { source }) => RemoteFailure::StorageDefensive {
|
||||
subject: source.subject,
|
||||
violation: source.violation,
|
||||
},
|
||||
}
|
||||
}
|
||||
#[derive(Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
enum WireReply {
|
||||
Success { payload: Box<RawValue> },
|
||||
Failure { error: RemoteFailure },
|
||||
}
|
||||
fn response_context(request: &CaptureRaftRequest, body: &[u8]) -> Result<Vec<u8>, CaptureRaftError> {
|
||||
let mut out = b"rustfs.capture-raft.response.v1\0".to_vec();
|
||||
out.try_reserve(
|
||||
request
|
||||
.body
|
||||
.len()
|
||||
.checked_add(body.len())
|
||||
.and_then(|n| n.checked_add(16))
|
||||
.ok_or(CaptureRaftError::Limits)?,
|
||||
)
|
||||
.map_err(std::io::Error::other)?;
|
||||
out.extend(
|
||||
u64::try_from(request.body.len())
|
||||
.map_err(|_| CaptureRaftError::Limits)?
|
||||
.to_be_bytes(),
|
||||
);
|
||||
out.extend_from_slice(&request.body);
|
||||
out.extend(u64::try_from(body.len()).map_err(|_| CaptureRaftError::Limits)?.to_be_bytes());
|
||||
out.extend_from_slice(body);
|
||||
Ok(out)
|
||||
}
|
||||
fn reply<P: Serialize>(
|
||||
request: &CaptureRaftRequest,
|
||||
result: Result<P, RemoteFailure>,
|
||||
limits: CaptureRaftLimits,
|
||||
) -> Result<CaptureRaftResponse, CaptureRaftError> {
|
||||
let wire = match result {
|
||||
Ok(value) => WireReply::Success {
|
||||
payload: raw(&value, limits.max_response_bytes)?,
|
||||
},
|
||||
Err(error) => WireReply::Failure { error },
|
||||
};
|
||||
let body = encoded(&wire, limits.max_response_bytes)?;
|
||||
let response_proof = http_auth::sign_tonic_rpc_response_proof(&response_context(request, &body)?)?;
|
||||
Ok(CaptureRaftResponse {
|
||||
body: body.into(),
|
||||
response_proof: response_proof.into(),
|
||||
})
|
||||
}
|
||||
pub(crate) fn verify_response<P: Serialize + DeserializeOwned>(
|
||||
request: &CaptureRaftRequest,
|
||||
response: CaptureRaftResponse,
|
||||
limits: CaptureRaftLimits,
|
||||
) -> Result<Result<P, RemoteFailure>, CaptureRaftError> {
|
||||
if request.body.len() > limits.max_request_bytes
|
||||
|| response.body.len() > limits.max_response_bytes
|
||||
|| response.response_proof.len() != 32
|
||||
{
|
||||
return Err(CaptureRaftError::Limits);
|
||||
}
|
||||
http_auth::verify_tonic_rpc_response_proof(&response_context(request, &response.body)?, &response.response_proof)?;
|
||||
let wire: WireReply = serde_json::from_slice(&response.body)?;
|
||||
if encoded(&wire, limits.max_response_bytes)?.as_slice() != response.body {
|
||||
return Err(CaptureRaftError::Protocol);
|
||||
}
|
||||
match wire {
|
||||
WireReply::Failure { error } => Ok(Err(error)),
|
||||
WireReply::Success { payload } => {
|
||||
let value: P = serde_json::from_str(payload.get())?;
|
||||
if encoded(&value, limits.max_response_bytes)?.as_slice() != payload.get().as_bytes() {
|
||||
return Err(CaptureRaftError::Protocol);
|
||||
}
|
||||
Ok(Ok(value))
|
||||
}
|
||||
}
|
||||
}
|
||||
fn restored_fatal(error: RemoteFailure) -> Result<Fatal<u64>, CaptureRaftError> {
|
||||
Ok(match error {
|
||||
RemoteFailure::Stopped => Fatal::Stopped,
|
||||
RemoteFailure::Panicked => Fatal::Panicked,
|
||||
RemoteFailure::StorageIo { operation } => {
|
||||
let (subject, verb) = operation
|
||||
.map(|op| (op.subject, op.verb))
|
||||
.unwrap_or((ErrorSubject::None, ErrorVerb::Read));
|
||||
Fatal::StorageError(
|
||||
StorageIOError::new(
|
||||
subject,
|
||||
verb,
|
||||
openraft::AnyError::new(&std::io::Error::other("remote Raft storage I/O failed; detailed source redacted")),
|
||||
)
|
||||
.into(),
|
||||
)
|
||||
}
|
||||
RemoteFailure::StorageDefensive { subject, violation } => Fatal::StorageError(StorageError::Defensive {
|
||||
source: DefensiveError {
|
||||
subject,
|
||||
violation,
|
||||
backtrace: None,
|
||||
},
|
||||
}),
|
||||
RemoteFailure::SnapshotMismatch { .. } => return Err(CaptureRaftError::Protocol),
|
||||
})
|
||||
}
|
||||
fn peers_valid(local: NodeIdentity, peers: &BTreeMap<u64, CaptureNode>) -> Result<(), CaptureRaftError> {
|
||||
if local.cluster_id == [0; 16] || !peers.contains_key(&local.node_id) {
|
||||
return Err(CaptureRaftError::Identity);
|
||||
}
|
||||
let mut addresses = std::collections::BTreeSet::new();
|
||||
for node in peers.values() {
|
||||
node.validate()?;
|
||||
if !addresses.insert(node.rpc_uri()) {
|
||||
return Err(CaptureRaftError::Identity);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
type WorkerPause = Arc<std::sync::Mutex<Option<(Arc<tokio::sync::Notify>, Arc<tokio::sync::Notify>)>>>;
|
||||
#[derive(Clone)]
|
||||
pub struct CaptureRaftRpc {
|
||||
owner: Arc<NativeRaftOwner>,
|
||||
peers: Arc<BTreeMap<u64, CaptureNode>>,
|
||||
audience: String,
|
||||
limits: CaptureRaftLimits,
|
||||
admission: Arc<tokio::sync::Semaphore>,
|
||||
accepting: Arc<std::sync::atomic::AtomicBool>,
|
||||
#[cfg(test)]
|
||||
pub(crate) worker_pause: WorkerPause,
|
||||
}
|
||||
impl CaptureRaftRpc {
|
||||
pub fn new(
|
||||
owner: Arc<NativeRaftOwner>,
|
||||
peers: BTreeMap<u64, CaptureNode>,
|
||||
limits: CaptureRaftLimits,
|
||||
) -> Result<Self, CaptureRaftError> {
|
||||
limits.validate()?;
|
||||
let identity = owner.identity();
|
||||
peers_valid(identity, &peers)?;
|
||||
let node = peers.get(&identity.node_id).ok_or(CaptureRaftError::Identity)?;
|
||||
let uri: http::Uri = node.rpc_uri().parse().map_err(|_| CaptureRaftError::Identity)?;
|
||||
let audience = http_auth::normalize_tonic_rpc_audience(uri.authority().ok_or(CaptureRaftError::Identity)?.as_str())?;
|
||||
Ok(Self {
|
||||
owner,
|
||||
peers: Arc::new(peers),
|
||||
audience,
|
||||
limits,
|
||||
#[cfg(test)]
|
||||
worker_pause: Arc::new(std::sync::Mutex::new(None)),
|
||||
accepting: Arc::new(std::sync::atomic::AtomicBool::new(true)),
|
||||
admission: Arc::new(tokio::sync::Semaphore::new(limits.max_inflight)),
|
||||
})
|
||||
}
|
||||
pub fn into_service(self) -> CaptureRaftServiceServer<Self> {
|
||||
let request = self.limits.max_request_bytes + 64;
|
||||
let response = self.limits.max_response_bytes + 128;
|
||||
CaptureRaftServiceServer::new(self)
|
||||
.max_decoding_message_size(request)
|
||||
.max_encoding_message_size(response)
|
||||
}
|
||||
fn prepare<P: Serialize + DeserializeOwned>(
|
||||
&self,
|
||||
request: &Request<CaptureRaftRequest>,
|
||||
method: Method,
|
||||
) -> Result<(WireRequest, P, tokio::sync::OwnedSemaphorePermit), Status> {
|
||||
if !self.accepting.load(std::sync::atomic::Ordering::Acquire) {
|
||||
return Err(Status::unavailable("capture Raft service stopping"));
|
||||
}
|
||||
let permit = self
|
||||
.admission
|
||||
.clone()
|
||||
.try_acquire_owned()
|
||||
.map_err(|_| Status::resource_exhausted("capture Raft admission full"))?;
|
||||
if !self.accepting.load(std::sync::atomic::Ordering::Acquire) {
|
||||
return Err(Status::unavailable("capture Raft service stopping"));
|
||||
}
|
||||
if request.get_ref().body.len() > self.limits.max_request_bytes {
|
||||
return Err(Status::resource_exhausted("capture Raft request limit"));
|
||||
}
|
||||
if request
|
||||
.metadata()
|
||||
.get(http_auth::RPC_REPLAY_SCOPE_VERSION_HEADER)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
!= Some("3")
|
||||
{
|
||||
return Err(auth_rejected("missing_replay_scope"));
|
||||
}
|
||||
http_auth::verify_tonic_rpc_signature_with_bootstrap(&self.audience, method.path(), request.metadata().as_ref(), false)
|
||||
.map_err(|_| auth_rejected("invalid_signature"))?;
|
||||
http_auth::verify_tonic_canonical_body_digest(request, &canonical_request(&request.get_ref().body))
|
||||
.map_err(|_| auth_rejected("body_digest_mismatch"))?;
|
||||
let wire = decode_wire(&request.get_ref().body, self.limits.max_request_bytes)
|
||||
.map_err(|_| Status::invalid_argument("capture Raft wire rejected"))?;
|
||||
let local = self.owner.identity();
|
||||
if wire.cluster != local.cluster_id
|
||||
|| wire.target != local.node_id
|
||||
|| wire.method != method
|
||||
|| !self.peers.contains_key(&wire.source)
|
||||
{
|
||||
return Err(Status::permission_denied("capture Raft identity rejected"));
|
||||
}
|
||||
let payload = decode_payload(&wire, self.limits.max_request_bytes)
|
||||
.map_err(|_| Status::invalid_argument("capture Raft payload rejected"))?;
|
||||
Ok((wire, payload, permit))
|
||||
}
|
||||
#[cfg(test)]
|
||||
async fn paused(pause: WorkerPause) {
|
||||
let gate = pause.lock().expect("worker pause lock").take();
|
||||
if let Some((entered, resume)) = gate {
|
||||
entered.notify_one();
|
||||
resume.notified().await;
|
||||
}
|
||||
}
|
||||
/// Stop admission and wait for all actual RPC tasks, including cancelled HTTP waiters.
|
||||
pub async fn shutdown(&self) -> Result<(), CaptureRaftError> {
|
||||
self.accepting.store(false, std::sync::atomic::Ordering::Release);
|
||||
let count = u32::try_from(self.limits.max_inflight).map_err(|_| CaptureRaftError::Limits)?;
|
||||
let _all = self
|
||||
.admission
|
||||
.clone()
|
||||
.acquire_many_owned(count)
|
||||
.await
|
||||
.map_err(|_| CaptureRaftError::ChannelUnavailable)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
fn signed_response<P: Serialize>(
|
||||
raw: &CaptureRaftRequest,
|
||||
result: Result<P, RemoteFailure>,
|
||||
limits: CaptureRaftLimits,
|
||||
audience: &str,
|
||||
challenge: Option<uuid::Uuid>,
|
||||
) -> Result<Response<CaptureRaftResponse>, Status> {
|
||||
let mut response =
|
||||
Response::new(reply(raw, result, limits).map_err(|_| Status::internal("capture Raft response unavailable"))?);
|
||||
if let Some(challenge) = challenge {
|
||||
response.metadata_mut().as_mut().extend(
|
||||
http_auth::tonic_boot_epoch_response_headers(audience, challenge)
|
||||
.map_err(|_| Status::internal("capture Raft epoch proof unavailable"))?,
|
||||
);
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
#[tonic::async_trait]
|
||||
impl CaptureRaftService for CaptureRaftRpc {
|
||||
async fn append_entries(&self, request: Request<CaptureRaftRequest>) -> Result<Response<CaptureRaftResponse>, Status> {
|
||||
let (wire, payload, permit) = self.prepare::<AppendEntriesRequest<CaptureTypeConfig>>(&request, Method::AppendEntries)?;
|
||||
validate_append(&payload, wire.source, self.limits)
|
||||
.map_err(|_| Status::invalid_argument("capture Raft append rejected"))?;
|
||||
let challenge = http_auth::tonic_boot_epoch_challenge(request.metadata().as_ref())
|
||||
.map_err(|_| Status::unauthenticated("capture Raft challenge rejected"))?;
|
||||
let raw = request.into_inner();
|
||||
let owner = self.owner.clone();
|
||||
let audience = self.audience.clone();
|
||||
let limits = self.limits;
|
||||
#[cfg(test)]
|
||||
let pause = self.worker_pause.clone();
|
||||
tokio::spawn(async move {
|
||||
let _permit = permit;
|
||||
#[cfg(test)]
|
||||
Self::paused(pause).await;
|
||||
let result = owner.raft().append_entries(payload).await.map_err(|error| match error {
|
||||
RaftError::Fatal(error) => fatal(error),
|
||||
RaftError::APIError(error) => match error {},
|
||||
});
|
||||
drop(owner);
|
||||
signed_response(&raw, result, limits, &audience, challenge)
|
||||
})
|
||||
.await
|
||||
.map_err(|_| Status::internal("capture Raft worker failed"))?
|
||||
}
|
||||
async fn vote(&self, request: Request<CaptureRaftRequest>) -> Result<Response<CaptureRaftResponse>, Status> {
|
||||
let (wire, payload, permit) = self.prepare::<VoteRequest<u64>>(&request, Method::Vote)?;
|
||||
if payload.vote.leader_id.node_id != wire.source {
|
||||
return Err(Status::invalid_argument("capture Raft candidate rejected"));
|
||||
}
|
||||
let challenge = http_auth::tonic_boot_epoch_challenge(request.metadata().as_ref())
|
||||
.map_err(|_| Status::unauthenticated("capture Raft challenge rejected"))?;
|
||||
let raw = request.into_inner();
|
||||
let owner = self.owner.clone();
|
||||
let audience = self.audience.clone();
|
||||
let limits = self.limits;
|
||||
#[cfg(test)]
|
||||
let pause = self.worker_pause.clone();
|
||||
tokio::spawn(async move {
|
||||
let _permit = permit;
|
||||
#[cfg(test)]
|
||||
Self::paused(pause).await;
|
||||
let result = owner.raft().vote(payload).await.map_err(|error| match error {
|
||||
RaftError::Fatal(error) => fatal(error),
|
||||
RaftError::APIError(error) => match error {},
|
||||
});
|
||||
drop(owner);
|
||||
signed_response(&raw, result, limits, &audience, challenge)
|
||||
})
|
||||
.await
|
||||
.map_err(|_| Status::internal("capture Raft worker failed"))?
|
||||
}
|
||||
async fn install_snapshot(&self, request: Request<CaptureRaftRequest>) -> Result<Response<CaptureRaftResponse>, Status> {
|
||||
let (wire, payload, permit) =
|
||||
self.prepare::<InstallSnapshotRequest<CaptureTypeConfig>>(&request, Method::InstallSnapshot)?;
|
||||
validate_snapshot(&payload, wire.source, self.limits)
|
||||
.map_err(|_| Status::invalid_argument("capture Raft snapshot rejected"))?;
|
||||
let challenge = http_auth::tonic_boot_epoch_challenge(request.metadata().as_ref())
|
||||
.map_err(|_| Status::unauthenticated("capture Raft challenge rejected"))?;
|
||||
let raw = request.into_inner();
|
||||
let owner = self.owner.clone();
|
||||
let audience = self.audience.clone();
|
||||
let limits = self.limits;
|
||||
#[cfg(test)]
|
||||
let pause = self.worker_pause.clone();
|
||||
tokio::spawn(async move {
|
||||
let _permit = permit;
|
||||
#[cfg(test)]
|
||||
Self::paused(pause).await;
|
||||
let result = owner.raft().install_snapshot(payload).await.map_err(|error| match error {
|
||||
RaftError::Fatal(error) => fatal(error),
|
||||
RaftError::APIError(error) => RemoteFailure::SnapshotMismatch { error },
|
||||
});
|
||||
drop(owner);
|
||||
signed_response(&raw, result, limits, &audience, challenge)
|
||||
})
|
||||
.await
|
||||
.map_err(|_| Status::internal("capture Raft worker failed"))?
|
||||
}
|
||||
}
|
||||
fn validate_append(
|
||||
request: &AppendEntriesRequest<CaptureTypeConfig>,
|
||||
source: u64,
|
||||
limits: CaptureRaftLimits,
|
||||
) -> Result<(), CaptureRaftError> {
|
||||
if request.vote.leader_id.node_id != source {
|
||||
return Err(CaptureRaftError::Identity);
|
||||
}
|
||||
if request.entries.len() > limits.max_entries {
|
||||
return Err(CaptureRaftError::RequestTooLarge);
|
||||
}
|
||||
for entry in &request.entries {
|
||||
if let openraft::EntryPayload::Membership(value) = &entry.payload {
|
||||
CaptureMembership::try_from(&openraft::StoredMembership::new(Some(entry.log_id), value.clone()))
|
||||
.map_err(|_| CaptureRaftError::Protocol)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
fn validate_snapshot(
|
||||
request: &InstallSnapshotRequest<CaptureTypeConfig>,
|
||||
source: u64,
|
||||
limits: CaptureRaftLimits,
|
||||
) -> Result<(), CaptureRaftError> {
|
||||
if request.vote.leader_id.node_id != source
|
||||
|| request.data.len() > limits.max_snapshot_chunk_bytes
|
||||
|| request
|
||||
.offset
|
||||
.checked_add(u64::try_from(request.data.len()).map_err(|_| CaptureRaftError::Limits)?)
|
||||
.is_none()
|
||||
|| request.meta.snapshot_id.is_empty()
|
||||
|| request.meta.snapshot_id.len() > 1024
|
||||
{
|
||||
return Err(CaptureRaftError::Limits);
|
||||
}
|
||||
CaptureMembership::try_from(&request.meta.last_membership).map_err(|_| CaptureRaftError::Protocol)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct CaptureRaftNetworkFactory {
|
||||
local: NodeIdentity,
|
||||
peers: Arc<BTreeMap<u64, CaptureNode>>,
|
||||
limits: CaptureRaftLimits,
|
||||
admission: Arc<tokio::sync::Semaphore>,
|
||||
}
|
||||
impl CaptureRaftNetworkFactory {
|
||||
pub fn new(
|
||||
local: NodeIdentity,
|
||||
peers: BTreeMap<u64, CaptureNode>,
|
||||
config: Arc<openraft::Config>,
|
||||
limits: CaptureRaftLimits,
|
||||
) -> Result<Self, CaptureRaftError> {
|
||||
limits.validate()?;
|
||||
peers_valid(local, &peers)?;
|
||||
if usize::try_from(config.max_payload_entries).map_err(|_| CaptureRaftError::Limits)? > limits.max_entries {
|
||||
return Err(CaptureRaftError::Limits);
|
||||
}
|
||||
if usize::try_from(config.snapshot_max_chunk_size).map_err(|_| CaptureRaftError::Limits)?
|
||||
> limits.max_snapshot_chunk_bytes
|
||||
{
|
||||
return Err(CaptureRaftError::Limits);
|
||||
}
|
||||
Ok(Self {
|
||||
local,
|
||||
peers: Arc::new(peers),
|
||||
limits,
|
||||
admission: Arc::new(tokio::sync::Semaphore::new(limits.max_inflight)),
|
||||
})
|
||||
}
|
||||
}
|
||||
pub struct CaptureRaftPeer {
|
||||
factory: CaptureRaftNetworkFactory,
|
||||
target: u64,
|
||||
node: CaptureNode,
|
||||
valid: bool,
|
||||
}
|
||||
impl RaftNetworkFactory<CaptureTypeConfig> for CaptureRaftNetworkFactory {
|
||||
type Network = CaptureRaftPeer;
|
||||
async fn new_client(&mut self, target: u64, node: &CaptureNode) -> Self::Network {
|
||||
CaptureRaftPeer {
|
||||
factory: self.clone(),
|
||||
target,
|
||||
node: node.clone(),
|
||||
valid: self.peers.get(&target) == Some(node),
|
||||
}
|
||||
}
|
||||
}
|
||||
impl CaptureRaftPeer {
|
||||
async fn exchange<P: Serialize, R: Serialize + DeserializeOwned>(
|
||||
&self,
|
||||
method: Method,
|
||||
payload: &P,
|
||||
option: &RPCOption,
|
||||
) -> Result<Result<R, RemoteFailure>, CaptureRaftError> {
|
||||
if !self.valid {
|
||||
return Err(CaptureRaftError::Identity);
|
||||
}
|
||||
let ttl = self.factory.limits.request_timeout.min(option.soft_ttl());
|
||||
tokio::time::timeout(ttl, async {
|
||||
let _permit = self
|
||||
.factory
|
||||
.admission
|
||||
.clone()
|
||||
.acquire_owned()
|
||||
.await
|
||||
.map_err(|_| CaptureRaftError::ChannelUnavailable)?;
|
||||
let raw =
|
||||
encode_request(self.factory.local, self.target, method, payload, self.factory.limits).map_err(
|
||||
|error| match error {
|
||||
CaptureRaftError::Limits => CaptureRaftError::RequestTooLarge,
|
||||
error => error,
|
||||
},
|
||||
)?;
|
||||
let mut request = Request::new(raw.clone());
|
||||
http_auth::set_tonic_canonical_body_digest(&mut request, &canonical_request(&raw.body))?;
|
||||
request.set_timeout(ttl);
|
||||
let mut client = client::capture_raft_time_out_client(
|
||||
self.node.rpc_uri(),
|
||||
self.factory.limits.connect_timeout.min(ttl),
|
||||
self.factory.limits.max_request_bytes + 64,
|
||||
self.factory.limits.max_response_bytes + 128,
|
||||
)
|
||||
.await?;
|
||||
let response = match method {
|
||||
Method::AppendEntries => client.append_entries(request).await,
|
||||
Method::Vote => client.vote(request).await,
|
||||
Method::InstallSnapshot => client.install_snapshot(request).await,
|
||||
}
|
||||
.map_err(|status| CaptureRaftError::Status(Box::new(status)))?;
|
||||
verify_response(&raw, response.into_inner(), self.factory.limits)
|
||||
})
|
||||
.await
|
||||
.map_err(|_| CaptureRaftError::Deadline)?
|
||||
}
|
||||
fn network_error<E: std::error::Error>(
|
||||
&self,
|
||||
error: CaptureRaftError,
|
||||
method: Method,
|
||||
option: &RPCOption,
|
||||
) -> RPCError<u64, CaptureNode, E> {
|
||||
match error {
|
||||
CaptureRaftError::Deadline => RPCError::Timeout(openraft::error::Timeout {
|
||||
action: match method {
|
||||
Method::AppendEntries => openraft::RPCTypes::AppendEntries,
|
||||
Method::Vote => openraft::RPCTypes::Vote,
|
||||
Method::InstallSnapshot => openraft::RPCTypes::InstallSnapshot,
|
||||
},
|
||||
id: self.factory.local.node_id,
|
||||
target: self.target,
|
||||
timeout: self.factory.limits.request_timeout.min(option.soft_ttl()),
|
||||
}),
|
||||
error @ CaptureRaftError::ChannelUnavailable => RPCError::Unreachable(Unreachable::new(&error)),
|
||||
error => RPCError::Network(openraft::error::NetworkError::new(&error)),
|
||||
}
|
||||
}
|
||||
fn append_error<E: std::error::Error>(
|
||||
&self,
|
||||
error: CaptureRaftError,
|
||||
count: usize,
|
||||
option: &RPCOption,
|
||||
) -> RPCError<u64, CaptureNode, E> {
|
||||
if matches!(error, CaptureRaftError::RequestTooLarge) && count > 1 {
|
||||
let hint = u64::try_from((count / 2).max(1).min(self.factory.limits.max_entries)).unwrap_or(1);
|
||||
return RPCError::PayloadTooLarge(openraft::error::PayloadTooLarge::new_entries_hint(hint));
|
||||
}
|
||||
self.network_error(error, Method::AppendEntries, option)
|
||||
}
|
||||
fn remote<E: std::error::Error>(&self, error: E) -> RPCError<u64, CaptureNode, E> {
|
||||
RPCError::RemoteError(RemoteError::new_with_node(self.target, self.node.clone(), error))
|
||||
}
|
||||
}
|
||||
impl RaftNetwork<CaptureTypeConfig> for CaptureRaftPeer {
|
||||
async fn append_entries(
|
||||
&mut self,
|
||||
request: AppendEntriesRequest<CaptureTypeConfig>,
|
||||
option: RPCOption,
|
||||
) -> Result<AppendEntriesResponse<u64>, RPCError<u64, CaptureNode, RaftError<u64>>> {
|
||||
let count = request.entries.len();
|
||||
validate_append(&request, self.factory.local.node_id, self.factory.limits)
|
||||
.map_err(|error| self.append_error(error, count, &option))?;
|
||||
match self
|
||||
.exchange(Method::AppendEntries, &request, &option)
|
||||
.await
|
||||
.map_err(|error| self.append_error(error, count, &option))?
|
||||
{
|
||||
Ok(value) => Ok(value),
|
||||
Err(error) => Err(self.remote(RaftError::Fatal(
|
||||
restored_fatal(error).map_err(|error| self.network_error(error, Method::AppendEntries, &option))?,
|
||||
))),
|
||||
}
|
||||
}
|
||||
async fn vote(
|
||||
&mut self,
|
||||
request: VoteRequest<u64>,
|
||||
option: RPCOption,
|
||||
) -> Result<VoteResponse<u64>, RPCError<u64, CaptureNode, RaftError<u64>>> {
|
||||
if request.vote.leader_id.node_id != self.factory.local.node_id {
|
||||
return Err(self.network_error(CaptureRaftError::Identity, Method::Vote, &option));
|
||||
}
|
||||
match self
|
||||
.exchange(Method::Vote, &request, &option)
|
||||
.await
|
||||
.map_err(|error| self.network_error(error, Method::Vote, &option))?
|
||||
{
|
||||
Ok(value) => Ok(value),
|
||||
Err(error) => Err(self.remote(RaftError::Fatal(
|
||||
restored_fatal(error).map_err(|error| self.network_error(error, Method::Vote, &option))?,
|
||||
))),
|
||||
}
|
||||
}
|
||||
async fn install_snapshot(
|
||||
&mut self,
|
||||
request: InstallSnapshotRequest<CaptureTypeConfig>,
|
||||
option: RPCOption,
|
||||
) -> Result<InstallSnapshotResponse<u64>, RPCError<u64, CaptureNode, RaftError<u64, InstallSnapshotError>>> {
|
||||
validate_snapshot(&request, self.factory.local.node_id, self.factory.limits)
|
||||
.map_err(|error| self.network_error(error, Method::InstallSnapshot, &option))?;
|
||||
match self
|
||||
.exchange(Method::InstallSnapshot, &request, &option)
|
||||
.await
|
||||
.map_err(|error| self.network_error(error, Method::InstallSnapshot, &option))?
|
||||
{
|
||||
Ok(value) => Ok(value),
|
||||
Err(error) => {
|
||||
let source = match error {
|
||||
RemoteFailure::SnapshotMismatch { error } => RaftError::APIError(error),
|
||||
error => RaftError::Fatal(
|
||||
restored_fatal(error).map_err(|error| self.network_error(error, Method::InstallSnapshot, &option))?,
|
||||
),
|
||||
};
|
||||
Err(self.remote(source))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn auth_rejected(reason: &'static str) -> Status {
|
||||
use rustfs_io_metrics::internode_metrics::{
|
||||
INTERNODE_OPERATION_GRPC_OTHER, INTERNODE_TRANSPORT_BACKEND_GRPC, global_internode_metrics,
|
||||
};
|
||||
global_internode_metrics().record_rpc_auth_failure_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_GRPC_OTHER,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
reason,
|
||||
);
|
||||
Status::unauthenticated("capture Raft authentication rejected")
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -535,6 +535,62 @@ impl tonic::service::Interceptor for TonicInterceptor {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) type CaptureRaftClient = rustfs_protos::proto_gen::node_service::capture_raft_service_client::CaptureRaftServiceClient<
|
||||
InterceptedService<AuthenticatedChannel, TonicInterceptor>,
|
||||
>;
|
||||
|
||||
/// Capture RPCs always perform the real authenticated epoch handshake, never seed replay state.
|
||||
pub(crate) async fn capture_raft_time_out_client(
|
||||
addr: &str,
|
||||
connect_timeout: std::time::Duration,
|
||||
request_max: usize,
|
||||
response_max: usize,
|
||||
) -> Result<CaptureRaftClient, super::capture_raft::CaptureRaftError> {
|
||||
use super::capture_raft::CaptureRaftError;
|
||||
let interceptor = TonicInterceptor::Signature(gen_tonic_signature_interceptor()).with_rpc_audience(addr)?;
|
||||
let channel = tokio::time::timeout(connect_timeout, async {
|
||||
match runtime_sources::cached_node_channel(addr).await {
|
||||
Some(channel) => Ok(channel),
|
||||
None => create_new_channel(addr)
|
||||
.await
|
||||
.map_err(|_| CaptureRaftError::ChannelUnavailable),
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| CaptureRaftError::Deadline)??;
|
||||
let audience = interceptor
|
||||
.replay_scope_audience()
|
||||
.ok_or(CaptureRaftError::Authentication)?
|
||||
.to_owned();
|
||||
let authenticated = ReplayScopeChannel::new(channel, Some(audience.clone()));
|
||||
let mut ping = NodeServiceClient::with_interceptor(authenticated.clone(), interceptor)
|
||||
.max_encoding_message_size(1024)
|
||||
.max_decoding_message_size(8192);
|
||||
ping.ping(tonic::Request::new(rustfs_protos::proto_gen::node_service::PingRequest {
|
||||
version: 1,
|
||||
body: bytes::Bytes::new(),
|
||||
}))
|
||||
.await
|
||||
.map_err(|status| CaptureRaftError::Status(Box::new(status)))?;
|
||||
{
|
||||
let peers = PEER_REPLAY_STATES.lock().map_err(|_| CaptureRaftError::Authentication)?;
|
||||
let state = peers.get(&audience).ok_or(CaptureRaftError::Authentication)?;
|
||||
if !matches!(state.cache_capability, Some(PeerReplayCapability::Capable { boot_epoch }) if Some(boot_epoch) == state.boot_epoch)
|
||||
{
|
||||
return Err(CaptureRaftError::Authentication);
|
||||
}
|
||||
}
|
||||
let interceptor = TonicInterceptor::Signature(gen_tonic_signature_interceptor()).with_rpc_audience(addr)?;
|
||||
Ok(
|
||||
rustfs_protos::proto_gen::node_service::capture_raft_service_client::CaptureRaftServiceClient::with_interceptor(
|
||||
authenticated,
|
||||
interceptor,
|
||||
)
|
||||
.max_encoding_message_size(request_max)
|
||||
.max_decoding_message_size(response_max),
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -13,6 +13,9 @@
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) mod background_monitor;
|
||||
pub mod capture_raft;
|
||||
#[cfg(test)]
|
||||
mod capture_raft_tests;
|
||||
pub(crate) mod client;
|
||||
pub(crate) mod context_propagation;
|
||||
pub(crate) mod http_auth;
|
||||
|
||||
@@ -58,13 +58,15 @@ use rustfs_protos::proto_gen::node_service::{
|
||||
BatchReadVersionRequest, BatchReadVersionResponse, CheckPartsRequest, CompareAndUpdateFileOutcome,
|
||||
CompareAndUpdateFileRequest, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest,
|
||||
DeleteVersionsResponse, DeleteVolumeRequest, DiskInfoRequest, ListDirRequest, ListVolumesRequest, MakeVolumeRequest,
|
||||
MakeVolumesRequest, PreparePartTransactionRequest, ReadAllRequest, ReadMetadataRequest, ReadMultipleRequest,
|
||||
ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest, RenameFileRequest,
|
||||
SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest,
|
||||
SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest,
|
||||
node_service_client::NodeServiceClient,
|
||||
MakeVolumesRequest, NativePreparationBeginRequest, NativePreparationFinalizeRequest, NativePreparationLocateRequest,
|
||||
NativePreparationProbeRequest, NativePreparationRecoverRequest, PreparePartTransactionRequest, ReadAllRequest,
|
||||
ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest,
|
||||
RenameDataRequest, RenameFileRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest,
|
||||
SnapshotLeaseRequest, SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest,
|
||||
WriteMetadataRequest, node_service_client::NodeServiceClient,
|
||||
};
|
||||
use serde::{Serialize, de::DeserializeOwned};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::{
|
||||
future::Future,
|
||||
io::Cursor,
|
||||
@@ -176,6 +178,172 @@ fn attach_mutation_body_digest<T>(
|
||||
crate::cluster::rpc::set_tonic_rolling_canonical_body_digest(request, &canonical_body).map_err(Error::other)
|
||||
}
|
||||
|
||||
fn native_response_proof(body: std::result::Result<Vec<u8>, std::num::TryFromIntError>, proof: &[u8]) -> Result<()> {
|
||||
let body = body.map_err(|_| Error::other("native preparation response is too large to authenticate"))?;
|
||||
if proof.len() != 32 {
|
||||
return Err(Error::other("native preparation response proof has invalid length"));
|
||||
}
|
||||
crate::cluster::rpc::verify_tonic_rpc_response_proof(&body, proof)
|
||||
.map_err(|_| Error::other("native preparation response proof is invalid"))
|
||||
}
|
||||
|
||||
fn native_response_error(error: Option<rustfs_protos::proto_gen::node_service::Error>) -> DiskError {
|
||||
let error = DiskError::from(error.unwrap_or_default());
|
||||
match error {
|
||||
DiskError::FileCorrupt => DiskError::Io(std::io::Error::new(std::io::ErrorKind::InvalidData, error)),
|
||||
error => error,
|
||||
}
|
||||
}
|
||||
|
||||
fn native_uuid_field(value: &[u8], field: &str) -> Result<Uuid> {
|
||||
let id = Uuid::from_slice(value).map_err(|_| Error::other(format!("native preparation returned invalid {field}")))?;
|
||||
if id.is_nil() {
|
||||
return Err(Error::other(format!("native preparation returned nil {field}")));
|
||||
}
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
fn native_version_field(value: &[u8]) -> Result<Uuid> {
|
||||
Uuid::from_slice(value).map_err(|_| Error::other("native preparation returned invalid version id"))
|
||||
}
|
||||
|
||||
fn native_object_length_from_metadata(input: &crate::disk::local::native_prepare::NativePrepareInput) -> Result<u64> {
|
||||
let raw = rustfs_filemeta::FileMeta::load(&input.metadata).map_err(Error::other)?;
|
||||
if raw.versions.len() != 1 {
|
||||
return Err(Error::other("native preparation metadata must have one version"));
|
||||
}
|
||||
let info = rustfs_filemeta::get_file_info(
|
||||
&input.metadata,
|
||||
&input.bucket,
|
||||
&input.key,
|
||||
&input.version.to_string(),
|
||||
rustfs_filemeta::FileInfoOpts {
|
||||
data: true,
|
||||
include_free_versions: false,
|
||||
include_part_checksums: true,
|
||||
},
|
||||
)
|
||||
.map_err(Error::other)?;
|
||||
if info.version_id != Some(input.version) || info.deleted || info.mark_deleted || info.size < 0 {
|
||||
return Err(Error::other("native preparation metadata identity mismatch"));
|
||||
}
|
||||
u64::try_from(info.size).map_err(Error::other)
|
||||
}
|
||||
|
||||
fn validate_native_metadata_for_version(
|
||||
metadata: &[u8],
|
||||
expected_digest: Option<&[u8; 32]>,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
version: Uuid,
|
||||
) -> Result<()> {
|
||||
validate_native_metadata(metadata, expected_digest)?;
|
||||
let raw = rustfs_filemeta::FileMeta::load(metadata).map_err(Error::other)?;
|
||||
if raw.versions.len() != 1 {
|
||||
return Err(Error::other("native preparation metadata must have one version"));
|
||||
}
|
||||
let info = rustfs_filemeta::get_file_info(
|
||||
metadata,
|
||||
bucket,
|
||||
key,
|
||||
&version.to_string(),
|
||||
rustfs_filemeta::FileInfoOpts {
|
||||
data: true,
|
||||
include_free_versions: false,
|
||||
include_part_checksums: true,
|
||||
},
|
||||
)
|
||||
.map_err(Error::other)?;
|
||||
if info.version_id != Some(version) || info.deleted || info.mark_deleted || info.size < 0 {
|
||||
return Err(Error::other("native preparation metadata identity mismatch"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn native_digest_field(value: &[u8], field: &str) -> Result<[u8; 32]> {
|
||||
value
|
||||
.try_into()
|
||||
.map_err(|_| Error::other(format!("native preparation returned invalid {field}")))
|
||||
}
|
||||
|
||||
fn native_receipt_from_wire(
|
||||
receipt: rustfs_protos::proto_gen::node_service::NativePreparationReceipt,
|
||||
expected_disk: Uuid,
|
||||
expected_store: Uuid,
|
||||
expected_preparation: Uuid,
|
||||
expected_bucket: &str,
|
||||
expected_key: &str,
|
||||
expected_incarnation: Uuid,
|
||||
expected_object: Option<(&[u8], u64)>,
|
||||
expected_version: Option<Uuid>,
|
||||
) -> Result<crate::disk::local::native_prepare::NativePreparationReceipt> {
|
||||
let disk = native_uuid_field(&receipt.disk_id, "disk id")?;
|
||||
let store = native_uuid_field(&receipt.store_id, "store id")?;
|
||||
let deployment = native_uuid_field(&receipt.deployment_id, "deployment id")?;
|
||||
let preparation = native_uuid_field(&receipt.preparation_id, "preparation id")?;
|
||||
let version = native_version_field(&receipt.version_id)?;
|
||||
let incarnation = native_uuid_field(&receipt.bucket_incarnation_id, "bucket incarnation")?;
|
||||
if disk != expected_disk
|
||||
|| store != expected_store
|
||||
|| deployment != expected_store
|
||||
|| preparation != expected_preparation
|
||||
|| incarnation != expected_incarnation
|
||||
|| receipt.bucket != expected_bucket
|
||||
|| receipt.key != expected_key
|
||||
|| expected_version.is_some_and(|expected| version != expected)
|
||||
{
|
||||
return Err(Error::other("native preparation receipt identity mismatch"));
|
||||
}
|
||||
let object_digest = native_digest_field(&receipt.object_digest, "object digest")?;
|
||||
if expected_object.is_some_and(|(digest, length)| digest != object_digest.as_slice() || length != receipt.object_length) {
|
||||
return Err(Error::other("native preparation receipt object mismatch"));
|
||||
}
|
||||
let set_digest = native_digest_field(&receipt.set_digest, "set digest")?;
|
||||
let distribution_digest = native_digest_field(&receipt.distribution_digest, "distribution digest")?;
|
||||
let metadata_digest = native_digest_field(&receipt.metadata_digest, "metadata digest")?;
|
||||
let shard_digest = native_digest_field(&receipt.shard_digest, "shard digest")?;
|
||||
let data_blocks = usize::try_from(receipt.data_blocks).map_err(Error::other)?;
|
||||
let parity_blocks = usize::try_from(receipt.parity_blocks).map_err(Error::other)?;
|
||||
let shard_index = usize::try_from(receipt.shard_index).map_err(Error::other)?;
|
||||
if data_blocks == 0
|
||||
|| parity_blocks == 0
|
||||
|| data_blocks
|
||||
.checked_add(parity_blocks)
|
||||
.is_none_or(|total| shard_index == 0 || shard_index > total)
|
||||
{
|
||||
return Err(Error::other("native preparation receipt has invalid erasure geometry"));
|
||||
}
|
||||
Ok(crate::disk::local::native_prepare::NativePreparationReceipt {
|
||||
preparation,
|
||||
version,
|
||||
disk,
|
||||
set_digest,
|
||||
distribution_digest,
|
||||
metadata_digest,
|
||||
shard_digest,
|
||||
shard_length: receipt.shard_length,
|
||||
bucket: receipt.bucket,
|
||||
key: receipt.key,
|
||||
deployment,
|
||||
incarnation,
|
||||
object_digest,
|
||||
object_length: receipt.object_length,
|
||||
data_blocks,
|
||||
parity_blocks,
|
||||
shard_index,
|
||||
})
|
||||
}
|
||||
|
||||
fn validate_native_metadata(metadata: &[u8], expected_digest: Option<&[u8; 32]>) -> Result<()> {
|
||||
if metadata.len() > rustfs_protos::NATIVE_PREPARATION_METADATA_MAX_BYTES {
|
||||
return Err(Error::other("native preparation metadata exceeds the protocol limit"));
|
||||
}
|
||||
if expected_digest.is_some_and(|expected| Sha256::digest(metadata).as_slice() != expected.as_slice()) {
|
||||
return Err(Error::other("native preparation metadata digest mismatch"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn decode_volume_infos(volume_infos: Vec<String>) -> Result<Vec<VolumeInfo>> {
|
||||
volume_infos
|
||||
.into_iter()
|
||||
@@ -896,6 +1064,339 @@ impl RemoteDisk {
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn native_begin_stream(
|
||||
&self,
|
||||
preparation: Uuid,
|
||||
store_id: Uuid,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
incarnation: Uuid,
|
||||
) -> Result<()> {
|
||||
let disk_id = self
|
||||
.get_disk_id()
|
||||
.await?
|
||||
.filter(|id| !id.is_nil())
|
||||
.ok_or(DiskError::InvalidPath)?;
|
||||
if preparation.is_nil() || store_id.is_nil() || incarnation.is_nil() || bucket.is_empty() || key.is_empty() {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
self.execute_with_timeout_for_op(
|
||||
"native_preparation_begin",
|
||||
|| async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(NativePreparationBeginRequest {
|
||||
disk: self.disk_ref().await,
|
||||
disk_id: disk_id.as_bytes().to_vec().into(),
|
||||
store_id: store_id.as_bytes().to_vec().into(),
|
||||
preparation_id: preparation.as_bytes().to_vec().into(),
|
||||
bucket: bucket.to_owned(),
|
||||
key: key.to_owned(),
|
||||
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
|
||||
});
|
||||
let canonical = rustfs_protos::canonical_native_preparation_begin_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical, "native_preparation_begin")?;
|
||||
let canonical_request = request.get_ref().clone();
|
||||
let response = client.native_preparation_begin(request).await?.into_inner();
|
||||
native_response_proof(
|
||||
rustfs_protos::canonical_native_preparation_begin_response_body(&canonical_request, &response),
|
||||
&response.response_proof,
|
||||
)?;
|
||||
if !response.success {
|
||||
return Err(native_response_error(response.error));
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn native_finalize(
|
||||
&self,
|
||||
input: crate::disk::local::native_prepare::NativePrepareInput,
|
||||
) -> Result<crate::disk::local::native_prepare::NativePreparationReceipt> {
|
||||
let disk_id = self
|
||||
.get_disk_id()
|
||||
.await?
|
||||
.filter(|id| !id.is_nil())
|
||||
.ok_or(DiskError::InvalidPath)?;
|
||||
if input.preparation.is_nil() || input.store_id.is_nil() || input.bucket_incarnation.is_nil() {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
validate_native_metadata(&input.metadata, None)?;
|
||||
let expected_object_digest = input.object_digest;
|
||||
let expected_object_length = native_object_length_from_metadata(&input)?;
|
||||
let expected_version = input.version;
|
||||
self.execute_with_timeout_for_op(
|
||||
"native_preparation_finalize",
|
||||
|| async move {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(NativePreparationFinalizeRequest {
|
||||
disk: self.disk_ref().await,
|
||||
disk_id: disk_id.as_bytes().to_vec().into(),
|
||||
store_id: input.store_id.as_bytes().to_vec().into(),
|
||||
preparation_id: input.preparation.as_bytes().to_vec().into(),
|
||||
bucket: input.bucket.clone(),
|
||||
key: input.key.clone(),
|
||||
bucket_incarnation_id: input.bucket_incarnation.as_bytes().to_vec().into(),
|
||||
object_digest: expected_object_digest.to_vec().into(),
|
||||
object_length: expected_object_length,
|
||||
version_id: expected_version.as_bytes().to_vec().into(),
|
||||
metadata: input.metadata.clone(),
|
||||
});
|
||||
let canonical = rustfs_protos::canonical_native_preparation_finalize_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical, "native_preparation_finalize")?;
|
||||
let canonical_request = request.get_ref().clone();
|
||||
let response = client.native_preparation_finalize(request).await?.into_inner();
|
||||
native_response_proof(
|
||||
rustfs_protos::canonical_native_preparation_finalize_response_body(&canonical_request, &response),
|
||||
&response.response_proof,
|
||||
)?;
|
||||
if !response.success {
|
||||
return Err(native_response_error(response.error));
|
||||
}
|
||||
let receipt = response
|
||||
.receipt
|
||||
.ok_or_else(|| Error::other("native finalize response omitted its receipt"))?;
|
||||
let receipt = native_receipt_from_wire(
|
||||
receipt,
|
||||
disk_id,
|
||||
input.store_id,
|
||||
input.preparation,
|
||||
&input.bucket,
|
||||
&input.key,
|
||||
input.bucket_incarnation,
|
||||
Some((&expected_object_digest, expected_object_length)),
|
||||
Some(expected_version),
|
||||
)?;
|
||||
validate_native_metadata_for_version(
|
||||
&input.metadata,
|
||||
Some(&receipt.metadata_digest),
|
||||
&input.bucket,
|
||||
&input.key,
|
||||
expected_version,
|
||||
)?;
|
||||
Ok(receipt)
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn native_probe(
|
||||
&self,
|
||||
preparation: Uuid,
|
||||
store_id: Uuid,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
incarnation: Uuid,
|
||||
) -> Result<crate::disk::local::native_prepare::NativePreparationProbe> {
|
||||
let disk_id = self
|
||||
.get_disk_id()
|
||||
.await?
|
||||
.filter(|id| !id.is_nil())
|
||||
.ok_or(DiskError::InvalidPath)?;
|
||||
if preparation.is_nil() || store_id.is_nil() || incarnation.is_nil() || bucket.is_empty() || key.is_empty() {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
self.execute_with_timeout_for_op(
|
||||
"native_preparation_probe",
|
||||
|| async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(NativePreparationProbeRequest {
|
||||
disk: self.disk_ref().await,
|
||||
disk_id: disk_id.as_bytes().to_vec().into(),
|
||||
store_id: store_id.as_bytes().to_vec().into(),
|
||||
preparation_id: preparation.as_bytes().to_vec().into(),
|
||||
bucket: bucket.to_owned(),
|
||||
key: key.to_owned(),
|
||||
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
|
||||
});
|
||||
let canonical = rustfs_protos::canonical_native_preparation_probe_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical, "native_preparation_probe")?;
|
||||
let canonical_request = request.get_ref().clone();
|
||||
let response = client.native_preparation_probe(request).await?.into_inner();
|
||||
native_response_proof(
|
||||
rustfs_protos::canonical_native_preparation_probe_response_body(&canonical_request, &response),
|
||||
&response.response_proof,
|
||||
)?;
|
||||
if !response.success {
|
||||
return Err(native_response_error(response.error));
|
||||
}
|
||||
validate_native_metadata(&response.metadata, None)?;
|
||||
if !response.exists {
|
||||
if response.prepared
|
||||
|| !response.metadata.is_empty()
|
||||
|| !response.version_id.is_empty()
|
||||
|| !response.object_digest.is_empty()
|
||||
|| response.object_length != 0
|
||||
|| !response.set_digest.is_empty()
|
||||
|| !response.distribution_digest.is_empty()
|
||||
{
|
||||
return Err(Error::other("native probe absent response carries object state"));
|
||||
}
|
||||
return Ok(crate::disk::local::native_prepare::NativePreparationProbe::Absent);
|
||||
}
|
||||
let version = native_version_field(&response.version_id)?;
|
||||
let object_digest = native_digest_field(&response.object_digest, "object digest")?;
|
||||
let set_digest = native_digest_field(&response.set_digest, "set digest")?;
|
||||
let distribution_digest = native_digest_field(&response.distribution_digest, "distribution digest")?;
|
||||
if !response.metadata.is_empty() {
|
||||
validate_native_metadata_for_version(&response.metadata, None, bucket, key, version)?;
|
||||
}
|
||||
Ok(crate::disk::local::native_prepare::NativePreparationProbe::Existing {
|
||||
version,
|
||||
prepared: response.prepared,
|
||||
object_digest,
|
||||
object_length: response.object_length,
|
||||
set_digest,
|
||||
distribution_digest,
|
||||
metadata: (!response.metadata.is_empty()).then_some(response.metadata),
|
||||
})
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn native_recover(
|
||||
&self,
|
||||
preparation: Uuid,
|
||||
store_id: Uuid,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
incarnation: Uuid,
|
||||
) -> Result<(crate::disk::local::native_prepare::NativePreparationReceipt, Bytes)> {
|
||||
let disk_id = self
|
||||
.get_disk_id()
|
||||
.await?
|
||||
.filter(|id| !id.is_nil())
|
||||
.ok_or(DiskError::InvalidPath)?;
|
||||
if preparation.is_nil() || store_id.is_nil() || incarnation.is_nil() || bucket.is_empty() || key.is_empty() {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
self.execute_with_timeout_for_op(
|
||||
"native_preparation_recover",
|
||||
|| async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(NativePreparationRecoverRequest {
|
||||
disk: self.disk_ref().await,
|
||||
disk_id: disk_id.as_bytes().to_vec().into(),
|
||||
store_id: store_id.as_bytes().to_vec().into(),
|
||||
preparation_id: preparation.as_bytes().to_vec().into(),
|
||||
bucket: bucket.to_owned(),
|
||||
key: key.to_owned(),
|
||||
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
|
||||
});
|
||||
let canonical = rustfs_protos::canonical_native_preparation_recover_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical, "native_preparation_recover")?;
|
||||
let canonical_request = request.get_ref().clone();
|
||||
let response = client.native_preparation_recover(request).await?.into_inner();
|
||||
native_response_proof(
|
||||
rustfs_protos::canonical_native_preparation_recover_response_body(&canonical_request, &response),
|
||||
&response.response_proof,
|
||||
)?;
|
||||
if !response.success {
|
||||
return Err(native_response_error(response.error));
|
||||
}
|
||||
let receipt = response
|
||||
.receipt
|
||||
.ok_or_else(|| Error::other("native recover response omitted its receipt"))?;
|
||||
let receipt =
|
||||
native_receipt_from_wire(receipt, disk_id, store_id, preparation, bucket, key, incarnation, None, None)?;
|
||||
validate_native_metadata_for_version(
|
||||
&response.metadata,
|
||||
Some(&receipt.metadata_digest),
|
||||
bucket,
|
||||
key,
|
||||
receipt.version,
|
||||
)?;
|
||||
Ok((receipt, response.metadata))
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn native_locate(
|
||||
&self,
|
||||
store_id: Uuid,
|
||||
preparation: Uuid,
|
||||
) -> Result<Option<crate::disk::local::native_prepare::NativePreparationDescription>> {
|
||||
let disk_id = self
|
||||
.get_disk_id()
|
||||
.await?
|
||||
.filter(|id| !id.is_nil())
|
||||
.ok_or(DiskError::InvalidPath)?;
|
||||
if preparation.is_nil() || store_id.is_nil() {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
self.execute_with_timeout_for_op(
|
||||
"native_preparation_locate",
|
||||
|| async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(NativePreparationLocateRequest {
|
||||
disk: self.disk_ref().await,
|
||||
disk_id: disk_id.as_bytes().to_vec().into(),
|
||||
store_id: store_id.as_bytes().to_vec().into(),
|
||||
preparation_id: preparation.as_bytes().to_vec().into(),
|
||||
});
|
||||
let canonical = rustfs_protos::canonical_native_preparation_locate_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical, "native_preparation_locate")?;
|
||||
let canonical_request = request.get_ref().clone();
|
||||
let response = client.native_preparation_locate(request).await?.into_inner();
|
||||
native_response_proof(
|
||||
rustfs_protos::canonical_native_preparation_locate_response_body(&canonical_request, &response),
|
||||
&response.response_proof,
|
||||
)?;
|
||||
if !response.success {
|
||||
return Err(native_response_error(response.error));
|
||||
}
|
||||
if !response.found {
|
||||
if !response.bucket.is_empty()
|
||||
|| !response.key.is_empty()
|
||||
|| !response.bucket_incarnation_id.is_empty()
|
||||
|| !response.metadata.is_empty()
|
||||
|| response.prepared
|
||||
{
|
||||
return Err(Error::other("native locate absent response carries preparation state"));
|
||||
}
|
||||
return Ok(None);
|
||||
}
|
||||
if response.bucket.is_empty()
|
||||
|| response.bucket.len() > 255
|
||||
|| response.key.is_empty()
|
||||
|| response.key.len() > 4096
|
||||
|| preparation.is_nil()
|
||||
{
|
||||
return Err(Error::other("native locate response has invalid description"));
|
||||
}
|
||||
let incarnation = native_uuid_field(&response.bucket_incarnation_id, "bucket incarnation")?;
|
||||
validate_native_metadata(&response.metadata, None)?;
|
||||
if response.metadata.is_empty() {
|
||||
return Err(Error::other("native locate response omitted metadata"));
|
||||
}
|
||||
let raw_metadata = rustfs_filemeta::FileMeta::load(&response.metadata)
|
||||
.map_err(|_| Error::other("native locate response metadata is invalid"))?;
|
||||
if raw_metadata.versions.len() != 1 {
|
||||
return Err(Error::other("native locate metadata must have one version"));
|
||||
}
|
||||
let version = raw_metadata.versions[0].header.version_id.unwrap_or_else(Uuid::nil);
|
||||
Ok(Some(crate::disk::local::native_prepare::NativePreparationDescription {
|
||||
preparation,
|
||||
bucket: response.bucket,
|
||||
key: response.key,
|
||||
incarnation,
|
||||
version,
|
||||
metadata: response.metadata,
|
||||
prepared: response.prepared,
|
||||
}))
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn ns_scanner_server_epoch(&self) -> Result<Option<Uuid>> {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
|
||||
@@ -449,6 +449,64 @@ where
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_observation<S>(api: Arc<S>, file: &str, max_bytes: usize) -> Result<(Vec<u8>, ObjectInfo)>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
let opts = ObjectOptions {
|
||||
suppress_read_repair: true,
|
||||
no_lock: false,
|
||||
skip_verify_bitrot: false,
|
||||
..Default::default()
|
||||
};
|
||||
crate::object_api::without_get_object_body_cache_hook(read_config_limited_preserve_empty_with_metadata_opts(
|
||||
api, file, &opts, max_bytes,
|
||||
))
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_observation_no_lock<S>(
|
||||
api: Arc<S>,
|
||||
file: &str,
|
||||
max_bytes: usize,
|
||||
guard: &crate::object_api::WriteCommitGuard,
|
||||
) -> Result<(Vec<u8>, ObjectInfo)>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
if !guard.matches(RUSTFS_META_BUCKET, file) {
|
||||
return Err(Error::InvalidArgument(
|
||||
RUSTFS_META_BUCKET.to_owned(),
|
||||
file.to_owned(),
|
||||
"locked observation requires a matching write owner".to_owned(),
|
||||
));
|
||||
}
|
||||
let lock_lost_error = || Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "locked_config_observation",
|
||||
bucket: RUSTFS_META_BUCKET.to_owned(),
|
||||
object: file.to_owned(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
};
|
||||
if guard.is_lock_lost() {
|
||||
return Err(lock_lost_error());
|
||||
}
|
||||
let opts = ObjectOptions {
|
||||
suppress_read_repair: true,
|
||||
no_lock: true,
|
||||
skip_verify_bitrot: false,
|
||||
..Default::default()
|
||||
};
|
||||
let result = crate::object_api::without_get_object_body_cache_hook(read_config_limited_preserve_empty_with_metadata_opts(
|
||||
api, file, &opts, max_bytes,
|
||||
))
|
||||
.await;
|
||||
if guard.is_lock_lost() {
|
||||
return Err(lock_lost_error());
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_limited_preserve_empty<S>(api: Arc<S>, file: &str, max_bytes: usize) -> Result<Vec<u8>>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
@@ -495,10 +553,22 @@ pub(crate) async fn read_config_preserve_empty<S>(api: Arc<S>, file: &str) -> Re
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
let (data, _obj) = read_config_with_metadata_inner(api, file, &ObjectOptions::default(), true, None).await?;
|
||||
let (data, _obj) = read_config_preserve_empty_with_metadata_opts(api, file, &ObjectOptions::default()).await?;
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
/// Preserve an empty config payload, its metadata, and caller-supplied read options.
|
||||
pub(crate) async fn read_config_preserve_empty_with_metadata_opts<S>(
|
||||
api: Arc<S>,
|
||||
file: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<(Vec<u8>, ObjectInfo)>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
read_config_with_metadata_inner(api, file, opts, true, None).await
|
||||
}
|
||||
|
||||
pub async fn read_config_no_lock<S>(api: Arc<S>, file: &str) -> Result<Vec<u8>>
|
||||
where
|
||||
S: ObjectIO<
|
||||
@@ -527,15 +597,13 @@ pub(crate) async fn read_config_no_lock_preserve_empty_with_metadata<S>(api: Arc
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
read_config_with_metadata_inner(
|
||||
read_config_preserve_empty_with_metadata_opts(
|
||||
api,
|
||||
file,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
true,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -2815,9 +2883,10 @@ mod tests {
|
||||
config_task_join_error, configs_semantically_equal, decode_server_config_blob, encode_server_config_blob,
|
||||
heal_config_descriptor, is_standard_object_server_config, lookup_configs, map_system_metadata_write_error,
|
||||
new_and_save_server_config, read_config, read_config_no_lock_preserve_empty_with_metadata, read_config_preserve_empty,
|
||||
read_config_with_metadata, read_config_without_migrate, read_server_config_snapshot, save_config_with_opts_inner,
|
||||
save_server_config, save_server_config_snapshot, save_server_config_snapshot_with_generation,
|
||||
server_config_transaction_lock_path, should_warn_ignored_scalar_section, storage_class_kvs_mut,
|
||||
read_config_preserve_empty_with_metadata_opts, read_config_with_metadata, read_config_without_migrate,
|
||||
read_server_config_snapshot, save_config_with_opts_inner, save_server_config, save_server_config_snapshot,
|
||||
save_server_config_snapshot_with_generation, server_config_transaction_lock_path, should_warn_ignored_scalar_section,
|
||||
storage_class_kvs_mut,
|
||||
};
|
||||
use crate::config::{audit, heal, notify, oidc, scanner};
|
||||
use crate::disk::{RUSTFS_META_BUCKET, endpoint::Endpoint};
|
||||
@@ -5065,6 +5134,7 @@ mod tests {
|
||||
heal_replacement: Option<Vec<u8>>,
|
||||
heal_calls: AtomicUsize,
|
||||
write_calls: AtomicUsize,
|
||||
last_get_no_lock: AtomicBool,
|
||||
last_put_no_lock: AtomicBool,
|
||||
last_put_preconditions: Mutex<Option<HTTPPreconditions>>,
|
||||
revision: AtomicUsize,
|
||||
@@ -5080,6 +5150,7 @@ mod tests {
|
||||
heal_replacement,
|
||||
heal_calls: AtomicUsize::new(0),
|
||||
write_calls: AtomicUsize::new(0),
|
||||
last_get_no_lock: AtomicBool::new(false),
|
||||
last_put_no_lock: AtomicBool::new(false),
|
||||
last_put_preconditions: Mutex::new(None),
|
||||
revision: AtomicUsize::new(1),
|
||||
@@ -5135,8 +5206,9 @@ mod tests {
|
||||
_object: &str,
|
||||
_range: Option<HTTPRangeSpec>,
|
||||
_h: HeaderMap,
|
||||
_opts: &ObjectOptions,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<GetObjectReader> {
|
||||
self.last_get_no_lock.store(opts.no_lock, Ordering::SeqCst);
|
||||
let data = match &*self.state.lock().expect("state lock poisoned") {
|
||||
RecoveryReadState::Missing => return Err(Error::ConfigNotFound),
|
||||
RecoveryReadState::Blob(data) => data.clone(),
|
||||
@@ -5411,6 +5483,329 @@ mod tests {
|
||||
assert_eq!(store.write_calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
struct ObservationStream {
|
||||
inner: Cursor<Vec<u8>>,
|
||||
consumed: Arc<AtomicUsize>,
|
||||
fail: bool,
|
||||
eof_gate: Option<(Option<tokio::sync::oneshot::Sender<()>>, tokio::sync::oneshot::Receiver<()>)>,
|
||||
}
|
||||
|
||||
impl AsyncRead for ObservationStream {
|
||||
fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
|
||||
if self.fail {
|
||||
return Poll::Ready(Err(std::io::Error::other("observation stream failed")));
|
||||
}
|
||||
if self.inner.position() == u64::try_from(self.inner.get_ref().len()).expect("fixture length")
|
||||
&& let Some((ready, release)) = self.eof_gate.as_mut()
|
||||
{
|
||||
if let Some(ready) = ready.take() {
|
||||
ready.send(()).expect("EOF observer remains active");
|
||||
}
|
||||
match std::future::Future::poll(Pin::new(release), cx) {
|
||||
Poll::Pending => return Poll::Pending,
|
||||
Poll::Ready(result) => result.expect("EOF gate released"),
|
||||
}
|
||||
self.eof_gate = None;
|
||||
}
|
||||
let before = buf.filled().len();
|
||||
let result = Pin::new(&mut self.inner).poll_read(cx, buf);
|
||||
self.consumed.fetch_add(buf.filled().len() - before, Ordering::SeqCst);
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
struct ObservationStore {
|
||||
body: Vec<u8>,
|
||||
declared_size: i64,
|
||||
error: Option<Error>,
|
||||
stream_error: bool,
|
||||
gets: AtomicUsize,
|
||||
eof_gate: std::sync::Mutex<Option<(Option<tokio::sync::oneshot::Sender<()>>, tokio::sync::oneshot::Receiver<()>)>>,
|
||||
consumed: Arc<AtomicUsize>,
|
||||
options: std::sync::Mutex<Option<(bool, bool, bool)>>,
|
||||
writes: AtomicUsize,
|
||||
}
|
||||
|
||||
impl Debug for ObservationStore {
|
||||
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("ObservationStore").finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl ObservationStore {
|
||||
fn new(body: Vec<u8>, declared_size: i64) -> Self {
|
||||
Self {
|
||||
body,
|
||||
declared_size,
|
||||
error: None,
|
||||
stream_error: false,
|
||||
gets: AtomicUsize::new(0),
|
||||
eof_gate: std::sync::Mutex::new(None),
|
||||
consumed: Arc::new(AtomicUsize::new(0)),
|
||||
options: std::sync::Mutex::new(None),
|
||||
writes: AtomicUsize::new(0),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::storage_api_contracts::object::ObjectIO for ObservationStore {
|
||||
type Error = Error;
|
||||
type RangeSpec = HTTPRangeSpec;
|
||||
type HeaderMap = HeaderMap;
|
||||
type ObjectOptions = ObjectOptions;
|
||||
type ObjectInfo = ObjectInfo;
|
||||
type GetObjectReader = GetObjectReader;
|
||||
type PutObjectReader = PutObjReader;
|
||||
|
||||
async fn get_object_reader(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
range: Option<HTTPRangeSpec>,
|
||||
_headers: HeaderMap,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<GetObjectReader> {
|
||||
assert_eq!(bucket, RUSTFS_META_BUCKET);
|
||||
assert_eq!(object, "config/observation.bin");
|
||||
self.gets.fetch_add(1, Ordering::SeqCst);
|
||||
assert!(range.is_none());
|
||||
*self.options.lock().expect("observation options") =
|
||||
Some((opts.suppress_read_repair, opts.no_lock, opts.skip_verify_bitrot));
|
||||
if let Some(error) = &self.error {
|
||||
return Err(error.clone());
|
||||
}
|
||||
Ok(GetObjectReader {
|
||||
stream: Box::new(ObservationStream {
|
||||
inner: Cursor::new(self.body.clone()),
|
||||
consumed: self.consumed.clone(),
|
||||
fail: self.stream_error,
|
||||
eof_gate: self.eof_gate.lock().expect("EOF gate").take(),
|
||||
}),
|
||||
object_info: ObjectInfo {
|
||||
size: self.declared_size,
|
||||
etag: Some("observation-etag".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
buffered_body: None,
|
||||
body_source: Default::default(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn put_object(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_object: &str,
|
||||
_data: &mut PutObjReader,
|
||||
_opts: &ObjectOptions,
|
||||
) -> Result<ObjectInfo> {
|
||||
self.writes.fetch_add(1, Ordering::SeqCst);
|
||||
Err(Error::other("observation must not write"))
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn locked_observation_preserves_options_and_typed_errors() {
|
||||
let object = "config/observation.bin";
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
|
||||
let wrapper = rustfs_lock::NamespaceLockWrapper::new(
|
||||
rustfs_lock::NamespaceLock::with_local_manager("locked-observation-options".to_owned(), manager.clone()),
|
||||
rustfs_lock::ObjectKey::new(RUSTFS_META_BUCKET, object),
|
||||
"locked-observation-options".to_owned(),
|
||||
);
|
||||
let guard = crate::object_api::WriteCommitGuard::acquire(&wrapper, std::time::Duration::from_secs(5))
|
||||
.await
|
||||
.expect("matching real owner");
|
||||
let store = Arc::new(ObservationStore::new(b"raw".to_vec(), 3));
|
||||
let result = super::read_config_observation_no_lock(store.clone(), object, 3, &guard).await;
|
||||
assert_eq!(result.is_ok(), true, "I06L_READ_FLAGS: {result:?}");
|
||||
assert_eq!(
|
||||
*store.options.lock().expect("options") == Some((true, true, false)),
|
||||
true,
|
||||
"I06L_READ_FLAGS"
|
||||
);
|
||||
assert_eq!(result.expect("raw observation").0, b"raw");
|
||||
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
|
||||
for (bucket, key) in [(RUSTFS_META_BUCKET, "config/other.bin"), ("wrong-bucket", object)] {
|
||||
let wrapper = rustfs_lock::NamespaceLockWrapper::new(
|
||||
rustfs_lock::NamespaceLock::with_local_manager("locked-observation-target".to_owned(), manager.clone()),
|
||||
rustfs_lock::ObjectKey::new(bucket, key),
|
||||
"locked-observation-target".to_owned(),
|
||||
);
|
||||
let wrong = crate::object_api::WriteCommitGuard::acquire(&wrapper, std::time::Duration::from_secs(5))
|
||||
.await
|
||||
.expect("wrong-target real owner");
|
||||
let store = Arc::new(ObservationStore::new(Vec::new(), 0));
|
||||
let result = super::read_config_observation_no_lock(store.clone(), object, 8, &wrong).await;
|
||||
assert_eq!(matches!(result, Err(Error::InvalidArgument(..))), true, "I06L_TARGET_REJECTED");
|
||||
assert_eq!(store.gets.load(Ordering::SeqCst) == 0, true, "I06L_TARGET_REJECTED");
|
||||
}
|
||||
for error in [
|
||||
Error::FileNotFound,
|
||||
Error::ObjectNotFound(RUSTFS_META_BUCKET.to_owned(), object.to_owned()),
|
||||
Error::ErasureReadQuorum,
|
||||
] {
|
||||
let expected = if error == Error::FileNotFound || matches!(error, Error::ObjectNotFound(..)) {
|
||||
Error::ConfigNotFound
|
||||
} else {
|
||||
error.clone()
|
||||
};
|
||||
let mut fixture = ObservationStore::new(Vec::new(), 0);
|
||||
fixture.error = Some(error);
|
||||
let store = Arc::new(fixture);
|
||||
assert_eq!(
|
||||
super::read_config_observation_no_lock(store.clone(), object, 8, &guard)
|
||||
.await
|
||||
.expect_err("typed error"),
|
||||
expected
|
||||
);
|
||||
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
let mut fixture = ObservationStore::new(Vec::new(), 0);
|
||||
fixture.stream_error = true;
|
||||
let store = Arc::new(fixture);
|
||||
let err = super::read_config_observation_no_lock(store.clone(), object, 8, &guard)
|
||||
.await
|
||||
.expect_err("stream failure");
|
||||
assert_eq!(err.to_string().contains("observation stream failed"), true, "I06L_TYPED_STREAM_ERROR");
|
||||
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
|
||||
let store = Arc::new(ObservationStore::new(vec![7; 64], 0));
|
||||
let result = super::read_config_observation_no_lock(store.clone(), object, 8, &guard).await;
|
||||
assert_eq!(matches!(result, Err(Error::CorruptedFormat)), true, "I06L_STREAM_BOUND");
|
||||
assert_eq!(store.consumed.load(Ordering::SeqCst) == 9, true, "I06L_STREAM_BOUND");
|
||||
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn locked_observation_rejects_actual_owner_loss_before_and_after_read() {
|
||||
let object = "config/observation.bin";
|
||||
for after_read in [false, true] {
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
|
||||
let client = Arc::new(RefreshControlledClient::new(manager));
|
||||
let wrapper = rustfs_lock::NamespaceLockWrapper::new(
|
||||
rustfs_lock::NamespaceLock::new("locked-observation-loss".to_owned(), client.clone()),
|
||||
rustfs_lock::ObjectKey::new(RUSTFS_META_BUCKET, object),
|
||||
"locked-observation-loss".to_owned(),
|
||||
);
|
||||
let guard = crate::object_api::WriteCommitGuard::acquire(&wrapper, std::time::Duration::from_secs(5))
|
||||
.await
|
||||
.expect("real distributed owner");
|
||||
let signal = guard.lock_lost_signal().expect("acquired distributed owner signal");
|
||||
let store = Arc::new(ObservationStore::new(b"raw".to_vec(), 3));
|
||||
if after_read {
|
||||
let (ready_tx, ready_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
|
||||
*store.eof_gate.lock().expect("EOF gate") = Some((Some(ready_tx), release_rx));
|
||||
let mut read = Box::pin(super::read_config_observation_no_lock(store.clone(), object, 3, &guard));
|
||||
tokio::time::timeout(std::time::Duration::from_secs(60), async {
|
||||
tokio::select! {
|
||||
ready = ready_rx => ready.expect("read reached EOF"),
|
||||
result = &mut read => assert_eq!(false, true, "I06L_LOSS_AFTER_READ: returned before EOF gate: {result:?}"),
|
||||
}
|
||||
}).await.expect("EOF watchdog");
|
||||
client.reject_refreshes();
|
||||
tokio::time::timeout(std::time::Duration::from_secs(60), signal.notified())
|
||||
.await
|
||||
.expect("actual owner loss watchdog");
|
||||
assert_eq!(guard.is_lock_lost(), true, "I06L_LOSS_AFTER_READ");
|
||||
release_tx.send(()).expect("release EOF");
|
||||
let result = tokio::time::timeout(std::time::Duration::from_secs(60), read)
|
||||
.await
|
||||
.expect("read watchdog");
|
||||
assert_eq!(
|
||||
matches!(
|
||||
result,
|
||||
Err(Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "locked_config_observation",
|
||||
..
|
||||
})
|
||||
),
|
||||
true,
|
||||
"I06L_LOSS_AFTER_READ"
|
||||
);
|
||||
assert_eq!(store.gets.load(Ordering::SeqCst), 1);
|
||||
} else {
|
||||
client.reject_refreshes();
|
||||
tokio::time::timeout(std::time::Duration::from_secs(60), signal.notified())
|
||||
.await
|
||||
.expect("actual owner loss watchdog");
|
||||
assert_eq!(guard.is_lock_lost(), true, "I06L_LOSS_BEFORE_GET");
|
||||
let result = super::read_config_observation_no_lock(store.clone(), object, 3, &guard).await;
|
||||
assert_eq!(
|
||||
matches!(
|
||||
result,
|
||||
Err(Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "locked_config_observation",
|
||||
..
|
||||
})
|
||||
),
|
||||
true,
|
||||
"I06L_LOSS_BEFORE_GET"
|
||||
);
|
||||
assert_eq!(store.gets.load(Ordering::SeqCst) == 0, true, "I06L_LOSS_BEFORE_GET");
|
||||
}
|
||||
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn observation_preserves_options_and_errors() {
|
||||
for body in [Vec::new(), b"raw config".to_vec()] {
|
||||
let store = Arc::new(ObservationStore::new(body.clone(), i64::try_from(body.len()).expect("fixture size")));
|
||||
let result = super::read_config_observation(store.clone(), "config/observation.bin", 32).await;
|
||||
assert!(result.is_ok(), "observation must preserve empty and nonempty objects: {result:?}");
|
||||
let (observed, info) = result.expect("successful observation");
|
||||
assert_eq!(observed, body);
|
||||
assert_eq!(info.etag.as_deref(), Some("observation-etag"));
|
||||
assert_eq!(
|
||||
*store.options.lock().expect("options"),
|
||||
Some((true, false, false)),
|
||||
"I05_REPAIR_SUPPRESSED"
|
||||
);
|
||||
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
for (source, expected) in [
|
||||
(Error::FileNotFound, Error::ConfigNotFound),
|
||||
(
|
||||
Error::ObjectNotFound(RUSTFS_META_BUCKET.to_string(), "config/observation.bin".to_string()),
|
||||
Error::ConfigNotFound,
|
||||
),
|
||||
(Error::ErasureReadQuorum, Error::ErasureReadQuorum),
|
||||
] {
|
||||
let mut fixture = ObservationStore::new(Vec::new(), 0);
|
||||
fixture.error = Some(source);
|
||||
let store = Arc::new(fixture);
|
||||
let result = super::read_config_observation(store.clone(), "config/observation.bin", 32).await;
|
||||
assert_eq!(result.expect_err("read error must propagate"), expected);
|
||||
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
let mut fixture = ObservationStore::new(Vec::new(), 0);
|
||||
fixture.stream_error = true;
|
||||
let store = Arc::new(fixture);
|
||||
let result = super::read_config_observation(store.clone(), "config/observation.bin", 32).await;
|
||||
let error = result.expect_err("stream error must propagate");
|
||||
assert!(error.to_string().contains("observation stream failed"));
|
||||
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn observation_enforces_byte_limit() {
|
||||
let store = Arc::new(ObservationStore::new(vec![7; 64], 0));
|
||||
let result = super::read_config_observation(store.clone(), "config/observation.bin", 8).await;
|
||||
assert_eq!(result.is_err(), true, "I05_BOUND_ENFORCED");
|
||||
assert_eq!(
|
||||
result.expect_err("bounded observation error"),
|
||||
Error::CorruptedFormat,
|
||||
"I05_BOUND_ENFORCED"
|
||||
);
|
||||
assert_eq!(store.consumed.load(Ordering::SeqCst), 9, "I05_BOUND_ENFORCED: drain exactly limit + 1");
|
||||
let store = Arc::new(ObservationStore::new(vec![7; 8], 8));
|
||||
let result = super::read_config_observation(store.clone(), "config/observation.bin", 8).await;
|
||||
assert!(result.is_ok(), "exact byte limit must succeed: {result:?}");
|
||||
assert_eq!(result.expect("exact limit").0, vec![7; 8]);
|
||||
assert_eq!(store.consumed.load(Ordering::SeqCst), 8);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_config_preserve_empty_distinguishes_empty_object() {
|
||||
let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Blob(Vec::new()), None));
|
||||
@@ -5431,6 +5826,60 @@ mod tests {
|
||||
assert!(data.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_config_preserve_empty_metadata_keeps_empty_and_options() {
|
||||
let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Blob(Vec::new()), None));
|
||||
for no_lock in [false, true] {
|
||||
let opts = ObjectOptions {
|
||||
no_lock,
|
||||
..Default::default()
|
||||
};
|
||||
let (data, info) = read_config_preserve_empty_with_metadata_opts(store.clone(), "config/empty.json", &opts)
|
||||
.await
|
||||
.expect("an existing empty config must retain its payload and metadata");
|
||||
assert!(data.is_empty());
|
||||
assert_eq!(info.size, 0);
|
||||
assert_eq!(info.actual_size, 0);
|
||||
assert_eq!(info.etag.as_deref(), Some("config-1"));
|
||||
assert_eq!(info.data_dir, Some(uuid::Uuid::from_u128(1)));
|
||||
assert_eq!(store.last_get_no_lock.load(Ordering::SeqCst), no_lock);
|
||||
}
|
||||
let err = read_config(store, "config/empty.json")
|
||||
.await
|
||||
.expect_err("the existing reader must still treat empty configs as missing");
|
||||
assert!(matches!(err, Error::ConfigNotFound));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_config_preserve_empty_metadata_keeps_payload_and_metadata() {
|
||||
let payload = b"original config bytes".to_vec();
|
||||
let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Blob(payload.clone()), None));
|
||||
let (data, info) =
|
||||
read_config_preserve_empty_with_metadata_opts(store, "config/nonempty.json", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("a nonempty config must retain its payload and metadata");
|
||||
assert_eq!(data, payload);
|
||||
let size = i64::try_from(payload.len()).expect("test payload size fits i64");
|
||||
assert_eq!(info.size, size);
|
||||
assert_eq!(info.actual_size, size);
|
||||
assert_eq!(info.etag.as_deref(), Some("config-1"));
|
||||
assert_eq!(info.data_dir, Some(uuid::Uuid::from_u128(1)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_config_preserve_empty_metadata_preserves_read_errors() {
|
||||
let missing = Arc::new(RecoveryMockStore::new(RecoveryReadState::Missing, None));
|
||||
let err = read_config_preserve_empty_with_metadata_opts(missing, "config/missing.json", &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("physical absence must remain ConfigNotFound");
|
||||
assert!(matches!(err, Error::ConfigNotFound));
|
||||
let quorum = Arc::new(RecoveryMockStore::new(RecoveryReadState::QuorumError, None));
|
||||
let err = read_config_preserve_empty_with_metadata_opts(quorum, "config/quorum.json", &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("a quorum failure must retain its typed read error");
|
||||
assert!(matches!(err, Error::ErasureReadQuorum));
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl StorageAdminApi for RecoveryMockStore {
|
||||
type BackendInfo = rustfs_madmin::BackendInfo;
|
||||
|
||||
@@ -72,38 +72,6 @@ type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
|
||||
|
||||
const LIST_MULTIPART_SETS_CONCURRENCY: usize = 4;
|
||||
|
||||
#[cfg(test)]
|
||||
type TestSetLockers = HashMap<usize, Vec<Arc<dyn rustfs_lock::LockClient>>>;
|
||||
|
||||
#[cfg(test)]
|
||||
static TEST_SET_LOCKERS: std::sync::LazyLock<std::sync::Mutex<TestSetLockers>> =
|
||||
std::sync::LazyLock::new(|| std::sync::Mutex::new(HashMap::new()));
|
||||
|
||||
/// Keep a fixture's real locker domain scoped to its owning instance.
|
||||
#[cfg(test)]
|
||||
pub(crate) struct ScopedSetLockers(Arc<InstanceContext>);
|
||||
|
||||
#[cfg(test)]
|
||||
impl ScopedSetLockers {
|
||||
pub(crate) fn install(ctx: Arc<InstanceContext>, lockers: Vec<Arc<dyn rustfs_lock::LockClient>>) -> Self {
|
||||
let key = Arc::as_ptr(&ctx) as usize;
|
||||
let mut domains = TEST_SET_LOCKERS.lock().expect("test set lockers mutex");
|
||||
assert!(!domains.contains_key(&key), "the instance already owns a test locker domain");
|
||||
domains.insert(key, lockers);
|
||||
Self(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for ScopedSetLockers {
|
||||
fn drop(&mut self) {
|
||||
TEST_SET_LOCKERS
|
||||
.lock()
|
||||
.expect("test set lockers mutex")
|
||||
.remove(&(Arc::as_ptr(&self.0) as usize));
|
||||
}
|
||||
}
|
||||
|
||||
fn is_idempotent_delete_prefix_error(err: &Error) -> bool {
|
||||
is_err_object_not_found(err) || is_err_strict_volume_not_found(err)
|
||||
}
|
||||
@@ -305,14 +273,6 @@ impl Sets {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
let pool_lockers = TEST_SET_LOCKERS
|
||||
.lock()
|
||||
.expect("test set lockers mutex")
|
||||
.get(&(Arc::as_ptr(&instance_ctx) as usize))
|
||||
.cloned()
|
||||
.unwrap_or_else(|| pool_lockers.clone());
|
||||
|
||||
let set_disks = SetDisks::new_with_instance_ctx(
|
||||
runtime_sources::local_node_name().await,
|
||||
Arc::new(RwLock::new(set_drive)),
|
||||
|
||||
@@ -2320,15 +2320,6 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
})
|
||||
}
|
||||
|
||||
async fn bucket_creation_witness(&self, volume: &str) -> Result<Option<super::os::NativeBucketCreationWitness>> {
|
||||
self.track_disk_health_with_op(
|
||||
"bucket_creation_witness",
|
||||
|| async { self.disk.bucket_creation_witness(volume).await },
|
||||
Duration::ZERO,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn make_volume(&self, volume: &str) -> Result<()> {
|
||||
// Scoped heal must drain directory creation before releasing its lifecycle owner.
|
||||
let timeout = if crate::store::bucket_heal_scope(volume).is_some() {
|
||||
|
||||
@@ -117,9 +117,6 @@ pub const O_APPEND: FileMode = 0x00400;
|
||||
// create_new: bool,
|
||||
|
||||
pub async fn open_file(path: impl AsRef<Path>, mode: FileMode) -> io::Result<File> {
|
||||
if mode & O_CREATE != 0 {
|
||||
super::os::record_native_publication(path.as_ref());
|
||||
}
|
||||
let base_opts = match mode & (O_RDONLY | O_WRONLY | O_RDWR) {
|
||||
O_RDONLY => get_readonly_options(),
|
||||
O_WRONLY => get_writeonly_options(),
|
||||
@@ -163,7 +160,6 @@ pub fn lstat_std(path: impl AsRef<Path>) -> io::Result<Metadata> {
|
||||
}
|
||||
|
||||
pub async fn make_dir_all(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
super::os::record_native_publication(path.as_ref());
|
||||
fs::create_dir_all(path.as_ref()).await
|
||||
}
|
||||
|
||||
@@ -215,18 +211,15 @@ pub fn remove_all_std(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
}
|
||||
|
||||
pub async fn mkdir(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
super::os::record_native_publication(path.as_ref());
|
||||
fs::create_dir(path.as_ref()).await
|
||||
}
|
||||
|
||||
pub async fn rename(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()> {
|
||||
super::os::record_native_publication(to.as_ref());
|
||||
fs::rename(from, to).await
|
||||
}
|
||||
|
||||
#[cfg(any(not(windows), test))]
|
||||
pub fn rename_std(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()> {
|
||||
super::os::record_native_publication(to.as_ref());
|
||||
std::fs::rename(from, to)
|
||||
}
|
||||
|
||||
|
||||
+397
-206
@@ -20,6 +20,7 @@ mod commit;
|
||||
#[cfg(all(unix, test))]
|
||||
#[path = "fd_cache_tests.rs"]
|
||||
mod fd_cache_tests;
|
||||
pub mod native_prepare;
|
||||
mod replacement_lease;
|
||||
#[cfg(any(target_os = "linux", test))]
|
||||
#[path = "uring_driver_budget.rs"]
|
||||
@@ -262,7 +263,6 @@ fn snapshot_part_transaction_file(src: &Path, backup: &Path, absent: &Path) -> s
|
||||
}
|
||||
|
||||
fn restore_part_transaction_file(current: &Path, backup: &Path, absent: &Path, restore: &Path) -> std::io::Result<()> {
|
||||
os::record_native_publication(current);
|
||||
match std::fs::symlink_metadata(backup) {
|
||||
Ok(metadata) if metadata.is_file() => {
|
||||
remove_file_if_exists(restore)?;
|
||||
@@ -288,7 +288,6 @@ async fn write_delete_rollback_file(
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
let backup_dir = object_dir.join(rollback_dir.to_string());
|
||||
os::record_native_publication(&backup_dir);
|
||||
let path = backup_dir.join(name);
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&path, namespace_owner).await;
|
||||
let data = data.to_vec();
|
||||
@@ -2150,9 +2149,6 @@ impl AsyncWrite for DirectWriter {
|
||||
fn open_direct_writer(file_path: &Path, state: &DirectIoWriteState) -> Result<Option<DirectWriter>> {
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
|
||||
#[cfg(test)]
|
||||
os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::DirectWriteOpen, file_path);
|
||||
os::record_native_publication(file_path);
|
||||
let open_result = std::fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.write(true)
|
||||
@@ -3925,7 +3921,6 @@ impl LocalIoBackend for StdBackend {
|
||||
if is_direct_io_write_enabled() && self.direct_io_write.supported.load(Ordering::Relaxed) {
|
||||
let write_state = self.direct_io_write.clone();
|
||||
let direct_path = file_path.clone();
|
||||
os::record_native_publication(&direct_path);
|
||||
let direct = tokio::task::spawn_blocking(move || open_direct_writer(&direct_path, &write_state))
|
||||
.await
|
||||
.map_err(|err| DiskError::other(format!("O_DIRECT open task failed: {err}")))??;
|
||||
@@ -5173,7 +5168,6 @@ async fn build_local_io_backend(root: PathBuf) -> Arc<dyn LocalIoBackend> {
|
||||
pub struct LocalDisk {
|
||||
pub root: PathBuf,
|
||||
publication_root: os::PublicationRoot,
|
||||
publication_history: Option<os::NativePublicationRegistration>,
|
||||
/// I/O root pinned to the mount instance that was opened while the disk
|
||||
/// was initialized. On Linux this is `/proc/self/fd/<dirfd>/.`; resolving
|
||||
/// paths beneath it keeps repair I/O on that mount even if the configured
|
||||
@@ -5542,10 +5536,6 @@ impl LocalDisk {
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let io_root = Self::open_mount_lease(&root)?;
|
||||
|
||||
let publication_history = os::register_native_publication_root(&root, &io_root);
|
||||
|
||||
#[cfg(all(test, target_os = "linux"))]
|
||||
os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::BeforeDataUsageLayout, &root);
|
||||
ensure_data_usage_layout(&io_root)
|
||||
.await
|
||||
.map_err(|e| e.narrow_to_disk().unwrap_or_else(DiskError::other))
|
||||
@@ -5681,7 +5671,6 @@ impl LocalDisk {
|
||||
let mut disk = Self {
|
||||
root: root.clone(),
|
||||
publication_root,
|
||||
publication_history,
|
||||
io_root: io_root.clone(),
|
||||
#[cfg(target_os = "linux")]
|
||||
mount_lease,
|
||||
@@ -6878,6 +6867,28 @@ impl LocalDisk {
|
||||
opt: &DeleteOptions,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
#[cfg(unix)]
|
||||
if let Some(native_path) = crate::disk::native_prepare_rpc::parse_native_stream_path(volume, path)? {
|
||||
return match native_path.file {
|
||||
crate::disk::native_prepare_rpc::NativeStreamFile::Staging(staging_id) => {
|
||||
let root = self.io_root().to_path_buf();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
native_prepare::remove_stream_staging_file(
|
||||
&root,
|
||||
native_path.preparation,
|
||||
native_path.data_dir,
|
||||
staging_id,
|
||||
)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?
|
||||
.map_err(to_file_error)?;
|
||||
Ok(())
|
||||
}
|
||||
crate::disk::native_prepare_rpc::NativeStreamFile::Part(_) => Err(DiskError::FileAccessDenied.into()),
|
||||
};
|
||||
}
|
||||
|
||||
let volume_dir = self.io_get_bucket_path(volume)?;
|
||||
if !skip_access_checks(volume)
|
||||
&& let Err(e) = cached_access(&volume_dir).await
|
||||
@@ -7451,7 +7462,6 @@ impl LocalDisk {
|
||||
};
|
||||
if let Some(rollback_dir) = rollback_dir {
|
||||
let rollback_path = object_dir.join(rollback_dir.to_string());
|
||||
os::record_native_publication(&rollback_path);
|
||||
if let Err(err) = fs::create_dir_all(&rollback_path).await {
|
||||
let err: DiskError = to_file_error(err).into();
|
||||
if reserved_version_delete {
|
||||
@@ -7974,7 +7984,6 @@ impl LocalDisk {
|
||||
}
|
||||
InternalBuf::Owned(buf) => {
|
||||
let path = file_path.to_path_buf();
|
||||
os::record_native_publication(&path);
|
||||
if let Some(parent) = path.parent()
|
||||
&& parent != skip_parent
|
||||
{
|
||||
@@ -7982,7 +7991,6 @@ impl LocalDisk {
|
||||
}
|
||||
|
||||
tokio::task::spawn_blocking(move || {
|
||||
os::record_native_publication(&path);
|
||||
#[cfg(test)]
|
||||
run_owned_file_write_before_open(&path);
|
||||
|
||||
@@ -9596,9 +9604,6 @@ impl DiskAPI for LocalDisk {
|
||||
}
|
||||
|
||||
async fn close(&self) -> Result<()> {
|
||||
if let Some(history) = self.publication_history.as_ref() {
|
||||
history.invalidate();
|
||||
}
|
||||
// This disk instance is being retired (e.g. replaced by renew_disk on
|
||||
// reconnect). Drop its cached descriptors so a replacement instance's
|
||||
// invalidations are never defeated by this one continuing to serve stale
|
||||
@@ -9756,7 +9761,6 @@ impl DiskAPI for LocalDisk {
|
||||
let file_path = self.io_get_object_path(volume, path)?;
|
||||
let path = path.to_string();
|
||||
let sync_metadata = effective_durability(volume).syncs_commit_metadata();
|
||||
os::record_native_publication(&file_path);
|
||||
return Ok(tokio::task::spawn_blocking(move || {
|
||||
// A persistent directory lock bounds metadata growth. Removing
|
||||
// per-target lock files can split flock ownership across inodes.
|
||||
@@ -10145,7 +10149,6 @@ impl DiskAPI for LocalDisk {
|
||||
.map_err(to_file_error)?;
|
||||
}
|
||||
}
|
||||
os::record_native_publication(&transaction_path);
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let source = std::fs::symlink_metadata(&src_file_path).map_err(to_file_error)?;
|
||||
if !source.is_file() {
|
||||
@@ -10217,7 +10220,6 @@ impl DiskAPI for LocalDisk {
|
||||
}
|
||||
let durability = effective_durability(volume);
|
||||
|
||||
os::record_native_publication(¤t_data_path);
|
||||
tokio::task::spawn_blocking(move || {
|
||||
match std::fs::symlink_metadata(&transaction_path) {
|
||||
Ok(metadata) if metadata.is_dir() => {}
|
||||
@@ -10477,6 +10479,33 @@ impl DiskAPI for LocalDisk {
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> {
|
||||
crate::hp_guard!("LocalDisk::rename_file");
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use crate::disk::native_prepare_rpc::{NativeStreamFile, parse_native_stream_path};
|
||||
|
||||
let src_native = parse_native_stream_path(src_volume, src_path)?;
|
||||
let dst_native = parse_native_stream_path(dst_volume, dst_path)?;
|
||||
if src_native.is_some() || dst_native.is_some() {
|
||||
let (Some(src), Some(dst)) = (src_native, dst_native) else {
|
||||
return Err(DiskError::InvalidPath.into());
|
||||
};
|
||||
let (NativeStreamFile::Staging(staging_id), NativeStreamFile::Part(part)) = (src.file, dst.file) else {
|
||||
return Err(DiskError::InvalidPath.into());
|
||||
};
|
||||
if src_volume != dst_volume || src.preparation != dst.preparation || src.data_dir != dst.data_dir {
|
||||
return Err(DiskError::InvalidPath.into());
|
||||
}
|
||||
let root = self.io_root().to_path_buf();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
native_prepare::publish_stream_part(&root, src.preparation, src.data_dir, staging_id, part)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?
|
||||
.map_err(to_file_error)?;
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
|
||||
let src_volume_dir = self.io_get_bucket_path(src_volume)?;
|
||||
let dst_volume_dir = self.io_get_bucket_path(dst_volume)?;
|
||||
if !skip_access_checks(src_volume) {
|
||||
@@ -10558,6 +10587,28 @@ impl DiskAPI for LocalDisk {
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(native_path) = crate::disk::native_prepare_rpc::parse_native_stream_path(volume, path)? {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use crate::disk::native_prepare_rpc::NativeStreamFile;
|
||||
|
||||
let root = self.io_root().to_path_buf();
|
||||
let file = tokio::task::spawn_blocking(move || match native_path.file {
|
||||
NativeStreamFile::Part(part) => {
|
||||
native_prepare::open_stream_part(&root, native_path.preparation, native_path.data_dir, part)
|
||||
}
|
||||
NativeStreamFile::Staging(id) => {
|
||||
native_prepare::open_stream_staging_file(&root, native_path.preparation, native_path.data_dir, id)
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?
|
||||
.map_err(to_file_error)?;
|
||||
return Ok(Box::new(tokio::fs::File::from_std(file)));
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
return Err(DiskError::UnsupportedDisk);
|
||||
}
|
||||
self.io_backend
|
||||
.open_write(volume, path, WriteMode::Truncate { size_hint: _file_size })
|
||||
.await
|
||||
@@ -10828,27 +10879,12 @@ impl DiskAPI for LocalDisk {
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn bucket_creation_witness(&self, volume: &str) -> Result<Option<os::NativeBucketCreationWitness>> {
|
||||
let Some(history) = self.publication_history.as_ref() else {
|
||||
return Ok(None);
|
||||
};
|
||||
let Some(disk_id) = self.get_disk_id().await? else {
|
||||
return Ok(None);
|
||||
};
|
||||
Ok(history.witness(volume, disk_id, Arc::clone(&self.io_backend)))
|
||||
}
|
||||
|
||||
async fn make_volume(&self, volume: &str) -> Result<()> {
|
||||
if !Self::is_valid_volname(volume) {
|
||||
return Err(Error::other("Invalid arguments specified"));
|
||||
}
|
||||
|
||||
let volume_dir = self.io_get_bucket_path(volume)?;
|
||||
#[cfg(all(test, not(windows)))]
|
||||
os::prepared_publication_test_hooks::run(
|
||||
os::prepared_publication_test_hooks::Stage::CreateDirectory,
|
||||
&self.root.join(volume),
|
||||
);
|
||||
|
||||
// Volume creation is a mutation boundary, so it must observe the live
|
||||
// filesystem rather than a potentially stale existence-cache entry.
|
||||
@@ -11590,186 +11626,120 @@ mod test {
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn owned_direct_child_open_attempt_consumes_native_history() {
|
||||
let dir = tempfile::tempdir().expect("native disk directory");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 disk path")).expect("disk endpoint");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("native local disk");
|
||||
let bucket = format!("owned-open-attempt-{}", Uuid::new_v4());
|
||||
let witness = disk
|
||||
.publication_history
|
||||
.as_ref()
|
||||
.expect("native registration")
|
||||
.witness(&bucket, Uuid::new_v4(), Arc::clone(&disk.io_backend))
|
||||
.expect("fresh history");
|
||||
|
||||
disk.write_all_public(&bucket, "direct-child", Bytes::from_static(b"payload"))
|
||||
#[serial_test::serial]
|
||||
async fn native_stream_reserved_split_never_truncates_attested_shard() {
|
||||
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
let (dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = "native-split-regression";
|
||||
let key = "object";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect_err("the direct child has no bucket parent");
|
||||
assert!(
|
||||
!os::validate_native_creation_witnesses(&[witness]),
|
||||
"even a failed O_CREATE in the real owned executor must permanently consume native history"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[tokio::test]
|
||||
async fn synchronous_direct_open_attempt_consumes_native_history() {
|
||||
let dir = tempfile::tempdir().expect("native disk directory");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 disk path")).expect("disk endpoint");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("native local disk");
|
||||
let bucket = format!("direct-io-open-attempt-{}", Uuid::new_v4());
|
||||
let witness = disk
|
||||
.publication_history
|
||||
.as_ref()
|
||||
.expect("native registration")
|
||||
.witness(&bucket, Uuid::new_v4(), Arc::clone(&disk.io_backend))
|
||||
.expect("fresh history");
|
||||
let path = disk.io_get_object_path(&bucket, "direct-child").expect("direct I/O path");
|
||||
|
||||
assert!(
|
||||
open_direct_writer(&path, &DirectIoWriteState::new()).is_err(),
|
||||
"the actual synchronous O_DIRECT entry must observe the missing bucket parent"
|
||||
);
|
||||
assert!(
|
||||
!os::validate_native_creation_witnesses(&[witness]),
|
||||
"a directly callable O_CREATE helper cannot retain retry permission after a failed publication attempt"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[tokio::test]
|
||||
async fn linux_queued_direct_open_cancellation_retains_started_history() {
|
||||
use crate::disk::os::prepared_publication_test_hooks as hooks;
|
||||
use std::sync::mpsc;
|
||||
|
||||
let dir = tempfile::tempdir().expect("native disk directory");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 disk path")).expect("disk endpoint");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("native local disk");
|
||||
let backend = Arc::new(StdBackend::new_without_fd_cache(disk.io_root().to_path_buf()));
|
||||
let actual_backend: Arc<dyn LocalIoBackend> = backend.clone();
|
||||
let history = disk.publication_history.as_ref().expect("native registration");
|
||||
let bucket = format!("native-queued-dio-{}", Uuid::new_v4());
|
||||
let disk_id = Uuid::new_v4();
|
||||
let witness = history
|
||||
.witness(&bucket, disk_id, Arc::clone(&actual_backend))
|
||||
.expect("fresh native history");
|
||||
let path = local_disk_object_path(disk.io_root(), "", &bucket).expect("first-level DIO path");
|
||||
let write_state = Arc::clone(&backend.direct_io_write);
|
||||
let idle_state_owners = Arc::strong_count(&write_state);
|
||||
let (entered_tx, entered_rx) = mpsc::channel();
|
||||
let (release_tx, release_rx) = mpsc::channel();
|
||||
let _hook = hooks::install_at(hooks::Stage::DirectWriteOpen, &path, move || {
|
||||
entered_tx.send(()).expect("signal real DIO executor entry");
|
||||
let _ = release_rx.recv();
|
||||
});
|
||||
|
||||
temp_env::async_with_vars([(ENV_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE, Some("true"))], async {
|
||||
let operation_backend = Arc::clone(&backend);
|
||||
let operation_bucket = bucket.clone();
|
||||
let operation = tokio::spawn(async move {
|
||||
operation_backend
|
||||
.open_write("", &operation_bucket, WriteMode::Truncate { size_hint: 0 })
|
||||
.await
|
||||
});
|
||||
tokio::task::spawn_blocking(move || entered_rx.recv_timeout(Duration::from_secs(30)))
|
||||
.expect("real bucket");
|
||||
let preparation = Uuid::new_v4();
|
||||
drop(
|
||||
store
|
||||
.prepare_native_object(
|
||||
bucket,
|
||||
key,
|
||||
preparation,
|
||||
&mut crate::object_api::PutObjReader::from_vec(vec![51; 2 * 1024 * 1024 + 7]),
|
||||
)
|
||||
.await
|
||||
.expect("DIO entry waiter")
|
||||
.expect("the real blocking executor must enter");
|
||||
assert!(
|
||||
!os::validate_native_creation_witnesses(std::slice::from_ref(&witness)),
|
||||
"Started must be recorded before the queued DIO worker performs its own history lookup"
|
||||
);
|
||||
assert!(!path.exists(), "the keyed pause must precede the actual O_DIRECT open");
|
||||
operation.abort();
|
||||
match operation.await {
|
||||
Err(error) if error.is_cancelled() => {}
|
||||
Err(error) => panic!("DIO waiter failed instead of cancelling: {error}"),
|
||||
Ok(_) => panic!("the paused DIO waiter must be cancelled"),
|
||||
}
|
||||
release_tx.send(()).expect("release the real DIO open");
|
||||
timeout(Duration::from_secs(30), async {
|
||||
while Arc::strong_count(&write_state) > idle_state_owners {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("the detached DIO executor must drain");
|
||||
})
|
||||
.await;
|
||||
|
||||
assert!(path.is_file(), "cancelling the waiter must not discard the queued native open");
|
||||
std::fs::remove_file(&path).expect("remove the file published by the cancelled DIO waiter");
|
||||
assert!(
|
||||
history.witness(&bucket, disk_id, actual_backend).is_none(),
|
||||
"executor drain and physical removal must not regrant native history"
|
||||
.expect("actual erasure/bitrot preparation"),
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[tokio::test]
|
||||
async fn linux_mount_lease_alias_publication_consumes_same_history() {
|
||||
let dir = tempfile::tempdir().expect("native disk directory");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 disk path")).expect("disk endpoint");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("native local disk");
|
||||
let history = disk.publication_history.as_ref().expect("native registration");
|
||||
let disk_id = Uuid::new_v4();
|
||||
for (alias, root) in [("root", disk.root.as_path()), ("io", disk.io_root())] {
|
||||
let bucket = format!("native-alias-{alias}-{}", Uuid::new_v4());
|
||||
let witness = history
|
||||
.witness(&bucket, disk_id, Arc::clone(&disk.io_backend))
|
||||
.expect("fresh native history through both real root aliases");
|
||||
super::super::fs::mkdir(root.join(&bucket))
|
||||
.await
|
||||
.expect("publish through the actual directory entry");
|
||||
let mut endpoint = Endpoint::try_from(dirs[0].path().to_str().expect("root")).expect("endpoint");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(0);
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("actual LocalDisk");
|
||||
let path = format!("native-preparations/{preparation}/{preparation}/part.1");
|
||||
let full = dirs[0].path().join(RUSTFS_META_BUCKET).join(&path);
|
||||
let before = std::fs::read(&full).expect("actual durable shard");
|
||||
for (volume, candidate) in [
|
||||
(
|
||||
format!("{RUSTFS_META_BUCKET}/native-preparations/{preparation}"),
|
||||
format!("{preparation}/part.1"),
|
||||
),
|
||||
(
|
||||
format!("{RUSTFS_META_BUCKET}/native-preparations"),
|
||||
format!("{preparation}/{preparation}/part.1"),
|
||||
),
|
||||
(RUSTFS_META_BUCKET.to_owned(), format!("./{path}")),
|
||||
(RUSTFS_META_BUCKET.to_owned(), format!("junk/../{path}")),
|
||||
(
|
||||
format!("{RUSTFS_META_BUCKET}/junk/../native-preparations/{preparation}"),
|
||||
format!("{preparation}/part.1"),
|
||||
),
|
||||
(
|
||||
format!("junk/../{RUSTFS_META_BUCKET}"),
|
||||
format!("native-preparations/{preparation}/{preparation}/part.1"),
|
||||
),
|
||||
(
|
||||
RUSTFS_META_BUCKET.to_owned(),
|
||||
format!("native-preparations//{preparation}/{preparation}/part.1"),
|
||||
),
|
||||
] {
|
||||
assert!(
|
||||
!os::validate_native_creation_witnesses(std::slice::from_ref(&witness)),
|
||||
"{alias}: publication through either physical-root alias must consume the same history"
|
||||
matches!(disk.create_file("", &volume, &candidate, 1).await, Err(DiskError::InvalidPath)),
|
||||
"NATIVE_RESERVED_SPLIT_MUST_REJECT_BEFORE_TRUNCATE: {volume}/{candidate}"
|
||||
);
|
||||
std::fs::remove_dir(disk.root.join(&bucket)).expect("remove the physically published directory");
|
||||
assert!(
|
||||
history.witness(&bucket, disk_id, Arc::clone(&disk.io_backend)).is_none(),
|
||||
"{alias}: removal through the other alias must not regrant native history"
|
||||
assert_eq!(
|
||||
std::fs::read(&full).expect("retained actual shard"),
|
||||
before,
|
||||
"NATIVE_RESERVED_SPLIT_MUST_PRESERVE_ATTESTED_SHARD"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[tokio::test]
|
||||
async fn linux_native_history_is_registered_before_data_usage_layout() {
|
||||
use crate::disk::os::prepared_publication_test_hooks as hooks;
|
||||
|
||||
let dir = tempfile::tempdir().expect("native disk directory");
|
||||
let root = std::fs::canonicalize(dir.path()).expect("canonical disk root");
|
||||
let bucket = format!("native-before-layout-{}", Uuid::new_v4());
|
||||
let source = root.join(format!("layout-source-{}", Uuid::new_v4()));
|
||||
std::fs::create_dir(&source).expect("prepare an actual directory for publication");
|
||||
let destination = root.join(&bucket);
|
||||
let publication_path = destination.clone();
|
||||
let _hook = hooks::install_at(hooks::Stage::BeforeDataUsageLayout, &root, move || {
|
||||
super::super::fs::rename_std(&source, &publication_path)
|
||||
.expect("publish through the native entry before data usage layout");
|
||||
});
|
||||
let endpoint = Endpoint::try_from(root.to_str().expect("UTF-8 disk path")).expect("disk endpoint");
|
||||
let disk = LocalDisk::new(&endpoint, false)
|
||||
let staged_id = Uuid::new_v4();
|
||||
let staged_path = format!("native-preparations/{preparation}/{preparation}/.rustfs-put-{staged_id}");
|
||||
let mut staged = disk
|
||||
.create_file("", RUSTFS_META_BUCKET, &staged_path, 7)
|
||||
.await
|
||||
.expect("native local disk with real startup layout");
|
||||
let history = disk.publication_history.as_ref().expect("native registration");
|
||||
let disk_id = Uuid::new_v4();
|
||||
.expect("create private native staging file");
|
||||
staged.write_all(b"staged!").await.expect("write staged payload");
|
||||
staged.shutdown().await.expect("close staged payload");
|
||||
disk.rename_file(
|
||||
RUSTFS_META_BUCKET,
|
||||
&staged_path,
|
||||
RUSTFS_META_BUCKET,
|
||||
&format!("native-preparations/{preparation}/{preparation}/part.2"),
|
||||
)
|
||||
.await
|
||||
.expect("publish private staging file without ordinary volume rename");
|
||||
assert_eq!(
|
||||
std::fs::read(
|
||||
dirs[0]
|
||||
.path()
|
||||
.join(RUSTFS_META_BUCKET)
|
||||
.join(format!("native-preparations/{preparation}/{preparation}/part.2"))
|
||||
)
|
||||
.expect("read published private part"),
|
||||
b"staged!"
|
||||
);
|
||||
|
||||
let untouched = format!("native-layout-untouched-{}", Uuid::new_v4());
|
||||
assert!(
|
||||
history.witness(&untouched, disk_id, Arc::clone(&disk.io_backend)).is_some(),
|
||||
"startup layout must keep an untouched bucket's native history known"
|
||||
);
|
||||
assert!(
|
||||
history.witness(&bucket, disk_id, Arc::clone(&disk.io_backend)).is_none(),
|
||||
"registration must capture real publication before startup layout finishes"
|
||||
);
|
||||
std::fs::remove_dir(destination).expect("remove the physically published directory");
|
||||
assert!(
|
||||
history.witness(&bucket, disk_id, Arc::clone(&disk.io_backend)).is_none(),
|
||||
"startup completion and physical removal must retain Started"
|
||||
);
|
||||
let abandoned_id = Uuid::new_v4();
|
||||
let abandoned_path = format!("native-preparations/{preparation}/{preparation}/.rustfs-put-{abandoned_id}");
|
||||
let mut abandoned = disk
|
||||
.create_file("", RUSTFS_META_BUCKET, &abandoned_path, 3)
|
||||
.await
|
||||
.expect("create private cleanup staging file");
|
||||
abandoned.write_all(b"tmp").await.expect("write cleanup staging file");
|
||||
abandoned.shutdown().await.expect("close cleanup staging file");
|
||||
disk.delete(
|
||||
RUSTFS_META_BUCKET,
|
||||
&abandoned_path,
|
||||
DeleteOptions {
|
||||
immediate: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("delete private staging file after failed publication");
|
||||
assert!(!dirs[0].path().join(RUSTFS_META_BUCKET).join(&abandoned_path).exists());
|
||||
assert!(store.recover_native_object(bucket, key, preparation).await.is_ok());
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
@@ -12697,6 +12667,227 @@ mod test {
|
||||
}
|
||||
}
|
||||
|
||||
// SIGKILL tests process death on a healthy filesystem, not power loss.
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn rename_data_process_kill_preserves_publication_boundary() {
|
||||
use std::io::Write as _;
|
||||
use std::os::unix::process::ExitStatusExt;
|
||||
use std::process::{Child, Command, ExitStatus, Stdio};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
const TEST: &str = "disk::local::test::rename_data_process_kill_preserves_publication_boundary";
|
||||
const ROLE: &str = "RUSTFS_TEST_PUBLICATION_KILL_ROLE";
|
||||
const ROOT: &str = "RUSTFS_TEST_PUBLICATION_KILL_ROOT";
|
||||
const NONCE: &str = "RUSTFS_TEST_PUBLICATION_KILL_NONCE";
|
||||
const BUCKET: &str = "bucket";
|
||||
const OBJECT: &str = "process-kill-object";
|
||||
const STAGED: &str = "process-kill-staged";
|
||||
const VERSION: Uuid = Uuid::from_u128(0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa);
|
||||
const OLD: Uuid = Uuid::from_u128(0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb);
|
||||
const NEW: Uuid = Uuid::from_u128(0xcccccccccccccccccccccccccccccccc);
|
||||
|
||||
struct ChildGuard(Child);
|
||||
impl Drop for ChildGuard {
|
||||
fn drop(&mut self) {
|
||||
let _ = self.0.kill();
|
||||
let _ = self.0.wait();
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_exit(child: &mut ChildGuard) -> ExitStatus {
|
||||
let deadline = Instant::now() + Duration::from_secs(60);
|
||||
loop {
|
||||
if child.0.try_wait().expect("poll owned child").is_some() {
|
||||
return child.0.wait().expect("reap owned child");
|
||||
}
|
||||
assert!(Instant::now() < deadline, "owned child exceeded exit deadline");
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
}
|
||||
|
||||
fn spawn(root: &std::path::Path, nonce: &str, role: &str) -> ChildGuard {
|
||||
let stdout = std::fs::File::create(root.join(format!("{role}.stdout"))).expect("child stdout file");
|
||||
let stderr = std::fs::File::create(root.join(format!("{role}.stderr"))).expect("child stderr file");
|
||||
ChildGuard(
|
||||
Command::new(std::env::current_exe().expect("test executable"))
|
||||
.args(["--exact", TEST, "--nocapture"])
|
||||
.env(ROLE, role)
|
||||
.env(ROOT, root)
|
||||
.env(NONCE, nonce)
|
||||
.stdout(Stdio::from(stdout))
|
||||
.stderr(Stdio::from(stderr))
|
||||
.spawn()
|
||||
.expect("spawn owned publication child"),
|
||||
)
|
||||
}
|
||||
|
||||
fn child_logs(root: &std::path::Path, role: &str) -> String {
|
||||
let stdout = std::fs::read_to_string(root.join(format!("{role}.stdout"))).expect("child stdout");
|
||||
let stderr = std::fs::read_to_string(root.join(format!("{role}.stderr"))).expect("child stderr");
|
||||
format!("{stdout} {stderr}")
|
||||
}
|
||||
|
||||
fn file_info(data_dir: Uuid) -> FileInfo {
|
||||
let mut fi = test_file_info(OBJECT, VERSION, Some(data_dir), None);
|
||||
fi.size = 8;
|
||||
fi.parts[0].size = 8;
|
||||
fi.parts[0].actual_size = 8;
|
||||
fi
|
||||
}
|
||||
|
||||
if let Some(role) = std::env::var_os(ROLE) {
|
||||
let role = role.to_str().expect("child role is UTF-8");
|
||||
assert!(matches!(role, "writer-before" | "writer-after" | "verify-before" | "verify-after"));
|
||||
let root = std::path::PathBuf::from(std::env::var_os(ROOT).expect("private child root"));
|
||||
let nonce = std::env::var(NONCE).expect("private child nonce");
|
||||
Uuid::parse_str(&nonce).expect("nonce is a UUID");
|
||||
assert_eq!(root.file_name().and_then(|name| name.to_str()), Some(nonce.as_str()));
|
||||
assert_eq!(std::fs::read_to_string(root.join("control-nonce")).expect("bound root nonce"), nonce);
|
||||
let endpoint = Endpoint::try_from(root.to_str().expect("private root UTF-8")).expect("private endpoint");
|
||||
let _strict = durability_mode_override::set(DurabilityMode::Strict);
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("child opens real disk");
|
||||
let object_dir = root.join(BUCKET).join(OBJECT);
|
||||
if role.starts_with("verify-") {
|
||||
let fi = disk
|
||||
.read_version("", BUCKET, OBJECT, &VERSION.to_string(), &ReadOptions::default())
|
||||
.await
|
||||
.expect("fresh verifier reads actual version");
|
||||
let (expected_dir, expected_bytes) = if role == "verify-before" {
|
||||
assert_eq!(
|
||||
fs::read(object_dir.join(STORAGE_FORMAT_FILE))
|
||||
.await
|
||||
.expect("read old metadata"),
|
||||
fs::read(root.join("old-meta")).await.expect("read old fixture"),
|
||||
"prepublication metadata must remain byte-for-byte old"
|
||||
);
|
||||
(OLD, b"old-data")
|
||||
} else {
|
||||
(NEW, b"new-data")
|
||||
};
|
||||
assert_eq!(fi.data_dir, Some(expected_dir), "{role}: actual reopened version data_dir");
|
||||
let part = object_dir
|
||||
.join(fi.data_dir.expect("referenced data dir").to_string())
|
||||
.join("part.1");
|
||||
assert_eq!(fs::read(part).await.expect("read referenced shard"), expected_bytes);
|
||||
return;
|
||||
}
|
||||
|
||||
let staged = root.join(RUSTFS_META_TMP_BUCKET).join(STAGED).join(NEW.to_string());
|
||||
fs::create_dir_all(&staged).await.expect("staged shard directory");
|
||||
fs::write(staged.join("part.1"), b"new-data")
|
||||
.await
|
||||
.expect("stage new shard bytes");
|
||||
let ready = root.join(format!("{role}.ready"));
|
||||
let pause = move || {
|
||||
let pending = ready.with_extension("pending");
|
||||
std::fs::write(&pending, &nonce).expect("write actual writer cut marker");
|
||||
std::fs::rename(pending, &ready).expect("announce complete writer cut marker");
|
||||
let (_sender, receiver) = std::sync::mpsc::channel::<()>();
|
||||
receiver.recv().expect("writer remains paused until SIGKILL");
|
||||
};
|
||||
let _hook = if role == "writer-before" {
|
||||
// Use the executor's exact path spelling, including canonical
|
||||
// macOS roots and Linux mount-pinned I/O roots.
|
||||
let destination = disk
|
||||
.io_get_object_path(BUCKET, &format!("{OBJECT}/{STORAGE_FORMAT_FILE}"))
|
||||
.expect("actual metadata publication path");
|
||||
assert_eq!(
|
||||
std::fs::canonicalize(&destination).expect("canonical publication destination"),
|
||||
disk.root.join(BUCKET).join(OBJECT).join(STORAGE_FORMAT_FILE),
|
||||
"hook destination must identify this disk's seeded metadata"
|
||||
);
|
||||
Some(os::prepared_publication_test_hooks::install_at(
|
||||
os::prepared_publication_test_hooks::Stage::PreparedRename,
|
||||
&destination,
|
||||
pause,
|
||||
))
|
||||
} else {
|
||||
disk.rename_data(RUSTFS_META_TMP_BUCKET, STAGED, file_info(NEW), BUCKET, OBJECT)
|
||||
.await
|
||||
.expect("Strict real rename_data completes before cut");
|
||||
pause();
|
||||
return;
|
||||
};
|
||||
disk.rename_data(RUSTFS_META_TMP_BUCKET, STAGED, file_info(NEW), BUCKET, OBJECT)
|
||||
.await
|
||||
.expect("before writer must remain in physical hook");
|
||||
panic!("prepublication hook unexpectedly returned");
|
||||
}
|
||||
|
||||
let temp = tempfile::tempdir().expect("parent owns private disk roots");
|
||||
for phase in ["before", "after"] {
|
||||
let nonce = Uuid::new_v4().to_string();
|
||||
let root = temp.path().join(&nonce);
|
||||
fs::create_dir(&root).await.expect("private scenario root");
|
||||
fs::write(root.join("control-nonce"), &nonce)
|
||||
.await
|
||||
.expect("bind private root");
|
||||
let endpoint = Endpoint::try_from(root.to_str().expect("private root UTF-8")).expect("private endpoint");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("seed real disk");
|
||||
ensure_test_volume(&disk, BUCKET).await;
|
||||
ensure_test_volume(&disk, RUSTFS_META_TMP_BUCKET).await;
|
||||
let object_dir = root.join(BUCKET).join(OBJECT);
|
||||
fs::create_dir_all(object_dir.join(OLD.to_string()))
|
||||
.await
|
||||
.expect("old shard directory");
|
||||
fs::write(object_dir.join(OLD.to_string()).join("part.1"), b"old-data")
|
||||
.await
|
||||
.expect("seed old shard bytes");
|
||||
let old_meta = test_meta(file_info(OLD));
|
||||
fs::write(object_dir.join(STORAGE_FORMAT_FILE), &old_meta)
|
||||
.await
|
||||
.expect("seed old metadata");
|
||||
fs::write(root.join("old-meta"), &old_meta)
|
||||
.await
|
||||
.expect("retain exact old fixture");
|
||||
drop(disk);
|
||||
|
||||
let writer_role = format!("writer-{phase}");
|
||||
let mut writer = spawn(&root, &nonce, &writer_role);
|
||||
let deadline = Instant::now() + Duration::from_secs(60);
|
||||
let ready = root.join(format!("{writer_role}.ready"));
|
||||
loop {
|
||||
if let Some(status) = writer.0.try_wait().expect("poll writer") {
|
||||
panic!(
|
||||
"writer exited before cut: {writer_role} status={status}: {}",
|
||||
child_logs(&root, &writer_role)
|
||||
);
|
||||
}
|
||||
match fs::read_to_string(&ready).await {
|
||||
Ok(marker) => {
|
||||
assert_eq!(marker, nonce, "cut marker belongs to this private writer");
|
||||
break;
|
||||
}
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
|
||||
Err(error) => panic!("read cut marker: {error}"),
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
panic!("writer cut deadline: {writer_role}: {}", child_logs(&root, &writer_role));
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
let pid = writer.0.id();
|
||||
writer.0.kill().expect("SIGKILL owned writer at verified cut");
|
||||
let status = wait_exit(&mut writer).await;
|
||||
assert_eq!(status.signal(), Some(9), "owned writer must actually die by SIGKILL");
|
||||
writeln!(std::io::stdout(), "{writer_role} pid={pid} signal={:?}", status.signal())
|
||||
.expect("record actual writer wait status");
|
||||
|
||||
let verifier_role = format!("verify-{phase}");
|
||||
let mut verifier = spawn(&root, &nonce, &verifier_role);
|
||||
let status = wait_exit(&mut verifier).await;
|
||||
let logs = child_logs(&root, &verifier_role);
|
||||
assert!(status.success(), "{verifier_role} status={status}: {logs}");
|
||||
let stdout = std::fs::read_to_string(root.join(format!("{verifier_role}.stdout"))).expect("verifier stdout");
|
||||
assert!(
|
||||
stdout.contains(TEST) && stdout.contains("1 passed"),
|
||||
"verifier must execute exact test: {logs}"
|
||||
);
|
||||
writeln!(std::io::stdout(), "{verifier_role} status={status}").expect("record actual verifier exit");
|
||||
}
|
||||
}
|
||||
|
||||
/// Crash-consistency harness for the rename_data commit sequence
|
||||
/// (rustfs/backlog#935 HP-14, test plan rustfs/backlog#896; hard rule from
|
||||
/// rustfs/backlog#878: "After partial commit, the object can only be an old or new version; it cannot be mixed").
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+147
-12
@@ -24,6 +24,7 @@ pub mod fs;
|
||||
pub mod health_state;
|
||||
mod io_admission;
|
||||
pub mod local;
|
||||
pub(crate) mod native_prepare_rpc;
|
||||
pub mod os;
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -230,6 +231,152 @@ pub enum Disk {
|
||||
}
|
||||
|
||||
impl Disk {
|
||||
#[cfg(unix)]
|
||||
pub(crate) async fn native_begin_stream(
|
||||
&self,
|
||||
preparation: Uuid,
|
||||
store_id: Uuid,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
incarnation: Uuid,
|
||||
fence: Arc<crate::object_api::ObjectLockConfigSnapshot>,
|
||||
) -> Result<()> {
|
||||
match self {
|
||||
Disk::Local(local) => {
|
||||
local
|
||||
.get_disk()
|
||||
.begin_native_stream(preparation, store_id, bucket, incarnation, fence)
|
||||
.await
|
||||
}
|
||||
Disk::Remote(remote) => {
|
||||
remote
|
||||
.native_begin_stream(preparation, store_id, bucket, key, incarnation)
|
||||
.await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
pub(crate) async fn native_begin_stream(
|
||||
&self,
|
||||
_preparation: Uuid,
|
||||
_store_id: Uuid,
|
||||
_bucket: &str,
|
||||
_key: &str,
|
||||
_incarnation: Uuid,
|
||||
_fence: Arc<crate::object_api::ObjectLockConfigSnapshot>,
|
||||
) -> Result<()> {
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(crate) async fn native_finalize(
|
||||
&self,
|
||||
input: local::native_prepare::NativePrepareInput,
|
||||
) -> Result<local::native_prepare::NativePreparationReceipt> {
|
||||
match self {
|
||||
Disk::Local(local) => local.get_disk().prepare_native(input).await,
|
||||
Disk::Remote(remote) => remote.native_finalize(input).await,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
pub(crate) async fn native_finalize(
|
||||
&self,
|
||||
_input: local::native_prepare::NativePrepareInput,
|
||||
) -> Result<local::native_prepare::NativePreparationReceipt> {
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(crate) async fn native_recover(
|
||||
&self,
|
||||
preparation: Uuid,
|
||||
store_id: Uuid,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
incarnation: Uuid,
|
||||
fence: Arc<crate::object_api::ObjectLockConfigSnapshot>,
|
||||
) -> Result<(local::native_prepare::NativePreparationReceipt, Bytes)> {
|
||||
match self {
|
||||
Disk::Local(local) => {
|
||||
local
|
||||
.get_disk()
|
||||
.recover_native_with_metadata(preparation, store_id, bucket, incarnation, fence)
|
||||
.await
|
||||
}
|
||||
Disk::Remote(remote) => remote.native_recover(preparation, store_id, bucket, key, incarnation).await,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
pub(crate) async fn native_recover(
|
||||
&self,
|
||||
_preparation: Uuid,
|
||||
_store_id: Uuid,
|
||||
_bucket: &str,
|
||||
_key: &str,
|
||||
_incarnation: Uuid,
|
||||
_fence: Arc<crate::object_api::ObjectLockConfigSnapshot>,
|
||||
) -> Result<(local::native_prepare::NativePreparationReceipt, Bytes)> {
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(crate) async fn native_probe(
|
||||
&self,
|
||||
preparation: Uuid,
|
||||
store_id: Uuid,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
incarnation: Uuid,
|
||||
fence: Arc<crate::object_api::ObjectLockConfigSnapshot>,
|
||||
) -> Result<local::native_prepare::NativePreparationProbe> {
|
||||
match self {
|
||||
Disk::Local(local) => {
|
||||
local
|
||||
.get_disk()
|
||||
.probe_native(preparation, store_id, bucket, key, incarnation, fence)
|
||||
.await
|
||||
}
|
||||
Disk::Remote(remote) => remote.native_probe(preparation, store_id, bucket, key, incarnation).await,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
pub(crate) async fn native_probe(
|
||||
&self,
|
||||
_preparation: Uuid,
|
||||
_store_id: Uuid,
|
||||
_bucket: &str,
|
||||
_key: &str,
|
||||
_incarnation: Uuid,
|
||||
_fence: Arc<crate::object_api::ObjectLockConfigSnapshot>,
|
||||
) -> Result<local::native_prepare::NativePreparationProbe> {
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(crate) async fn native_locate(
|
||||
&self,
|
||||
store_id: Uuid,
|
||||
preparation: Uuid,
|
||||
) -> Result<Option<local::native_prepare::NativePreparationDescription>> {
|
||||
match self {
|
||||
Disk::Local(local) => local.get_disk().locate_native_preparation(store_id, preparation).await,
|
||||
Disk::Remote(remote) => remote.native_locate(store_id, preparation).await,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
pub(crate) async fn native_locate(
|
||||
&self,
|
||||
_store_id: Uuid,
|
||||
_preparation: Uuid,
|
||||
) -> Result<Option<local::native_prepare::NativePreparationDescription>> {
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
}
|
||||
|
||||
pub(crate) async fn set_disk_id_state(&self, id: Option<Uuid>) -> Result<()> {
|
||||
match self {
|
||||
Disk::Local(local_disk) => {
|
||||
@@ -327,13 +474,6 @@ impl DiskAPI for Disk {
|
||||
}
|
||||
}
|
||||
|
||||
async fn bucket_creation_witness(&self, volume: &str) -> Result<Option<os::NativeBucketCreationWitness>> {
|
||||
match self {
|
||||
Disk::Local(local_disk) => local_disk.bucket_creation_witness(volume).await,
|
||||
Disk::Remote(_) => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn make_volume(&self, volume: &str) -> Result<()> {
|
||||
match self {
|
||||
@@ -1128,11 +1268,6 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
|
||||
// NSScanner
|
||||
|
||||
// Volume operations.
|
||||
/// Unknown or remote backends cannot attest process-local publication history.
|
||||
async fn bucket_creation_witness(&self, _volume: &str) -> Result<Option<os::NativeBucketCreationWitness>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn make_volume(&self, volume: &str) -> Result<()>;
|
||||
async fn make_volumes(&self, volume: Vec<&str>) -> Result<()>;
|
||||
async fn list_volumes(&self) -> Result<Vec<VolumeInfo>>;
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::disk::{
|
||||
RUSTFS_META_BUCKET,
|
||||
error::{DiskError, Result},
|
||||
};
|
||||
use rustfs_utils::path::path_join_buf;
|
||||
use uuid::Uuid;
|
||||
|
||||
const NATIVE_STREAM_PREFIX: &str = "native-preparations/";
|
||||
const NATIVE_STREAM_ROOT: &str = "native-preparations";
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum NativeStreamFile {
|
||||
Part(usize),
|
||||
Staging(Uuid),
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct NativeStreamPath {
|
||||
pub(crate) preparation: Uuid,
|
||||
pub(crate) data_dir: Uuid,
|
||||
pub(crate) file: NativeStreamFile,
|
||||
}
|
||||
|
||||
fn targets_native_stream_namespace(volume: &str, path: &str) -> bool {
|
||||
if !volume.contains(RUSTFS_META_BUCKET) && !path.contains(NATIVE_STREAM_ROOT) {
|
||||
return false;
|
||||
}
|
||||
let normalized = path_join_buf(&[volume, path]);
|
||||
let prefix = format!("{RUSTFS_META_BUCKET}/{NATIVE_STREAM_PREFIX}");
|
||||
normalized == format!("{RUSTFS_META_BUCKET}/{NATIVE_STREAM_ROOT}") || normalized.starts_with(&prefix)
|
||||
}
|
||||
|
||||
/// Recognize only the reserved per-preparation shard path used by the native
|
||||
/// stream writer. Malformed paths under that namespace must not fall through to
|
||||
/// the general disk writer.
|
||||
pub(crate) fn parse_native_stream_path(volume: &str, path: &str) -> Result<Option<NativeStreamPath>> {
|
||||
if !targets_native_stream_namespace(volume, path) {
|
||||
return Ok(None);
|
||||
}
|
||||
if volume != RUSTFS_META_BUCKET {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
if !path.starts_with(NATIVE_STREAM_PREFIX) {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
|
||||
let mut components = path.split('/');
|
||||
if components.next() != Some("native-preparations") {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
let preparation = components
|
||||
.next()
|
||||
.and_then(|value| Uuid::parse_str(value).ok())
|
||||
.filter(|value| !value.is_nil())
|
||||
.ok_or(DiskError::InvalidPath)?;
|
||||
let data_dir = components
|
||||
.next()
|
||||
.and_then(|value| Uuid::parse_str(value).ok())
|
||||
.filter(|value| !value.is_nil())
|
||||
.ok_or(DiskError::InvalidPath)?;
|
||||
let file_name = components.next().ok_or(DiskError::InvalidPath)?;
|
||||
let file = file_name
|
||||
.strip_prefix("part.")
|
||||
.and_then(|value| value.parse::<usize>().ok())
|
||||
.filter(|value| *value > 0)
|
||||
.filter(|part| file_name == format!("part.{part}"))
|
||||
.map(NativeStreamFile::Part)
|
||||
.or_else(|| {
|
||||
file_name
|
||||
.strip_prefix(".rustfs-put-")
|
||||
.and_then(|value| Uuid::parse_str(value).ok())
|
||||
.filter(|value| !value.is_nil())
|
||||
.filter(|id| file_name == format!(".rustfs-put-{id}"))
|
||||
.map(NativeStreamFile::Staging)
|
||||
});
|
||||
|
||||
if components.next().is_some()
|
||||
|| file.is_none()
|
||||
|| path != format!("native-preparations/{preparation}/{data_dir}/{file_name}")
|
||||
{
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
Ok(Some(NativeStreamPath {
|
||||
preparation,
|
||||
data_dir,
|
||||
file: file.expect("validated native stream filename"),
|
||||
}))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn native_stream_path_requires_canonical_identity_and_part() {
|
||||
let preparation = Uuid::from_u128(1);
|
||||
let data_dir = Uuid::from_u128(2);
|
||||
let valid = format!("native-preparations/{preparation}/{data_dir}/part.1");
|
||||
assert!(
|
||||
parse_native_stream_path(RUSTFS_META_BUCKET, &valid)
|
||||
.expect("valid path")
|
||||
.is_some()
|
||||
);
|
||||
let staging = format!("native-preparations/{preparation}/{data_dir}/.rustfs-put-{data_dir}");
|
||||
assert!(
|
||||
parse_native_stream_path(RUSTFS_META_BUCKET, &staging)
|
||||
.expect("valid staging path")
|
||||
.is_some()
|
||||
);
|
||||
assert!(parse_native_stream_path("bucket", &valid).expect("other volume").is_none());
|
||||
assert!(
|
||||
parse_native_stream_path(RUSTFS_META_BUCKET, "ordinary/object")
|
||||
.expect("ordinary path")
|
||||
.is_none()
|
||||
);
|
||||
for invalid in [
|
||||
"native-preparations/00000000-0000-0000-0000-000000000000/00000000-0000-0000-0000-000000000002/part.1".to_owned(),
|
||||
format!("native-preparations/{preparation}/{data_dir}/part.0"),
|
||||
format!("native-preparations/{preparation}/{data_dir}/part.01"),
|
||||
format!("native-preparations/{preparation}/{data_dir}/.rustfs-put-{}", Uuid::nil()),
|
||||
format!("native-preparations/{preparation}/{data_dir}/.rustfs-put-not-a-uuid"),
|
||||
format!("native-preparations/{preparation}/{data_dir}/part.1/extra"),
|
||||
format!("native-preparations/{preparation}/../part.1"),
|
||||
] {
|
||||
assert!(matches!(
|
||||
parse_native_stream_path(RUSTFS_META_BUCKET, &invalid),
|
||||
Err(DiskError::InvalidPath)
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -21,7 +21,7 @@ use futures::TryStreamExt;
|
||||
use parking_lot::Mutex;
|
||||
use rustfs_utils::path::SLASH_SEPARATOR;
|
||||
use std::{
|
||||
collections::{HashMap, HashSet, VecDeque},
|
||||
collections::{HashMap, VecDeque},
|
||||
io,
|
||||
path::{Component, Path, PathBuf},
|
||||
sync::{Arc, LazyLock, Weak},
|
||||
@@ -276,14 +276,6 @@ pub(crate) mod prepared_publication_test_hooks {
|
||||
Rollback,
|
||||
#[cfg(test)]
|
||||
DirFsync,
|
||||
#[cfg(test)]
|
||||
CreateDirectory,
|
||||
#[cfg(test)]
|
||||
NativeRootIdentity,
|
||||
#[cfg(all(test, target_os = "linux"))]
|
||||
DirectWriteOpen,
|
||||
#[cfg(all(test, target_os = "linux"))]
|
||||
BeforeDataUsageLayout,
|
||||
}
|
||||
|
||||
type Hook = Box<dyn FnOnce() + Send>;
|
||||
@@ -1389,226 +1381,6 @@ fn fsync_spawn_blocking<T: Send + 'static>(f: impl FnOnce() -> T + Send + 'stati
|
||||
}
|
||||
static DISK_VOLUME_MUTATION_LOCKS: LazyLock<Mutex<HashMap<PathBuf, Weak<RwLock<()>>>>> =
|
||||
LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
#[cfg(windows)]
|
||||
type NativeRootIdentity = (u64, [u8; 16]);
|
||||
#[cfg(not(windows))]
|
||||
type NativeRootIdentity = (u64, u64);
|
||||
|
||||
// Publication history outlives disk handles, cleanup, and metadata snapshots.
|
||||
// Reusing a pathname or descriptor for another physical root is not new proof.
|
||||
#[derive(Debug, Default)]
|
||||
struct NativePublicationHistory {
|
||||
roots: HashMap<NativeRootIdentity, NativeRootHistory>,
|
||||
aliases: HashMap<PathBuf, HashSet<NativeRootIdentity>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct NativeRootHistory {
|
||||
unknown: bool,
|
||||
started: HashSet<String>,
|
||||
}
|
||||
|
||||
static NATIVE_PUBLICATION_HISTORY: LazyLock<Mutex<NativePublicationHistory>> =
|
||||
LazyLock::new(|| Mutex::new(NativePublicationHistory::default()));
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct NativePublicationRegistration {
|
||||
identity: NativeRootIdentity,
|
||||
root: PathBuf,
|
||||
io_root: PathBuf,
|
||||
}
|
||||
|
||||
/// Opaque, process-local observation. Unsupported disks cannot construct it.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct NativeBucketCreationWitness {
|
||||
registration: NativePublicationRegistration,
|
||||
backend: Arc<dyn super::local::LocalIoBackend>,
|
||||
bucket: String,
|
||||
disk_id: uuid::Uuid,
|
||||
}
|
||||
|
||||
fn native_root_identity(path: &Path) -> Option<NativeRootIdentity> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
#[cfg(test)]
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::NativeRootIdentity, path);
|
||||
let metadata = std::fs::metadata(path).ok()?;
|
||||
metadata.is_dir().then(|| (metadata.dev(), metadata.ino()))
|
||||
}
|
||||
#[cfg(windows)]
|
||||
{
|
||||
// Resolve and validate the same directory anchor used for publication;
|
||||
// keep its volume and complete 128-bit file ID in the history key.
|
||||
let (_, directory) = open_windows_publication_root(path).ok()?;
|
||||
windows_file_identity(directory.handle.as_ref()).ok()
|
||||
}
|
||||
#[cfg(all(not(unix), not(windows)))]
|
||||
{
|
||||
let _ = path;
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn register_native_publication_root(root: &Path, io_root: &Path) -> Option<NativePublicationRegistration> {
|
||||
let identity = native_root_identity(root)?;
|
||||
if native_root_identity(io_root) != Some(identity) {
|
||||
return None;
|
||||
}
|
||||
let mut history = NATIVE_PUBLICATION_HISTORY.lock();
|
||||
history.roots.entry(identity).or_default();
|
||||
for alias in [root, io_root] {
|
||||
let alias: PathBuf = alias.components().collect();
|
||||
let previous = history.aliases.entry(alias).or_default();
|
||||
let collisions: Vec<_> = previous.iter().copied().filter(|old| *old != identity).collect();
|
||||
previous.insert(identity);
|
||||
if !collisions.is_empty() {
|
||||
history.roots.get_mut(&identity)?.unknown = true;
|
||||
for old in collisions {
|
||||
history.roots.get_mut(&old)?.unknown = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
Some(NativePublicationRegistration {
|
||||
identity,
|
||||
root: root.to_path_buf(),
|
||||
io_root: io_root.to_path_buf(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Record before queueing filesystem work. Cancellation never removes Started.
|
||||
pub(crate) fn record_native_publication(path: &Path) {
|
||||
let aliases: Vec<_> = {
|
||||
let history = NATIVE_PUBLICATION_HISTORY.lock();
|
||||
path.ancestors()
|
||||
.filter(|alias| history.aliases.contains_key(*alias))
|
||||
.map(Path::to_path_buf)
|
||||
.collect()
|
||||
};
|
||||
// Physical lookups must never hold the process-wide publication mutex.
|
||||
let observations: Vec<_> = aliases
|
||||
.into_iter()
|
||||
.filter_map(|alias| {
|
||||
let relative = path.strip_prefix(&alias).ok()?;
|
||||
let mut components = relative.components().filter(|part| !matches!(part, Component::CurDir));
|
||||
let first = components.next()?;
|
||||
let bucket = match first {
|
||||
Component::Normal(bucket) if components.all(|part| matches!(part, Component::Normal(_))) => {
|
||||
bucket.to_str().map(str::to_owned)
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
let physical = native_root_identity(&alias);
|
||||
Some((alias, physical, bucket))
|
||||
})
|
||||
.collect();
|
||||
let mut history = NATIVE_PUBLICATION_HISTORY.lock();
|
||||
for (alias, physical, bucket) in observations {
|
||||
// Aliases only accumulate identities. Include registration concurrent
|
||||
// with the stat, rather than using a stale captured identity list.
|
||||
let identities = history.aliases.get(&alias).cloned().unwrap_or_default();
|
||||
if physical.is_none() || identities.iter().any(|identity| Some(*identity) != physical) {
|
||||
for identity in identities {
|
||||
history.roots.entry(identity).or_default().unknown = true;
|
||||
}
|
||||
if let Some(identity) = physical {
|
||||
history.roots.entry(identity).or_default().unknown = true;
|
||||
}
|
||||
} else if let Some(identity) = physical {
|
||||
let root = history.roots.entry(identity).or_default();
|
||||
match bucket {
|
||||
Some(bucket) => {
|
||||
root.started.insert(bucket);
|
||||
}
|
||||
None => root.unknown = true,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl NativePublicationRegistration {
|
||||
pub(crate) fn invalidate(&self) {
|
||||
if let Some(root) = NATIVE_PUBLICATION_HISTORY.lock().roots.get_mut(&self.identity) {
|
||||
root.unknown = true;
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn witness(
|
||||
&self,
|
||||
bucket: &str,
|
||||
disk_id: uuid::Uuid,
|
||||
backend: Arc<dyn super::local::LocalIoBackend>,
|
||||
) -> Option<NativeBucketCreationWitness> {
|
||||
if disk_id.is_nil()
|
||||
|| !matches!(Path::new(bucket).components().next(), Some(Component::Normal(_)))
|
||||
|| Path::new(bucket).components().count() != 1
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let witness = NativeBucketCreationWitness {
|
||||
registration: self.clone(),
|
||||
backend,
|
||||
bucket: bucket.to_owned(),
|
||||
disk_id,
|
||||
};
|
||||
validate_native_creation_witnesses(std::slice::from_ref(&witness)).then_some(witness)
|
||||
}
|
||||
}
|
||||
|
||||
impl NativeBucketCreationWitness {
|
||||
pub(crate) fn matches(&self, other: &Self) -> bool {
|
||||
self.registration.identity == other.registration.identity
|
||||
&& Arc::ptr_eq(&self.backend, &other.backend)
|
||||
&& self.disk_id == other.disk_id
|
||||
&& self.bucket == other.bucket
|
||||
}
|
||||
|
||||
fn physical_root_is_current(&self) -> bool {
|
||||
let root = native_root_identity(&self.registration.root);
|
||||
let io_root = native_root_identity(&self.registration.io_root);
|
||||
if root == Some(self.registration.identity) && io_root == Some(self.registration.identity) {
|
||||
return true;
|
||||
}
|
||||
let mut history = NATIVE_PUBLICATION_HISTORY.lock();
|
||||
history.roots.entry(self.registration.identity).or_default().unknown = true;
|
||||
for identity in [root, io_root].into_iter().flatten() {
|
||||
history.roots.entry(identity).or_default().unknown = true;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn valid_in(&self, history: &NativePublicationHistory) -> bool {
|
||||
history
|
||||
.roots
|
||||
.get(&self.registration.identity)
|
||||
.is_some_and(|root| !root.unknown && !root.started.contains(&self.bucket))
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn validate_native_creation_witnesses(witnesses: &[NativeBucketCreationWitness]) -> bool {
|
||||
if witnesses.is_empty() || !witnesses.iter().all(NativeBucketCreationWitness::physical_root_is_current) {
|
||||
return false;
|
||||
}
|
||||
let history = NATIVE_PUBLICATION_HISTORY.lock();
|
||||
witnesses.iter().all(|witness| witness.valid_in(&history))
|
||||
}
|
||||
|
||||
pub(crate) fn consume_native_creation_witnesses(witnesses: &[NativeBucketCreationWitness]) -> bool {
|
||||
if witnesses.is_empty() || !witnesses.iter().all(NativeBucketCreationWitness::physical_root_is_current) {
|
||||
return false;
|
||||
}
|
||||
let mut history = NATIVE_PUBLICATION_HISTORY.lock();
|
||||
if !witnesses.iter().all(|witness| witness.valid_in(&history)) {
|
||||
return false;
|
||||
}
|
||||
for witness in witnesses {
|
||||
if let Some(root) = history.roots.get_mut(&witness.registration.identity) {
|
||||
root.started.insert(witness.bucket.clone());
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
type NamespaceMutationLock = AsyncMutex<()>;
|
||||
type NamespaceMutationLockRegistry = HashMap<PathBuf, Weak<NamespaceMutationLock>>;
|
||||
static DISK_NAMESPACE_MUTATION_LOCKS: LazyLock<Mutex<NamespaceMutationLockRegistry>> =
|
||||
@@ -2311,7 +2083,6 @@ pub(in crate::disk) async fn rename_with_namespace_owner(
|
||||
dst: &Path,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> io::Result<()> {
|
||||
record_native_publication(dst);
|
||||
if namespace_owner.is_none() {
|
||||
return tokio::fs::rename(src, dst).await;
|
||||
}
|
||||
@@ -2333,19 +2104,12 @@ pub(in crate::disk) async fn create_dir_all_with_namespace_owner(
|
||||
path: &Path,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> io::Result<()> {
|
||||
record_native_publication(path);
|
||||
if namespace_owner.is_none() {
|
||||
return tokio::fs::create_dir_all(path).await;
|
||||
}
|
||||
let path = path.to_path_buf();
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(&path, namespace_owner).await;
|
||||
run_blocking_namespace_operation(lease, move || {
|
||||
#[cfg(all(test, not(windows)))]
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::CreateDirectory, &path);
|
||||
record_native_publication(&path);
|
||||
std::fs::create_dir_all(path)
|
||||
})
|
||||
.await
|
||||
run_blocking_namespace_operation(lease, move || std::fs::create_dir_all(path)).await
|
||||
}
|
||||
|
||||
#[tracing::instrument(name = "rename_all", level = "debug", skip_all)]
|
||||
@@ -2356,7 +2120,6 @@ pub(crate) async fn rename_all_with_owner(
|
||||
publication_root: &PublicationRoot,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
record_native_publication(dst_file_path.as_ref());
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(dst_file_path.as_ref(), namespace_owner).await;
|
||||
rename_all_with_lease(src_file_path, dst_file_path, base_dir, publication_root, lease).await
|
||||
}
|
||||
@@ -2368,7 +2131,6 @@ pub(crate) async fn rename_all_with_lease(
|
||||
publication_root: &PublicationRoot,
|
||||
lease: Arc<NamespaceMutationLease>,
|
||||
) -> Result<()> {
|
||||
record_native_publication(dst_file_path.as_ref());
|
||||
reliable_rename_inner_with_lease(
|
||||
src_file_path.as_ref().to_path_buf(),
|
||||
dst_file_path.as_ref().to_path_buf(),
|
||||
@@ -2393,7 +2155,6 @@ pub(crate) async fn rename_reconciled_metadata(
|
||||
owner: Option<Arc<dyn Send + Sync>>,
|
||||
authority: Arc<crate::bucket::lifecycle::legacy_transition_state_reconcile::TransitionStateReconcileAuthority>,
|
||||
) -> Result<()> {
|
||||
record_native_publication(&destination);
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(&destination, owner).await;
|
||||
run_blocking_namespace_operation(lease, move || {
|
||||
let preparation = prepare_rename_with_retry(&source, &destination, &base_dir, &publication_root)?;
|
||||
@@ -2638,7 +2399,6 @@ pub(crate) async fn rename_all_ignore_missing_source_with_owner(
|
||||
publication_root: &PublicationRoot,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
record_native_publication(dst_file_path.as_ref());
|
||||
let src_file_path = src_file_path.as_ref();
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(dst_file_path.as_ref(), namespace_owner).await;
|
||||
match reliable_rename_inner_with_lease(
|
||||
@@ -2693,7 +2453,6 @@ async fn reliable_rename_inner(
|
||||
publication_root: &PublicationRoot,
|
||||
warn_on_missing_source: bool,
|
||||
) -> io::Result<()> {
|
||||
record_native_publication(dst_file_path.as_ref());
|
||||
let src_file_path = src_file_path.as_ref().to_path_buf();
|
||||
let dst_file_path = dst_file_path.as_ref().to_path_buf();
|
||||
let base_dir = base_dir.as_ref().to_path_buf();
|
||||
@@ -2717,7 +2476,6 @@ async fn reliable_rename_inner_with_lease(
|
||||
warn_on_missing_source: bool,
|
||||
lease: Arc<NamespaceMutationLease>,
|
||||
) -> io::Result<()> {
|
||||
record_native_publication(&dst_file_path);
|
||||
let operation = {
|
||||
let src_file_path = src_file_path.clone();
|
||||
let dst_file_path = dst_file_path.clone();
|
||||
@@ -3120,7 +2878,6 @@ fn rename_into_existing_parent(
|
||||
dst_file_path: &Path,
|
||||
parent_guard: Option<&ExistingBaseDirectoryGuard>,
|
||||
) -> io::Result<()> {
|
||||
record_native_publication(dst_file_path);
|
||||
use rustix::fs::{Mode, OFlags, open, renameat};
|
||||
|
||||
let Some(parent_guard) = parent_guard else {
|
||||
@@ -3169,7 +2926,6 @@ fn rename_into_existing_parent(
|
||||
parent_guard: Option<&ExistingBaseDirectoryGuard>,
|
||||
source: &winapi_util::Handle,
|
||||
) -> io::Result<()> {
|
||||
record_native_publication(dst_file_path);
|
||||
use std::{
|
||||
mem::size_of,
|
||||
os::windows::{ffi::OsStrExt, io::AsRawHandle},
|
||||
@@ -4458,7 +4214,6 @@ pub(crate) fn mkdir_all_below_existing_base_std(
|
||||
base_dir: &Path,
|
||||
publication_root: &PublicationRoot,
|
||||
) -> io::Result<ExistingBaseDirectoryGuard> {
|
||||
record_native_publication(dir_path);
|
||||
let relative = dir_path
|
||||
.strip_prefix(base_dir)
|
||||
.map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "rename destination must remain below its base directory"))?;
|
||||
@@ -5963,77 +5718,6 @@ mod tests {
|
||||
assert_eq!(err.raw_os_error(), Some(disk_full));
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
fn windows_native_root_identity_matches_directory_handles() {
|
||||
let temp = tempdir().expect("create native identity fixture");
|
||||
let first = temp.path().join("first");
|
||||
let second = temp.path().join("second");
|
||||
std::fs::create_dir(&first).expect("create first root");
|
||||
std::fs::create_dir(&second).expect("create second root");
|
||||
let (_, first_anchor) = open_windows_publication_root(&first).expect("open first directory anchor");
|
||||
let (_, second_anchor) = open_windows_publication_root(&second).expect("open second directory anchor");
|
||||
let first_identity = windows_file_identity(first_anchor.handle.as_ref()).expect("query full first identity");
|
||||
let second_identity = windows_file_identity(second_anchor.handle.as_ref()).expect("query full second identity");
|
||||
|
||||
assert_eq!(native_root_identity(&first), Some(first_identity));
|
||||
assert_eq!(native_root_identity(&first.join(".")), Some(first_identity));
|
||||
assert_eq!(native_root_identity(&second), Some(second_identity));
|
||||
assert_ne!(first_identity, second_identity, "distinct physical roots must have distinct identities");
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[tokio::test]
|
||||
async fn windows_native_creation_witness_retains_started_history() {
|
||||
let temp = tempdir().expect("create native witness fixture");
|
||||
let root = std::fs::canonicalize(temp.path()).expect("canonical native root");
|
||||
let _publication_root = PublicationRoot::new(&root).expect("retain native directory anchor");
|
||||
let registration = register_native_publication_root(&root, &root.join(".")).expect("register same physical root");
|
||||
let backend: Arc<dyn super::super::local::LocalIoBackend> = Arc::new(super::super::local::StdBackend::new(root.clone()));
|
||||
let disk_id = uuid::Uuid::new_v4();
|
||||
let witness = registration
|
||||
.witness("published", disk_id, Arc::clone(&backend))
|
||||
.expect("fresh native witness");
|
||||
let bucket = root.join("published");
|
||||
|
||||
super::super::fs::mkdir(&bucket)
|
||||
.await
|
||||
.expect("publish through the actual native entry");
|
||||
assert!(bucket.is_dir(), "the production entry must create the directory");
|
||||
assert!(!validate_native_creation_witnesses(std::slice::from_ref(&witness)));
|
||||
std::fs::remove_dir(&bucket).expect("remove published directory");
|
||||
assert!(registration.witness("published", disk_id, Arc::clone(&backend)).is_none());
|
||||
let reconnected = register_native_publication_root(&root, &root).expect("reconnect same physical root");
|
||||
assert!(reconnected.witness("published", disk_id, Arc::clone(&backend)).is_none());
|
||||
|
||||
let unstarted = reconnected
|
||||
.witness("unstarted", disk_id, Arc::clone(&backend))
|
||||
.expect("untouched bucket keeps its witness");
|
||||
assert!(consume_native_creation_witnesses(std::slice::from_ref(&unstarted)));
|
||||
assert!(!validate_native_creation_witnesses(std::slice::from_ref(&unstarted)));
|
||||
assert!(reconnected.witness("unstarted", disk_id, backend).is_none());
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
fn windows_native_creation_witness_rejects_unknown_roots() {
|
||||
let temp = tempdir().expect("create unknown identity fixture");
|
||||
let root = temp.path().join("root");
|
||||
let other = temp.path().join("other");
|
||||
let file = temp.path().join("file");
|
||||
std::fs::create_dir(&root).expect("create native root");
|
||||
std::fs::create_dir(&other).expect("create distinct native root");
|
||||
std::fs::write(&file, b"not a directory").expect("create regular file");
|
||||
|
||||
assert!(native_root_identity(&temp.path().join("missing")).is_none());
|
||||
assert!(native_root_identity(&file).is_none());
|
||||
assert!(register_native_publication_root(&root, &other).is_none());
|
||||
let registration = register_native_publication_root(&root, &root).expect("register actual native root");
|
||||
let backend: Arc<dyn super::super::local::LocalIoBackend> = Arc::new(super::super::local::StdBackend::new(root));
|
||||
registration.invalidate();
|
||||
assert!(registration.witness("unstarted", uuid::Uuid::new_v4(), backend).is_none());
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
fn windows_rename_all_legacy_file_identity_distinguishes_sources() {
|
||||
@@ -8053,272 +7737,3 @@ mod tests {
|
||||
assert_eq!(err.kind(), io::ErrorKind::NotFound);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, unix))]
|
||||
mod native_creation_history_tests {
|
||||
use super::*;
|
||||
|
||||
struct NativeFixture {
|
||||
_temp: tempfile::TempDir,
|
||||
root: PathBuf,
|
||||
registration: NativePublicationRegistration,
|
||||
backend: Arc<dyn super::super::local::LocalIoBackend>,
|
||||
disk_id: uuid::Uuid,
|
||||
}
|
||||
|
||||
impl NativeFixture {
|
||||
fn new() -> Self {
|
||||
let temp = tempfile::tempdir().expect("native root");
|
||||
let root = std::fs::canonicalize(temp.path()).expect("canonical root");
|
||||
let registration = register_native_publication_root(&root, &root).expect("native registration");
|
||||
let backend = Arc::new(super::super::local::StdBackend::new(root.clone()));
|
||||
Self {
|
||||
_temp: temp,
|
||||
root,
|
||||
registration,
|
||||
backend,
|
||||
disk_id: uuid::Uuid::new_v4(),
|
||||
}
|
||||
}
|
||||
|
||||
fn witness(&self, bucket: &str) -> Option<NativeBucketCreationWitness> {
|
||||
self.registration.witness(bucket, self.disk_id, Arc::clone(&self.backend))
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn every_native_directory_publication_entry_invalidates_retry() {
|
||||
let fixture = NativeFixture::new();
|
||||
for operation in [
|
||||
"recursive",
|
||||
"single",
|
||||
"open",
|
||||
"rename",
|
||||
"rename_std",
|
||||
"fallback",
|
||||
"owned_mkdir",
|
||||
"owned_rename",
|
||||
"handle_relative",
|
||||
] {
|
||||
let bucket = format!("native-{operation}-{}", uuid::Uuid::new_v4());
|
||||
let path = fixture.root.join(&bucket);
|
||||
let witness = fixture.witness(&bucket).expect("fresh native witness");
|
||||
match operation {
|
||||
"recursive" => super::super::fs::make_dir_all(path.join("object/child"))
|
||||
.await
|
||||
.expect("recursive mkdir"),
|
||||
"single" => super::super::fs::mkdir(&path).await.expect("single mkdir"),
|
||||
"open" => {
|
||||
let file = super::super::fs::open_file(&path, super::super::fs::O_CREATE | super::super::fs::O_WRONLY)
|
||||
.await
|
||||
.expect("create file");
|
||||
drop(file);
|
||||
}
|
||||
"rename" | "rename_std" | "owned_rename" => {
|
||||
let source = fixture.root.join(format!("source-{}", uuid::Uuid::new_v4()));
|
||||
std::fs::create_dir(&source).expect("rename source");
|
||||
match operation {
|
||||
"rename" => super::super::fs::rename(&source, &path).await.expect("async rename"),
|
||||
"rename_std" => super::super::fs::rename_std(&source, &path).expect("sync rename"),
|
||||
_ => rename_with_namespace_owner(&source, &path, Some(Arc::new(())))
|
||||
.await
|
||||
.expect("owned rename"),
|
||||
}
|
||||
}
|
||||
"fallback" => make_dir_all(path.join("object/child"), &path)
|
||||
.await
|
||||
.expect("implicit volume fallback"),
|
||||
"owned_mkdir" => create_dir_all_with_namespace_owner(&path.join("child"), Some(Arc::new(())))
|
||||
.await
|
||||
.expect("owned recursive mkdir"),
|
||||
"handle_relative" => {
|
||||
let publication_root = PublicationRoot::new(&fixture.root).expect("publication root");
|
||||
let guard = mkdir_all_below_existing_base_std(&path.join("child"), &fixture.root, &publication_root)
|
||||
.expect("first-level handle-relative mkdir");
|
||||
drop(guard);
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
assert!(
|
||||
!validate_native_creation_witnesses(std::slice::from_ref(&witness)),
|
||||
"{operation}: old proof remains live"
|
||||
);
|
||||
if path.is_dir() {
|
||||
std::fs::remove_dir_all(&path).expect("remove published directory");
|
||||
} else {
|
||||
std::fs::remove_file(&path).expect("remove published file");
|
||||
}
|
||||
assert!(fixture.witness(&bucket).is_none(), "{operation}: removal must not regrant proof");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelled_namespace_waiter_cannot_restore_native_proof() {
|
||||
let fixture = NativeFixture::new();
|
||||
let bucket = format!("native-cancel-{}", uuid::Uuid::new_v4());
|
||||
let path = fixture.root.join(&bucket);
|
||||
let witness = fixture.witness(&bucket).expect("fresh witness");
|
||||
let lease = acquire_namespace_mutation_lease(&path).await;
|
||||
let mut attempt = Box::pin(create_dir_all_with_namespace_owner(&path, Some(Arc::new(()))));
|
||||
assert!(matches!(futures::poll!(&mut attempt), std::task::Poll::Pending));
|
||||
assert!(
|
||||
!validate_native_creation_witnesses(std::slice::from_ref(&witness)),
|
||||
"Started must precede admission wait"
|
||||
);
|
||||
assert!(!path.exists(), "the namespace wait must still precede the syscall");
|
||||
drop(attempt);
|
||||
drop(lease);
|
||||
assert!(fixture.witness(&bucket).is_none(), "cancellation cannot undo an admitted attempt");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn queued_mkdir_cancellation_retains_lease_and_consumed_history() {
|
||||
use futures::FutureExt;
|
||||
use std::sync::mpsc;
|
||||
|
||||
let fixture = NativeFixture::new();
|
||||
let bucket = format!("native-queued-cancel-{}", uuid::Uuid::new_v4());
|
||||
let path = fixture.root.join(&bucket);
|
||||
let witness = fixture.witness(&bucket).expect("fresh witness");
|
||||
let (entered_tx, entered_rx) = mpsc::channel();
|
||||
let (release_tx, release_rx) = mpsc::channel();
|
||||
let _hook = prepared_publication_test_hooks::install_at(
|
||||
prepared_publication_test_hooks::Stage::CreateDirectory,
|
||||
&path,
|
||||
move || {
|
||||
entered_tx.send(()).expect("signal real executor entry");
|
||||
let _ = release_rx.recv();
|
||||
},
|
||||
);
|
||||
let operation_path = path.clone();
|
||||
let operation =
|
||||
tokio::spawn(async move { create_dir_all_with_namespace_owner(&operation_path, Some(Arc::new(()))).await });
|
||||
tokio::task::spawn_blocking(move || entered_rx.recv_timeout(std::time::Duration::from_secs(30)))
|
||||
.await
|
||||
.expect("entry waiter")
|
||||
.expect("the real blocking executor must enter");
|
||||
operation.abort();
|
||||
assert!(operation.await.expect_err("cancel the async waiter").is_cancelled());
|
||||
assert!(
|
||||
!validate_native_creation_witnesses(std::slice::from_ref(&witness)),
|
||||
"queued publication is permanently Started"
|
||||
);
|
||||
assert!(
|
||||
acquire_namespace_mutation_lease(&path).now_or_never().is_none(),
|
||||
"the actual blocked executor must still own the namespace lease"
|
||||
);
|
||||
release_tx.send(()).expect("release the syscall");
|
||||
let drained = tokio::time::timeout(std::time::Duration::from_secs(30), acquire_namespace_mutation_lease(&path))
|
||||
.await
|
||||
.expect("the lease must drain with the syscall");
|
||||
drop(drained);
|
||||
assert!(path.is_dir(), "the cancelled waiter cannot discard the already queued syscall");
|
||||
std::fs::remove_dir_all(&path).expect("remove the published directory");
|
||||
assert!(fixture.witness(&bucket).is_none(), "drain and removal cannot restore proof");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn publication_detects_retargeted_physical_alias_without_registration() {
|
||||
let source = NativeFixture::new();
|
||||
let destination = NativeFixture::new();
|
||||
let bucket = format!("native-retarget-{}", uuid::Uuid::new_v4());
|
||||
let witness = destination.witness(&bucket).expect("destination witness");
|
||||
let retired = source.root.with_extension("retired");
|
||||
std::fs::rename(&source.root, &retired).expect("move the original physical root");
|
||||
std::os::unix::fs::symlink(&destination.root, &source.root).expect("retarget the registered alias");
|
||||
super::super::fs::mkdir(source.root.join(&bucket))
|
||||
.await
|
||||
.expect("publish through the changed alias");
|
||||
assert!(
|
||||
!validate_native_creation_witnesses(std::slice::from_ref(&witness)),
|
||||
"publication-time alias drift must invalidate the actual destination root"
|
||||
);
|
||||
std::fs::remove_dir(destination.root.join(&bucket)).expect("remove the published bucket");
|
||||
assert!(destination.witness(&bucket).is_none(), "removal must not undo alias uncertainty");
|
||||
std::fs::remove_file(&source.root).expect("remove the symlink");
|
||||
std::fs::rename(retired, &source.root).expect("restore the test root");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blocked_physical_identity_lookup_does_not_block_other_publication() {
|
||||
use std::sync::mpsc;
|
||||
let stalled = NativeFixture::new();
|
||||
let healthy = NativeFixture::new();
|
||||
let witness = stalled.witness("unpublished").expect("stalled root witness");
|
||||
let (entered_tx, entered_rx) = mpsc::channel();
|
||||
let (release_tx, release_rx) = mpsc::channel();
|
||||
let _hook = prepared_publication_test_hooks::install_at(
|
||||
prepared_publication_test_hooks::Stage::NativeRootIdentity,
|
||||
&stalled.root,
|
||||
move || {
|
||||
entered_tx.send(()).expect("signal physical lookup");
|
||||
let _ = release_rx.recv();
|
||||
},
|
||||
);
|
||||
let validator = std::thread::spawn(move || validate_native_creation_witnesses(&[witness]));
|
||||
entered_rx
|
||||
.recv_timeout(std::time::Duration::from_secs(30))
|
||||
.expect("enter the actual identity lookup");
|
||||
let (published_tx, published_rx) = mpsc::channel();
|
||||
let path = healthy.root.join("published");
|
||||
let publisher = std::thread::spawn(move || {
|
||||
record_native_publication(&path);
|
||||
published_tx.send(()).expect("signal independent publication");
|
||||
});
|
||||
let published = published_rx.recv_timeout(std::time::Duration::from_secs(5));
|
||||
release_tx.send(()).expect("release the identity lookup");
|
||||
validator.join().expect("identity lookup thread");
|
||||
publisher.join().expect("publication thread");
|
||||
assert!(
|
||||
published.is_ok(),
|
||||
"a slow physical lookup must not retain the process-wide publication mutex"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_consume_is_atomic_and_history_survives_reconnect() {
|
||||
let fixture = NativeFixture::new();
|
||||
let first = format!("native-first-{}", uuid::Uuid::new_v4());
|
||||
let second = format!("native-second-{}", uuid::Uuid::new_v4());
|
||||
let first_witness = fixture.witness(&first).expect("first witness");
|
||||
let second_witness = fixture.witness(&second).expect("second witness");
|
||||
record_native_publication(&fixture.root.join(&second));
|
||||
assert!(!consume_native_creation_witnesses(&[first_witness.clone(), second_witness]));
|
||||
assert!(
|
||||
validate_native_creation_witnesses(std::slice::from_ref(&first_witness)),
|
||||
"failed cohort consume must be atomic"
|
||||
);
|
||||
assert!(consume_native_creation_witnesses(std::slice::from_ref(&first_witness)));
|
||||
assert!(!validate_native_creation_witnesses(std::slice::from_ref(&first_witness)));
|
||||
let reconnected = register_native_publication_root(&fixture.root, &fixture.root).expect("reconnected registration");
|
||||
let backend: Arc<dyn super::super::local::LocalIoBackend> =
|
||||
Arc::new(super::super::local::StdBackend::new(fixture.root.clone()));
|
||||
assert!(
|
||||
reconnected.witness(&first, fixture.disk_id, backend).is_none(),
|
||||
"reconnect must retain Started"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn physical_alias_reuse_is_unknown_and_backend_identity_is_real() {
|
||||
let temp = tempfile::tempdir().expect("alias parent");
|
||||
let root = temp.path().join("root");
|
||||
std::fs::create_dir(&root).expect("first physical root");
|
||||
let old = register_native_publication_root(&root, &root).expect("first registration");
|
||||
let backend: Arc<dyn super::super::local::LocalIoBackend> = Arc::new(super::super::local::StdBackend::new(root.clone()));
|
||||
let id = uuid::Uuid::new_v4();
|
||||
let witness = old.witness("same-name", id, Arc::clone(&backend)).expect("old witness");
|
||||
let other_backend: Arc<dyn super::super::local::LocalIoBackend> =
|
||||
Arc::new(super::super::local::StdBackend::new(root.clone()));
|
||||
let other = old.witness("same-name", id, other_backend).expect("new backend witness");
|
||||
assert!(!witness.matches(&other), "a different actual backend cannot reuse a pending observation");
|
||||
std::fs::rename(&root, temp.path().join("retired-root")).expect("retire physical root");
|
||||
std::fs::create_dir(&root).expect("replacement root at the same alias");
|
||||
let replacement = register_native_publication_root(&root, &root).expect("replacement registration");
|
||||
assert!(replacement.witness("same-name", id, backend).is_none(), "alias reuse must remain Unknown");
|
||||
assert!(
|
||||
!validate_native_creation_witnesses(&[witness]),
|
||||
"old physical identity must not validate at a new root"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,8 +17,8 @@
|
||||
use crate::bucket::error::BucketMetadataError;
|
||||
use crate::disk::error::DiskError;
|
||||
use crate::storage_api_contracts::{error::StorageErrorCode, range::HTTPRangeError};
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
use rustfs_utils::path::decode_dir_object;
|
||||
use s3s::S3ErrorCode;
|
||||
|
||||
pub type Error = StorageError;
|
||||
pub type Result<T> = core::result::Result<T, Error>;
|
||||
|
||||
@@ -442,12 +442,19 @@ impl WritePlan {
|
||||
}
|
||||
|
||||
if let Some(encryption) = self.encryption {
|
||||
let etag_nonce = match encryption.mode {
|
||||
WriteEncryptionMode::Singlepart { base_nonce } | WriteEncryptionMode::MultipartLegacy { base_nonce, .. } => {
|
||||
base_nonce
|
||||
}
|
||||
WriteEncryptionMode::SinglepartObjectKey | WriteEncryptionMode::MultipartObjectKey { .. } => [0u8; 12],
|
||||
};
|
||||
let protected_reader = rustfs_rio::EtagReader::new_encrypted(reader, encryption.key_bytes, etag_nonce)?;
|
||||
reader = match encryption.mode {
|
||||
WriteEncryptionMode::SinglepartObjectKey => HashReader::from_reader(
|
||||
#[cfg(feature = "rio-v2")]
|
||||
EncryptReader::new_with_object_key(reader, encryption.key_bytes),
|
||||
EncryptReader::new_with_object_key(protected_reader, encryption.key_bytes),
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
EncryptReader::new(reader, encryption.key_bytes, [0u8; 12]),
|
||||
EncryptReader::new(protected_reader, encryption.key_bytes, [0u8; 12]),
|
||||
HashReader::SIZE_PRESERVE_LAYER,
|
||||
actual_size,
|
||||
None,
|
||||
@@ -457,12 +464,12 @@ impl WritePlan {
|
||||
WriteEncryptionMode::Singlepart { base_nonce } => {
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
let encrypt_reader = if encryption_frame_v2_enabled() {
|
||||
EncryptReader::new_v2(reader, encryption.key_bytes, base_nonce)
|
||||
EncryptReader::new_v2(protected_reader, encryption.key_bytes, base_nonce)
|
||||
} else {
|
||||
EncryptReader::new(reader, encryption.key_bytes, base_nonce)
|
||||
EncryptReader::new(protected_reader, encryption.key_bytes, base_nonce)
|
||||
};
|
||||
#[cfg(feature = "rio-v2")]
|
||||
let encrypt_reader = EncryptReader::new(reader, encryption.key_bytes, base_nonce);
|
||||
let encrypt_reader = EncryptReader::new(protected_reader, encryption.key_bytes, base_nonce);
|
||||
HashReader::from_reader(encrypt_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false)?
|
||||
}
|
||||
WriteEncryptionMode::MultipartLegacy {
|
||||
@@ -471,20 +478,25 @@ impl WritePlan {
|
||||
} => {
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
let encrypt_reader = if encryption_frame_v2_enabled() {
|
||||
EncryptReader::new_multipart_v2(reader, encryption.key_bytes, base_nonce, multipart_part_number)
|
||||
EncryptReader::new_multipart_v2(protected_reader, encryption.key_bytes, base_nonce, multipart_part_number)
|
||||
} else {
|
||||
EncryptReader::new_multipart(reader, encryption.key_bytes, base_nonce, multipart_part_number)
|
||||
EncryptReader::new_multipart(protected_reader, encryption.key_bytes, base_nonce, multipart_part_number)
|
||||
};
|
||||
#[cfg(feature = "rio-v2")]
|
||||
let encrypt_reader =
|
||||
EncryptReader::new_multipart(reader, encryption.key_bytes, base_nonce, multipart_part_number);
|
||||
EncryptReader::new_multipart(protected_reader, encryption.key_bytes, base_nonce, multipart_part_number);
|
||||
HashReader::from_reader(encrypt_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false)?
|
||||
}
|
||||
WriteEncryptionMode::MultipartObjectKey { multipart_part_number } => HashReader::from_reader(
|
||||
#[cfg(feature = "rio-v2")]
|
||||
EncryptReader::new_multipart_with_object_key(reader, encryption.key_bytes, multipart_part_number),
|
||||
EncryptReader::new_multipart_with_object_key(protected_reader, encryption.key_bytes, multipart_part_number),
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
EncryptReader::new_multipart(reader, encryption.key_bytes, [0u8; 12], multipart_part_number as usize),
|
||||
EncryptReader::new_multipart(
|
||||
protected_reader,
|
||||
encryption.key_bytes,
|
||||
[0u8; 12],
|
||||
multipart_part_number as usize,
|
||||
),
|
||||
HashReader::SIZE_PRESERVE_LAYER,
|
||||
actual_size,
|
||||
None,
|
||||
@@ -497,6 +509,7 @@ impl WritePlan {
|
||||
// `ignore_value` deliberately avoids a second hasher over compressed or
|
||||
// encrypted bytes. The inner reader still validates the plaintext request
|
||||
// checksum while this outer reader exposes the request checksum context.
|
||||
reader.encrypted_etag = encrypted;
|
||||
reader.add_non_trailing_checksum(checksum, true)?;
|
||||
reader.set_trailer(trailer);
|
||||
|
||||
@@ -513,6 +526,36 @@ mod tests {
|
||||
use std::io::Cursor;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
#[tokio::test]
|
||||
async fn write_plan_encryption_persists_protected_etags_for_every_mode() {
|
||||
let data = b"Salary 2026: 85000 EUR";
|
||||
let size = i64::try_from(data.len()).expect("test data size");
|
||||
let mut control =
|
||||
HashReader::from_stream(Cursor::new(data), size, size, None, None, false).expect("create control reader");
|
||||
control.read_to_end(&mut Vec::new()).await.expect("read control plaintext");
|
||||
let md5 = control.try_resolve_etag().expect("control MD5");
|
||||
for encryption in [
|
||||
WriteEncryption::singlepart([1; 32], [1; 12]),
|
||||
WriteEncryption::singlepart_object_key([1; 32]),
|
||||
WriteEncryption::multipart([1; 32], [1; 12], 1),
|
||||
WriteEncryption::multipart_object_key([1; 32], 1),
|
||||
] {
|
||||
let plaintext = HashReader::from_stream(Cursor::new(data), size, size, Some(md5.clone()), None, false)
|
||||
.expect("create checked reader");
|
||||
let mut reader = WritePlan::new()
|
||||
.with_encryption(encryption)
|
||||
.apply(plaintext, size)
|
||||
.expect("apply encryption");
|
||||
let mut ciphertext = Vec::new();
|
||||
reader.read_to_end(&mut ciphertext).await.expect("stream ciphertext");
|
||||
assert_ne!(ciphertext, data);
|
||||
let tag = reader.try_resolve_etag().expect("resolve persisted encrypted ETag");
|
||||
assert_ne!(tag, md5, "stored ETag must not reveal the plaintext MD5");
|
||||
assert_eq!(tag.len(), 32, "multipart ETags require a 16-byte hexadecimal part tag");
|
||||
assert_eq!(reader.try_resolve_etag().as_ref(), Some(&tag));
|
||||
}
|
||||
}
|
||||
|
||||
async fn assert_non_trailing_checksum_survives(plan: WritePlan) {
|
||||
let plaintext = b"checksum-context-through-write-plan".repeat(256);
|
||||
let actual_size = plaintext.len() as i64;
|
||||
|
||||
@@ -29,7 +29,7 @@ use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions};
|
||||
use crate::storage_api_contracts::range::HTTPRangeSpec;
|
||||
use rustfs_filemeta::ObjectPartInfo;
|
||||
use rustfs_rio::HashReader;
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
use s3s::S3ErrorCode;
|
||||
|
||||
//#[derive(Clone)]
|
||||
pub struct PutObjReader {
|
||||
@@ -231,45 +231,6 @@ mod tests {
|
||||
assert_eq!(result.2, 1024 * 1024);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_new_getobjectreader_rejects_a_range_past_the_object_as_invalid_range() {
|
||||
let oi = multipart_object_info();
|
||||
let past_the_end = Some(HTTPRangeSpec {
|
||||
is_suffix_length: false,
|
||||
start: oi.size,
|
||||
end: oi.size + 10,
|
||||
});
|
||||
|
||||
let err = new_getobjectreader(&past_the_end, &oi, &ObjectOptions::default(), &HeaderMap::new())
|
||||
.err()
|
||||
.expect("a range starting at the object size must be rejected");
|
||||
|
||||
assert_eq!(err.code, S3ErrorCode::InvalidRange);
|
||||
assert_eq!(err.code.as_str(), "InvalidRange");
|
||||
assert_eq!(err.key, None);
|
||||
assert_eq!(err.bucket_name, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_new_getobjectreader_rejects_an_unknown_size_without_a_range_as_invalid_range() {
|
||||
let oi = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
size: -1,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let err = new_getobjectreader(&None, &oi, &ObjectOptions::default(), &HeaderMap::new())
|
||||
.err()
|
||||
.expect("an object of unknown size cannot be read without a range");
|
||||
|
||||
assert_eq!(err.code, S3ErrorCode::InvalidRange);
|
||||
assert_eq!(err.code.as_str(), "InvalidRange");
|
||||
assert_eq!(err.message, "Invalid range");
|
||||
assert_eq!(err.key.as_deref(), Some("object"));
|
||||
assert_eq!(err.bucket_name.as_deref(), Some("bucket"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_to_s3s_etag() {
|
||||
// Test unquoted ETag - should become strong etag
|
||||
|
||||
@@ -68,7 +68,7 @@ impl WriteCommitGuard {
|
||||
self.guards.iter().any(|guard| guard.is_lock_lost() || guard.is_released())
|
||||
}
|
||||
|
||||
fn matches(&self, bucket: &str, object: &str) -> bool {
|
||||
pub(crate) fn matches(&self, bucket: &str, object: &str) -> bool {
|
||||
self.bucket.as_ref() == bucket && self.object.as_ref() == object
|
||||
}
|
||||
}
|
||||
|
||||
@@ -82,9 +82,8 @@ use crate::{
|
||||
use rustfs_filemeta::FileInfo;
|
||||
use rustfs_rio::HashReader;
|
||||
use rustfs_s3_client::{admin_handler_utils::AdminError, provider_versions::ProviderVersionCapabilities};
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
use rustfs_utils::path::{SLASH_SEPARATOR, path_join};
|
||||
use s3s::dto::BucketLifecycleConfiguration;
|
||||
use s3s::{S3ErrorCode, dto::BucketLifecycleConfiguration};
|
||||
|
||||
use super::{
|
||||
tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_PERM_ERR},
|
||||
|
||||
@@ -878,6 +878,7 @@ pub(crate) use ops::heal::AbsenceProofRequest;
|
||||
#[cfg(test)]
|
||||
pub(crate) use ops::heal::DanglingDeleteFailure;
|
||||
pub(crate) use ops::heal::HealedObjectAbsence;
|
||||
pub(crate) use ops::native_prepare::same_identity;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) use ops::hermetic_set_disks_isolated;
|
||||
@@ -1095,6 +1096,311 @@ mod prepared_get_object_metadata_tests {
|
||||
use http::HeaderMap;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
const OBSERVATION_HOOK_BODY_LEN: usize = 1024 * 1024 + 1;
|
||||
|
||||
struct ObservationBodyHook {
|
||||
object: &'static str,
|
||||
calls: Arc<AtomicU64>,
|
||||
body: bytes::Bytes,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::object_api::GetObjectBodyCacheHook for ObservationBodyHook {
|
||||
async fn lookup(&self, bucket: &str, object: &str, info: &ObjectInfo) -> Option<bytes::Bytes> {
|
||||
if bucket == crate::disk::RUSTFS_META_BUCKET && object == self.object {
|
||||
assert_eq!(info.size, i64::try_from(self.body.len()).expect("hook fixture size"));
|
||||
self.calls.fetch_add(1, Ordering::SeqCst);
|
||||
Some(self.body.clone())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct ObservationHookGuard;
|
||||
|
||||
impl Drop for ObservationHookGuard {
|
||||
fn drop(&mut self) {
|
||||
crate::object_api::unregister_get_object_body_cache_hook();
|
||||
}
|
||||
}
|
||||
|
||||
async fn ordinary_observation_get(set: &SetDisks, object: &str) -> Vec<u8> {
|
||||
let mut reader = set
|
||||
.get_object_reader(crate::disk::RUSTFS_META_BUCKET, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("ordinary GET positive control");
|
||||
let mut bytes = Vec::new();
|
||||
reader.stream.read_to_end(&mut bytes).await.expect("ordinary GET body");
|
||||
bytes
|
||||
}
|
||||
|
||||
fn observation_register_hook(object: &'static str, body: Vec<u8>) -> (ObservationHookGuard, Arc<AtomicU64>) {
|
||||
let calls = Arc::new(AtomicU64::new(0));
|
||||
crate::object_api::register_get_object_body_cache_hook(Arc::new(ObservationBodyHook {
|
||||
object,
|
||||
calls: calls.clone(),
|
||||
body: bytes::Bytes::from(body),
|
||||
}));
|
||||
(ObservationHookGuard, calls)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn locked_observation_uses_real_outer_guard() {
|
||||
let object = "config/locked-observation-real.bin";
|
||||
let payload = vec![b'p'; OBSERVATION_HOOK_BODY_LEN];
|
||||
let (_dirs, set) = make_local_set_disks(4, 2).await;
|
||||
let mut reader = PutObjReader::from_vec(payload.clone());
|
||||
let written = set
|
||||
.put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("persist locked observation fixture");
|
||||
let (_hook, calls) = observation_register_hook(object, vec![b'x'; payload.len()]);
|
||||
assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; payload.len()]);
|
||||
assert_eq!(calls.swap(0, Ordering::SeqCst), 1);
|
||||
let namespace = set
|
||||
.new_ns_lock(crate::disk::RUSTFS_META_BUCKET, object)
|
||||
.await
|
||||
.expect("namespace");
|
||||
let guard = crate::object_api::WriteCommitGuard::acquire(&namespace, std::time::Duration::from_secs(5))
|
||||
.await
|
||||
.expect("matching real owner");
|
||||
let mut contender = Box::pin(crate::object_api::WriteCommitGuard::acquire(
|
||||
&namespace,
|
||||
std::time::Duration::from_secs(30),
|
||||
));
|
||||
assert_eq!(futures::poll!(contender.as_mut()).is_pending(), true, "I06L_OWNER_RETAINED");
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(10),
|
||||
crate::config::com::read_config_observation_no_lock(set.clone(), object, payload.len(), &guard),
|
||||
)
|
||||
.await
|
||||
.expect("locked observation hang watchdog");
|
||||
assert_eq!(result.is_ok(), true, "I06L_NO_NESTED_LOCK: {result:?}");
|
||||
let (bytes, info) = result.expect("locked observation");
|
||||
assert_eq!(bytes == payload, true, "I06L_HOOK_BYPASS");
|
||||
assert_eq!(calls.load(Ordering::SeqCst) == 0, true, "I06L_HOOK_BYPASS");
|
||||
assert_eq!(info.size, written.size);
|
||||
assert_eq!(info.etag, written.etag);
|
||||
assert_eq!(info.data_dir, written.data_dir);
|
||||
assert_eq!(info.name, object);
|
||||
assert_eq!(futures::poll!(contender.as_mut()).is_pending(), true, "I06L_OWNER_RETAINED");
|
||||
drop(guard);
|
||||
let next = tokio::time::timeout(std::time::Duration::from_secs(10), contender)
|
||||
.await
|
||||
.expect("owner release watchdog")
|
||||
.expect("competing owner acquired");
|
||||
drop(next);
|
||||
assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; payload.len()]);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn locked_observation_preserves_empty_missing_and_bound() {
|
||||
let (_dirs, set) = make_local_set_disks(4, 2).await;
|
||||
for (object, body) in [
|
||||
("config/locked-empty.bin", Vec::new()),
|
||||
("config/locked-bound.bin", b"raw".to_vec()),
|
||||
] {
|
||||
let mut reader = PutObjReader::from_vec(body.clone());
|
||||
let written = set
|
||||
.put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("persist boundary fixture");
|
||||
let namespace = set
|
||||
.new_ns_lock(crate::disk::RUSTFS_META_BUCKET, object)
|
||||
.await
|
||||
.expect("namespace");
|
||||
let guard = crate::object_api::WriteCommitGuard::acquire(&namespace, std::time::Duration::from_secs(5))
|
||||
.await
|
||||
.expect("matching boundary owner");
|
||||
let result = crate::config::com::read_config_observation_no_lock(set.clone(), object, body.len(), &guard).await;
|
||||
assert_eq!(result.is_ok(), true, "I06L_EMPTY_DISTINCT: {result:?}");
|
||||
let (bytes, info) = result.expect("boundary observation");
|
||||
assert_eq!(bytes, body);
|
||||
assert_eq!(info.size, written.size);
|
||||
if !body.is_empty() {
|
||||
let result =
|
||||
crate::config::com::read_config_observation_no_lock(set.clone(), object, body.len() - 1, &guard).await;
|
||||
assert_eq!(matches!(result, Err(Error::CorruptedFormat)), true, "I06L_BOUND_ENFORCED");
|
||||
}
|
||||
}
|
||||
let object = "config/locked-absent.bin";
|
||||
let namespace = set
|
||||
.new_ns_lock(crate::disk::RUSTFS_META_BUCKET, object)
|
||||
.await
|
||||
.expect("namespace");
|
||||
let guard = crate::object_api::WriteCommitGuard::acquire(&namespace, std::time::Duration::from_secs(5))
|
||||
.await
|
||||
.expect("matching missing owner");
|
||||
let result = crate::config::com::read_config_observation_no_lock(set, object, 0, &guard).await;
|
||||
assert_eq!(matches!(result, Err(Error::ConfigNotFound)), true, "I06L_EMPTY_DISTINCT");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn observation_real_backend_boundaries() {
|
||||
let object = "config/observation-real.bin";
|
||||
let payload = b"persisted config";
|
||||
let (_dirs, set) = make_local_set_disks(4, 2).await;
|
||||
let mut reader = PutObjReader::from_vec(payload.to_vec());
|
||||
let written = set
|
||||
.put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("persist real config fixture");
|
||||
let result = crate::config::com::read_config_observation(set.clone(), object, payload.len()).await;
|
||||
assert_eq!(result.is_ok(), true, "I05_OBSERVATION_SUCCESS: {result:?}");
|
||||
let (bytes, info) = result.expect("real observation success");
|
||||
assert_eq!(bytes, payload);
|
||||
assert_eq!(info.size, written.size);
|
||||
assert_eq!(info.etag, written.etag);
|
||||
assert_eq!(info.data_dir, written.data_dir);
|
||||
assert_eq!(info.name, object);
|
||||
let over = crate::config::com::read_config_observation(set.clone(), object, payload.len() - 1).await;
|
||||
assert_eq!(over.is_err(), true, "I05_BOUND_ENFORCED");
|
||||
assert_eq!(over.expect_err("over limit must fail"), Error::CorruptedFormat, "I05_BOUND_ENFORCED");
|
||||
let absent = crate::config::com::read_config_observation(set.clone(), "config/observation-absent.bin", 32).await;
|
||||
assert_eq!(absent.expect_err("absent config"), Error::ConfigNotFound);
|
||||
let empty_object = "config/observation-empty.bin";
|
||||
let mut empty = PutObjReader::from_vec(Vec::new());
|
||||
set.put_object(crate::disk::RUSTFS_META_BUCKET, empty_object, &mut empty, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("persist empty config");
|
||||
let empty = crate::config::com::read_config_observation(set, empty_object, 0).await;
|
||||
assert!(empty.is_ok(), "existing empty config must succeed: {empty:?}");
|
||||
let (bytes, info) = empty.expect("empty observation");
|
||||
assert!(bytes.is_empty());
|
||||
assert_eq!(info.size, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn observation_bypasses_registered_body_hook() {
|
||||
let object = "config/observation-hook.bin";
|
||||
let (_dirs, set) = make_local_set_disks(4, 2).await;
|
||||
let mut reader = PutObjReader::from_vec(vec![b'p'; OBSERVATION_HOOK_BODY_LEN]);
|
||||
set.put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("persist real config fixture");
|
||||
let (_hook, calls) = observation_register_hook(object, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
|
||||
assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
|
||||
assert_eq!(calls.swap(0, Ordering::SeqCst), 1);
|
||||
let result = crate::config::com::read_config_observation(set, object, OBSERVATION_HOOK_BODY_LEN).await;
|
||||
assert_eq!(result.is_ok(), true, "I05_HOOK_BYPASS: {result:?}");
|
||||
assert_eq!(
|
||||
result.expect("observed persisted body").0,
|
||||
vec![b'p'; OBSERVATION_HOOK_BODY_LEN],
|
||||
"I05_HOOK_BYPASS"
|
||||
);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0, "I05_HOOK_BYPASS");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn observation_hook_scope_restores_after_error() {
|
||||
let object = "config/observation-hook-error.bin";
|
||||
let (_dirs, set) = make_local_set_disks(4, 2).await;
|
||||
let mut reader = PutObjReader::from_vec(vec![b'p'; OBSERVATION_HOOK_BODY_LEN]);
|
||||
set.put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("persist real config fixture");
|
||||
let (_hook, calls) = observation_register_hook(object, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
|
||||
let result = crate::config::com::read_config_observation(set.clone(), object, OBSERVATION_HOOK_BODY_LEN - 1).await;
|
||||
assert_eq!(result.expect_err("bounded observation error"), Error::CorruptedFormat);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
struct PendingObservationStore {
|
||||
set: Arc<SetDisks>,
|
||||
ready: std::sync::Mutex<Option<tokio::sync::oneshot::Sender<()>>>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for PendingObservationStore {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("PendingObservationStore").finish()
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::storage_api_contracts::object::ObjectIO for PendingObservationStore {
|
||||
type Error = Error;
|
||||
type RangeSpec = HTTPRangeSpec;
|
||||
type HeaderMap = HeaderMap;
|
||||
type ObjectOptions = ObjectOptions;
|
||||
type ObjectInfo = ObjectInfo;
|
||||
type GetObjectReader = GetObjectReader;
|
||||
type PutObjectReader = PutObjReader;
|
||||
|
||||
async fn get_object_reader(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
range: Option<HTTPRangeSpec>,
|
||||
headers: HeaderMap,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<GetObjectReader> {
|
||||
let mut reader = self.set.get_object_reader(bucket, object, range, headers, opts).await?;
|
||||
let mut body = Vec::new();
|
||||
reader.stream.read_to_end(&mut body).await.expect("pending producer real GET");
|
||||
assert_eq!(
|
||||
body,
|
||||
vec![b'p'; OBSERVATION_HOOK_BODY_LEN],
|
||||
"I05_HOOK_BYPASS: producer must enter its own scope"
|
||||
);
|
||||
self.ready
|
||||
.lock()
|
||||
.expect("pending signal")
|
||||
.take()
|
||||
.expect("one GET")
|
||||
.send(())
|
||||
.expect("producer entry signal");
|
||||
std::future::pending::<Result<GetObjectReader>>().await
|
||||
}
|
||||
|
||||
async fn put_object(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_object: &str,
|
||||
_data: &mut PutObjReader,
|
||||
_opts: &ObjectOptions,
|
||||
) -> Result<ObjectInfo> {
|
||||
assert!(false, "observation must not PUT");
|
||||
Err(Error::other("unexpected observation PUT"))
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn observation_hook_scope_restores_after_cancel() {
|
||||
let object = "config/observation-hook-cancel.bin";
|
||||
let (_dirs, set) = make_local_set_disks(4, 2).await;
|
||||
let mut reader = PutObjReader::from_vec(vec![b'p'; OBSERVATION_HOOK_BODY_LEN]);
|
||||
set.put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("persist real config fixture");
|
||||
let (_hook, calls) = observation_register_hook(object, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
|
||||
assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
|
||||
assert_eq!(calls.swap(0, Ordering::SeqCst), 1);
|
||||
let (ready_tx, ready_rx) = tokio::sync::oneshot::channel();
|
||||
let adapter = Arc::new(PendingObservationStore {
|
||||
set: set.clone(),
|
||||
ready: std::sync::Mutex::new(Some(ready_tx)),
|
||||
});
|
||||
let mut scope = Box::pin(crate::config::com::read_config_observation(adapter, object, OBSERVATION_HOOK_BODY_LEN));
|
||||
tokio::select! {
|
||||
result = ready_rx => { result.expect("producer reached pending GET"); }
|
||||
result = &mut scope => { assert!(false, "producer must remain pending until cancelled: {result:?}"); }
|
||||
}
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
drop(scope);
|
||||
assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
const READ_VERSION_BARRIER_GUARD: std::time::Duration = std::time::Duration::from_secs(10);
|
||||
|
||||
fn object_with_initial_data_shards(bucket: &str, prefix: &str) -> String {
|
||||
|
||||
@@ -24,6 +24,7 @@ pub(crate) mod heal_walk;
|
||||
pub(crate) mod list;
|
||||
pub(crate) mod locking;
|
||||
pub(crate) mod multipart;
|
||||
pub(crate) mod native_prepare;
|
||||
pub(crate) mod object;
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -3004,6 +3004,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
// etag
|
||||
let etag = resolve_complete_etag(opts, &uploaded_parts);
|
||||
|
||||
// An upload can contain parts from an older writer or a peer. Until
|
||||
// every part's format is attested, keep its ETag out of keyless lists.
|
||||
rustfs_utils::http::metadata_compat::remove_str(&mut fi.metadata, rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG);
|
||||
|
||||
fi.metadata.insert("etag".to_owned(), etag);
|
||||
|
||||
let persist_encryption_original_size = should_persist_encryption_original_size(&fi.metadata);
|
||||
|
||||
@@ -0,0 +1,783 @@
|
||||
//! Encode native candidates privately; only LocalDisk can attest durable shards.
|
||||
use crate::{
|
||||
disk::{
|
||||
Disk,
|
||||
error::Error as DiskError,
|
||||
local::native_prepare::{NativePreparationProbe, NativePreparationReceipt, NativePrepareInput},
|
||||
},
|
||||
erasure::coding::{BitrotReader, Erasure, encode::IntegrityEncodeMode},
|
||||
error::{Error, Result},
|
||||
object_api::ObjectLockConfigSnapshot,
|
||||
set_disk::{SetDisks, resolve_write_layout},
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use rustfs_filemeta::{FileInfo, FileInfoOpts, FileMeta};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::{io::Cursor, sync::Arc};
|
||||
use tokio::sync::mpsc;
|
||||
use uuid::Uuid;
|
||||
|
||||
pub(crate) fn same_identity(a: &NativePreparationReceipt, b: &NativePreparationReceipt) -> bool {
|
||||
a.preparation_id() == b.preparation_id()
|
||||
&& a.version_id() == b.version_id()
|
||||
&& a.bucket() == b.bucket()
|
||||
&& a.key() == b.key()
|
||||
&& a.deployment_id() == b.deployment_id()
|
||||
&& a.bucket_incarnation() == b.bucket_incarnation()
|
||||
&& a.object_digest() == b.object_digest()
|
||||
&& a.object_length() == b.object_length()
|
||||
&& a.set_digest() == b.set_digest()
|
||||
&& a.distribution_digest() == b.distribution_digest()
|
||||
&& a.erasure_geometry().0 == b.erasure_geometry().0
|
||||
&& a.erasure_geometry().1 == b.erasure_geometry().1
|
||||
}
|
||||
|
||||
pub(super) struct DigestReader<R> {
|
||||
inner: R,
|
||||
digest: Sha256,
|
||||
}
|
||||
impl<R> DigestReader<R> {
|
||||
pub(super) fn new(inner: R) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
digest: Sha256::new(),
|
||||
}
|
||||
}
|
||||
pub(super) fn into_inner_and_digest(self) -> (R, [u8; 32]) {
|
||||
(self.inner, self.digest.finalize().into())
|
||||
}
|
||||
}
|
||||
impl<R: tokio::io::AsyncRead + Unpin> tokio::io::AsyncRead for DigestReader<R> {
|
||||
fn poll_read(
|
||||
mut self: std::pin::Pin<&mut Self>,
|
||||
cx: &mut std::task::Context<'_>,
|
||||
buf: &mut tokio::io::ReadBuf<'_>,
|
||||
) -> std::task::Poll<std::io::Result<()>> {
|
||||
let before = buf.filled().len();
|
||||
let result = std::pin::Pin::new(&mut self.inner).poll_read(cx, buf);
|
||||
if matches!(result, std::task::Poll::Ready(Ok(()))) {
|
||||
self.digest.update(&buf.filled()[before..]);
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
struct DigestSink {
|
||||
digest: Sha256,
|
||||
length: usize,
|
||||
}
|
||||
impl tokio::io::AsyncWrite for DigestSink {
|
||||
fn poll_write(
|
||||
mut self: std::pin::Pin<&mut Self>,
|
||||
_: &mut std::task::Context<'_>,
|
||||
bytes: &[u8],
|
||||
) -> std::task::Poll<std::io::Result<usize>> {
|
||||
match self.length.checked_add(bytes.len()) {
|
||||
Some(length) => {
|
||||
self.length = length;
|
||||
self.digest.update(bytes);
|
||||
std::task::Poll::Ready(Ok(bytes.len()))
|
||||
}
|
||||
None => std::task::Poll::Ready(Err(std::io::Error::other("native content length overflow"))),
|
||||
}
|
||||
}
|
||||
fn poll_flush(self: std::pin::Pin<&mut Self>, _: &mut std::task::Context<'_>) -> std::task::Poll<std::io::Result<()>> {
|
||||
std::task::Poll::Ready(Ok(()))
|
||||
}
|
||||
fn poll_shutdown(self: std::pin::Pin<&mut Self>, _: &mut std::task::Context<'_>) -> std::task::Poll<std::io::Result<()>> {
|
||||
std::task::Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
|
||||
impl SetDisks {
|
||||
fn native_expected_set_digest(&self, store_id: Uuid) -> Result<[u8; 32]> {
|
||||
let set = self
|
||||
.format
|
||||
.erasure
|
||||
.sets
|
||||
.get(self.set_index)
|
||||
.ok_or_else(|| Error::other("native target set missing"))?;
|
||||
if self.format.id != store_id
|
||||
|| set.len() != self.set_drive_count
|
||||
|| set.iter().any(Uuid::is_nil)
|
||||
|| set.iter().enumerate().any(|(slot, id)| set[slot + 1..].contains(id))
|
||||
{
|
||||
return Err(Error::other("invalid native target physical set"));
|
||||
}
|
||||
Ok(Sha256::digest(serde_json::to_vec(set)?).into())
|
||||
}
|
||||
|
||||
/// Attest already encoded private shards while the caller owns the actual PUT commit guards.
|
||||
pub(crate) async fn attest_native_stream(
|
||||
&self,
|
||||
store_id: Uuid,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
preparation: Uuid,
|
||||
incarnation: Uuid,
|
||||
candidates: Vec<(crate::disk::DiskStore, FileInfo)>,
|
||||
object_digest: [u8; 32],
|
||||
object_length: u64,
|
||||
fence: Arc<ObjectLockConfigSnapshot>,
|
||||
) -> Result<crate::store::native_prepare::NativeQuorumPreparation> {
|
||||
self.attest_native_stream_owned(
|
||||
store_id,
|
||||
bucket,
|
||||
key,
|
||||
preparation,
|
||||
incarnation,
|
||||
candidates,
|
||||
object_digest,
|
||||
object_length,
|
||||
fence,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
async fn attest_native_stream_owned(
|
||||
&self,
|
||||
store_id: Uuid,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
preparation: Uuid,
|
||||
incarnation: Uuid,
|
||||
candidates: Vec<(crate::disk::DiskStore, FileInfo)>,
|
||||
object_digest: [u8; 32],
|
||||
object_length: u64,
|
||||
fence: Arc<ObjectLockConfigSnapshot>,
|
||||
operation_owner: Option<Arc<rustfs_lock::NamespaceLockGuard>>,
|
||||
) -> Result<crate::store::native_prepare::NativeQuorumPreparation> {
|
||||
let expected_set = self.native_expected_set_digest(store_id)?;
|
||||
if !fence.is_valid_for_destructive_put(store_id, bucket, incarnation) {
|
||||
return Err(Error::other("native adoption bucket fence mismatch"));
|
||||
}
|
||||
let layout = resolve_write_layout(
|
||||
&self.storage_class_config_snapshot(),
|
||||
self.pool_index,
|
||||
self.set_drive_count,
|
||||
self.default_parity_count,
|
||||
None,
|
||||
false,
|
||||
)?;
|
||||
let (send, mut receive) = mpsc::unbounded_channel();
|
||||
for (disk, info) in candidates {
|
||||
if info.size != i64::try_from(object_length).map_err(Error::other)?
|
||||
|| (info.name != key && info.name != format!("{bucket}/{key}"))
|
||||
|| info.inline_data()
|
||||
|| info.deleted
|
||||
|| info.mark_deleted
|
||||
{
|
||||
return Err(Error::other("native adoption metadata identity mismatch"));
|
||||
}
|
||||
let version = info
|
||||
.version_id
|
||||
.filter(|id| !id.is_nil())
|
||||
.ok_or_else(|| Error::other("native adoption version missing"))?;
|
||||
let mut metadata = FileMeta::new();
|
||||
metadata.add_version(info)?;
|
||||
let input = NativePrepareInput {
|
||||
preparation,
|
||||
store_id,
|
||||
bucket: bucket.into(),
|
||||
bucket_incarnation: incarnation,
|
||||
object_digest,
|
||||
key: key.into(),
|
||||
version,
|
||||
metadata: metadata.marshal_msg()?.into(),
|
||||
bucket_fence: fence.clone(),
|
||||
};
|
||||
let send = send.clone();
|
||||
let owner = operation_owner.clone();
|
||||
tokio::spawn(async move {
|
||||
let result = match (disk.as_ref(), owner) {
|
||||
(Disk::Local(local), Some(owner)) => local.get_disk().prepare_native_with_owner(input, owner).await,
|
||||
(_, owner) => {
|
||||
let _owner = owner;
|
||||
disk.native_finalize(input).await
|
||||
}
|
||||
};
|
||||
let _ = send.send(result);
|
||||
});
|
||||
}
|
||||
drop(send);
|
||||
let mut receipts = Vec::new();
|
||||
while let Some(result) = receive.recv().await {
|
||||
let receipt = match result {
|
||||
Ok(receipt) => receipt,
|
||||
Err(DiskError::Io(error)) if error.kind() == std::io::ErrorKind::InvalidData => return Err(error.into()),
|
||||
Err(_) => continue,
|
||||
};
|
||||
if receipt.set_digest() != &expected_set
|
||||
|| receipts.first().is_some_and(|old| !same_identity(old, &receipt))
|
||||
|| receipts
|
||||
.iter()
|
||||
.any(|old: &NativePreparationReceipt| old.disk_id() == receipt.disk_id())
|
||||
{
|
||||
return Err(Error::other("native adoption quorum identity mismatch"));
|
||||
}
|
||||
receipts.push(receipt);
|
||||
}
|
||||
crate::store::native_prepare::NativeQuorumPreparation::from_receipts(layout.write_quorum, receipts)
|
||||
}
|
||||
|
||||
pub(crate) async fn prepare_native_stream(
|
||||
&self,
|
||||
store_id: Uuid,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
preparation: Uuid,
|
||||
incarnation: Uuid,
|
||||
data: &mut crate::object_api::PutObjReader,
|
||||
fence: Arc<ObjectLockConfigSnapshot>,
|
||||
operation_owner: Arc<rustfs_lock::NamespaceLockGuard>,
|
||||
) -> Result<(usize, Vec<NativePreparationReceipt>)> {
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
let disks = self.get_disks_internal().await;
|
||||
let layout = resolve_write_layout(
|
||||
&self.storage_class_config_snapshot(),
|
||||
self.pool_index,
|
||||
disks.len(),
|
||||
self.default_parity_count,
|
||||
None,
|
||||
false,
|
||||
)?;
|
||||
let mut info = FileInfo::new(key, layout.data_drives, layout.parity_drives);
|
||||
info.name = key.to_owned();
|
||||
info.version_id = Some(preparation);
|
||||
info.data_dir = Some(preparation);
|
||||
info.size = data.size();
|
||||
info.versioned = true;
|
||||
info.mod_time = Some(time::OffsetDateTime::now_utc());
|
||||
let erasure = Arc::new(Erasure::try_new_with_options(
|
||||
info.erasure.data_blocks,
|
||||
info.erasure.parity_blocks,
|
||||
info.erasure.block_size,
|
||||
false,
|
||||
)?);
|
||||
let mut shuffled = Self::shuffle_disks_owned(disks.clone(), &info.erasure.distribution);
|
||||
let path = format!("native-preparations/{preparation}/{preparation}/part.1");
|
||||
let mut writers = Vec::with_capacity(shuffled.len());
|
||||
for disk in &mut shuffled {
|
||||
let result = match disk {
|
||||
Some(disk) => {
|
||||
if let Err(error) = disk
|
||||
.native_begin_stream(preparation, store_id, bucket, key, incarnation, fence.clone())
|
||||
.await
|
||||
{
|
||||
if matches!(&error, DiskError::Io(error) if error.kind() == std::io::ErrorKind::InvalidData) {
|
||||
return Err(error.into());
|
||||
}
|
||||
writers.push(None);
|
||||
continue;
|
||||
}
|
||||
crate::io_support::bitrot::create_bitrot_writer(
|
||||
false,
|
||||
Some(disk),
|
||||
RUSTFS_META_BUCKET,
|
||||
&path,
|
||||
erasure.shard_file_size(info.size),
|
||||
erasure.shard_size(),
|
||||
rustfs_utils::HashAlgorithm::HighwayHash256S,
|
||||
)
|
||||
.await
|
||||
}
|
||||
None => Err(DiskError::DiskNotFound),
|
||||
};
|
||||
match result {
|
||||
Ok(writer) => writers.push(Some(writer)),
|
||||
Err(_) => {
|
||||
*disk = None;
|
||||
writers.push(None);
|
||||
}
|
||||
}
|
||||
}
|
||||
let stream = std::mem::replace(
|
||||
&mut data.stream,
|
||||
rustfs_rio::HashReader::from_stream(Cursor::new(Vec::new()), 0, 0, None, None, false)?,
|
||||
);
|
||||
let reader = DigestReader::new(stream);
|
||||
let (reader, size, _, _) = erasure
|
||||
.encode_with_shard_integrity(reader, &mut writers, layout.write_quorum, 1, IntegrityEncodeMode::Streaming, false)
|
||||
.await?;
|
||||
let (stream, digest) = reader.into_inner_and_digest();
|
||||
data.stream = stream;
|
||||
if info.size >= 0 && i64::try_from(size).map_err(Error::other)? != info.size {
|
||||
return Err(Error::other("native stream length mismatch"));
|
||||
}
|
||||
info.size = i64::try_from(size).map_err(Error::other)?;
|
||||
info.add_object_part(1, String::new(), size, None, info.size, None, None);
|
||||
let mut candidates = Vec::new();
|
||||
for (slot, disk) in disks.into_iter().enumerate() {
|
||||
let index = *info
|
||||
.erasure
|
||||
.distribution
|
||||
.get(slot)
|
||||
.ok_or_else(|| Error::other("native stream distribution missing"))?;
|
||||
if writers
|
||||
.get(
|
||||
index
|
||||
.checked_sub(1)
|
||||
.ok_or_else(|| Error::other("invalid native stream index"))?,
|
||||
)
|
||||
.and_then(Option::as_ref)
|
||||
.is_none()
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let Some(disk) = disk else {
|
||||
continue;
|
||||
};
|
||||
let mut per_disk = info.clone();
|
||||
per_disk.erasure.index = index;
|
||||
candidates.push((disk, per_disk));
|
||||
}
|
||||
let owner = self
|
||||
.attest_native_stream_owned(
|
||||
store_id,
|
||||
bucket,
|
||||
key,
|
||||
preparation,
|
||||
incarnation,
|
||||
candidates,
|
||||
digest,
|
||||
u64::try_from(size).map_err(Error::other)?,
|
||||
fence,
|
||||
Some(operation_owner),
|
||||
)
|
||||
.await?;
|
||||
Ok((layout.write_quorum, owner.into_receipts()))
|
||||
}
|
||||
|
||||
pub(crate) async fn prepare_native_bytes(
|
||||
&self,
|
||||
store_id: Uuid,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
preparation: Uuid,
|
||||
incarnation: Uuid,
|
||||
body: Bytes,
|
||||
fence: Arc<ObjectLockConfigSnapshot>,
|
||||
saved_metadata: Vec<Option<Bytes>>,
|
||||
operation_owner: Arc<rustfs_lock::NamespaceLockGuard>,
|
||||
) -> Result<(usize, Vec<NativePreparationReceipt>)> {
|
||||
let disks = self.get_disks_internal().await;
|
||||
if disks.iter().flatten().any(|disk| matches!(disk.as_ref(), Disk::Remote(_))) {
|
||||
return Err(Error::other("native remote preparation unsupported"));
|
||||
}
|
||||
let layout = resolve_write_layout(
|
||||
&self.storage_class_config_snapshot(),
|
||||
self.pool_index,
|
||||
disks.len(),
|
||||
self.default_parity_count,
|
||||
None,
|
||||
false,
|
||||
)?;
|
||||
let mut info = FileInfo::new(&format!("{bucket}/{key}"), layout.data_drives, layout.parity_drives);
|
||||
info.name = key.to_owned();
|
||||
// This is the actual private xl.meta version, not a claim about S3 readability.
|
||||
info.version_id = Some(preparation);
|
||||
info.data_dir = Some(preparation);
|
||||
info.size = i64::try_from(body.len()).map_err(Error::other)?;
|
||||
info.versioned = true;
|
||||
// Mint one timestamp for new preparation; retries use the persisted writer value.
|
||||
let mut modification_time = None;
|
||||
for metadata in saved_metadata.iter().flatten() {
|
||||
let saved = rustfs_filemeta::get_file_info(
|
||||
metadata,
|
||||
bucket,
|
||||
key,
|
||||
&preparation.to_string(),
|
||||
FileInfoOpts {
|
||||
data: false,
|
||||
include_free_versions: false,
|
||||
include_part_checksums: true,
|
||||
},
|
||||
)?;
|
||||
let time = saved
|
||||
.mod_time
|
||||
.ok_or_else(|| Error::other("persisted native modification time missing"))?;
|
||||
if modification_time.is_some_and(|old| old != time) {
|
||||
return Err(Error::other("conflicting native modification times"));
|
||||
}
|
||||
modification_time = Some(time);
|
||||
}
|
||||
info.mod_time = Some(modification_time.unwrap_or_else(time::OffsetDateTime::now_utc));
|
||||
info.add_object_part(1, String::new(), body.len(), None, info.size, None, None);
|
||||
let erasure = Arc::new(Erasure::try_new_with_options(
|
||||
info.erasure.data_blocks,
|
||||
info.erasure.parity_blocks,
|
||||
info.erasure.block_size,
|
||||
false,
|
||||
)?);
|
||||
if body.len() > erasure.block_size || body.len() > 1024 * 1024 {
|
||||
return Err(Error::other("native preparation requires bounded single-block inline data"));
|
||||
}
|
||||
if !self
|
||||
.storage_class_config_snapshot()
|
||||
.should_inline(erasure.shard_file_size(info.size), layout.data_drives, true)
|
||||
{
|
||||
return Err(Error::other("native preparation exceeds configured inline admission"));
|
||||
}
|
||||
let digest: [u8; 32] = Sha256::digest(&body).into();
|
||||
let mut writers = std::iter::repeat_with(|| None).take(disks.len()).collect::<Vec<_>>();
|
||||
let (_, size, shards, _) = erasure
|
||||
.encode_with_shard_integrity(
|
||||
Cursor::new(body),
|
||||
&mut writers,
|
||||
layout.write_quorum,
|
||||
1,
|
||||
IntegrityEncodeMode::Inline(usize::try_from(info.size).map_err(Error::other)?),
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
if i64::try_from(size).map_err(Error::other)? != info.size {
|
||||
return Err(Error::other("native encoder length mismatch"));
|
||||
}
|
||||
let shards = shards.ok_or_else(|| Error::other("native inline encoder omitted shards"))?;
|
||||
let (send, mut receive) = mpsc::unbounded_channel();
|
||||
for (slot, disk) in disks.into_iter().enumerate() {
|
||||
let Some(disk) = disk else {
|
||||
continue;
|
||||
};
|
||||
let Disk::Local(wrapper) = disk.as_ref() else {
|
||||
return Err(Error::other("native remote preparation owner is not supported"));
|
||||
};
|
||||
let local = wrapper.get_disk();
|
||||
let mut per_disk = info.clone();
|
||||
let index = *info
|
||||
.erasure
|
||||
.distribution
|
||||
.get(slot)
|
||||
.ok_or_else(|| Error::other("native disk distribution missing"))?;
|
||||
per_disk.erasure.index = index;
|
||||
per_disk.data = Some(
|
||||
shards
|
||||
.get(
|
||||
index
|
||||
.checked_sub(1)
|
||||
.ok_or_else(|| Error::other("invalid native shard index"))?,
|
||||
)
|
||||
.cloned()
|
||||
.ok_or_else(|| Error::other("native encoder omitted assigned shard"))?,
|
||||
);
|
||||
per_disk.set_inline_data();
|
||||
let metadata = if let Some(bytes) = saved_metadata.get(slot).and_then(Option::as_ref) {
|
||||
let saved = rustfs_filemeta::get_file_info(
|
||||
bytes,
|
||||
bucket,
|
||||
key,
|
||||
&preparation.to_string(),
|
||||
FileInfoOpts {
|
||||
data: true,
|
||||
include_free_versions: false,
|
||||
include_part_checksums: true,
|
||||
},
|
||||
)?;
|
||||
if saved.version_id != per_disk.version_id
|
||||
|| saved.deleted
|
||||
|| saved.mark_deleted
|
||||
|| saved.uses_legacy_checksum != per_disk.uses_legacy_checksum
|
||||
|| saved.mod_time != per_disk.mod_time
|
||||
|| saved.size != per_disk.size
|
||||
|| saved.erasure.distribution != per_disk.erasure.distribution
|
||||
|| saved.erasure.index != index
|
||||
|| saved.erasure.block_size != per_disk.erasure.block_size
|
||||
|| saved.erasure.data_blocks != layout.data_drives
|
||||
|| saved.erasure.parity_blocks != layout.parity_drives
|
||||
|| saved.data != per_disk.data
|
||||
{
|
||||
return Err(Error::other("persisted native shard does not encode the request content"));
|
||||
}
|
||||
bytes.clone()
|
||||
} else {
|
||||
let mut encoded = FileMeta::new();
|
||||
encoded.add_version(per_disk)?;
|
||||
Bytes::from(encoded.marshal_msg()?)
|
||||
};
|
||||
let input = NativePrepareInput {
|
||||
preparation,
|
||||
store_id,
|
||||
bucket: bucket.to_owned(),
|
||||
bucket_incarnation: incarnation,
|
||||
object_digest: digest,
|
||||
key: key.to_owned(),
|
||||
version: preparation,
|
||||
metadata,
|
||||
bucket_fence: fence.clone(),
|
||||
};
|
||||
let sender = send.clone();
|
||||
let parent = operation_owner.clone();
|
||||
// No per-disk lock is acquired here. NP01 owns its actual lock chain.
|
||||
// Detached tasks retain their owners after quorum or caller cancellation;
|
||||
// an unobserved reply leaves durable facts for probe/recovery, never rollback.
|
||||
tokio::spawn(async move {
|
||||
let result = match local.prepare_native_with_owner(input, parent).await {
|
||||
Ok(receipt) => Ok(Some(receipt)),
|
||||
Err(DiskError::Io(error)) if error.kind() == std::io::ErrorKind::InvalidData => Err(Error::from(error)),
|
||||
Err(_) => Ok(None),
|
||||
};
|
||||
let _ = sender.send(result);
|
||||
});
|
||||
}
|
||||
drop(send);
|
||||
let mut receipts = Vec::with_capacity(layout.write_quorum);
|
||||
while let Some(result) = receive.recv().await {
|
||||
if let Some(receipt) = result? {
|
||||
if let Some(first) = receipts.first() {
|
||||
if !same_identity(first, &receipt) {
|
||||
return Err(Error::other("conflicting native preparation identities"));
|
||||
}
|
||||
}
|
||||
if receipts
|
||||
.iter()
|
||||
.any(|r: &NativePreparationReceipt| r.disk_id() == receipt.disk_id())
|
||||
{
|
||||
return Err(Error::other("duplicate native physical disk"));
|
||||
}
|
||||
receipts.push(receipt);
|
||||
if receipts.len() >= layout.write_quorum {
|
||||
return Ok((layout.write_quorum, receipts));
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(Error::ErasureWriteQuorum)
|
||||
}
|
||||
|
||||
pub(crate) async fn recover_native_bytes(
|
||||
&self,
|
||||
store_id: Uuid,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
preparation: Uuid,
|
||||
incarnation: Uuid,
|
||||
fence: Arc<ObjectLockConfigSnapshot>,
|
||||
prepared_slots: Vec<bool>,
|
||||
) -> Result<(usize, Vec<NativePreparationReceipt>, Vec<(Uuid, Bytes)>)> {
|
||||
let disks = self.get_disks_internal().await;
|
||||
let expected_set = self.native_expected_set_digest(store_id)?;
|
||||
let layout = resolve_write_layout(
|
||||
&self.storage_class_config_snapshot(),
|
||||
self.pool_index,
|
||||
disks.len(),
|
||||
self.default_parity_count,
|
||||
None,
|
||||
false,
|
||||
)?;
|
||||
let mut receipts: Vec<NativePreparationReceipt> = Vec::new();
|
||||
let mut verified_metadata = Vec::new();
|
||||
let mut sources: Vec<Option<(crate::disk::DiskStore, FileInfo)>> =
|
||||
std::iter::repeat_with(|| None).take(disks.len()).collect();
|
||||
let mut erasure = None;
|
||||
let mut modification_time = None;
|
||||
let mut legacy_codec = None;
|
||||
let mut block_size = None;
|
||||
// Inspect every available disk: an identity conflict cannot be majority-voted away.
|
||||
for (slot, disk) in disks.into_iter().enumerate() {
|
||||
if !prepared_slots.get(slot).copied().unwrap_or(false) {
|
||||
continue;
|
||||
}
|
||||
let disk = disk.ok_or_else(|| Error::other("prepared native disk unavailable"))?;
|
||||
let (receipt, metadata) = match disk
|
||||
.native_recover(preparation, store_id, bucket, key, incarnation, fence.clone())
|
||||
.await
|
||||
{
|
||||
Ok(value) => value,
|
||||
Err(error) => return Err(error.into()),
|
||||
};
|
||||
if receipt.set_digest() != &expected_set {
|
||||
return Err(Error::other("native recovery receipt target physical set mismatch"));
|
||||
}
|
||||
if receipt.key() != key
|
||||
|| receipt.preparation_id() != preparation
|
||||
|| receipt.bucket() != bucket
|
||||
|| receipt.bucket_incarnation() != incarnation
|
||||
|| receipt.deployment_id() != store_id
|
||||
{
|
||||
return Err(Error::other("native recovery request identity mismatch"));
|
||||
}
|
||||
if let Some(first) = receipts.first() {
|
||||
if !same_identity(first, &receipt) {
|
||||
return Err(Error::other("conflicting native recovery identities"));
|
||||
}
|
||||
}
|
||||
if receipts.iter().any(|r| r.disk_id() == receipt.disk_id()) {
|
||||
return Err(Error::other("duplicate native recovery disk"));
|
||||
}
|
||||
verified_metadata.push((receipt.disk_id(), metadata.clone()));
|
||||
let info = rustfs_filemeta::get_file_info(
|
||||
&metadata,
|
||||
bucket,
|
||||
key,
|
||||
&receipt.version_id().to_string(),
|
||||
FileInfoOpts {
|
||||
data: true,
|
||||
include_free_versions: false,
|
||||
include_part_checksums: true,
|
||||
},
|
||||
)?;
|
||||
let time = info
|
||||
.mod_time
|
||||
.ok_or_else(|| Error::other("recovered native modification time missing"))?;
|
||||
if modification_time.is_some_and(|old| old != time) {
|
||||
return Err(Error::other("conflicting recovered native modification times"));
|
||||
}
|
||||
modification_time = Some(time);
|
||||
if info.erasure.data_blocks != layout.data_drives || info.erasure.parity_blocks != layout.parity_drives {
|
||||
return Err(Error::other("native recovery layout changed"));
|
||||
}
|
||||
if block_size.is_some_and(|old| old != info.erasure.block_size) {
|
||||
return Err(Error::other("conflicting recovered native block sizes"));
|
||||
}
|
||||
block_size = Some(info.erasure.block_size);
|
||||
if legacy_codec.is_some_and(|old| old != info.uses_legacy_checksum) {
|
||||
return Err(Error::other("conflicting recovered native codecs"));
|
||||
}
|
||||
legacy_codec = Some(info.uses_legacy_checksum);
|
||||
let selected = Erasure::try_new_with_options(
|
||||
info.erasure.data_blocks,
|
||||
info.erasure.parity_blocks,
|
||||
info.erasure.block_size,
|
||||
info.uses_legacy_checksum,
|
||||
)?;
|
||||
let index = info
|
||||
.erasure
|
||||
.index
|
||||
.checked_sub(1)
|
||||
.ok_or_else(|| Error::other("invalid recovered native index"))?;
|
||||
let slot = sources
|
||||
.get_mut(index)
|
||||
.ok_or_else(|| Error::other("recovered native index outside set"))?;
|
||||
if slot.is_some() {
|
||||
return Err(Error::other("duplicate recovered native shard"));
|
||||
}
|
||||
if let Some((_, previous)) = sources.iter().flatten().next() {
|
||||
if previous.parts != info.parts || previous.data_dir != info.data_dir {
|
||||
return Err(Error::other("conflicting native recovered part manifest"));
|
||||
}
|
||||
}
|
||||
sources[index] = Some((disk.clone(), info));
|
||||
erasure = Some(selected);
|
||||
receipts.push(receipt);
|
||||
}
|
||||
if receipts.len() < layout.write_quorum {
|
||||
return Err(Error::ErasureWriteQuorum);
|
||||
}
|
||||
let first = receipts.first().ok_or(Error::ErasureWriteQuorum)?;
|
||||
let length = usize::try_from(first.object_length()).map_err(Error::other)?;
|
||||
let mut output = DigestSink {
|
||||
digest: Sha256::new(),
|
||||
length: 0,
|
||||
};
|
||||
let codec = erasure.ok_or(Error::ErasureWriteQuorum)?;
|
||||
let parts = sources
|
||||
.iter()
|
||||
.flatten()
|
||||
.next()
|
||||
.ok_or(Error::ErasureWriteQuorum)?
|
||||
.1
|
||||
.parts
|
||||
.clone();
|
||||
let mut size = 0_usize;
|
||||
for part in parts {
|
||||
let mut readers = Vec::with_capacity(sources.len());
|
||||
for source in &sources {
|
||||
let Some((disk, info)) = source else {
|
||||
readers.push(None);
|
||||
continue;
|
||||
};
|
||||
let checksum = info.erasure.get_checksum_info(part.number).algorithm;
|
||||
let checksum = if info.uses_legacy_checksum && checksum == rustfs_utils::HashAlgorithm::HighwayHash256S {
|
||||
rustfs_utils::HashAlgorithm::HighwayHash256SLegacy
|
||||
} else {
|
||||
checksum
|
||||
};
|
||||
let reader = if info.inline_data() {
|
||||
crate::io_support::bitrot::ShardReader::InMemory(Cursor::new(
|
||||
info.data.clone().ok_or_else(|| Error::other("native inline shard missing"))?,
|
||||
))
|
||||
} else {
|
||||
use crate::disk::DiskAPI;
|
||||
let data_dir = info.data_dir.ok_or_else(|| Error::other("native stream data dir missing"))?;
|
||||
let shard_size = usize::try_from(codec.shard_file_size(i64::try_from(part.size).map_err(Error::other)?))
|
||||
.map_err(Error::other)?;
|
||||
let physical = if matches!(
|
||||
checksum,
|
||||
rustfs_utils::HashAlgorithm::HighwayHash256S | rustfs_utils::HashAlgorithm::HighwayHash256SLegacy
|
||||
) {
|
||||
shard_size
|
||||
.div_ceil(codec.shard_size())
|
||||
.checked_mul(checksum.size())
|
||||
.and_then(|hashes| shard_size.checked_add(hashes))
|
||||
.ok_or_else(|| Error::other("native physical shard length overflow"))?
|
||||
} else {
|
||||
shard_size
|
||||
};
|
||||
crate::io_support::bitrot::ShardReader::Stream(
|
||||
disk.read_file_stream(
|
||||
crate::disk::RUSTFS_META_BUCKET,
|
||||
&format!("native-preparations/{preparation}/{data_dir}/part.{}", part.number),
|
||||
0,
|
||||
physical,
|
||||
)
|
||||
.await?,
|
||||
)
|
||||
};
|
||||
readers.push(Some(BitrotReader::new(reader, codec.shard_size(), checksum, false)));
|
||||
}
|
||||
let (decoded, failure) = codec.decode(&mut output, readers, 0, part.size, part.size).await;
|
||||
if let Some(error) = failure {
|
||||
return Err(error.into());
|
||||
}
|
||||
if decoded != part.size {
|
||||
return Err(Error::other("native recovered part length mismatch"));
|
||||
}
|
||||
size = size
|
||||
.checked_add(decoded)
|
||||
.ok_or_else(|| Error::other("native decoded length overflow"))?;
|
||||
}
|
||||
if size != length || output.length != length || output.digest.finalize().as_slice() != first.object_digest() {
|
||||
return Err(Error::other("native recovered content digest mismatch"));
|
||||
}
|
||||
Ok((layout.write_quorum, receipts, verified_metadata))
|
||||
}
|
||||
}
|
||||
|
||||
impl SetDisks {
|
||||
pub(crate) async fn probe_native_candidates(
|
||||
&self,
|
||||
store_id: Uuid,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
preparation: Uuid,
|
||||
incarnation: Uuid,
|
||||
fence: Arc<ObjectLockConfigSnapshot>,
|
||||
match_object: bool,
|
||||
) -> Vec<std::result::Result<NativePreparationProbe, DiskError>> {
|
||||
let mut observations = Vec::new();
|
||||
for disk in self.get_disks_internal().await {
|
||||
observations.push(match disk {
|
||||
Some(disk) => {
|
||||
if match_object {
|
||||
match disk.native_locate(store_id, preparation).await {
|
||||
Ok(Some(record))
|
||||
if record.bucket != bucket || record.key != key || record.incarnation != incarnation =>
|
||||
{
|
||||
observations.push(Ok(NativePreparationProbe::Absent));
|
||||
continue;
|
||||
}
|
||||
Err(error) => {
|
||||
observations.push(Err(error));
|
||||
continue;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
disk.native_probe(preparation, store_id, bucket, key, incarnation, fence.clone())
|
||||
.await
|
||||
}
|
||||
None => Err(DiskError::InvalidPath),
|
||||
});
|
||||
}
|
||||
observations
|
||||
}
|
||||
}
|
||||
@@ -3847,6 +3847,16 @@ impl SetDisks {
|
||||
.map(crate::io_support::rio::compression_index_storage_bytes);
|
||||
|
||||
let mut etag = data.stream.try_resolve_etag().unwrap_or_default();
|
||||
if data.stream.encrypted_etag {
|
||||
if opts.preserve_etag.as_ref().is_none_or(|preserved| preserved == &etag) {
|
||||
insert_str(&mut user_defined, rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG, "v1".to_owned());
|
||||
} else {
|
||||
rustfs_utils::http::metadata_compat::remove_str(
|
||||
&mut user_defined,
|
||||
rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG,
|
||||
);
|
||||
}
|
||||
}
|
||||
if let Some(ref tag) = opts.preserve_etag {
|
||||
etag = tag.clone();
|
||||
}
|
||||
@@ -10610,6 +10620,65 @@ mod object_encryption_resolver_wiring_tests {
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
#[tokio::test]
|
||||
async fn encrypted_put_attests_only_etags_produced_by_its_encryption_reader() {
|
||||
use super::hermetic_set_disks_support::hermetic_set_disks_isolated;
|
||||
use crate::io_support::rio::{WriteEncryption, WritePlan};
|
||||
use crate::storage_api_contracts::object::ObjectIO as _;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
let (_temp_dirs, disks, set_disks) = hermetic_set_disks_isolated(4).await;
|
||||
let bucket = "encrypted-etag-proof";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("create bucket volume");
|
||||
}
|
||||
let plaintext = b"Salary 2026: 85000 EUR";
|
||||
let size = plaintext.len() as i64;
|
||||
let mut control = HashReader::from_stream(Cursor::new(plaintext), size, size, None, None, false).expect("control reader");
|
||||
control.read_to_end(&mut Vec::new()).await.expect("read control plaintext");
|
||||
let plaintext_md5 = control.try_resolve_etag().expect("control MD5");
|
||||
for (object, preserved) in [("new", None), ("preserved", Some(plaintext_md5.clone()))] {
|
||||
let reader =
|
||||
HashReader::from_stream(Cursor::new(plaintext), size, size, None, None, false).expect("plaintext reader");
|
||||
let reader = WritePlan::new()
|
||||
.with_encryption(WriteEncryption::singlepart([1; 32], [2; 12]))
|
||||
.apply(reader, size)
|
||||
.expect("encrypted reader");
|
||||
let mut opts = ObjectOptions {
|
||||
preserve_etag: preserved.clone(),
|
||||
user_defined: HashMap::from([
|
||||
("x-amz-server-side-encryption-customer-algorithm".to_owned(), "AES256".to_owned()),
|
||||
(rustfs_utils::http::SSEC_ORIGINAL_SIZE_HEADER.to_owned(), size.to_string()),
|
||||
]),
|
||||
..Default::default()
|
||||
};
|
||||
// A copied producer marker cannot attest to a preserved plaintext tag.
|
||||
insert_str(&mut opts.user_defined, rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG, "v1".to_owned());
|
||||
set_disks
|
||||
.put_object(bucket, object, &mut PutObjReader::new(reader), &opts)
|
||||
.await
|
||||
.expect("store encrypted bytes");
|
||||
let stored = set_disks
|
||||
.get_object_info(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read persisted metadata");
|
||||
if let Some(preserved) = preserved {
|
||||
assert_eq!(stored.etag.as_deref(), Some(preserved.as_str()));
|
||||
assert!(!rustfs_utils::http::etag_can_be_listed(&stored.user_defined));
|
||||
} else {
|
||||
assert_ne!(stored.etag.as_ref(), Some(&plaintext_md5));
|
||||
assert_eq!(
|
||||
rustfs_utils::http::get_consistent_str(
|
||||
&stored.user_defined,
|
||||
rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG
|
||||
),
|
||||
Some("v1")
|
||||
);
|
||||
assert!(rustfs_utils::http::etag_can_be_listed(&stored.user_defined));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct CapturedLogs(Arc<Mutex<Vec<u8>>>);
|
||||
|
||||
|
||||
+229
-990
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -448,8 +448,8 @@ const MAX_LIST_OBJECTS_METADATA_FAST_STALENESS_MS: u64 = 60_000;
|
||||
const LIST_OBJECTS_INDEX_PROVIDER_WALKER_KEY_ONLY: &str = "walker_key_only";
|
||||
const LIST_OBJECTS_INDEX_PROVIDER_PERSISTENT_KEY_ONLY: &str = "persistent_key_only";
|
||||
const LIST_OBJECTS_INDEX_PROVIDER_PERSISTENT_KEY_ONLY_DEFAULT_GENERATION: &str = "persistent-key-only";
|
||||
const PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION: u8 = 2;
|
||||
const PERSISTENT_KEY_ONLY_INDEX_HEADER: &str = "# rustfs-listobjects-key-only-v2";
|
||||
const PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION: u8 = 3;
|
||||
const PERSISTENT_KEY_ONLY_INDEX_HEADER: &str = "# rustfs-listobjects-key-only-v3";
|
||||
const PERSISTENT_KEY_ONLY_INDEX_BUCKET_HEADER: &str = "# bucket=";
|
||||
const PERSISTENT_KEY_ONLY_INDEX_GENERATION_HEADER: &str = "# generation=";
|
||||
const PERSISTENT_KEY_ONLY_INDEX_CHECKPOINT_HEADER: &str = "# checkpoint_high_water_mark=";
|
||||
@@ -656,7 +656,10 @@ impl PersistentListMetadataObject {
|
||||
name: object.name.clone(),
|
||||
size: object.size,
|
||||
mod_time: object.mod_time,
|
||||
etag: object.etag.clone(),
|
||||
etag: object
|
||||
.etag
|
||||
.clone()
|
||||
.filter(|_| rustfs_utils::http::etag_can_be_listed(&object.user_defined)),
|
||||
storage_class: object.storage_class.clone(),
|
||||
}
|
||||
}
|
||||
@@ -2361,23 +2364,43 @@ fn list_objects_paginate(
|
||||
(objects, prefixes, is_truncated, next_marker, next_version_idmarker)
|
||||
}
|
||||
|
||||
fn list_objects_paginate_versions<'a>(
|
||||
mut get_objects: Vec<ObjectInfo>,
|
||||
delimiter: &Option<String>,
|
||||
struct VersionedListPagination<'a, I> {
|
||||
delimiter: &'a Option<String>,
|
||||
max_keys: i32,
|
||||
disk_has_more: bool,
|
||||
cache_id: Option<&str>,
|
||||
marker: Option<&str>,
|
||||
(last_scanned_key, raw_keys): (Option<&str>, impl Iterator<Item = &'a str>),
|
||||
) -> (Vec<ObjectInfo>, Vec<String>, bool, Option<String>, Option<String>) {
|
||||
filter_versions_common_prefixes_after_marker(&mut get_objects, delimiter.as_deref(), marker);
|
||||
let prefix_at_page_boundary =
|
||||
versioned_prefix_page_boundary_key(&get_objects, raw_keys, delimiter.as_deref(), max_keys, disk_has_more);
|
||||
cache_id: Option<&'a str>,
|
||||
marker: Option<&'a str>,
|
||||
last_scanned_key: Option<&'a str>,
|
||||
raw_keys: I,
|
||||
}
|
||||
|
||||
let (objects, prefixes, is_truncated, mut next_marker, mut next_version_idmarker) =
|
||||
list_objects_paginate(get_objects, delimiter, max_keys, disk_has_more, cache_id, true, last_scanned_key);
|
||||
fn list_objects_paginate_versions<'a, I>(
|
||||
mut get_objects: Vec<ObjectInfo>,
|
||||
pagination: VersionedListPagination<'a, I>,
|
||||
) -> (Vec<ObjectInfo>, Vec<String>, bool, Option<String>, Option<String>)
|
||||
where
|
||||
I: Iterator<Item = &'a str>,
|
||||
{
|
||||
filter_versions_common_prefixes_after_marker(&mut get_objects, pagination.delimiter.as_deref(), pagination.marker);
|
||||
let prefix_at_page_boundary = versioned_prefix_page_boundary_key(
|
||||
&get_objects,
|
||||
pagination.raw_keys,
|
||||
pagination.delimiter.as_deref(),
|
||||
pagination.max_keys,
|
||||
pagination.disk_has_more,
|
||||
);
|
||||
|
||||
let (objects, prefixes, is_truncated, mut next_marker, mut next_version_idmarker) = list_objects_paginate(
|
||||
get_objects,
|
||||
pagination.delimiter,
|
||||
pagination.max_keys,
|
||||
pagination.disk_has_more,
|
||||
pagination.cache_id,
|
||||
true,
|
||||
pagination.last_scanned_key,
|
||||
);
|
||||
if is_truncated && let Some(raw_key) = prefix_at_page_boundary {
|
||||
next_marker = Some(append_list_cache_id_to_marker(raw_key, cache_id));
|
||||
next_marker = Some(append_list_cache_id_to_marker(raw_key, pagination.cache_id));
|
||||
next_version_idmarker = None;
|
||||
}
|
||||
|
||||
@@ -4325,12 +4348,15 @@ impl ECStore {
|
||||
|
||||
let (objects, prefixes, is_truncated, next_marker, next_version_idmarker) = list_objects_paginate_versions(
|
||||
get_objects,
|
||||
&delimiter,
|
||||
max_keys,
|
||||
disk_has_more,
|
||||
next_cache_id.as_deref(),
|
||||
opts.marker.as_deref(),
|
||||
(last_scanned_key.as_deref(), entries.entries().iter().map(|entry| entry.name.as_str())),
|
||||
VersionedListPagination {
|
||||
delimiter: &delimiter,
|
||||
max_keys,
|
||||
disk_has_more,
|
||||
cache_id: next_cache_id.as_deref(),
|
||||
marker: opts.marker.as_deref(),
|
||||
last_scanned_key: last_scanned_key.as_deref(),
|
||||
raw_keys: entries.entries().iter().map(|entry| entry.name.as_str()),
|
||||
},
|
||||
);
|
||||
|
||||
Ok(ListObjectVersionsInfo {
|
||||
@@ -5752,12 +5778,15 @@ impl Sets {
|
||||
|
||||
let (objects, prefixes, is_truncated, next_marker, next_version_idmarker) = list_objects_paginate_versions(
|
||||
get_objects,
|
||||
&delimiter,
|
||||
max_keys,
|
||||
disk_has_more,
|
||||
next_cache_id.as_deref(),
|
||||
opts.marker.as_deref(),
|
||||
(last_scanned_key.as_deref(), entries.entries().iter().map(|entry| entry.name.as_str())),
|
||||
VersionedListPagination {
|
||||
delimiter: &delimiter,
|
||||
max_keys,
|
||||
disk_has_more,
|
||||
cache_id: next_cache_id.as_deref(),
|
||||
marker: opts.marker.as_deref(),
|
||||
last_scanned_key: last_scanned_key.as_deref(),
|
||||
raw_keys: entries.entries().iter().map(|entry| entry.name.as_str()),
|
||||
},
|
||||
);
|
||||
|
||||
Ok(ListObjectVersionsInfo {
|
||||
@@ -6565,12 +6594,15 @@ impl SetDisks {
|
||||
|
||||
let (objects, prefixes, is_truncated, next_marker, next_version_idmarker) = list_objects_paginate_versions(
|
||||
get_objects,
|
||||
&delimiter,
|
||||
max_keys,
|
||||
disk_has_more,
|
||||
next_cache_id.as_deref(),
|
||||
opts.marker.as_deref(),
|
||||
(last_scanned_key.as_deref(), entries.entries().iter().map(|entry| entry.name.as_str())),
|
||||
VersionedListPagination {
|
||||
delimiter: &delimiter,
|
||||
max_keys,
|
||||
disk_has_more,
|
||||
cache_id: next_cache_id.as_deref(),
|
||||
marker: opts.marker.as_deref(),
|
||||
last_scanned_key: last_scanned_key.as_deref(),
|
||||
raw_keys: entries.entries().iter().map(|entry| entry.name.as_str()),
|
||||
},
|
||||
);
|
||||
|
||||
Ok(ListObjectVersionsInfo {
|
||||
@@ -8844,12 +8876,15 @@ mod test {
|
||||
let get_objects = fold_delimiter_page(&window, "", "-");
|
||||
let (objects, prefixes, is_truncated, next_marker, _v) = super::list_objects_paginate_versions(
|
||||
get_objects,
|
||||
&delimiter,
|
||||
max_keys,
|
||||
disk_has_more,
|
||||
None,
|
||||
marker.as_deref(),
|
||||
(last_scanned.as_deref(), window.iter().map(String::as_str)),
|
||||
super::VersionedListPagination {
|
||||
delimiter: &delimiter,
|
||||
max_keys,
|
||||
disk_has_more,
|
||||
cache_id: None,
|
||||
marker: marker.as_deref(),
|
||||
last_scanned_key: last_scanned.as_deref(),
|
||||
raw_keys: window.iter().map(String::as_str),
|
||||
},
|
||||
);
|
||||
|
||||
assert!(
|
||||
@@ -8922,12 +8957,15 @@ mod test {
|
||||
let get_objects = fold_delimiter_page(&window, "", "-");
|
||||
let (objects, prefixes, is_truncated, next_marker, _) = super::list_objects_paginate_versions(
|
||||
get_objects,
|
||||
&delimiter,
|
||||
max_keys,
|
||||
disk_has_more,
|
||||
None,
|
||||
marker.as_deref(),
|
||||
(last_scanned.as_deref(), window.iter().map(String::as_str)),
|
||||
super::VersionedListPagination {
|
||||
delimiter: &delimiter,
|
||||
max_keys,
|
||||
disk_has_more,
|
||||
cache_id: None,
|
||||
marker: marker.as_deref(),
|
||||
last_scanned_key: last_scanned.as_deref(),
|
||||
raw_keys: window.iter().map(String::as_str),
|
||||
},
|
||||
);
|
||||
|
||||
assert_eq!(objects.len() + prefixes.len(), 1, "a page must respect max-keys");
|
||||
@@ -9170,6 +9208,38 @@ mod test {
|
||||
assert_eq!(parsed, object);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn persistent_listing_snapshots_cannot_restore_legacy_encrypted_fingerprints() {
|
||||
let mut object = ObjectInfo {
|
||||
name: "encrypted".to_owned(),
|
||||
etag: Some("fbfcbcc6d2f035411e6268c8b119593e".to_owned()),
|
||||
user_defined: Arc::new(HashMap::from([(
|
||||
"x-amz-server-side-encryption-customer-algorithm".to_owned(),
|
||||
"AES256".to_owned(),
|
||||
)])),
|
||||
..Default::default()
|
||||
};
|
||||
let snapshot = PersistentListMetadataObject::from_object_info(&object);
|
||||
assert!(snapshot.to_object_info("bucket").etag.is_none());
|
||||
let mut old_snapshot = snapshot.clone();
|
||||
old_snapshot.etag = object.etag.clone();
|
||||
let old_contents = format!(
|
||||
"# rustfs-listobjects-key-only-v2\n{}\n",
|
||||
encode_persistent_list_metadata_object(&old_snapshot)
|
||||
);
|
||||
let old_index = parse_persistent_key_only_index(&old_contents);
|
||||
assert_ne!(old_index.format_version, PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION);
|
||||
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
Arc::make_mut(&mut object.user_defined),
|
||||
rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG,
|
||||
"v1".to_owned(),
|
||||
);
|
||||
assert_eq!(PersistentListMetadataObject::from_object_info(&object).etag, object.etag);
|
||||
object.user_defined = Arc::new(HashMap::new());
|
||||
assert_eq!(PersistentListMetadataObject::from_object_info(&object).etag, object.etag);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn persistent_key_only_index_parses_legacy_keys_and_metadata_snapshots() {
|
||||
let object = PersistentListMetadataObject {
|
||||
|
||||
@@ -437,6 +437,11 @@ mod init;
|
||||
pub(crate) mod init_format;
|
||||
pub(crate) mod list_objects;
|
||||
mod multipart;
|
||||
pub mod native_prepare;
|
||||
#[cfg(all(test, unix))]
|
||||
mod native_prepare_multi_pool_tests;
|
||||
#[cfg(all(test, unix))]
|
||||
mod native_prepare_tests;
|
||||
mod object;
|
||||
#[cfg(feature = "test-util")]
|
||||
pub use object::DeleteAfterObjectLockSnapshotBarrier;
|
||||
|
||||
@@ -0,0 +1,609 @@
|
||||
//! Private native object preparation; a data quorum is not publication authority.
|
||||
use super::ECStore;
|
||||
use crate::bucket::utils::check_put_object_args;
|
||||
use crate::disk::DiskAPI;
|
||||
use crate::{
|
||||
disk::local::native_prepare::{
|
||||
NativePreparationDescription, NativePreparationProbe, NativePreparationReceipt, NativePrepareInput,
|
||||
},
|
||||
error::{Error, Result},
|
||||
object_api::PutObjReader,
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Identity supplied to a node-local native preparation operation.
|
||||
///
|
||||
/// `store_id` and `bucket_incarnation` are expectations to validate, not
|
||||
/// authorization tokens. The receiving ECStore establishes its own fence.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct NativePreparationTarget {
|
||||
pub store_id: Uuid,
|
||||
pub disk_id: Uuid,
|
||||
pub bucket: String,
|
||||
pub key: String,
|
||||
pub preparation: Uuid,
|
||||
pub bucket_incarnation: Uuid,
|
||||
}
|
||||
|
||||
/// Metadata and content identity for finalizing a shard already streamed to a disk.
|
||||
pub struct NativePreparationFinalizeRequest {
|
||||
pub target: NativePreparationTarget,
|
||||
pub version: Uuid,
|
||||
pub object_digest: [u8; 32],
|
||||
pub metadata: Bytes,
|
||||
}
|
||||
|
||||
/// Verified local recovery data returned by a target owner.
|
||||
pub struct NativePreparationRecovery {
|
||||
pub receipt: NativePreparationReceipt,
|
||||
pub metadata: Bytes,
|
||||
}
|
||||
|
||||
pub struct NativeQuorumPreparation {
|
||||
receipts: Vec<NativePreparationReceipt>,
|
||||
}
|
||||
impl NativeQuorumPreparation {
|
||||
pub(crate) fn into_receipts(self) -> Vec<NativePreparationReceipt> {
|
||||
self.receipts
|
||||
}
|
||||
pub(crate) fn receipts(&self) -> &[NativePreparationReceipt] {
|
||||
&self.receipts
|
||||
}
|
||||
|
||||
pub fn preparation_id(&self) -> Uuid {
|
||||
self.receipts[0].preparation_id()
|
||||
}
|
||||
pub fn version_id(&self) -> Uuid {
|
||||
self.receipts[0].version_id()
|
||||
}
|
||||
pub fn object_digest(&self) -> [u8; 32] {
|
||||
*self.receipts[0].object_digest()
|
||||
}
|
||||
pub fn object_length(&self) -> u64 {
|
||||
self.receipts[0].object_length()
|
||||
}
|
||||
pub fn receipt_count(&self) -> usize {
|
||||
self.receipts().len()
|
||||
}
|
||||
pub fn disk_ids(&self) -> impl Iterator<Item = Uuid> + '_ {
|
||||
self.receipts.iter().map(NativePreparationReceipt::disk_id)
|
||||
}
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
struct Identity {
|
||||
version: Uuid,
|
||||
digest: [u8; 32],
|
||||
length: u64,
|
||||
set: [u8; 32],
|
||||
distribution: [u8; 32],
|
||||
}
|
||||
struct Observation {
|
||||
set: std::sync::Arc<crate::set_disk::SetDisks>,
|
||||
identity: Option<Identity>,
|
||||
metadata: Vec<Option<bytes::Bytes>>,
|
||||
prepared: Vec<bool>,
|
||||
unknown: bool,
|
||||
}
|
||||
impl NativeQuorumPreparation {
|
||||
pub(crate) fn from_receipts(quorum: usize, receipts: Vec<NativePreparationReceipt>) -> Result<Self> {
|
||||
let first = receipts.first().ok_or_else(|| Error::other("empty native quorum"))?;
|
||||
let distinct: std::collections::HashSet<_> = receipts.iter().map(NativePreparationReceipt::disk_id).collect();
|
||||
if receipts.len() < quorum
|
||||
|| distinct.len() != receipts.len()
|
||||
|| receipts.iter().any(|r| !crate::set_disk::same_identity(first, r))
|
||||
{
|
||||
return Err(Error::other("inconsistent native quorum"));
|
||||
}
|
||||
Ok(Self { receipts })
|
||||
}
|
||||
}
|
||||
impl ECStore {
|
||||
/// Locate a durable preparation on the exact local disk named by the caller.
|
||||
/// The description is local locator data and is not a preparation receipt.
|
||||
#[cfg(unix)]
|
||||
pub async fn locate_native_preparation_on_disk(
|
||||
&self,
|
||||
disk_id: Uuid,
|
||||
store_id: Uuid,
|
||||
preparation: Uuid,
|
||||
) -> Result<Option<NativePreparationDescription>> {
|
||||
if store_id != self.id || store_id.is_nil() || disk_id.is_nil() || preparation.is_nil() {
|
||||
return Err(Error::other("native preparation locator identity mismatch"));
|
||||
}
|
||||
let disk = self.resolve_native_local_disk(disk_id).await?;
|
||||
disk.locate_native_preparation(store_id, preparation)
|
||||
.await
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
pub async fn locate_native_preparation_on_disk(
|
||||
&self,
|
||||
_disk_id: Uuid,
|
||||
_store_id: Uuid,
|
||||
_preparation: Uuid,
|
||||
) -> Result<Option<NativePreparationDescription>> {
|
||||
Err(Error::UnsupportedDisk)
|
||||
}
|
||||
|
||||
async fn validate_native_target(
|
||||
&self,
|
||||
target: &NativePreparationTarget,
|
||||
) -> Result<std::sync::Arc<crate::object_api::ObjectLockConfigSnapshot>> {
|
||||
if target.store_id != self.id
|
||||
|| self.id.is_nil()
|
||||
|| target.disk_id.is_nil()
|
||||
|| target.preparation.is_nil()
|
||||
|| target.bucket_incarnation.is_nil()
|
||||
{
|
||||
return Err(Error::other("native preparation target identity mismatch"));
|
||||
}
|
||||
check_put_object_args(&target.bucket, &target.key)?;
|
||||
|
||||
let fence = self.object_lock_config_snapshot(&target.bucket).await?;
|
||||
let incarnation = self.bucket_incarnation_id_from_disk(&target.bucket).await?;
|
||||
if incarnation != target.bucket_incarnation || !fence.is_valid_for_destructive_put(self.id, &target.bucket, incarnation) {
|
||||
return Err(Error::other("native preparation target bucket fence mismatch"));
|
||||
}
|
||||
Ok(fence)
|
||||
}
|
||||
|
||||
async fn resolve_native_local_disk(&self, disk_id: Uuid) -> Result<std::sync::Arc<crate::disk::local::LocalDisk>> {
|
||||
use crate::disk::{Disk, DiskAPI};
|
||||
|
||||
let mut matches = Vec::new();
|
||||
let mut lookup_error = None;
|
||||
for pool in &self.pools {
|
||||
for set in &pool.disk_set {
|
||||
let disks = set.disks.read().await.clone();
|
||||
for disk in disks.into_iter().flatten() {
|
||||
let Disk::Local(local) = disk.as_ref() else {
|
||||
continue;
|
||||
};
|
||||
let local_disk = local.get_disk();
|
||||
match local_disk.get_disk_id().await {
|
||||
Ok(Some(id)) if id == disk_id => matches.push(local_disk),
|
||||
Ok(_) => {}
|
||||
Err(error) => {
|
||||
lookup_error.get_or_insert(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if matches.len() != 1 {
|
||||
if matches.is_empty()
|
||||
&& let Some(error) = lookup_error
|
||||
{
|
||||
return Err(error.into());
|
||||
}
|
||||
return Err(Error::other("native preparation target disk is not uniquely local"));
|
||||
}
|
||||
matches
|
||||
.pop()
|
||||
.ok_or_else(|| Error::other("native preparation target disk disappeared"))
|
||||
}
|
||||
|
||||
/// Create private stream staging on the exact local disk named by the caller.
|
||||
/// The caller holds the distributed object lock across all disk RPCs; this owner must not
|
||||
/// reacquire that non-reentrant lock. Store and bucket identities are checked against fresh
|
||||
/// local state, while disk operations take their own physical mutation locks.
|
||||
pub async fn begin_native_preparation_on_disk(&self, target: &NativePreparationTarget) -> Result<()> {
|
||||
let fence = self.validate_native_target(target).await?;
|
||||
let disk = self.resolve_native_local_disk(target.disk_id).await?;
|
||||
disk.begin_native_stream(target.preparation, self.id, &target.bucket, target.bucket_incarnation, fence)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Verify and durably attest the stream already written to the named local disk.
|
||||
/// This does not select placement or encode the object.
|
||||
pub async fn finalize_native_preparation_on_disk(
|
||||
&self,
|
||||
request: NativePreparationFinalizeRequest,
|
||||
) -> Result<NativePreparationReceipt> {
|
||||
let target = &request.target;
|
||||
let fence = self.validate_native_target(target).await?;
|
||||
let disk = self.resolve_native_local_disk(target.disk_id).await?;
|
||||
disk.prepare_native(NativePrepareInput {
|
||||
preparation: target.preparation,
|
||||
store_id: self.id,
|
||||
bucket: target.bucket.clone(),
|
||||
bucket_incarnation: target.bucket_incarnation,
|
||||
object_digest: request.object_digest,
|
||||
key: target.key.clone(),
|
||||
version: request.version,
|
||||
metadata: request.metadata,
|
||||
bucket_fence: fence,
|
||||
})
|
||||
.await
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
/// Inspect the named local disk for a preparation without trusting a remote fence.
|
||||
pub async fn probe_native_preparation_on_disk(&self, target: &NativePreparationTarget) -> Result<NativePreparationProbe> {
|
||||
let fence = self.validate_native_target(target).await?;
|
||||
let disk = self.resolve_native_local_disk(target.disk_id).await?;
|
||||
disk.probe_native(target.preparation, self.id, &target.bucket, &target.key, target.bucket_incarnation, fence)
|
||||
.await
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
/// Recover and revalidate durable preparation state from the named local disk.
|
||||
pub async fn recover_native_preparation_on_disk(
|
||||
&self,
|
||||
target: &NativePreparationTarget,
|
||||
) -> Result<NativePreparationRecovery> {
|
||||
let fence = self.validate_native_target(target).await?;
|
||||
let disk = self.resolve_native_local_disk(target.disk_id).await?;
|
||||
let (receipt, metadata) = disk
|
||||
.recover_native_with_metadata(target.preparation, self.id, &target.bucket, target.bucket_incarnation, fence)
|
||||
.await?;
|
||||
Ok(NativePreparationRecovery { receipt, metadata })
|
||||
}
|
||||
|
||||
async fn native_observations(
|
||||
&self,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
preparation: Uuid,
|
||||
incarnation: Uuid,
|
||||
fence: std::sync::Arc<crate::object_api::ObjectLockConfigSnapshot>,
|
||||
) -> Result<Vec<Observation>> {
|
||||
self.native_observations_scan(bucket, key, preparation, incarnation, fence, false)
|
||||
.await
|
||||
}
|
||||
async fn native_observations_scan(
|
||||
&self,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
preparation: Uuid,
|
||||
incarnation: Uuid,
|
||||
fence: std::sync::Arc<crate::object_api::ObjectLockConfigSnapshot>,
|
||||
all_sets: bool,
|
||||
) -> Result<Vec<Observation>> {
|
||||
let mut pools = Vec::new();
|
||||
for pool in &self.pools {
|
||||
let sets = if all_sets {
|
||||
pool.disk_set.clone()
|
||||
} else {
|
||||
vec![pool.get_disks_by_key(key)]
|
||||
};
|
||||
for set in sets {
|
||||
let candidates = set
|
||||
.probe_native_candidates(self.id, bucket, key, preparation, incarnation, fence.clone(), all_sets)
|
||||
.await;
|
||||
let mut observed = Observation {
|
||||
set,
|
||||
identity: None,
|
||||
metadata: Vec::new(),
|
||||
prepared: Vec::new(),
|
||||
unknown: false,
|
||||
};
|
||||
for candidate in candidates {
|
||||
let mut bytes = None;
|
||||
let mut prepared = false;
|
||||
match candidate {
|
||||
Ok(crate::disk::local::native_prepare::NativePreparationProbe::Absent) => {}
|
||||
Ok(crate::disk::local::native_prepare::NativePreparationProbe::Existing {
|
||||
version,
|
||||
prepared: ready,
|
||||
object_digest,
|
||||
object_length,
|
||||
set_digest,
|
||||
distribution_digest,
|
||||
metadata,
|
||||
}) => {
|
||||
let identity = Identity {
|
||||
version,
|
||||
digest: object_digest,
|
||||
length: object_length,
|
||||
set: set_digest,
|
||||
distribution: distribution_digest,
|
||||
};
|
||||
if observed.identity.as_ref().is_some_and(|old| old != &identity) {
|
||||
return Err(Error::other("conflicting native preparation identity"));
|
||||
}
|
||||
observed.identity = Some(identity);
|
||||
bytes = metadata;
|
||||
prepared = ready;
|
||||
}
|
||||
Err(crate::disk::error::Error::Io(error)) if error.kind() == std::io::ErrorKind::InvalidData => {
|
||||
return Err(Error::from(error));
|
||||
}
|
||||
Err(_) => observed.unknown = true,
|
||||
}
|
||||
observed.metadata.push(bytes);
|
||||
observed.prepared.push(prepared);
|
||||
}
|
||||
pools.push(observed);
|
||||
}
|
||||
}
|
||||
Ok(pools)
|
||||
}
|
||||
fn existing_native_pool(observations: &[Observation]) -> Result<Option<usize>> {
|
||||
let existing: Vec<_> = observations
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, p)| p.identity.is_some())
|
||||
.map(|(i, _)| i)
|
||||
.collect();
|
||||
if existing.len() > 1 {
|
||||
return Err(Error::other("native preparation exists in multiple pools"));
|
||||
}
|
||||
let selected = existing.first().copied();
|
||||
if observations.len() > 1 && observations.iter().enumerate().any(|(i, p)| p.unknown && Some(i) != selected) {
|
||||
return Err(Error::other("native preparation pool attribution is unknown"));
|
||||
}
|
||||
Ok(selected)
|
||||
}
|
||||
pub async fn prepare_native_object(
|
||||
&self,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
preparation: Uuid,
|
||||
data: &mut PutObjReader,
|
||||
) -> Result<NativeQuorumPreparation> {
|
||||
use sha2::{Digest, Sha256};
|
||||
use tokio::io::AsyncReadExt;
|
||||
check_put_object_args(bucket, key)?;
|
||||
if preparation.is_nil() || self.pools.is_empty() || data.size() < -1 {
|
||||
return Err(Error::other("unsupported native preparation"));
|
||||
}
|
||||
let mut remote = false;
|
||||
for pool in &self.pools {
|
||||
if pool
|
||||
.get_disks_by_key(key)
|
||||
.disks
|
||||
.read()
|
||||
.await
|
||||
.iter()
|
||||
.flatten()
|
||||
.any(|disk| !disk.is_local())
|
||||
{
|
||||
remote = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if remote || data.size() < 0 || data.size() > 1024 * 1024 {
|
||||
let fence = self.object_lock_config_snapshot(bucket).await?;
|
||||
let incarnation = self.bucket_incarnation_id(bucket).await?;
|
||||
let owner = std::sync::Arc::new(
|
||||
self.handle_new_ns_lock(bucket, key)
|
||||
.await?
|
||||
.get_write_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await?,
|
||||
);
|
||||
let observations = self
|
||||
.native_observations_scan(bucket, key, preparation, incarnation, fence.clone(), true)
|
||||
.await?;
|
||||
if let Some(selected) = Self::existing_native_pool(&observations)? {
|
||||
let identity = observations[selected]
|
||||
.identity
|
||||
.as_ref()
|
||||
.ok_or_else(|| Error::other("missing native identity"))?;
|
||||
let mut hash = Sha256::new();
|
||||
let mut length = 0_u64;
|
||||
let mut buffer = vec![0_u8; 64 * 1024];
|
||||
loop {
|
||||
let size = data.stream.read(&mut buffer).await?;
|
||||
if size == 0 {
|
||||
break;
|
||||
}
|
||||
length = length
|
||||
.checked_add(u64::try_from(size).map_err(Error::other)?)
|
||||
.ok_or_else(|| Error::other("native retry length overflow"))?;
|
||||
hash.update(&buffer[..size]);
|
||||
}
|
||||
if hash.finalize().as_slice() != identity.digest || length != identity.length {
|
||||
return Err(Error::other("native streaming retry content conflict"));
|
||||
}
|
||||
drop(owner);
|
||||
return self.recover_native_object(bucket, key, preparation).await;
|
||||
}
|
||||
let selected = if self.pools.len() == 1 {
|
||||
0
|
||||
} else {
|
||||
self.get_pool_idx_no_lock(bucket, key, data.size().max(0)).await?
|
||||
};
|
||||
let set = self.pools[selected].get_disks_by_key(key);
|
||||
let (quorum, receipts) =
|
||||
Box::pin(set.prepare_native_stream(self.id, bucket, key, preparation, incarnation, data, fence, owner)).await?;
|
||||
return NativeQuorumPreparation::from_receipts(quorum, receipts);
|
||||
}
|
||||
let length = data.size();
|
||||
let mut body = Vec::new();
|
||||
(&mut data.stream).take(1024 * 1024 + 1).read_to_end(&mut body).await?;
|
||||
if body.len() as i64 != length {
|
||||
return Err(Error::other("native reader length mismatch"));
|
||||
}
|
||||
let digest: [u8; 32] = Sha256::digest(&body).into();
|
||||
let fence = self.object_lock_config_snapshot(bucket).await?;
|
||||
let incarnation = self.bucket_incarnation_id(bucket).await?;
|
||||
let owner = std::sync::Arc::new(
|
||||
self.handle_new_ns_lock(bucket, key)
|
||||
.await?
|
||||
.get_write_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await?,
|
||||
);
|
||||
let mut observations = self
|
||||
.native_observations(bucket, key, preparation, incarnation, fence.clone())
|
||||
.await?;
|
||||
let selected = match Self::existing_native_pool(&observations)? {
|
||||
Some(index) => index,
|
||||
None if observations.len() == 1 => 0,
|
||||
None => self.get_pool_idx_no_lock(bucket, key, length).await?,
|
||||
};
|
||||
let observed = observations.remove(selected);
|
||||
if observed
|
||||
.identity
|
||||
.as_ref()
|
||||
.is_some_and(|identity| identity.digest != digest || identity.length != body.len() as u64)
|
||||
{
|
||||
return Err(Error::other("native preparation content conflict"));
|
||||
}
|
||||
let (quorum, receipts) = self.pools[selected]
|
||||
.get_disks_by_key(key)
|
||||
.prepare_native_bytes(
|
||||
self.id,
|
||||
bucket,
|
||||
key,
|
||||
preparation,
|
||||
incarnation,
|
||||
body.into(),
|
||||
fence,
|
||||
observed.metadata,
|
||||
owner,
|
||||
)
|
||||
.await?;
|
||||
NativeQuorumPreparation::from_receipts(quorum, receipts)
|
||||
}
|
||||
pub async fn recover_native_object(&self, bucket: &str, key: &str, preparation: Uuid) -> Result<NativeQuorumPreparation> {
|
||||
check_put_object_args(bucket, key)?;
|
||||
if preparation.is_nil() || self.pools.is_empty() {
|
||||
return Err(Error::other("invalid native preparation"));
|
||||
}
|
||||
let _owner = self
|
||||
.handle_new_ns_lock(bucket, key)
|
||||
.await?
|
||||
.get_write_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await?;
|
||||
self.recover_native_object_unlocked(bucket, key, preparation)
|
||||
.await
|
||||
.map(|(owner, _)| owner)
|
||||
}
|
||||
/// Caller owns the distributed namespace lock; each disk still acquires its actual physical recovery owners.
|
||||
pub(crate) async fn recover_native_object_unlocked(
|
||||
&self,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
preparation: Uuid,
|
||||
) -> Result<(NativeQuorumPreparation, Vec<(Uuid, bytes::Bytes)>)> {
|
||||
check_put_object_args(bucket, key)?;
|
||||
if preparation.is_nil() || self.pools.is_empty() {
|
||||
return Err(Error::other("invalid native preparation"));
|
||||
}
|
||||
let fence = self.object_lock_config_snapshot(bucket).await?;
|
||||
let incarnation = self.bucket_incarnation_id(bucket).await?;
|
||||
let observations = self
|
||||
.native_observations_scan(bucket, key, preparation, incarnation, fence.clone(), true)
|
||||
.await?;
|
||||
let selected = Self::existing_native_pool(&observations)?.ok_or_else(|| Error::other("native preparation absent"))?;
|
||||
let (quorum, receipts, metadata) = observations[selected]
|
||||
.set
|
||||
.recover_native_bytes(
|
||||
self.id,
|
||||
bucket,
|
||||
key,
|
||||
preparation,
|
||||
incarnation,
|
||||
fence,
|
||||
observations[selected].prepared.clone(),
|
||||
)
|
||||
.await?;
|
||||
Ok((NativeQuorumPreparation::from_receipts(quorum, receipts)?, metadata))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, unix))]
|
||||
mod target_owner_tests {
|
||||
use super::*;
|
||||
use crate::{
|
||||
bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks},
|
||||
disk::DiskAPI,
|
||||
object_api::PutObjReader,
|
||||
storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions},
|
||||
};
|
||||
use uuid::Uuid;
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn target_owner_apis_use_fresh_local_identity_and_exact_disk() {
|
||||
let (dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = "native-target-owner";
|
||||
let key = "object.bin";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create target bucket");
|
||||
|
||||
let preparation = Uuid::new_v4();
|
||||
let body = b"target-owner facade identity";
|
||||
let prepared = store
|
||||
.prepare_native_object(bucket, key, preparation, &mut PutObjReader::from_vec(body.to_vec()))
|
||||
.await
|
||||
.expect("prepare native object");
|
||||
let disk = store.pools[0].get_disks_by_key(key).disks.read().await[0]
|
||||
.clone()
|
||||
.expect("first local disk");
|
||||
let disk_id = disk.get_disk_id().await.expect("disk identity").expect("formatted disk");
|
||||
assert!(prepared.disk_ids().any(|id| id == disk_id));
|
||||
let target = NativePreparationTarget {
|
||||
store_id: store.id,
|
||||
disk_id,
|
||||
bucket: bucket.into(),
|
||||
key: key.into(),
|
||||
preparation,
|
||||
bucket_incarnation: store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("durable bucket incarnation"),
|
||||
};
|
||||
|
||||
let metadata_path = dirs[0]
|
||||
.path()
|
||||
.join(".rustfs.sys/native-preparations")
|
||||
.join(preparation.to_string())
|
||||
.join("xl.meta");
|
||||
let metadata = Bytes::from(std::fs::read(metadata_path).expect("persisted shard metadata"));
|
||||
let unrelated_format = dirs[1].path().join(".rustfs.sys/format.json");
|
||||
let saved_unrelated_format = std::fs::read(&unrelated_format).expect("unrelated disk format");
|
||||
std::fs::write(&unrelated_format, b"invalid format").expect("damage unrelated disk format");
|
||||
let caller_owner = store
|
||||
.handle_new_ns_lock(bucket, key)
|
||||
.await
|
||||
.expect("caller namespace lock")
|
||||
.get_write_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await
|
||||
.expect("caller owns namespace lock");
|
||||
let receipt = store
|
||||
.finalize_native_preparation_on_disk(NativePreparationFinalizeRequest {
|
||||
target: target.clone(),
|
||||
version: preparation,
|
||||
object_digest: prepared.object_digest(),
|
||||
metadata: metadata.clone(),
|
||||
})
|
||||
.await
|
||||
.expect("target-owner idempotent finalize");
|
||||
assert_eq!(receipt.disk_id(), disk_id);
|
||||
assert_eq!(receipt.object_digest(), &prepared.object_digest());
|
||||
|
||||
assert!(matches!(
|
||||
tokio::time::timeout(
|
||||
std::time::Duration::from_secs(2),
|
||||
store.probe_native_preparation_on_disk(&target),
|
||||
)
|
||||
.await
|
||||
.expect("target-owner must not reacquire the caller's non-reentrant lock")
|
||||
.expect("target-owner probe"),
|
||||
NativePreparationProbe::Existing { prepared: true, metadata: Some(found), .. } if found == metadata
|
||||
));
|
||||
let recovered = store
|
||||
.recover_native_preparation_on_disk(&target)
|
||||
.await
|
||||
.expect("target-owner recovery");
|
||||
assert_eq!(recovered.receipt.disk_id(), disk_id);
|
||||
assert_eq!(recovered.metadata, metadata);
|
||||
|
||||
let mut wrong_incarnation = target.clone();
|
||||
wrong_incarnation.bucket_incarnation = Uuid::new_v4();
|
||||
assert!(store.begin_native_preparation_on_disk(&wrong_incarnation).await.is_err());
|
||||
let mut wrong_disk = target;
|
||||
wrong_disk.disk_id = Uuid::new_v4();
|
||||
assert!(store.begin_native_preparation_on_disk(&wrong_disk).await.is_err());
|
||||
drop(caller_owner);
|
||||
std::fs::write(unrelated_format, saved_unrelated_format).expect("restore unrelated disk format");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,278 @@
|
||||
//! Real private preparation placement across two physical local pools.
|
||||
use super::ECStore;
|
||||
use crate::{
|
||||
bucket::metadata_sys::init_bucket_metadata_sys,
|
||||
disk::endpoint::Endpoint,
|
||||
layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
|
||||
object_api::PutObjReader,
|
||||
runtime::instance::InstanceContext,
|
||||
storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions},
|
||||
store::init_local_disks_with_instance_ctx,
|
||||
};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::sync::Arc;
|
||||
use tempfile::TempDir;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use uuid::Uuid;
|
||||
|
||||
const BUCKET: &str = "native-multi-pool-preparation";
|
||||
const KEY: &str = "private/placement.bin";
|
||||
const CONTENT: &[u8] = b"actual two-pool reader bytes\0remain bound to durable placement";
|
||||
const DRIVES: usize = 4;
|
||||
|
||||
async fn reopen(dirs: &[TempDir]) -> Arc<ECStore> {
|
||||
assert_eq!(dirs.len(), 2 * DRIVES);
|
||||
let pools = (0..2)
|
||||
.map(|pool| {
|
||||
let endpoints = (0..DRIVES)
|
||||
.map(|slot| {
|
||||
let mut endpoint = Endpoint::try_from(dirs[pool * DRIVES + slot].path().to_str().expect("local path"))
|
||||
.expect("actual local endpoint");
|
||||
endpoint.set_pool_index(pool);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(slot);
|
||||
endpoint
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: DRIVES,
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: format!("native-multi-pool-{pool}"),
|
||||
platform: "test".into(),
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let pools = EndpointServerPools(pools);
|
||||
let context = Arc::new(InstanceContext::new());
|
||||
init_local_disks_with_instance_ctx(&context, pools.clone())
|
||||
.await
|
||||
.expect("initialize real formats for both pools");
|
||||
let store =
|
||||
ECStore::new_with_instance_ctx("127.0.0.1:0".parse().expect("local address"), pools, CancellationToken::new(), context)
|
||||
.await
|
||||
.expect("open actual two-pool store");
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
store
|
||||
}
|
||||
|
||||
fn private_path(dir: &TempDir, preparation: Uuid) -> std::path::PathBuf {
|
||||
dir.path()
|
||||
.join(".rustfs.sys/native-preparations")
|
||||
.join(preparation.to_string())
|
||||
}
|
||||
|
||||
async fn create_bucket(store: &Arc<ECStore>) {
|
||||
let store = store.clone();
|
||||
tokio::spawn(async move { store.make_bucket(BUCKET, &MakeBucketOptions::default()).await })
|
||||
.await
|
||||
.expect("join actual bucket creation")
|
||||
.expect("create actual bucket");
|
||||
}
|
||||
|
||||
fn assert_only_pool(dirs: &[TempDir], preparation: Uuid, pool: usize) {
|
||||
let other = 1 - pool;
|
||||
for dir in &dirs[other * DRIVES..(other + 1) * DRIVES] {
|
||||
assert!(
|
||||
!private_path(dir, preparation).exists(),
|
||||
"NP02_POOL_PLACEMENT_BOUND: retry must not create private facts in another pool"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async fn await_pending_owners(store: &ECStore) {
|
||||
// The preparation's parent owner outlives its early Wdata return. Acquire the
|
||||
// same real object lock before changing files in this test's fixture.
|
||||
let lock = store.handle_new_ns_lock(BUCKET, KEY).await.expect("actual object lock");
|
||||
let owner = lock
|
||||
.get_write_lock(std::time::Duration::from_secs(30))
|
||||
.await
|
||||
.expect("pending disk owners finish before fixture mutation");
|
||||
drop(owner);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn native_object_prepare_multi_pool_restart_pending_and_unknown_preserve_placement() {
|
||||
let dirs = (0..2 * DRIVES)
|
||||
.map(|_| tempfile::tempdir().expect("physical local disk"))
|
||||
.collect::<Vec<_>>();
|
||||
let store = reopen(&dirs).await;
|
||||
create_bucket(&store).await;
|
||||
let preparation = Uuid::new_v4();
|
||||
let prepared = store
|
||||
.prepare_native_object(BUCKET, KEY, preparation, &mut PutObjReader::from_vec(CONTENT.to_vec()))
|
||||
.await
|
||||
.expect("NP02_POOL_PLACEMENT_BOUND: two-pool native preparation is supported");
|
||||
let version = prepared.version_id();
|
||||
assert_eq!(prepared.object_digest(), <[u8; 32]>::from(Sha256::digest(CONTENT)));
|
||||
let present = (0..2)
|
||||
.filter(|pool| {
|
||||
dirs[pool * DRIVES..(pool + 1) * DRIVES]
|
||||
.iter()
|
||||
.any(|dir| private_path(dir, preparation).exists())
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(present.len(), 1, "NP02_POOL_PLACEMENT_BOUND: one actual durable pool");
|
||||
let pool = present[0];
|
||||
assert_only_pool(&dirs, preparation, pool);
|
||||
drop(prepared);
|
||||
await_pending_owners(&store).await;
|
||||
// Recovery has to rediscover private placement from physical files in a fresh context.
|
||||
let reopened = reopen(&dirs).await;
|
||||
let recovered = reopened
|
||||
.recover_native_object(BUCKET, KEY, preparation)
|
||||
.await
|
||||
.expect("NP02_POOL_PLACEMENT_BOUND: restart recovers original private pool");
|
||||
assert_eq!(recovered.version_id(), version, "NP02_POOL_PLACEMENT_BOUND");
|
||||
assert_eq!(recovered.object_digest(), <[u8; 32]>::from(Sha256::digest(CONTENT)));
|
||||
drop(recovered);
|
||||
await_pending_owners(&reopened).await;
|
||||
let marker = dirs[pool * DRIVES..(pool + 1) * DRIVES]
|
||||
.iter()
|
||||
.map(|dir| private_path(dir, preparation).join("PREPARED"))
|
||||
.find(|path| path.exists())
|
||||
.expect("actual persisted acknowledgment");
|
||||
let original_marker = std::fs::read(&marker).expect("retain original marker bytes");
|
||||
std::fs::write(&marker, b"corrupt existing acknowledgment").expect("corrupt own actual marker");
|
||||
let corrupt = reopened
|
||||
.prepare_native_object(BUCKET, KEY, preparation, &mut PutObjReader::from_vec(CONTENT.to_vec()))
|
||||
.await;
|
||||
assert!(
|
||||
corrupt.is_err(),
|
||||
"NP02_POOL_PLACEMENT_BOUND: a corrupted existing marker is not absent placement"
|
||||
);
|
||||
assert_only_pool(&dirs, preparation, pool);
|
||||
await_pending_owners(&reopened).await;
|
||||
std::fs::write(&marker, original_marker).expect("restore original actual marker");
|
||||
// Keep durable intents while removing acknowledgments and xl.meta.
|
||||
// Retry must restore each writer's original raw bytes in the same pool.
|
||||
let mut original_metadata = Vec::new();
|
||||
for dir in &dirs[pool * DRIVES..(pool + 1) * DRIVES] {
|
||||
let marker = private_path(dir, preparation).join("PREPARED");
|
||||
if marker.exists() {
|
||||
std::fs::remove_file(marker).expect("remove own fixture acknowledgment");
|
||||
}
|
||||
let metadata = private_path(dir, preparation).join("xl.meta");
|
||||
if metadata.exists() {
|
||||
let bytes = std::fs::read(&metadata).expect("retain original raw metadata");
|
||||
std::fs::remove_file(&metadata).expect("leave only original durable intent");
|
||||
original_metadata.push((metadata, bytes));
|
||||
}
|
||||
}
|
||||
let repaired = reopened
|
||||
.prepare_native_object(BUCKET, KEY, preparation, &mut PutObjReader::from_vec(CONTENT.to_vec()))
|
||||
.await
|
||||
.expect("NP02_POOL_PLACEMENT_BOUND: pending intents repair in their existing pool");
|
||||
assert_eq!(repaired.version_id(), version, "NP02_POOL_PLACEMENT_BOUND");
|
||||
drop(repaired);
|
||||
await_pending_owners(&reopened).await;
|
||||
assert!(
|
||||
original_metadata.len() >= 3,
|
||||
"NP02_POOL_PLACEMENT_BOUND: retain actual write quorum metadata"
|
||||
);
|
||||
for (path, bytes) in original_metadata {
|
||||
assert_eq!(
|
||||
std::fs::read(path).expect("repaired actual metadata file"),
|
||||
bytes,
|
||||
"NP02_POOL_PLACEMENT_BOUND: top-level retry restores original raw bytes from intent"
|
||||
);
|
||||
}
|
||||
assert_only_pool(&dirs, preparation, pool);
|
||||
// Preserve each original directory while making actual filesystem inspection fail.
|
||||
// Public object absence must not turn this unknown private outcome into a new placement.
|
||||
for dir in &dirs[pool * DRIVES..(pool + 1) * DRIVES] {
|
||||
let root = dir.path().join(".rustfs.sys/native-preparations");
|
||||
std::fs::rename(&root, dir.path().join(".rustfs.sys/native-preparations-preserved"))
|
||||
.expect("retain original private facts");
|
||||
std::fs::write(root, b"actual namespace obstruction").expect("install real I/O failure");
|
||||
}
|
||||
let retry = reopened
|
||||
.prepare_native_object(BUCKET, KEY, preparation, &mut PutObjReader::from_vec(CONTENT.to_vec()))
|
||||
.await;
|
||||
assert!(
|
||||
retry.is_err(),
|
||||
"NP02_POOL_PLACEMENT_BOUND: unknown original pool cannot migrate to healthy pool"
|
||||
);
|
||||
assert_only_pool(&dirs, preparation, pool);
|
||||
}
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn native_object_prepare_coherent_replay_rejects_changed_block_size() {
|
||||
let (dirs, store) = Box::pin(crate::bucket::metadata_sys::test_support::isolated_store_over_temp_disks()).await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
create_bucket(&store).await;
|
||||
let preparation = Uuid::new_v4();
|
||||
let prepared = store
|
||||
.prepare_native_object(BUCKET, KEY, preparation, &mut PutObjReader::from_vec(CONTENT.to_vec()))
|
||||
.await
|
||||
.expect("original real native preparation");
|
||||
drop(prepared);
|
||||
await_pending_owners(&store).await;
|
||||
let private = private_path(&dirs[0], preparation);
|
||||
let original = std::fs::read(private.join("xl.meta")).expect("original actual writer metadata");
|
||||
let mut info = rustfs_filemeta::get_file_info(
|
||||
&original,
|
||||
BUCKET,
|
||||
KEY,
|
||||
&preparation.to_string(),
|
||||
rustfs_filemeta::FileInfoOpts {
|
||||
data: true,
|
||||
include_free_versions: false,
|
||||
include_part_checksums: true,
|
||||
},
|
||||
)
|
||||
.expect("strictly decode real writer metadata");
|
||||
let original_block_size = info.erasure.block_size;
|
||||
info.erasure.block_size /= 2;
|
||||
assert!(info.erasure.block_size > CONTENT.len());
|
||||
assert_ne!(info.erasure.block_size, original_block_size);
|
||||
let mut meta = rustfs_filemeta::FileMeta::new();
|
||||
meta.add_version(info).expect("real metadata writer for changed geometry");
|
||||
let changed = meta.marshal_msg().expect("encode actual changed raw metadata");
|
||||
let mut encoded = std::fs::read_to_string(private.join("intent.json")).expect("actual canonical durable intent");
|
||||
let original_intent: serde_json::Value = serde_json::from_str(&encoded).expect("decode actual intent");
|
||||
// Preserve the original canonical field order. Change only the three raw-byte
|
||||
// bindings, so this is a coherent candidate rather than a checksum mismatch.
|
||||
for (field, value) in [
|
||||
("metadata_digest", serde_json::json!(Sha256::digest(&changed).to_vec())),
|
||||
("metadata_length", serde_json::json!(changed.len())),
|
||||
("recovery_metadata", serde_json::json!(base64_simd::STANDARD.encode_to_string(&changed))),
|
||||
] {
|
||||
let old = format!("\"{field}\":{}", serde_json::to_string(&original_intent[field]).expect("original field"));
|
||||
let new = format!("\"{field}\":{}", serde_json::to_string(&value).expect("changed field"));
|
||||
assert_eq!(encoded.matches(&old).count(), 1, "only one original canonical binding");
|
||||
encoded = encoded.replacen(&old, &new, 1);
|
||||
}
|
||||
std::fs::write(private.join("xl.meta"), &changed).expect("replace own actual shard metadata");
|
||||
std::fs::write(private.join("intent.json"), encoded.as_bytes()).expect("retain coherent actual intent");
|
||||
std::fs::write(private.join("PREPARED"), Sha256::digest(encoded.as_bytes())).expect("coherent existing marker");
|
||||
let set = store.pools[0].get_disks_by_key(KEY);
|
||||
let disk = set.disks.read().await[0].clone().expect("actual first physical disk");
|
||||
let crate::disk::Disk::Local(local) = disk.as_ref() else {
|
||||
panic!("local fixture disk");
|
||||
};
|
||||
let fence = store
|
||||
.object_lock_config_snapshot(BUCKET)
|
||||
.await
|
||||
.expect("fresh real lifecycle fence");
|
||||
let incarnation = store.bucket_incarnation_id(BUCKET).await.expect("actual bucket incarnation");
|
||||
local
|
||||
.get_disk()
|
||||
.recover_native(preparation, store.id, BUCKET, incarnation, fence)
|
||||
.await
|
||||
.expect("NP02_GEOMETRY_IDENTITY_BOUND: changed short shard remains a valid real NP01 candidate");
|
||||
assert!(
|
||||
store
|
||||
.prepare_native_object(BUCKET, KEY, preparation, &mut PutObjReader::from_vec(CONTENT.to_vec()))
|
||||
.await
|
||||
.is_err(),
|
||||
"NP02_GEOMETRY_IDENTITY_BOUND: top-level retry rejects a different actual block geometry"
|
||||
);
|
||||
await_pending_owners(&store).await;
|
||||
assert!(
|
||||
store.recover_native_object(BUCKET, KEY, preparation).await.is_err(),
|
||||
"NP02_GEOMETRY_IDENTITY_BOUND: top-level recovery rejects mixed actual block geometry"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,981 @@
|
||||
use super::ECStore;
|
||||
use crate::{
|
||||
bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks},
|
||||
disk::endpoint::Endpoint,
|
||||
layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
|
||||
object_api::{ObjectOptions, PutObjReader},
|
||||
runtime::instance::InstanceContext,
|
||||
storage_api_contracts::{
|
||||
bucket::{BucketOperations as _, MakeBucketOptions},
|
||||
object::ObjectOperations as _,
|
||||
},
|
||||
store::init_local_disks_with_instance_ctx,
|
||||
};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::{collections::HashSet, sync::Arc};
|
||||
use tempfile::TempDir;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use uuid::Uuid;
|
||||
|
||||
const BUCKET: &str = "native-quorum-preparation";
|
||||
const KEY: &str = "nested/object.bin";
|
||||
const CONTENT: &[u8] = b"native preparation content\0with an immutable identity";
|
||||
|
||||
async fn fixture() -> (Vec<TempDir>, Arc<ECStore>) {
|
||||
let (dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
store
|
||||
.make_bucket(BUCKET, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create actual four-disk bucket");
|
||||
(dirs, store)
|
||||
}
|
||||
|
||||
async fn multi_pool_fixture() -> (Vec<TempDir>, Arc<ECStore>) {
|
||||
let dirs = (0..8)
|
||||
.map(|_| tempfile::tempdir().expect("real local disk directory"))
|
||||
.collect::<Vec<_>>();
|
||||
let pools = EndpointServerPools(
|
||||
(0..2)
|
||||
.map(|pool_index| {
|
||||
let endpoints = (0..4)
|
||||
.map(|disk_index| {
|
||||
let dir = &dirs[pool_index * 4 + disk_index];
|
||||
let mut endpoint = Endpoint::try_from(dir.path().to_str().expect("disk path")).expect("endpoint");
|
||||
endpoint.set_pool_index(pool_index);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(disk_index);
|
||||
endpoint
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: 4,
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: "native-multi-pool-test".into(),
|
||||
platform: "test".into(),
|
||||
}
|
||||
})
|
||||
.collect(),
|
||||
);
|
||||
let instance = Arc::new(InstanceContext::new());
|
||||
init_local_disks_with_instance_ctx(&instance, pools.clone())
|
||||
.await
|
||||
.expect("initialize both physical pools");
|
||||
let store =
|
||||
ECStore::new_with_instance_ctx("127.0.0.1:0".parse().expect("test address"), pools, CancellationToken::new(), instance)
|
||||
.await
|
||||
.expect("initialize multi-pool store");
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
store
|
||||
.make_bucket(BUCKET, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket across both pools");
|
||||
(dirs, store)
|
||||
}
|
||||
|
||||
async fn prepare_stream_in_pool(store: &Arc<ECStore>, pool_index: usize, preparation: Uuid, body: Vec<u8>) {
|
||||
let incarnation = store.bucket_incarnation_id(BUCKET).await.expect("bucket incarnation");
|
||||
let fence = store.object_lock_config_snapshot(BUCKET).await.expect("bucket fence");
|
||||
let owner = Arc::new(
|
||||
store
|
||||
.handle_new_ns_lock(BUCKET, KEY)
|
||||
.await
|
||||
.expect("object namespace lock")
|
||||
.get_write_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await
|
||||
.expect("object write lock"),
|
||||
);
|
||||
let mut reader = PutObjReader::from_vec(body);
|
||||
let (quorum, receipts) = store.pools[pool_index]
|
||||
.get_disks_by_key(KEY)
|
||||
.prepare_native_stream(store.id, BUCKET, KEY, preparation, incarnation, &mut reader, fence, owner)
|
||||
.await
|
||||
.expect("prepare on the specified physical pool");
|
||||
assert!(receipts.len() >= quorum, "NATIVE_CROSS_POOL_PREPARE_REQUIRES_WRITE_QUORUM");
|
||||
}
|
||||
|
||||
async fn assert_not_published(store: &ECStore) {
|
||||
let result = store.get_object_info(BUCKET, KEY, &ObjectOptions::default()).await;
|
||||
assert!(
|
||||
matches!(result, Err(ref error) if crate::error::is_err_object_not_found(error)),
|
||||
"NP02_PREPARATION_NOT_AUTHORITY: native preparation must remain invisible to ordinary object HEAD"
|
||||
);
|
||||
}
|
||||
|
||||
fn fail_preparation_disk(dir: &TempDir) {
|
||||
// Keep the physical disk and its format present; only native preparation I/O fails.
|
||||
std::fs::write(dir.path().join(".rustfs.sys/native-preparations"), b"not a directory")
|
||||
.expect("install real per-disk namespace obstruction");
|
||||
}
|
||||
|
||||
async fn reopen(dirs: &[TempDir]) -> Arc<ECStore> {
|
||||
let endpoints = dirs
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, dir)| {
|
||||
let mut endpoint = Endpoint::try_from(dir.path().to_str().expect("disk path")).expect("existing local endpoint");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(index / 4);
|
||||
endpoint.set_disk_index(index % 4);
|
||||
endpoint
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let pools = EndpointServerPools(vec![PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: dirs.len() / 4,
|
||||
drives_per_set: 4,
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: "native-quorum-reopen".into(),
|
||||
platform: "test".into(),
|
||||
}]);
|
||||
let context = Arc::new(InstanceContext::new());
|
||||
init_local_disks_with_instance_ctx(&context, pools.clone())
|
||||
.await
|
||||
.expect("reopen existing disk formats");
|
||||
let store =
|
||||
ECStore::new_with_instance_ctx("127.0.0.1:0".parse().expect("local address"), pools, CancellationToken::new(), context)
|
||||
.await
|
||||
.expect("restart store over the same physical disks");
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
store
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn native_object_prepare_one_disk_failure_retains_write_quorum() {
|
||||
let (dirs, store) = fixture().await;
|
||||
fail_preparation_disk(&dirs[0]);
|
||||
let preparation = Uuid::new_v4();
|
||||
let owner = store
|
||||
.prepare_native_object(BUCKET, KEY, preparation, &mut PutObjReader::from_vec(CONTENT.to_vec()))
|
||||
.await;
|
||||
assert!(
|
||||
owner.is_ok(),
|
||||
"NP02_WDATA_BOUND: N4/W3 must tolerate one actual disk I/O failure: {:?}",
|
||||
owner.as_ref().err()
|
||||
);
|
||||
let owner = owner.expect("checked preparation result");
|
||||
assert_eq!(owner.preparation_id(), preparation, "NP02_WDATA_BOUND");
|
||||
assert_eq!(
|
||||
owner.object_digest(),
|
||||
<[u8; 32]>::from(Sha256::digest(CONTENT)),
|
||||
"NP02_OPERATION_IDENTITY_BOUND: digest computed from actual reader bytes"
|
||||
);
|
||||
assert_eq!(owner.receipt_count(), 3, "NP02_WDATA_BOUND: three real durable receipts");
|
||||
assert_eq!(
|
||||
owner.disk_ids().collect::<HashSet<_>>().len(),
|
||||
3,
|
||||
"NP02_WDATA_BOUND: distinct physical disks"
|
||||
);
|
||||
assert_eq!(owner.object_length(), u64::try_from(CONTENT.len()).expect("small content"));
|
||||
assert!(!owner.version_id().is_nil(), "NP02_WDATA_BOUND: real immutable version");
|
||||
drop(owner);
|
||||
assert_not_published(&store).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn native_object_prepare_below_write_quorum_has_no_authority() {
|
||||
let (dirs, store) = fixture().await;
|
||||
fail_preparation_disk(&dirs[0]);
|
||||
fail_preparation_disk(&dirs[1]);
|
||||
let preparation = Uuid::new_v4();
|
||||
let prepared = store
|
||||
.prepare_native_object(BUCKET, KEY, preparation, &mut PutObjReader::from_vec(CONTENT.to_vec()))
|
||||
.await;
|
||||
assert!(
|
||||
prepared.is_err(),
|
||||
"NP02_PREPARATION_NOT_AUTHORITY: W-1 must not construct preparation authority"
|
||||
);
|
||||
drop(prepared);
|
||||
let recovered = store.recover_native_object(BUCKET, KEY, preparation).await;
|
||||
assert!(
|
||||
recovered.is_err(),
|
||||
"NP02_PREPARATION_NOT_AUTHORITY: partial durable facts are not quorum authority"
|
||||
);
|
||||
drop(recovered);
|
||||
assert_not_published(&store).await;
|
||||
// Recovery after a new store context must apply the same threshold to those facts.
|
||||
drop(store);
|
||||
let restarted = reopen(&dirs).await;
|
||||
assert!(
|
||||
restarted.recover_native_object(BUCKET, KEY, preparation).await.is_err(),
|
||||
"NP02_PREPARATION_NOT_AUTHORITY: restart cannot promote W-1 facts"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn native_object_prepare_restart_preserves_operation_and_content_identity() {
|
||||
let (dirs, store) = fixture().await;
|
||||
let preparation = Uuid::new_v4();
|
||||
let owner = store
|
||||
.prepare_native_object(BUCKET, KEY, preparation, &mut PutObjReader::from_vec(CONTENT.to_vec()))
|
||||
.await;
|
||||
assert!(
|
||||
owner.is_ok(),
|
||||
"NP02_OPERATION_IDENTITY_BOUND: prepare actual native object: {:?}",
|
||||
owner.as_ref().err()
|
||||
);
|
||||
let owner = owner.expect("checked preparation result");
|
||||
let expected = (owner.preparation_id(), owner.version_id(), owner.object_digest(), owner.object_length());
|
||||
assert!(owner.receipt_count() >= 3, "NP02_WDATA_BOUND: durable write quorum");
|
||||
assert_eq!(
|
||||
owner.disk_ids().collect::<HashSet<_>>().len(),
|
||||
owner.receipt_count(),
|
||||
"NP02_WDATA_BOUND: independent disk facts"
|
||||
);
|
||||
drop(owner);
|
||||
drop(store);
|
||||
let restarted = reopen(&dirs).await;
|
||||
let recovered = restarted.recover_native_object(BUCKET, KEY, preparation).await;
|
||||
assert!(
|
||||
recovered.is_ok(),
|
||||
"NP02_OPERATION_IDENTITY_BOUND: recover real disk records in new store context"
|
||||
);
|
||||
let recovered = recovered.expect("checked recovery result");
|
||||
assert_eq!(
|
||||
(
|
||||
recovered.preparation_id(),
|
||||
recovered.version_id(),
|
||||
recovered.object_digest(),
|
||||
recovered.object_length()
|
||||
),
|
||||
expected,
|
||||
"NP02_OPERATION_IDENTITY_BOUND: recovery retains exact operation, version and content"
|
||||
);
|
||||
assert!(recovered.receipt_count() >= 3, "NP02_WDATA_BOUND: recovered write quorum");
|
||||
assert_eq!(
|
||||
recovered.disk_ids().collect::<HashSet<_>>().len(),
|
||||
recovered.receipt_count(),
|
||||
"NP02_WDATA_BOUND"
|
||||
);
|
||||
drop(recovered);
|
||||
let replay = restarted
|
||||
.prepare_native_object(BUCKET, KEY, preparation, &mut PutObjReader::from_vec(CONTENT.to_vec()))
|
||||
.await;
|
||||
assert!(replay.is_ok(), "NP02_OPERATION_IDENTITY_BOUND: retry original preparation and content");
|
||||
let replay = replay.expect("checked replay result");
|
||||
assert_eq!(
|
||||
(
|
||||
replay.preparation_id(),
|
||||
replay.version_id(),
|
||||
replay.object_digest(),
|
||||
replay.object_length()
|
||||
),
|
||||
expected,
|
||||
"NP02_OPERATION_IDENTITY_BOUND: retry of the local preparation must not invent a new version"
|
||||
);
|
||||
drop(replay);
|
||||
let mut changed_content = CONTENT.to_vec();
|
||||
changed_content[0] ^= 1;
|
||||
assert!(
|
||||
restarted
|
||||
.prepare_native_object(BUCKET, KEY, preparation, &mut PutObjReader::from_vec(changed_content))
|
||||
.await
|
||||
.is_err(),
|
||||
"NP02_OPERATION_IDENTITY_BOUND: a local preparation cannot be rebound to different content"
|
||||
);
|
||||
assert!(
|
||||
restarted
|
||||
.recover_native_object(BUCKET, "another-key", preparation)
|
||||
.await
|
||||
.is_err(),
|
||||
"NP02_OPERATION_IDENTITY_BOUND: recovery is bound to the original key"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn native_object_prepare_cancelled_writer_retains_pending_disk_owner() {
|
||||
let (dirs, store) = fixture().await;
|
||||
fail_preparation_disk(&dirs[0]);
|
||||
let preparation = Uuid::new_v4();
|
||||
let (entered, release) = crate::disk::local::native_prepare::pause_native_preparation(preparation, "PREPARED:file");
|
||||
// LocalDisk canonicalizes its root before constructing metadata lock keys.
|
||||
let owner_paths = dirs[1..]
|
||||
.iter()
|
||||
.map(|dir| {
|
||||
std::fs::canonicalize(dir.path())
|
||||
.expect("actual canonical disk root")
|
||||
.join(BUCKET)
|
||||
.join(KEY)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(owner_paths.len(), 3, "three healthy actual local disk owners");
|
||||
let writing_store = store.clone();
|
||||
let writer = tokio::spawn(async move {
|
||||
writing_store
|
||||
.prepare_native_object(BUCKET, KEY, preparation, &mut PutObjReader::from_vec(CONTENT.to_vec()))
|
||||
.await
|
||||
});
|
||||
let reached_io = tokio::task::spawn_blocking(move || entered.recv_timeout(std::time::Duration::from_secs(30)))
|
||||
.await
|
||||
.expect("observe actual disk barrier");
|
||||
// Release before asserting so a failed test never strands the real blocking I/O.
|
||||
if reached_io.is_err() {
|
||||
let _ = release.send(());
|
||||
writer.abort();
|
||||
let _ = writer.await;
|
||||
crate::disk::local::native_prepare::clear_native_preparation_probe(preparation);
|
||||
panic!("NP02_PENDING_OWNER_BOUND: preparation must reach actual disk I/O");
|
||||
}
|
||||
writer.abort();
|
||||
let cancelled = matches!(writer.await, Err(error) if error.is_cancelled());
|
||||
let mut held_owners = 0;
|
||||
for path in &owner_paths {
|
||||
if tokio::time::timeout(
|
||||
std::time::Duration::from_millis(50),
|
||||
crate::disk::os::acquire_metadata_mutation_lease(path, None),
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
held_owners += 1;
|
||||
}
|
||||
}
|
||||
release.send(()).expect("release actual disk I/O after cancelled waiter");
|
||||
for path in &owner_paths {
|
||||
let owner = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(5),
|
||||
crate::disk::os::acquire_metadata_mutation_lease(path, None),
|
||||
)
|
||||
.await
|
||||
.expect("completed actual I/O releases metadata owner");
|
||||
drop(owner);
|
||||
}
|
||||
crate::disk::local::native_prepare::clear_native_preparation_probe(preparation);
|
||||
assert!(
|
||||
cancelled,
|
||||
"NP02_PENDING_OWNER_BOUND: writer remains pending below W3 until the actual disk barrier"
|
||||
);
|
||||
assert!(
|
||||
held_owners >= 1,
|
||||
"NP02_PENDING_OWNER_BOUND: cancellation cannot release an in-flight real disk owner"
|
||||
);
|
||||
assert_not_published(&store).await;
|
||||
drop(store);
|
||||
let restarted = reopen(&dirs).await;
|
||||
let recovered = restarted.recover_native_object(BUCKET, KEY, preparation).await;
|
||||
assert!(
|
||||
recovered.is_ok(),
|
||||
"NP02_PENDING_OWNER_BOUND: retain actual durable facts after cancelled waiter"
|
||||
);
|
||||
let recovered = recovered.expect("checked recovery result");
|
||||
let version = recovered.version_id();
|
||||
assert_eq!(recovered.preparation_id(), preparation, "NP02_OPERATION_IDENTITY_BOUND");
|
||||
assert_eq!(
|
||||
recovered.object_digest(),
|
||||
<[u8; 32]>::from(Sha256::digest(CONTENT)),
|
||||
"NP02_OPERATION_IDENTITY_BOUND"
|
||||
);
|
||||
assert_eq!(recovered.object_length(), u64::try_from(CONTENT.len()).expect("small content"));
|
||||
assert_eq!(recovered.receipt_count(), 3, "NP02_WDATA_BOUND");
|
||||
drop(recovered);
|
||||
let replay = restarted
|
||||
.prepare_native_object(BUCKET, KEY, preparation, &mut PutObjReader::from_vec(CONTENT.to_vec()))
|
||||
.await;
|
||||
assert!(
|
||||
replay.is_ok(),
|
||||
"NP02_OPERATION_IDENTITY_BOUND: retry cancelled operation from persisted facts"
|
||||
);
|
||||
assert_eq!(replay.expect("checked replay").version_id(), version, "NP02_OPERATION_IDENTITY_BOUND");
|
||||
assert_not_published(&restarted).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn native_object_prepare_nonzero_set_reopens_the_routed_identity() {
|
||||
let dirs = (0..8)
|
||||
.map(|_| tempfile::tempdir().expect("real local disk directory"))
|
||||
.collect::<Vec<_>>();
|
||||
let store = reopen(&dirs).await;
|
||||
store
|
||||
.make_bucket(BUCKET, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("real two-set bucket");
|
||||
let selected_key = (0..1024)
|
||||
.map(|index| format!("routed/{index}.bin"))
|
||||
.find(|key| store.pools[0].get_disks_by_key(key).set_index == 1)
|
||||
.expect("find a key routed by the actual algorithm to nonzero set");
|
||||
let other_key = (0..1024)
|
||||
.map(|index| format!("separate/{index}.bin"))
|
||||
.find(|key| store.pools[0].get_disks_by_key(key).set_index == 0)
|
||||
.expect("find another key routed to the other real set");
|
||||
let target_disks = store.pools[0].format.erasure.sets[1].iter().copied().collect::<HashSet<_>>();
|
||||
let preparation = Uuid::new_v4();
|
||||
let prepared = store
|
||||
.prepare_native_object(BUCKET, &selected_key, preparation, &mut PutObjReader::from_vec(CONTENT.to_vec()))
|
||||
.await;
|
||||
assert!(
|
||||
prepared.is_ok(),
|
||||
"NP02_OPERATION_IDENTITY_BOUND: nonzero routed set prepares real data: {:?}",
|
||||
prepared.as_ref().err()
|
||||
);
|
||||
let prepared = prepared.expect("checked nonzero set preparation");
|
||||
let expected = (
|
||||
prepared.preparation_id(),
|
||||
prepared.version_id(),
|
||||
prepared.object_digest(),
|
||||
prepared.object_length(),
|
||||
);
|
||||
assert!(
|
||||
prepared.receipt_count() >= 3,
|
||||
"NP02_WDATA_BOUND: quorum is scoped to the selected physical set"
|
||||
);
|
||||
let actual_disks = prepared.disk_ids().collect::<HashSet<_>>();
|
||||
assert_eq!(actual_disks.len(), prepared.receipt_count(), "NP02_WDATA_BOUND");
|
||||
assert!(
|
||||
actual_disks.is_subset(&target_disks),
|
||||
"NP02_OPERATION_IDENTITY_BOUND: receipts must come from routed set one"
|
||||
);
|
||||
drop(prepared);
|
||||
assert!(
|
||||
dirs[..4].iter().all(|dir| !dir
|
||||
.path()
|
||||
.join(".rustfs.sys/native-preparations")
|
||||
.join(preparation.to_string())
|
||||
.exists()),
|
||||
"NP02_OPERATION_IDENTITY_BOUND: preparation must not write unrelated set zero"
|
||||
);
|
||||
let mut other_content = CONTENT.to_vec();
|
||||
other_content[0] ^= 1;
|
||||
let other = store
|
||||
.prepare_native_object(BUCKET, &other_key, preparation, &mut PutObjReader::from_vec(other_content.clone()))
|
||||
.await;
|
||||
assert!(
|
||||
other.is_ok(),
|
||||
"NP02_OPERATION_IDENTITY_BOUND: preparation UUID is local set/key identity, not global operation authority"
|
||||
);
|
||||
drop(other);
|
||||
for key in [&selected_key, &other_key] {
|
||||
let head = store.get_object_info(BUCKET, key, &ObjectOptions::default()).await;
|
||||
assert!(
|
||||
matches!(head, Err(ref error) if crate::error::is_err_object_not_found(error)),
|
||||
"NP02_PREPARATION_NOT_AUTHORITY"
|
||||
);
|
||||
}
|
||||
drop(store);
|
||||
let restarted = reopen(&dirs).await;
|
||||
assert_eq!(
|
||||
restarted.pools[0].get_disks_by_key(&selected_key).set_index,
|
||||
1,
|
||||
"NP02_OPERATION_IDENTITY_BOUND: durable routing after reopen"
|
||||
);
|
||||
let recovered = restarted.recover_native_object(BUCKET, &selected_key, preparation).await;
|
||||
assert!(recovered.is_ok(), "NP02_OPERATION_IDENTITY_BOUND: reopen routed set preparation");
|
||||
let recovered = recovered.expect("checked routed recovery");
|
||||
assert_eq!(
|
||||
(
|
||||
recovered.preparation_id(),
|
||||
recovered.version_id(),
|
||||
recovered.object_digest(),
|
||||
recovered.object_length()
|
||||
),
|
||||
expected,
|
||||
"NP02_OPERATION_IDENTITY_BOUND: other set facts cannot replace selected key content"
|
||||
);
|
||||
assert!(
|
||||
recovered.disk_ids().collect::<HashSet<_>>().is_subset(&target_disks),
|
||||
"NP02_OPERATION_IDENTITY_BOUND"
|
||||
);
|
||||
drop(recovered);
|
||||
let other = restarted.recover_native_object(BUCKET, &other_key, preparation).await;
|
||||
assert!(
|
||||
other.is_ok(),
|
||||
"NP02_OPERATION_IDENTITY_BOUND: recover independent preparation in other set"
|
||||
);
|
||||
assert_eq!(
|
||||
other.expect("checked other set recovery").object_digest(),
|
||||
<[u8; 32]>::from(Sha256::digest(&other_content)),
|
||||
"NP02_OPERATION_IDENTITY_BOUND: select actual key/set facts"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn native_stream_prepare_large_restart() {
|
||||
let (dirs, store) = fixture().await;
|
||||
let body = (0..(5 * 1024 * 1024 + 37))
|
||||
.map(|index| u8::try_from(index % 251).expect("bounded byte"))
|
||||
.collect::<Vec<_>>();
|
||||
let expected: [u8; 32] = Sha256::digest(&body).into();
|
||||
let preparation = Uuid::new_v4();
|
||||
let prepared = store
|
||||
.prepare_native_object(BUCKET, KEY, preparation, &mut PutObjReader::from_vec(body.clone()))
|
||||
.await
|
||||
.expect("actual multi-block streaming preparation");
|
||||
assert_eq!(prepared.object_digest(), expected);
|
||||
assert_eq!(prepared.object_length(), u64::try_from(body.len()).expect("bounded content length"));
|
||||
assert!(prepared.receipt_count() >= 3);
|
||||
assert_not_published(&store).await;
|
||||
let mut records = Vec::new();
|
||||
for disk in &dirs {
|
||||
let path = disk
|
||||
.path()
|
||||
.join(".rustfs.sys/native-preparations")
|
||||
.join(preparation.to_string());
|
||||
let metadata = std::fs::read(path.join("xl.meta")).expect("exact streaming metadata");
|
||||
let record: serde_json::Value =
|
||||
serde_json::from_slice(&std::fs::read(path.join("intent.json")).expect("durable locator"))
|
||||
.expect("strict streaming intent JSON");
|
||||
assert_eq!(record["format"], 3);
|
||||
let disk_format: crate::layout::format::FormatV3 =
|
||||
serde_json::from_slice(&std::fs::read(disk.path().join(".rustfs.sys/format.json")).expect("actual disk format"))
|
||||
.expect("typed disk format");
|
||||
let disk_id: Uuid = serde_json::from_value(record["disk"].clone()).expect("record disk identity");
|
||||
let expected_set = disk_format
|
||||
.erasure
|
||||
.sets
|
||||
.iter()
|
||||
.find(|set| set.contains(&disk_id))
|
||||
.expect("disk belongs to one ordered set");
|
||||
let saved_set: Vec<Uuid> = serde_json::from_value(record["set_disks"].clone()).expect("persistent disk set");
|
||||
assert_eq!(&saved_set, expected_set, "persist the full ordered physical set");
|
||||
assert_eq!(record["data_dir"], preparation.to_string());
|
||||
assert!(
|
||||
std::fs::metadata(path.join(preparation.to_string()).join("part.1"))
|
||||
.expect("real non-inline shard")
|
||||
.len()
|
||||
> 0
|
||||
);
|
||||
records.push((path, metadata));
|
||||
}
|
||||
drop(prepared);
|
||||
drop(store);
|
||||
let restarted = reopen(&dirs).await;
|
||||
let recovered = restarted
|
||||
.recover_native_object(BUCKET, KEY, preparation)
|
||||
.await
|
||||
.expect("actual streaming reopen recovery");
|
||||
assert_eq!(recovered.object_digest(), expected);
|
||||
assert_eq!(recovered.object_length(), u64::try_from(body.len()).expect("length"));
|
||||
assert_eq!(recovered.version_id(), preparation);
|
||||
for (path, bytes) in records {
|
||||
assert_eq!(std::fs::read(path.join("xl.meta")).expect("preserved metadata"), bytes);
|
||||
}
|
||||
assert_not_published(&restarted).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn native_stream_prepare_locator_conflict() {
|
||||
let (dirs, store) = fixture().await;
|
||||
let preparation = Uuid::new_v4();
|
||||
let body = vec![17; 2 * 1024 * 1024 + 1];
|
||||
drop(
|
||||
store
|
||||
.prepare_native_object(BUCKET, KEY, preparation, &mut PutObjReader::from_vec(body))
|
||||
.await
|
||||
.expect("real durable streaming preparation"),
|
||||
);
|
||||
for disk in &dirs {
|
||||
let path = disk
|
||||
.path()
|
||||
.join(".rustfs.sys/native-preparations")
|
||||
.join(preparation.to_string());
|
||||
let mut record: serde_json::Value =
|
||||
serde_json::from_slice(&std::fs::read(path.join("intent.json")).expect("record")).expect("JSON");
|
||||
record["set_disks"]
|
||||
.as_array_mut()
|
||||
.expect("saved physical disk order")
|
||||
.swap(0, 1);
|
||||
// Preserve the actual record serialization order; receipt marker signs exact canonical record bytes.
|
||||
let bytes = crate::disk::local::native_prepare::canonical_test_record(&record);
|
||||
std::fs::write(path.join("intent.json"), &bytes).expect("install conflicting durable locator");
|
||||
std::fs::write(path.join("PREPARED"), Sha256::digest(&bytes)).expect("matching record marker");
|
||||
}
|
||||
assert!(
|
||||
store.recover_native_object(BUCKET, KEY, preparation).await.is_err(),
|
||||
"NATIVE_STREAM_REQUIRES_SAVED_PHYSICAL_SET"
|
||||
);
|
||||
assert_not_published(&store).await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn native_stream_prepare_locator_conflict_across_pools_and_unknown_disk() {
|
||||
std::thread::Builder::new()
|
||||
.stack_size(32 * 1024 * 1024)
|
||||
.spawn(|| {
|
||||
let runtime = tokio::runtime::Builder::new_multi_thread()
|
||||
.enable_all()
|
||||
.worker_threads(2)
|
||||
.thread_stack_size(32 * 1024 * 1024)
|
||||
.build()
|
||||
.expect("large-stack test runtime");
|
||||
runtime.block_on(async {
|
||||
use crate::disk::DiskAPI as _;
|
||||
|
||||
let (dirs, store) = multi_pool_fixture().await;
|
||||
let preparation = Uuid::new_v4();
|
||||
prepare_stream_in_pool(&store, 0, preparation, vec![41; 2 * 1024 * 1024 + 3]).await;
|
||||
prepare_stream_in_pool(&store, 1, preparation, vec![42; 2 * 1024 * 1024 + 3]).await;
|
||||
|
||||
let pool_zero: serde_json::Value = serde_json::from_slice(
|
||||
&std::fs::read(
|
||||
dirs[0]
|
||||
.path()
|
||||
.join(".rustfs.sys/native-preparations")
|
||||
.join(preparation.to_string())
|
||||
.join("intent.json"),
|
||||
)
|
||||
.expect("pool zero durable locator"),
|
||||
)
|
||||
.expect("pool zero locator JSON");
|
||||
let pool_one: serde_json::Value = serde_json::from_slice(
|
||||
&std::fs::read(
|
||||
dirs[4]
|
||||
.path()
|
||||
.join(".rustfs.sys/native-preparations")
|
||||
.join(preparation.to_string())
|
||||
.join("intent.json"),
|
||||
)
|
||||
.expect("pool one durable locator"),
|
||||
)
|
||||
.expect("pool one locator JSON");
|
||||
assert_ne!(
|
||||
pool_zero["object_digest"], pool_one["object_digest"],
|
||||
"NATIVE_CROSS_POOL_CONFLICT_FIXTURE_MUST_HAVE_DIFFERENT_CONTENT"
|
||||
);
|
||||
assert!(
|
||||
store.recover_native_object(BUCKET, KEY, preparation).await.is_err(),
|
||||
"NATIVE_STREAM_MUST_REJECT_PREPARATION_IN_MULTIPLE_POOLS"
|
||||
);
|
||||
assert_not_published(&store).await;
|
||||
|
||||
let unknown_preparation = Uuid::new_v4();
|
||||
prepare_stream_in_pool(&store, 0, unknown_preparation, vec![51; 2 * 1024 * 1024 + 5]).await;
|
||||
let unknown_disk = store.pools[0].get_disks_by_key(KEY).disks.read().await[0]
|
||||
.as_ref()
|
||||
.expect("pool zero physical disk")
|
||||
.get_disk_id()
|
||||
.await
|
||||
.expect("read pool zero physical identity")
|
||||
.expect("known pool zero physical identity");
|
||||
let disk_root = dirs
|
||||
.iter()
|
||||
.find(|dir| {
|
||||
serde_json::from_slice::<crate::layout::format::FormatV3>(
|
||||
&std::fs::read(dir.path().join(".rustfs.sys/format.json")).expect("read physical format"),
|
||||
)
|
||||
.is_ok_and(|format| format.erasure.this == unknown_disk)
|
||||
})
|
||||
.expect("resolve pool zero physical root");
|
||||
let prep_dir = disk_root
|
||||
.path()
|
||||
.join(".rustfs.sys/native-preparations")
|
||||
.join(unknown_preparation.to_string());
|
||||
let record_path = prep_dir.join("intent.json");
|
||||
let mut record: serde_json::Value =
|
||||
serde_json::from_slice(&std::fs::read(&record_path).expect("durable locator")).expect("locator JSON");
|
||||
let unknown_identity = Uuid::new_v4();
|
||||
assert_ne!(unknown_disk, unknown_identity);
|
||||
record["disk"] = serde_json::Value::String(unknown_identity.to_string());
|
||||
let bytes = crate::disk::local::native_prepare::canonical_test_record(&record);
|
||||
std::fs::write(&record_path, &bytes).expect("install unknown disk identity");
|
||||
std::fs::write(prep_dir.join("PREPARED"), Sha256::digest(&bytes))
|
||||
.expect("retain valid marker for unknown identity");
|
||||
|
||||
assert!(
|
||||
store.recover_native_object(BUCKET, KEY, unknown_preparation).await.is_err(),
|
||||
"NATIVE_STREAM_MUST_FAIL_CLOSED_ON_UNKNOWN_DISK_IDENTITY"
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.prepare_native_object(
|
||||
BUCKET,
|
||||
KEY,
|
||||
unknown_preparation,
|
||||
&mut PutObjReader::from_vec(vec![51; 2 * 1024 * 1024 + 5]),
|
||||
)
|
||||
.await
|
||||
.is_err(),
|
||||
"NATIVE_STREAM_MUST_NOT_REALLOCATE_WHEN_DISK_IDENTITY_IS_UNKNOWN"
|
||||
);
|
||||
assert!(
|
||||
dirs[4..].iter().all(|dir| !dir
|
||||
.path()
|
||||
.join(".rustfs.sys/native-preparations")
|
||||
.join(unknown_preparation.to_string())
|
||||
.exists()),
|
||||
"NATIVE_STREAM_UNKNOWN_DISK_MUST_NOT_RECEIVE_REASSIGNED_PREPARATION"
|
||||
);
|
||||
assert_not_published(&store).await;
|
||||
});
|
||||
})
|
||||
.expect("spawn large-stack native preparation test")
|
||||
.join()
|
||||
.expect("native preparation test thread");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn native_stream_prepare_crash_barriers() {
|
||||
use crate::disk::local::native_prepare::{clear_native_preparation_probe, fail_native_preparation};
|
||||
for (stage, after, recoverable) in [
|
||||
("part.1:file", false, false),
|
||||
("intent.json:file", true, false),
|
||||
("PREPARED:directory", true, true),
|
||||
] {
|
||||
let (dirs, store) = fixture().await;
|
||||
let preparation = Uuid::new_v4();
|
||||
fail_native_preparation(preparation, stage, after);
|
||||
let result = store
|
||||
.prepare_native_object(BUCKET, KEY, preparation, &mut PutObjReader::from_vec(vec![23; 2 * 1024 * 1024 + 3]))
|
||||
.await;
|
||||
assert!(result.is_err(), "injected disk barrier must not return a receipt: {stage}");
|
||||
clear_native_preparation_probe(preparation);
|
||||
assert_not_published(&store).await;
|
||||
drop(store);
|
||||
let restarted = reopen(&dirs).await;
|
||||
let recovered = restarted.recover_native_object(BUCKET, KEY, preparation).await;
|
||||
assert_eq!(recovered.is_ok(), recoverable, "durable boundary: {stage}");
|
||||
assert_not_published(&restarted).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn native_stream_prepare_multi_part_restart() {
|
||||
use crate::disk::Disk;
|
||||
use rustfs_filemeta::FileInfoOpts;
|
||||
let (dirs, store) = fixture().await;
|
||||
let body = vec![29; 3 * 1024 * 1024 + 13];
|
||||
let encoded_preparation = Uuid::new_v4();
|
||||
drop(
|
||||
store
|
||||
.prepare_native_object(BUCKET, KEY, encoded_preparation, &mut PutObjReader::from_vec(body.clone()))
|
||||
.await
|
||||
.expect("existing ordinary streaming encoder fixture"),
|
||||
);
|
||||
let preparation = Uuid::new_v4();
|
||||
let incarnation = store.bucket_incarnation_id(BUCKET).await.expect("actual incarnation");
|
||||
let fence = store.object_lock_config_snapshot(BUCKET).await.expect("actual bucket fence");
|
||||
let set = store.pools[0].get_disks_by_key(KEY);
|
||||
let disks = set.disks.read().await.clone();
|
||||
let guard = store
|
||||
.handle_new_ns_lock(BUCKET, KEY)
|
||||
.await
|
||||
.expect("actual commit namespace")
|
||||
.get_write_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await
|
||||
.expect("owned commit lock");
|
||||
let mut candidates = Vec::new();
|
||||
for (dir, disk) in dirs.iter().zip(disks) {
|
||||
let original = dir
|
||||
.path()
|
||||
.join(".rustfs.sys/native-preparations")
|
||||
.join(encoded_preparation.to_string());
|
||||
let metadata = std::fs::read(original.join("xl.meta")).expect("encoder metadata");
|
||||
let mut info = rustfs_filemeta::get_file_info(
|
||||
&metadata,
|
||||
BUCKET,
|
||||
KEY,
|
||||
&encoded_preparation.to_string(),
|
||||
FileInfoOpts {
|
||||
data: false,
|
||||
include_free_versions: false,
|
||||
include_part_checksums: true,
|
||||
},
|
||||
)
|
||||
.expect("exact FileInfo");
|
||||
let first = info.erasure.block_size;
|
||||
let codec = crate::erasure::coding::Erasure::try_new_with_options(
|
||||
info.erasure.data_blocks,
|
||||
info.erasure.parity_blocks,
|
||||
first,
|
||||
info.uses_legacy_checksum,
|
||||
)
|
||||
.expect("existing erasure geometry");
|
||||
let split = crate::erasure::coding::bitrot_shard_file_size(
|
||||
usize::try_from(codec.shard_file_size(i64::try_from(first).expect("part size"))).expect("shard size"),
|
||||
codec.shard_size(),
|
||||
info.erasure.get_checksum_info(1).algorithm,
|
||||
);
|
||||
let shard = std::fs::read(original.join(encoded_preparation.to_string()).join("part.1")).expect("actual encoded shard");
|
||||
let disk = disk.expect("actual disk");
|
||||
let Disk::Local(local) = disk.as_ref() else {
|
||||
panic!("local fixture");
|
||||
};
|
||||
local
|
||||
.get_disk()
|
||||
.begin_native_stream(preparation, store.id, BUCKET, incarnation, fence.clone())
|
||||
.await
|
||||
.expect("actual private owner");
|
||||
let destination = dir
|
||||
.path()
|
||||
.join(".rustfs.sys/native-preparations")
|
||||
.join(preparation.to_string())
|
||||
.join(preparation.to_string());
|
||||
std::fs::create_dir(&destination).expect("private part fixture directory");
|
||||
std::fs::write(destination.join("part.1"), &shard[..split]).expect("first actual encoded blocks");
|
||||
std::fs::write(destination.join("part.2"), &shard[split..]).expect("remaining actual encoded blocks");
|
||||
info.version_id = Some(preparation);
|
||||
info.data_dir = Some(preparation);
|
||||
info.parts.clear();
|
||||
info.add_object_part(1, String::new(), first, None, i64::try_from(first).expect("size"), None, None);
|
||||
info.add_object_part(
|
||||
2,
|
||||
String::new(),
|
||||
body.len() - first,
|
||||
None,
|
||||
i64::try_from(body.len() - first).expect("size"),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
candidates.push((disk.clone(), info));
|
||||
}
|
||||
let owner = set
|
||||
.attest_native_stream(
|
||||
store.id,
|
||||
BUCKET,
|
||||
KEY,
|
||||
preparation,
|
||||
incarnation,
|
||||
candidates,
|
||||
Sha256::digest(&body).into(),
|
||||
u64::try_from(body.len()).expect("length"),
|
||||
fence,
|
||||
)
|
||||
.await
|
||||
.expect("actual adoption verifies/fsyncs both parts without rereading upload or taking namespace");
|
||||
assert!(owner.receipt_count() >= 3);
|
||||
drop(owner);
|
||||
drop(guard);
|
||||
drop(set);
|
||||
drop(store);
|
||||
let restarted = reopen(&dirs).await;
|
||||
let recovered = restarted
|
||||
.recover_native_object(BUCKET, KEY, preparation)
|
||||
.await
|
||||
.expect("actual multi-part recovery decode");
|
||||
assert_eq!(recovered.object_digest(), <[u8; 32]>::from(Sha256::digest(&body)));
|
||||
assert_eq!(recovered.object_length(), u64::try_from(body.len()).expect("length"));
|
||||
assert_not_published(&restarted).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn native_stream_prepare_rejects_overwrite() {
|
||||
use crate::disk::{DiskAPI, RUSTFS_META_BUCKET};
|
||||
let (dirs, store) = fixture().await;
|
||||
let preparation = Uuid::new_v4();
|
||||
drop(
|
||||
store
|
||||
.prepare_native_object(BUCKET, KEY, preparation, &mut PutObjReader::from_vec(vec![31; 2 * 1024 * 1024 + 11]))
|
||||
.await
|
||||
.expect("attested private stream"),
|
||||
);
|
||||
let disk = store.pools[0].get_disks_by_key(KEY).disks.read().await[0]
|
||||
.clone()
|
||||
.expect("actual disk");
|
||||
let path = format!("native-preparations/{preparation}/{preparation}/part.1");
|
||||
let full = dirs[0].path().join(RUSTFS_META_BUCKET).join(&path);
|
||||
let before = std::fs::read(&full).expect("attested shard bytes");
|
||||
assert!(
|
||||
disk.create_file("", RUSTFS_META_BUCKET, &path, 1).await.is_err(),
|
||||
"NATIVE_STREAM_ATTESTED_PART_MUST_NOT_BE_TRUNCATED"
|
||||
);
|
||||
assert_eq!(std::fs::read(&full).expect("retained shard"), before);
|
||||
assert!(store.recover_native_object(BUCKET, KEY, preparation).await.is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn native_stream_prepare_record_locator() {
|
||||
use crate::disk::Disk;
|
||||
let (dirs, store) = fixture().await;
|
||||
let preparation = Uuid::new_v4();
|
||||
drop(
|
||||
store
|
||||
.prepare_native_object(BUCKET, KEY, preparation, &mut PutObjReader::from_vec(vec![37; 2 * 1024 * 1024 + 9]))
|
||||
.await
|
||||
.expect("actual prepared locator"),
|
||||
);
|
||||
let disk = store.pools[0].get_disks_by_key(KEY).disks.read().await[0]
|
||||
.clone()
|
||||
.expect("actual disk");
|
||||
let Disk::Local(local) = disk.as_ref() else {
|
||||
panic!("local fixture");
|
||||
};
|
||||
let listed = local
|
||||
.get_disk()
|
||||
.list_native_preparations(store.id, None, 1)
|
||||
.await
|
||||
.expect("descriptor-relative list");
|
||||
assert_eq!(listed.records.len(), 1);
|
||||
assert!(listed.unfinished.is_empty());
|
||||
assert_eq!(listed.next, None);
|
||||
let record = &listed.records[0];
|
||||
assert_eq!(record.preparation, preparation);
|
||||
assert_eq!(record.bucket, BUCKET);
|
||||
assert_eq!(record.key, KEY);
|
||||
assert_eq!(record.version, preparation);
|
||||
assert!(record.prepared);
|
||||
let path = dirs[0]
|
||||
.path()
|
||||
.join(".rustfs.sys/native-preparations")
|
||||
.join(preparation.to_string());
|
||||
assert_eq!(
|
||||
record.metadata.as_ref(),
|
||||
std::fs::read(path.join("xl.meta")).expect("retained exact metadata")
|
||||
);
|
||||
let direct = local
|
||||
.get_disk()
|
||||
.locate_native_preparation(store.id, preparation)
|
||||
.await
|
||||
.expect("direct locator")
|
||||
.expect("present record");
|
||||
assert_eq!(direct.metadata, record.metadata);
|
||||
assert!(
|
||||
local
|
||||
.get_disk()
|
||||
.locate_native_preparation(store.id, Uuid::new_v4())
|
||||
.await
|
||||
.expect("healthy absence")
|
||||
.is_none()
|
||||
);
|
||||
assert!(local.get_disk().list_native_preparations(store.id, None, 65).await.is_err());
|
||||
// Sender-owned namespace guard must not be acquired again by the recovery receiver.
|
||||
let guard = store
|
||||
.handle_new_ns_lock(BUCKET, KEY)
|
||||
.await
|
||||
.expect("namespace")
|
||||
.get_write_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await
|
||||
.expect("actual sender lock");
|
||||
let (owner, metadata) = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(10),
|
||||
store.recover_native_object_unlocked(BUCKET, KEY, preparation),
|
||||
)
|
||||
.await
|
||||
.expect("no recursive namespace lock")
|
||||
.expect("actual disk recovery under sender lock");
|
||||
assert!(metadata.iter().any(|(id, bytes)| {
|
||||
owner
|
||||
.receipts()
|
||||
.iter()
|
||||
.any(|receipt| receipt.disk_id() == *id && receipt.metadata_digest() == &<[u8; 32]>::from(Sha256::digest(bytes)))
|
||||
}));
|
||||
drop(guard);
|
||||
// Moving a published part must not erase its durable location facts; locate grants no readiness.
|
||||
std::fs::rename(
|
||||
path.join(preparation.to_string()).join("part.1"),
|
||||
dirs[0].path().join("published-part-test"),
|
||||
)
|
||||
.expect("publication progress fixture");
|
||||
let located = local
|
||||
.get_disk()
|
||||
.locate_native_preparation(store.id, preparation)
|
||||
.await
|
||||
.expect("retained locator after part move")
|
||||
.expect("retained record");
|
||||
assert_eq!(located.metadata, record.metadata);
|
||||
assert!(
|
||||
store.recover_native_object(BUCKET, KEY, preparation).await.is_err(),
|
||||
"prepared marker is not publication authority"
|
||||
);
|
||||
}
|
||||
@@ -5351,13 +5351,41 @@ impl ECStore {
|
||||
tier_journal_api: Option<Arc<ECStore>>,
|
||||
) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
|
||||
Box::pin(async move {
|
||||
let count = objects.len();
|
||||
let mut deleted = vec![DeletedObject::default(); count];
|
||||
let mut errors: Vec<Option<Error>> = (0..count).map(|_| None).collect();
|
||||
let mut accounting = vec![None; count];
|
||||
let mut valid_objects = Vec::with_capacity(count);
|
||||
let mut valid_indices = Vec::with_capacity(count);
|
||||
// Validate the literal key before encoding it or taking locks. A
|
||||
// dot-segment key must never reach filesystem path resolution.
|
||||
for (index, object) in objects.into_iter().enumerate() {
|
||||
match check_del_obj_args(bucket, &object.object_name) {
|
||||
Ok(()) => {
|
||||
valid_indices.push(index);
|
||||
valid_objects.push(object);
|
||||
}
|
||||
Err(err) => errors[index] = Some(err),
|
||||
}
|
||||
}
|
||||
if valid_objects.is_empty() {
|
||||
return (deleted, errors, accounting);
|
||||
}
|
||||
let mut opts = opts;
|
||||
let receipt_sink = install_tier_free_version_receipt_sink(&mut opts);
|
||||
let result = self
|
||||
.handle_delete_objects_with_journal_and_accounting_inner(bucket, objects, opts, tier_journal_api)
|
||||
let (valid_deleted, valid_errors, valid_accounting) = self
|
||||
.handle_delete_objects_with_journal_and_accounting_inner(bucket, valid_objects, opts, tier_journal_api)
|
||||
.await;
|
||||
enqueue_recorded_tier_free_versions(self, receipt_sink).await;
|
||||
result
|
||||
for (index, ((object, error), receipt)) in valid_indices
|
||||
.into_iter()
|
||||
.zip(valid_deleted.into_iter().zip(valid_errors).zip(valid_accounting))
|
||||
{
|
||||
deleted[index] = object;
|
||||
errors[index] = error;
|
||||
accounting[index] = receipt;
|
||||
}
|
||||
(deleted, errors, accounting)
|
||||
})
|
||||
.await
|
||||
}
|
||||
@@ -9094,6 +9122,58 @@ mod tests {
|
||||
(dirs, store)
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn batch_delete_rejects_dot_segments_and_keeps_valid_result_positions() {
|
||||
let bucket = "batch-delete-key-validation";
|
||||
let (_dirs, store) = multipool_version_test_store(bucket).await;
|
||||
let options = ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
};
|
||||
let victim = store
|
||||
.put_object(bucket, "victim/object", &mut PutObjReader::from_vec(b"victim".to_vec()), &options)
|
||||
.await
|
||||
.expect("write protected-prefix control object");
|
||||
store
|
||||
.put_object(bucket, "allowed/object", &mut PutObjReader::from_vec(b"allowed".to_vec()), &options)
|
||||
.await
|
||||
.expect("write valid deletion control object");
|
||||
let keys = [
|
||||
"allowed/../victim/object",
|
||||
"allowed/object",
|
||||
"a/./b",
|
||||
"../other-bucket/object",
|
||||
];
|
||||
let objects = keys
|
||||
.iter()
|
||||
.map(|key| ObjectToDelete {
|
||||
object_name: (*key).to_owned(),
|
||||
..Default::default()
|
||||
})
|
||||
.collect();
|
||||
let (deleted, errors) = store.handle_delete_objects(bucket, objects, options).await;
|
||||
assert_eq!(deleted.len(), keys.len());
|
||||
assert_eq!(errors.len(), keys.len());
|
||||
for index in [0, 2, 3] {
|
||||
assert!(
|
||||
matches!(&errors[index], Some(StorageError::ObjectNameInvalid(_, _))),
|
||||
"invalid key {}: {:?}",
|
||||
keys[index],
|
||||
errors[index]
|
||||
);
|
||||
assert!(!deleted[index].delete_marker);
|
||||
}
|
||||
assert!(errors[1].is_none(), "valid deletion must still succeed: {:?}", errors[1]);
|
||||
assert!(deleted[1].delete_marker);
|
||||
assert_eq!(deleted[1].object_name, "allowed/object");
|
||||
let after = store
|
||||
.get_object_info(bucket, "victim/object", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("the resolved victim path must remain visible");
|
||||
assert_eq!(after.version_id, victim.version_id);
|
||||
assert_eq!(after.etag, victim.etag);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn retired_marker_converges_after_bucket_recreation_on_sixteen_disks() {
|
||||
let bucket = "retired-marker-c11";
|
||||
|
||||
@@ -1,497 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Byte-level goldens for the 13 persisted bucket-configuration XML families
|
||||
//! (rustfs/backlog#2744).
|
||||
//!
|
||||
//! Responsible for: pinning the exact bytes that `deserialize` followed by
|
||||
//! `serialize` produces for every family in its three source shapes (AWS
|
||||
//! documentation shape, MinIO-written shape, RustFS-written shape), and the
|
||||
//! exact bytes RustFS writes for a typed value. Only bytes are compared; a
|
||||
//! structural comparison would let a codec replacement change the persisted
|
||||
//! encoding without failing here.
|
||||
//!
|
||||
//! Not responsible for: validating configurations, touching the bucket
|
||||
//! metadata store, or choosing a canonical shape.
|
||||
//!
|
||||
//! Upstream: the bucket-config `deserialize`/`serialize` pair, reached through the
|
||||
//! integration-test facade `tests/storage_api.rs`, and the s3s DTOs they are generic over. Downstream: the DTO/codec
|
||||
//! replacement (T1.4) and the persisted-bytes comparison (T3.7), which must
|
||||
//! keep every golden green without regenerating it. Regeneration is reserved
|
||||
//! to `scripts/gen_bucket_config_goldens.sh`, which sets
|
||||
//! `BUCKET_CONFIG_GOLDENS_WRITE`: in that mode every test writes its files
|
||||
//! and then fails on purpose, so a write run never prints a green line.
|
||||
|
||||
mod storage_api;
|
||||
|
||||
use s3s::dto::{
|
||||
AbortIncompleteMultipartUpload, AccelerateConfiguration, BucketAccelerateStatus, BucketLifecycleConfiguration,
|
||||
BucketLoggingStatus, BucketVersioningStatus, CORSConfiguration, CORSRule, Condition, DefaultRetention, DelMarkerExpiration,
|
||||
DeleteMarkerReplication, DeleteMarkerReplicationStatus, DeleteReplication, DeleteReplicationStatus, Destination,
|
||||
ErrorDocument, Event, ExcludedPrefix, ExistingObjectReplication, ExistingObjectReplicationStatus, ExpirationStatus,
|
||||
FilterRule, FilterRuleName, IndexDocument, LifecycleExpiration, LifecycleRule, LifecycleRuleAndOperator, LifecycleRuleFilter,
|
||||
LoggingEnabled, NoncurrentVersionExpiration, NotificationConfiguration, NotificationConfigurationFilter,
|
||||
ObjectLockConfiguration, ObjectLockEnabled, ObjectLockRetentionMode, ObjectLockRule, Payer, PublicAccessBlockConfiguration,
|
||||
QueueConfiguration, Redirect, ReplicaModifications, ReplicaModificationsStatus, ReplicationConfiguration, ReplicationRule,
|
||||
ReplicationRuleAndOperator, ReplicationRuleFilter, ReplicationRuleStatus, RequestPaymentConfiguration, RoutingRule,
|
||||
S3KeyFilter, ServerSideEncryption, ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration,
|
||||
ServerSideEncryptionRule, SourceSelectionCriteria, StorageClass, Tag, Tagging, Transition, TransitionStorageClass,
|
||||
VersioningConfiguration, WebsiteConfiguration,
|
||||
};
|
||||
use s3s::xml;
|
||||
use std::path::{Path, PathBuf};
|
||||
use storage_api::bucket_config_codec::{deserialize, serialize};
|
||||
|
||||
/// Set only by `scripts/gen_bucket_config_goldens.sh`. Every pair test then
|
||||
/// writes its `.out.xml` (and, for the `rustfs` shape, its `.in.xml`) and
|
||||
/// fails on purpose, so a write run can never be mistaken for a verification.
|
||||
const WRITE_ENV: &str = "BUCKET_CONFIG_GOLDENS_WRITE";
|
||||
|
||||
fn goldens_dir() -> PathBuf {
|
||||
Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("tests")
|
||||
.join("fixtures")
|
||||
.join("bucket-config-goldens")
|
||||
}
|
||||
|
||||
fn pair_path(family: &str, shape: &str, side: &str) -> PathBuf {
|
||||
goldens_dir().join(family).join(format!("{shape}.{side}.xml"))
|
||||
}
|
||||
|
||||
fn read(path: &Path) -> Vec<u8> {
|
||||
std::fs::read(path).unwrap_or_else(|err| panic!("read {}: {err}", path.display()))
|
||||
}
|
||||
|
||||
fn write(path: &Path, bytes: &[u8]) {
|
||||
std::fs::write(path, bytes).unwrap_or_else(|err| panic!("write {}: {err}", path.display()));
|
||||
}
|
||||
|
||||
fn contains(haystack: &[u8], needle: &[u8]) -> bool {
|
||||
haystack.windows(needle.len()).any(|window| window == needle)
|
||||
}
|
||||
|
||||
fn round_trip<T>(input: &[u8], label: &str) -> Vec<u8>
|
||||
where
|
||||
T: for<'xml> xml::Deserialize<'xml> + xml::Serialize,
|
||||
{
|
||||
let parsed: T = deserialize(input).unwrap_or_else(|err| panic!("{label}: deserialize failed: {err}"));
|
||||
serialize(&parsed).unwrap_or_else(|err| panic!("{label}: serialize failed: {err}"))
|
||||
}
|
||||
|
||||
/// Byte comparison with a readable report. Parsed values are never compared.
|
||||
fn assert_bytes_eq(actual: &[u8], expected: &[u8], context: &str) {
|
||||
if actual != expected {
|
||||
panic!(
|
||||
"{context}\n--- expected ({} bytes) ---\n{}\n--- actual ({} bytes) ---\n{}\n",
|
||||
expected.len(),
|
||||
String::from_utf8_lossy(expected),
|
||||
actual.len(),
|
||||
String::from_utf8_lossy(actual),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Checks one `<family>/<shape>` pair. `typed` is the value RustFS writes for
|
||||
/// the `rustfs` shape; the `aws` and `minio` shapes pass `None`.
|
||||
fn check_pair<T>(family: &str, shape: &str, typed: Option<&T>)
|
||||
where
|
||||
T: for<'xml> xml::Deserialize<'xml> + xml::Serialize,
|
||||
{
|
||||
let in_path = pair_path(family, shape, "in");
|
||||
let out_path = pair_path(family, shape, "out");
|
||||
let label = format!("{family}/{shape}");
|
||||
|
||||
if std::env::var_os(WRITE_ENV).is_some() {
|
||||
if let Some(value) = typed {
|
||||
let written = serialize(value).unwrap_or_else(|err| panic!("{label}: typed value failed to serialize: {err}"));
|
||||
write(&in_path, &written);
|
||||
}
|
||||
let input = read(&in_path);
|
||||
write(&out_path, &round_trip::<T>(&input, &label));
|
||||
panic!(
|
||||
"{WRITE_ENV} is set: wrote {} without verifying anything; rerun without {WRITE_ENV}",
|
||||
out_path.display()
|
||||
);
|
||||
}
|
||||
|
||||
let input = read(&in_path);
|
||||
let expected = read(&out_path);
|
||||
if let Some(value) = typed {
|
||||
let written = serialize(value).unwrap_or_else(|err| panic!("{label}: typed value failed to serialize: {err}"));
|
||||
assert_bytes_eq(
|
||||
&written,
|
||||
&input,
|
||||
&format!(
|
||||
"{label}: RustFS no longer writes the pinned bytes for the typed value ({})",
|
||||
in_path.display()
|
||||
),
|
||||
);
|
||||
}
|
||||
let actual = round_trip::<T>(&input, &label);
|
||||
assert_bytes_eq(
|
||||
&actual,
|
||||
&expected,
|
||||
&format!("{label}: deserialize->serialize bytes drifted from {}", out_path.display()),
|
||||
);
|
||||
let again = round_trip::<T>(&expected, &label);
|
||||
assert_bytes_eq(
|
||||
&again,
|
||||
&expected,
|
||||
&format!("{label}: the golden is not a fixed point of deserialize->serialize"),
|
||||
);
|
||||
}
|
||||
|
||||
/// The typed values behind every `rustfs.in.xml`: what RustFS writes today
|
||||
/// through `serialize` when it persists a configuration it built itself.
|
||||
/// MinIO-only members (`DelMarkerExpiration`, `ExcludeFolders`,
|
||||
/// `ExcludedPrefixes`, `DeleteReplication`) are included where RustFS accepts
|
||||
/// them, so the `rustfs` shape is the one that pins their encoding.
|
||||
mod rustfs_values {
|
||||
use super::*;
|
||||
|
||||
fn tag(key: &str, value: &str) -> Tag {
|
||||
Tag {
|
||||
key: Some(key.to_owned()),
|
||||
value: Some(value.to_owned()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn notification() -> NotificationConfiguration {
|
||||
NotificationConfiguration {
|
||||
queue_configurations: Some(vec![QueueConfiguration {
|
||||
events: vec![
|
||||
Event::from("s3:ObjectCreated:*".to_owned()),
|
||||
Event::from("s3:ObjectRemoved:*".to_owned()),
|
||||
],
|
||||
filter: Some(NotificationConfigurationFilter {
|
||||
key: Some(S3KeyFilter {
|
||||
filter_rules: Some(vec![FilterRule {
|
||||
name: Some(FilterRuleName::from_static(FilterRuleName::PREFIX)),
|
||||
value: Some("uploads/".to_owned()),
|
||||
}]),
|
||||
}),
|
||||
}),
|
||||
id: Some("rustfs-webhook".to_owned()),
|
||||
queue_arn: "arn:rustfs:sqs::primary:webhook".to_owned(),
|
||||
}]),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn lifecycle() -> BucketLifecycleConfiguration {
|
||||
BucketLifecycleConfiguration {
|
||||
rules: vec![
|
||||
LifecycleRule {
|
||||
abort_incomplete_multipart_upload: Some(AbortIncompleteMultipartUpload {
|
||||
days_after_initiation: Some(3),
|
||||
}),
|
||||
del_marker_expiration: Some(DelMarkerExpiration { days: Some(7) }),
|
||||
expiration: Some(LifecycleExpiration {
|
||||
days: Some(30),
|
||||
..Default::default()
|
||||
}),
|
||||
filter: Some(LifecycleRuleFilter {
|
||||
prefix: Some("tmp/".to_owned()),
|
||||
..Default::default()
|
||||
}),
|
||||
id: Some("expire-tmp".to_owned()),
|
||||
noncurrent_version_expiration: Some(NoncurrentVersionExpiration {
|
||||
newer_noncurrent_versions: Some(2),
|
||||
noncurrent_days: Some(7),
|
||||
}),
|
||||
noncurrent_version_transitions: None,
|
||||
prefix: None,
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
transitions: None,
|
||||
},
|
||||
LifecycleRule {
|
||||
abort_incomplete_multipart_upload: None,
|
||||
del_marker_expiration: None,
|
||||
expiration: None,
|
||||
filter: Some(LifecycleRuleFilter {
|
||||
and: Some(LifecycleRuleAndOperator {
|
||||
prefix: Some("archive/".to_owned()),
|
||||
tags: Some(vec![tag("tier", "cold")]),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
}),
|
||||
id: Some("tier-cold".to_owned()),
|
||||
noncurrent_version_expiration: None,
|
||||
noncurrent_version_transitions: None,
|
||||
prefix: None,
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
transitions: Some(vec![Transition {
|
||||
days: Some(90),
|
||||
storage_class: Some(TransitionStorageClass::from_static("COLDTIER")),
|
||||
..Default::default()
|
||||
}]),
|
||||
},
|
||||
],
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn object_lock() -> ObjectLockConfiguration {
|
||||
ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
rule: Some(ObjectLockRule {
|
||||
default_retention: Some(DefaultRetention {
|
||||
days: Some(30),
|
||||
mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::GOVERNANCE)),
|
||||
years: None,
|
||||
}),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn versioning() -> VersioningConfiguration {
|
||||
VersioningConfiguration {
|
||||
exclude_folders: Some(true),
|
||||
excluded_prefixes: Some(vec![
|
||||
ExcludedPrefix {
|
||||
prefix: Some("tmp/".to_owned()),
|
||||
},
|
||||
ExcludedPrefix {
|
||||
prefix: Some("cache/".to_owned()),
|
||||
},
|
||||
]),
|
||||
mfa_delete: None,
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn encryption() -> ServerSideEncryptionConfiguration {
|
||||
ServerSideEncryptionConfiguration {
|
||||
rules: vec![ServerSideEncryptionRule {
|
||||
apply_server_side_encryption_by_default: Some(ServerSideEncryptionByDefault {
|
||||
kms_master_key_id: Some("rustfs-bucket-key".to_owned()),
|
||||
sse_algorithm: ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS),
|
||||
}),
|
||||
blocked_encryption_types: None,
|
||||
bucket_key_enabled: Some(false),
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn tagging() -> Tagging {
|
||||
Tagging {
|
||||
tag_set: vec![tag("team", "storage"), tag("env", "staging")],
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn replication() -> ReplicationConfiguration {
|
||||
ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![ReplicationRule {
|
||||
delete_marker_replication: Some(DeleteMarkerReplication {
|
||||
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)),
|
||||
}),
|
||||
delete_replication: Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
}),
|
||||
destination: Destination {
|
||||
bucket: "arn:rustfs:replication::0b6f3c2a-5e6a-4d2b-9c1e-7a8f9d0e1b2c:interop-dr".to_owned(),
|
||||
storage_class: Some(StorageClass::from_static(StorageClass::STANDARD)),
|
||||
..Default::default()
|
||||
},
|
||||
existing_object_replication: Some(ExistingObjectReplication {
|
||||
status: ExistingObjectReplicationStatus::from_static(ExistingObjectReplicationStatus::ENABLED),
|
||||
}),
|
||||
filter: Some(ReplicationRuleFilter {
|
||||
and: Some(ReplicationRuleAndOperator {
|
||||
prefix: Some("docs/".to_owned()),
|
||||
tags: Some(vec![tag("replicate", "yes")]),
|
||||
}),
|
||||
..Default::default()
|
||||
}),
|
||||
id: Some("rustfs-dr".to_owned()),
|
||||
prefix: None,
|
||||
priority: Some(1),
|
||||
source_selection_criteria: Some(SourceSelectionCriteria {
|
||||
replica_modifications: Some(ReplicaModifications {
|
||||
status: ReplicaModificationsStatus::from_static(ReplicaModificationsStatus::ENABLED),
|
||||
}),
|
||||
sse_kms_encrypted_objects: None,
|
||||
}),
|
||||
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn cors() -> CORSConfiguration {
|
||||
CORSConfiguration {
|
||||
cors_rules: vec![CORSRule {
|
||||
allowed_headers: Some(vec!["Authorization".to_owned(), "Content-Type".to_owned()]),
|
||||
allowed_methods: vec!["GET".to_owned(), "PUT".to_owned()],
|
||||
allowed_origins: vec!["https://app.example.internal".to_owned()],
|
||||
expose_headers: Some(vec!["ETag".to_owned(), "x-amz-request-id".to_owned()]),
|
||||
id: Some("rustfs-app".to_owned()),
|
||||
max_age_seconds: Some(3600),
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn logging() -> BucketLoggingStatus {
|
||||
BucketLoggingStatus {
|
||||
logging_enabled: Some(LoggingEnabled {
|
||||
target_bucket: "rustfs-access-logs".to_owned(),
|
||||
target_grants: None,
|
||||
target_object_key_format: None,
|
||||
target_prefix: "buckets/interop/".to_owned(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn website() -> WebsiteConfiguration {
|
||||
WebsiteConfiguration {
|
||||
error_document: Some(ErrorDocument {
|
||||
key: "404.html".to_owned(),
|
||||
}),
|
||||
index_document: Some(IndexDocument {
|
||||
suffix: "index.html".to_owned(),
|
||||
}),
|
||||
redirect_all_requests_to: None,
|
||||
routing_rules: Some(vec![RoutingRule {
|
||||
condition: Some(Condition {
|
||||
http_error_code_returned_equals: None,
|
||||
key_prefix_equals: Some("legacy/".to_owned()),
|
||||
}),
|
||||
redirect: Redirect {
|
||||
host_name: None,
|
||||
http_redirect_code: Some("301".to_owned()),
|
||||
protocol: None,
|
||||
replace_key_prefix_with: Some("current/".to_owned()),
|
||||
replace_key_with: None,
|
||||
},
|
||||
}]),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn accelerate() -> AccelerateConfiguration {
|
||||
AccelerateConfiguration {
|
||||
status: Some(BucketAccelerateStatus::from_static(BucketAccelerateStatus::SUSPENDED)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn request_payment() -> RequestPaymentConfiguration {
|
||||
RequestPaymentConfiguration {
|
||||
payer: Payer::from_static(Payer::BUCKET_OWNER),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn public_access_block() -> PublicAccessBlockConfiguration {
|
||||
PublicAccessBlockConfiguration {
|
||||
block_public_acls: Some(true),
|
||||
block_public_policy: Some(false),
|
||||
ignore_public_acls: Some(true),
|
||||
restrict_public_buckets: Some(false),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One test per `<family>/<shape>` pair: 13 families x 3 shapes = 39.
|
||||
mod bucket_config_goldens {
|
||||
use super::*;
|
||||
|
||||
macro_rules! golden {
|
||||
($name:ident, $family:literal, $shape:literal, $ty:ty) => {
|
||||
#[test]
|
||||
fn $name() {
|
||||
check_pair::<$ty>($family, $shape, None);
|
||||
}
|
||||
};
|
||||
($name:ident, $family:literal, rustfs = $value:expr) => {
|
||||
#[test]
|
||||
fn $name() {
|
||||
check_pair($family, "rustfs", Some(&$value));
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
golden!(notification_aws, "notification", "aws", NotificationConfiguration);
|
||||
golden!(notification_minio, "notification", "minio", NotificationConfiguration);
|
||||
golden!(notification_rustfs, "notification", rustfs = rustfs_values::notification());
|
||||
|
||||
golden!(lifecycle_aws, "lifecycle", "aws", BucketLifecycleConfiguration);
|
||||
golden!(lifecycle_minio, "lifecycle", "minio", BucketLifecycleConfiguration);
|
||||
golden!(lifecycle_rustfs, "lifecycle", rustfs = rustfs_values::lifecycle());
|
||||
|
||||
golden!(object_lock_aws, "object_lock", "aws", ObjectLockConfiguration);
|
||||
golden!(object_lock_minio, "object_lock", "minio", ObjectLockConfiguration);
|
||||
golden!(object_lock_rustfs, "object_lock", rustfs = rustfs_values::object_lock());
|
||||
|
||||
golden!(versioning_aws, "versioning", "aws", VersioningConfiguration);
|
||||
golden!(versioning_minio, "versioning", "minio", VersioningConfiguration);
|
||||
golden!(versioning_rustfs, "versioning", rustfs = rustfs_values::versioning());
|
||||
|
||||
golden!(encryption_aws, "encryption", "aws", ServerSideEncryptionConfiguration);
|
||||
golden!(encryption_minio, "encryption", "minio", ServerSideEncryptionConfiguration);
|
||||
golden!(encryption_rustfs, "encryption", rustfs = rustfs_values::encryption());
|
||||
|
||||
golden!(tagging_aws, "tagging", "aws", Tagging);
|
||||
golden!(tagging_minio, "tagging", "minio", Tagging);
|
||||
golden!(tagging_rustfs, "tagging", rustfs = rustfs_values::tagging());
|
||||
|
||||
/// `replication/aws.in.xml` is the unknown-element sample from
|
||||
/// `crates/replication/src/config.rs`
|
||||
/// (`s3_xml_parser_discards_unknown_replication_elements_before_validation`).
|
||||
/// Beyond the byte golden, the pair proves the root-level `FutureTopLevel`
|
||||
/// element is skipped on read and never re-emitted on write.
|
||||
#[test]
|
||||
fn replication_aws() {
|
||||
check_pair::<ReplicationConfiguration>("replication", "aws", None);
|
||||
let input = read(&pair_path("replication", "aws", "in"));
|
||||
let output = read(&pair_path("replication", "aws", "out"));
|
||||
assert!(
|
||||
contains(&input, b"<FutureTopLevel>future</FutureTopLevel>"),
|
||||
"the sample must still carry the unknown element"
|
||||
);
|
||||
assert!(
|
||||
!contains(&output, b"FutureTopLevel"),
|
||||
"an unknown element must not be re-emitted: {}",
|
||||
String::from_utf8_lossy(&output)
|
||||
);
|
||||
}
|
||||
golden!(replication_minio, "replication", "minio", ReplicationConfiguration);
|
||||
golden!(replication_rustfs, "replication", rustfs = rustfs_values::replication());
|
||||
|
||||
golden!(cors_aws, "cors", "aws", CORSConfiguration);
|
||||
golden!(cors_minio, "cors", "minio", CORSConfiguration);
|
||||
golden!(cors_rustfs, "cors", rustfs = rustfs_values::cors());
|
||||
|
||||
golden!(logging_aws, "logging", "aws", BucketLoggingStatus);
|
||||
golden!(logging_minio, "logging", "minio", BucketLoggingStatus);
|
||||
golden!(logging_rustfs, "logging", rustfs = rustfs_values::logging());
|
||||
|
||||
golden!(website_aws, "website", "aws", WebsiteConfiguration);
|
||||
golden!(website_minio, "website", "minio", WebsiteConfiguration);
|
||||
golden!(website_rustfs, "website", rustfs = rustfs_values::website());
|
||||
|
||||
golden!(accelerate_aws, "accelerate", "aws", AccelerateConfiguration);
|
||||
golden!(accelerate_minio, "accelerate", "minio", AccelerateConfiguration);
|
||||
golden!(accelerate_rustfs, "accelerate", rustfs = rustfs_values::accelerate());
|
||||
|
||||
golden!(request_payment_aws, "request_payment", "aws", RequestPaymentConfiguration);
|
||||
golden!(request_payment_minio, "request_payment", "minio", RequestPaymentConfiguration);
|
||||
golden!(request_payment_rustfs, "request_payment", rustfs = rustfs_values::request_payment());
|
||||
|
||||
golden!(public_access_block_aws, "public_access_block", "aws", PublicAccessBlockConfiguration);
|
||||
golden!(public_access_block_minio, "public_access_block", "minio", PublicAccessBlockConfiguration);
|
||||
golden!(
|
||||
public_access_block_rustfs,
|
||||
"public_access_block",
|
||||
rustfs = rustfs_values::public_access_block()
|
||||
);
|
||||
}
|
||||
@@ -1,58 +0,0 @@
|
||||
# Persisted bucket-configuration XML goldens (rustfs/backlog#2744)
|
||||
|
||||
Byte-level goldens for the 13 XML families that `BucketMetadata` persists as
|
||||
`*_config_xml`. They pin what `rustfs_ecstore::api::bucket::utils::deserialize`
|
||||
followed by `serialize` emits today, so that the DTO/codec replacement (T1.4)
|
||||
and the persisted-bytes comparison (T3.7) can be proven byte-identical: an
|
||||
older binary must keep reading what a newer one writes, and the reverse.
|
||||
|
||||
Generated at commit: `6b1554003ebf8f2037ffb7da9c9b906527e758da` (2026-10-07T17:40:56Z)
|
||||
|
||||
Checked by `crates/ecstore/tests/bucket_config_goldens.rs`
|
||||
(`cargo nextest run -p rustfs-ecstore bucket_config_goldens`, 39 tests).
|
||||
Generated once by `scripts/gen_bucket_config_goldens.sh`, which records the
|
||||
commit above and refuses to run again unless `--force` is passed. Do not
|
||||
regenerate after a DTO or codec replacement: at that point a diff here is the
|
||||
finding, not noise.
|
||||
|
||||
## Layout
|
||||
|
||||
`<family>/<shape>.in.xml` is an input; `<family>/<shape>.out.xml` is the exact
|
||||
bytes of `serialize(deserialize(in))` on the generating commit. The test
|
||||
asserts `bytes == out` (never structural equality), that `out` is a fixed point
|
||||
of the same round trip, and for the `rustfs` shape that `serialize(typed
|
||||
value)` still equals `rustfs.in.xml`.
|
||||
|
||||
| Shape | Source |
|
||||
| --- | --- |
|
||||
| `aws` | Written for this corpus from the public S3 XML schema in the shape of the AWS documentation examples: XML declaration, `xmlns` on the root, indented. No AWS prose is copied. |
|
||||
| `minio` | The bytes a MinIO server persists. Seven families are carved verbatim out of `../minio/bucket_metadata.blob.hex` (MinIO `RELEASE.2025-07-23T15-54-02Z`); the rest are hand-shaped as compact Go `encoding/xml` output with the S3 namespace on the root, because MinIO does not persist those families. |
|
||||
| `rustfs` | The bytes RustFS writes today through `serialize` for a typed value (`rustfs_values` in the test). This shape carries the MinIO-only members RustFS accepts (`DelMarkerExpiration`, `ExcludeFolders`, `ExcludedPrefixes`, `DeleteReplication`). |
|
||||
|
||||
| Family | Root element | `minio.in.xml` provenance |
|
||||
| --- | --- | --- |
|
||||
| `notification` | `NotificationConfiguration` | MinIO fixture |
|
||||
| `lifecycle` | `LifecycleConfiguration` | MinIO fixture (carries the MinIO `ExpiryUpdatedAt` extension) |
|
||||
| `object_lock` | `ObjectLockConfiguration` | MinIO fixture |
|
||||
| `versioning` | `VersioningConfiguration` | MinIO fixture |
|
||||
| `encryption` | `ServerSideEncryptionConfiguration` | MinIO fixture |
|
||||
| `tagging` | `Tagging` | MinIO fixture |
|
||||
| `replication` | `ReplicationConfiguration` | MinIO fixture |
|
||||
| `cors` | `CORSConfiguration` | hand-shaped |
|
||||
| `logging` | `BucketLoggingStatus` | hand-shaped |
|
||||
| `website` | `WebsiteConfiguration` | hand-shaped |
|
||||
| `accelerate` | `AccelerateConfiguration` | hand-shaped |
|
||||
| `request_payment` | `RequestPaymentConfiguration` | hand-shaped |
|
||||
| `public_access_block` | `PublicAccessBlockConfiguration` | hand-shaped |
|
||||
|
||||
## Unknown elements
|
||||
|
||||
`replication/aws.in.xml` is the unknown-element sample from
|
||||
`crates/replication/src/config.rs`
|
||||
(`s3_xml_parser_discards_unknown_replication_elements_before_validation`),
|
||||
re-indented to two spaces. Its golden proves the root-level `FutureTopLevel`
|
||||
element is skipped on read and never re-emitted on write, which is the
|
||||
persisted-configuration exception recorded in the root `AGENTS.md`. On the
|
||||
generating commit every one of the 13 root elements skips unknown children;
|
||||
nested members (`Rule`, `Filter`, `Destination`, ...) reject them. A
|
||||
replacement codec must keep that boundary where it is.
|
||||
@@ -1,4 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<AccelerateConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Status>Enabled</Status>
|
||||
</AccelerateConfiguration>
|
||||
@@ -1 +0,0 @@
|
||||
<AccelerateConfiguration><Status>Enabled</Status></AccelerateConfiguration>
|
||||
@@ -1 +0,0 @@
|
||||
<AccelerateConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Suspended</Status></AccelerateConfiguration>
|
||||
@@ -1 +0,0 @@
|
||||
<AccelerateConfiguration><Status>Suspended</Status></AccelerateConfiguration>
|
||||
@@ -1 +0,0 @@
|
||||
<AccelerateConfiguration><Status>Suspended</Status></AccelerateConfiguration>
|
||||
@@ -1 +0,0 @@
|
||||
<AccelerateConfiguration><Status>Suspended</Status></AccelerateConfiguration>
|
||||
@@ -1,17 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<CORSConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<CORSRule>
|
||||
<ID>AllowWebApp</ID>
|
||||
<AllowedOrigin>https://www.example.com</AllowedOrigin>
|
||||
<AllowedMethod>PUT</AllowedMethod>
|
||||
<AllowedMethod>POST</AllowedMethod>
|
||||
<AllowedMethod>DELETE</AllowedMethod>
|
||||
<AllowedHeader>*</AllowedHeader>
|
||||
<ExposeHeader>x-amz-server-side-encryption</ExposeHeader>
|
||||
<MaxAgeSeconds>3000</MaxAgeSeconds>
|
||||
</CORSRule>
|
||||
<CORSRule>
|
||||
<AllowedOrigin>*</AllowedOrigin>
|
||||
<AllowedMethod>GET</AllowedMethod>
|
||||
</CORSRule>
|
||||
</CORSConfiguration>
|
||||
@@ -1 +0,0 @@
|
||||
<CORSConfiguration><CORSRule><AllowedHeader>*</AllowedHeader><AllowedMethod>PUT</AllowedMethod><AllowedMethod>POST</AllowedMethod><AllowedMethod>DELETE</AllowedMethod><AllowedOrigin>https://www.example.com</AllowedOrigin><ExposeHeader>x-amz-server-side-encryption</ExposeHeader><ID>AllowWebApp</ID><MaxAgeSeconds>3000</MaxAgeSeconds></CORSRule><CORSRule><AllowedMethod>GET</AllowedMethod><AllowedOrigin>*</AllowedOrigin></CORSRule></CORSConfiguration>
|
||||
@@ -1 +0,0 @@
|
||||
<CORSConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><CORSRule><ID>console</ID><AllowedOrigin>https://console.example.internal</AllowedOrigin><AllowedMethod>GET</AllowedMethod><AllowedMethod>HEAD</AllowedMethod><AllowedHeader>Authorization</AllowedHeader><AllowedHeader>Content-Type</AllowedHeader><ExposeHeader>ETag</ExposeHeader><MaxAgeSeconds>600</MaxAgeSeconds></CORSRule></CORSConfiguration>
|
||||
@@ -1 +0,0 @@
|
||||
<CORSConfiguration><CORSRule><AllowedHeader>Authorization</AllowedHeader><AllowedHeader>Content-Type</AllowedHeader><AllowedMethod>GET</AllowedMethod><AllowedMethod>HEAD</AllowedMethod><AllowedOrigin>https://console.example.internal</AllowedOrigin><ExposeHeader>ETag</ExposeHeader><ID>console</ID><MaxAgeSeconds>600</MaxAgeSeconds></CORSRule></CORSConfiguration>
|
||||
@@ -1 +0,0 @@
|
||||
<CORSConfiguration><CORSRule><AllowedHeader>Authorization</AllowedHeader><AllowedHeader>Content-Type</AllowedHeader><AllowedMethod>GET</AllowedMethod><AllowedMethod>PUT</AllowedMethod><AllowedOrigin>https://app.example.internal</AllowedOrigin><ExposeHeader>ETag</ExposeHeader><ExposeHeader>x-amz-request-id</ExposeHeader><ID>rustfs-app</ID><MaxAgeSeconds>3600</MaxAgeSeconds></CORSRule></CORSConfiguration>
|
||||
@@ -1 +0,0 @@
|
||||
<CORSConfiguration><CORSRule><AllowedHeader>Authorization</AllowedHeader><AllowedHeader>Content-Type</AllowedHeader><AllowedMethod>GET</AllowedMethod><AllowedMethod>PUT</AllowedMethod><AllowedOrigin>https://app.example.internal</AllowedOrigin><ExposeHeader>ETag</ExposeHeader><ExposeHeader>x-amz-request-id</ExposeHeader><ID>rustfs-app</ID><MaxAgeSeconds>3600</MaxAgeSeconds></CORSRule></CORSConfiguration>
|
||||
@@ -1,10 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<ServerSideEncryptionConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Rule>
|
||||
<ApplyServerSideEncryptionByDefault>
|
||||
<SSEAlgorithm>aws:kms</SSEAlgorithm>
|
||||
<KMSMasterKeyID>arn:aws:kms:us-east-1:123456789012:key/1234abcd-12ab-34cd-56ef-1234567890ab</KMSMasterKeyID>
|
||||
</ApplyServerSideEncryptionByDefault>
|
||||
<BucketKeyEnabled>true</BucketKeyEnabled>
|
||||
</Rule>
|
||||
</ServerSideEncryptionConfiguration>
|
||||
@@ -1 +0,0 @@
|
||||
<ServerSideEncryptionConfiguration><Rule><ApplyServerSideEncryptionByDefault><KMSMasterKeyID>arn:aws:kms:us-east-1:123456789012:key/1234abcd-12ab-34cd-56ef-1234567890ab</KMSMasterKeyID><SSEAlgorithm>aws:kms</SSEAlgorithm></ApplyServerSideEncryptionByDefault><BucketKeyEnabled>true</BucketKeyEnabled></Rule></ServerSideEncryptionConfiguration>
|
||||
@@ -1 +0,0 @@
|
||||
<ServerSideEncryptionConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>AES256</SSEAlgorithm></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user