mirror of
https://github.com/rustfs/rustfs.git
synced 2026-10-29 15:29:35 +00:00
Compare commits
140 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| cfe75fddd9 | |||
| fe7854500c | |||
| 75ab3052a9 | |||
| 368cdfd214 | |||
| 90a1eb814e | |||
| b97a952cda | |||
| 0a8e78238b | |||
| fe04c79894 | |||
| 5c4a285a7b | |||
| 9a8faf7c86 | |||
| 0a2f58c7a9 | |||
| 4d97501327 | |||
| 3b2af12c64 | |||
| c73ddde848 | |||
| ef72a37fd7 | |||
| e46cf750dd | |||
| fe955fa3ee | |||
| 64578c067e | |||
| 78715f0921 | |||
| 3284aafd73 | |||
| 9286442f87 | |||
| b8aad1c168 | |||
| 5fcb229760 | |||
| 94baef7462 | |||
| 60d90f085a | |||
| dfa5d50565 | |||
| a902886737 | |||
| 8e53b87421 | |||
| f7631e91b1 | |||
| f9bc4e921f | |||
| 781584307a | |||
| 59d641a5fe | |||
| c618b57870 | |||
| bbe1d54098 | |||
| 9e8ea4dcdb | |||
| 63cf8e2f2f | |||
| b7fe40f4b3 | |||
| 5b872b8e56 | |||
| ca5be1ef95 | |||
| 9373e8fa3e | |||
| cb32965522 | |||
| fb6cf49a67 | |||
| 8c97049e68 | |||
| 17d4fe27cc | |||
| fb0c7972b2 | |||
| eba7967755 | |||
| 451c1c31ad | |||
| 8780dd670e | |||
| f29bf2c902 | |||
| fd03436090 | |||
| 1f1e7b1c97 | |||
| 45a952609e | |||
| 323190a95a | |||
| 38cdda9292 | |||
| 85c84fa73d | |||
| 51402087c5 | |||
| a994984fe2 | |||
| 48eb0e402c | |||
| 446479791e | |||
| 99af59a914 | |||
| b77bd26874 | |||
| 956c1cf027 | |||
| 8b9d44110e | |||
| 44ce8d114f | |||
| 769b0d7578 | |||
| 03e775947d | |||
| 25e73d3bd6 | |||
| a464b45367 | |||
| 5972a09cc3 | |||
| 8a5e730bc9 | |||
| 1df874b724 | |||
| 68dcad8387 | |||
| 8474feaa26 | |||
| d386f7741d | |||
| 5b4432620e | |||
| 1a450bf228 | |||
| 371241afcb | |||
| 61b31e7b45 | |||
| 0d1d1487f3 | |||
| 3824779520 | |||
| a9670ebb4d | |||
| 99ae115731 | |||
| 545e794945 | |||
| 855586255f | |||
| bf47a919ac | |||
| bdd23da034 | |||
| 89a277c3d9 | |||
| 3e87004032 | |||
| 76386572f6 | |||
| 34917c7947 | |||
| f0b2080e0c | |||
| 9c6b0e182b | |||
| ab9af5523c | |||
| b18c5afc8d | |||
| df988cfd86 | |||
| 04bc5dffd6 | |||
| 681c95f359 | |||
| 32d98ff8b2 | |||
| 6b1554003e | |||
| 19978b2cb6 | |||
| db2abcf0a6 | |||
| 14eb28dcd6 | |||
| 2f0143ef15 | |||
| 3b92708011 | |||
| 8fea8ef600 | |||
| 477f768f7e | |||
| 47ed6717de | |||
| 480312e5ad | |||
| 1c5bbc0c47 | |||
| 14adc5ed3f | |||
| 714706328b | |||
| cd0b895ad7 | |||
| 073b4d838e | |||
| 95268a3b9e | |||
| f7c4ac11b2 | |||
| 9d9e92bab8 | |||
| 0ad0a6d8de | |||
| 051a54f129 | |||
| c71e4f72ea | |||
| 003570b61e | |||
| 0c3efda7bd | |||
| 2c7e7d195c | |||
| 0764785754 | |||
| 182b2bfe98 | |||
| 00a549504c | |||
| 83fa2611f4 | |||
| e22414fe68 | |||
| 45f97bccf9 | |||
| 046fb21d86 | |||
| 4783adff36 | |||
| 68ad1a1d7b | |||
| 7c289001ed | |||
| 7b1ffbfdc1 | |||
| 562e03670e | |||
| 5e1bd498ce | |||
| 5aca750a6e | |||
| a760d567b6 | |||
| 47d208325e | |||
| d087416ff9 | |||
| 309128f034 |
@@ -50,6 +50,7 @@ Update this file only when an advisory adds or changes a reusable lesson, affect
|
||||
|
||||
- `GHSA-g8w9-qw9q-fghr` and `GHSA-xm99-m3gq-83g8`: a valid presigned `PutObject` accepted extra `x-amz-*` headers omitted from `SignedHeaders`; `x-amz-copy-source` could turn that upload capability into a cross-bucket read performed as the signer. Lesson: a presigned URL is a bounded capability; inspect all received security-sensitive headers and reject unsigned ones before selecting the S3 operation or reaching storage.
|
||||
- `GHSA-3ppv-fx5m-m749`: explicit `versionId` reads and copy sources authorized `s3:GetObject` instead of `s3:GetObjectVersion`. Lesson: version-specific object access must select version-specific actions for direct reads, `CopyObject`, and `UploadPartCopy`, with tests proving the backend is not reached on denial.
|
||||
- `GHSA-gjph-xcgw-4p7w`: `PutObject` and browser POST accepted a caller-supplied `versionId` and replaced an unlocked historical version with only `s3:PutObject`. Lesson: only replication-authorized writes may choose an existing version ID; ordinary uploads must not rewrite version history.
|
||||
- `GHSA-x298-9x87-fvjq`: anonymous `ListObjectVersions` fell back to `ListBucket` and returned before public-access-block gates. Lesson: compatibility fallbacks must converge on the same post-authorization checks as direct grants, especially `RestrictPublicBuckets` and anonymous data-plane denies.
|
||||
- `GHSA-mx42-j6wv-px98`: `UploadPartCopy` missed source authorization and allowed cross-bucket object exfiltration. Lesson: multipart copy must enforce the same source and destination contract as `CopyObject`.
|
||||
- `GHSA-wfxj-ph3v-7mjf`: `UploadPartCopy` checked source and destination independently but missed destination copy-source policy constraints. Lesson: source read and destination write checks are not sufficient when policy constrains allowed copy sources.
|
||||
@@ -60,12 +61,14 @@ Update this file only when an advisory adds or changes a reusable lesson, affect
|
||||
- `GHSA-3g29-xff2-92vp`: FTP `RETR` and `SIZE`/`MDTM` read paths authenticated the user but skipped IAM before calling storage. Lesson: non-HTTP protocol frontends must enforce the same per-operation authorization as the S3 API before backend access.
|
||||
- `GHSA-g3vq-vv42-f647`: FTPS `MKD` called `create_bucket` without checking `s3:CreateBucket`. Lesson: protocol command handlers need action-specific checks even when sibling handlers already authorize correctly.
|
||||
- `GHSA-3p3x-734c-h5vx`: FTPS and WebDAV compared secret keys with early-return string equality, while FTPS also returned distinguishable invalid-user and invalid-password failures. Lesson: password-style protocol auth needs constant-time secret comparison, indistinguishable failures where practical, and rate limiting.
|
||||
- `GHSA-rhg6-jrf2-29c4`: WebDAV, FTP/FTPS, and SFTP used identity-policy checks with empty conditions and bypassed S3 bucket-policy explicit denies and condition-key denies. Lesson: alternate listeners must evaluate the same effective policy and trustworthy request context before storage access.
|
||||
|
||||
### Filesystem paths and object key traversal
|
||||
|
||||
- `GHSA-pq29-69jg-9mxc`: RPC `read_file_stream` joined untrusted paths under a volume directory without canonical boundary checks. Lesson: `PathBuf::join` plus length checks are not path security.
|
||||
- `GHSA-8r6f-hmq2-28rg`: object keys containing traversal sequences bypassed bucket/object authorization when mapped to filesystem paths. Lesson: reject traversal at object-key parsing and verify final storage paths remain under the expected bucket/key root.
|
||||
- `GHSA-f4vq-9ffr-m8m3`: Snowball auto-extract accepted archive entries such as `../victim-bucket/object`, authorized the raw attacker-bucket path, then storage path cleaning crossed bucket boundaries. Lesson: archive entries become object keys and need traversal rejection plus consistent authz/storage normalization before writes.
|
||||
- `GHSA-g7fq-6xjm-mxf4`: versioned `DeleteObjects` accepted dot-segment keys, authorized the literal prefix, then wrote delete markers under a resolved sibling prefix. Lesson: validate each batch key before authorization and storage normalization, using the same object-name rules as single delete.
|
||||
|
||||
### Secrets, defaults, and cryptographic misuse
|
||||
|
||||
@@ -106,6 +109,7 @@ Update this file only when an advisory adds or changes a reusable lesson, affect
|
||||
### Object Lock and retention invariants
|
||||
|
||||
- `GHSA-j548-9grx-fh4f`: Object Lock enforcement treated unreadable, fabricated, or unparsable bucket metadata as absent configuration and allowed retained objects to be deleted or expired. Lesson: retention must fail closed unless Object Lock absence is authoritative, and every delete, lifecycle, scanner, force-delete, and default-retention path needs the same state distinction.
|
||||
- `GHSA-28w4-rp2p-636w` and `GHSA-c7hx-vr97-45rg`: an authorized `DeleteBucket` with `x-rustfs-force-delete: true` removed active COMPLIANCE-retained versions even though ordinary version deletion was denied. Lesson: bucket force-delete must enforce retention across every version, including against root/owner callers.
|
||||
|
||||
### Serde deserialization and input validation
|
||||
|
||||
@@ -135,16 +139,17 @@ rg -n "deny_unknown_fields|serde.default|as u32|as usize|as i32" rustfs crates
|
||||
## Minimum Regression Test Expectations
|
||||
|
||||
- Authz fixes: include unauthenticated, valid low-privilege, wrong-action, correct-action, owner, non-owner, and root/admin cases as applicable.
|
||||
- Protocol frontend authz fixes: include denied `RETR`, `SIZE`/`MDTM`, `MKD`, bucket probe, and sibling allowed-operation cases, and assert denied paths do not reach the storage backend.
|
||||
- Protocol frontend authz fixes: include denied `RETR`, `SIZE`/`MDTM`, `MKD`, bucket probe, and sibling allowed-operation cases; test bucket-policy explicit denies and condition-key denies on each enabled listener against an S3 control, and assert denied paths do not reach storage.
|
||||
- IAM fixes: include import/update/list service-account cases with attacker-controlled parent, claims, access key, secret key, and policy.
|
||||
- Copy/upload fixes: include cross-bucket, cross-user, source-denied, destination-denied, copy-source-condition, and multipart completion cases.
|
||||
- Presigned upload fixes: include a valid presign with extra unsigned tagging, redirect, storage-class, and cross-bucket copy-source headers; require rejection before source or destination storage access, and verify explicitly signed equivalents still work.
|
||||
- Version-action fixes: include historical UUID, explicit current version, `null`, range, partNumber, presigned, STS/session, service-account, anonymous bucket-policy, copy source, and multipart-copy source cases.
|
||||
- Version-write fixes: a `PutObject`-only principal must not replace an existing version through `PutObject` or browser POST `?versionId=`; verify replication-authorized writes and versioned, suspended, and unversioned buckets separately.
|
||||
- Policy-condition fixes: include reserved-key header collisions, missing keys, partially overlapping multi-value sets, plugin mode, and built-in policy mode.
|
||||
- Path fixes: include encoded traversal, absolute path, nested traversal, archive entries with `..`, valid object keys that resemble traversal text but should be rejected, and canonical bucket/prefix boundary checks.
|
||||
- Path fixes: include encoded traversal, absolute path, nested traversal, archive entries with `..`, valid object keys that resemble traversal text but should be rejected, and canonical bucket/prefix boundary checks; test per-key `DeleteObjects` errors and prefix-scoped denials on versioned buckets.
|
||||
- Logging fixes: assert redacted output for structs and response bodies that may contain credentials.
|
||||
- IAM export fixes: assert exported archives omit plaintext user and service-account secrets unless the format deliberately encrypts or seals them.
|
||||
- RPC auth fixes: include captured metadata replay across two concrete methods, stale timestamps, wrong path, wrong method surrogate, wrong secret, and valid same-method calls.
|
||||
- Browser/CORS fixes: assert no credentials on reflected/default origins, correct behavior for explicit allowlists, and no same-origin script execution for previewed object content.
|
||||
- SSE fixes: inspect stored bytes and verify API metadata, read-back behavior, and on-disk ciphertext together; for ETags, upload equal plaintext under different keys and assert list-only callers cannot derive or correlate the plaintext digest, including multipart objects.
|
||||
- Object Lock fixes: include unreadable metadata, fabricated metadata defaults, unparsable config, confirmed absent config, COMPLIANCE/GOVERNANCE retention, lifecycle expiry, scanner sweeps, and force-delete paths.
|
||||
- Object Lock fixes: include unreadable metadata, fabricated metadata defaults, unparsable config, confirmed absent config, COMPLIANCE/GOVERNANCE retention, lifecycle expiry, scanner sweeps, and force-delete paths; an authorized force-delete must preserve active COMPLIANCE versions.
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=98dff0961c6eb21d6128bfd2d4a6100a624ddf1fb7db05dba360a5d3d0a310c2
|
||||
sha256-linux=9ff0da7406de9b89c0e848ff29f2238b30ba9da89c7e9df97c246ce86631350f
|
||||
sha256-linux=890f5925b87e1bf3710c356253f3b2276d901e156df9ccc04ec9315374de6d22
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=55534a97fbd376f64c8f6c341d319017d11ff77cad6da8629a1a7f6a874e0315
|
||||
sha256-linux=c06fb8c19aed6f388b9dc61cb8251b7a44f8561a9bf764ad2b9e635598f8dc17
|
||||
sha256-darwin=a2cb67d91eb0d093928371d6e84a84b2dd41b2ba975a28094e81879bd65838ad
|
||||
sha256-linux=82db0561cf9b111f11782d647d0859c9d2a44c5c68481b5e39c48505828bd392
|
||||
|
||||
@@ -61,6 +61,11 @@
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::core::io_primitives::tests::rename_rollback_incomplete_preserves_overwrite_data_dirs_and_staging"
|
||||
},
|
||||
{
|
||||
"invariant": "rollback-closed-inspection-channel",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::core::io_primitives::tests::rename_rollback_closed_heal_receiver_preserves_recovery_material"
|
||||
},
|
||||
{
|
||||
"invariant": "rollback-cancellation",
|
||||
"suite": "rustfs-ecstore",
|
||||
@@ -86,6 +91,11 @@
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::ops::object::put_object_tmp_cleanup_tests::tail_drained_put_waits_for_tail_and_allows_immediate_cas"
|
||||
},
|
||||
{
|
||||
"invariant": "seeded-put-tail-pressure",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::ops::object::put_object_tmp_cleanup_tests::seeded_put_completion_pressure_drains_every_real_tail"
|
||||
},
|
||||
{
|
||||
"invariant": "borrowed-lock-tail",
|
||||
"suite": "rustfs-ecstore",
|
||||
@@ -151,6 +161,11 @@
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::core::metadata_quorum::tests::metadata_observation_all_four_slot_states_and_arrival_orders_preserve_reduction"
|
||||
},
|
||||
{
|
||||
"invariant": "metadata-observation-mixed-current-identity-arrivals-preserve-selection",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::core::metadata_quorum::tests::metadata_observation_mixed_current_identity_arrivals_preserve_selection"
|
||||
},
|
||||
{
|
||||
"invariant": "metadata-observation-pending-newer-version-and-same-time-different-directory-force-full-wait",
|
||||
"suite": "rustfs-ecstore",
|
||||
@@ -166,6 +181,21 @@
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::core::metadata_quorum::tests::metadata_observation_null_versions_markers_and_invalid_success_keep_their_meaning"
|
||||
},
|
||||
{
|
||||
"invariant": "metadata-cache-real-get-and-range-preserve-body-across-read-modes",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::read::metadata_cache_tests::metadata_cache_real_get_and_range_preserve_body_across_read_modes"
|
||||
},
|
||||
{
|
||||
"invariant": "late-metadata-refresh-rejects-changed-generation-and-mapped-shards",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::read::metadata_cache_tests::late_metadata_refresh_rejects_changed_generation_and_mapped_shards"
|
||||
},
|
||||
{
|
||||
"invariant": "late-full-get-rejects-changed-object-generation",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::prepared_get_object_metadata_tests::non_inline_two_phase_get_rejects_generation_change_before_late_refresh"
|
||||
},
|
||||
{
|
||||
"invariant": "list-objects-shared-corpus-static-namespace",
|
||||
"suite": "rustfs-ecstore",
|
||||
@@ -260,6 +290,46 @@
|
||||
"invariant": "logical-prefix-limit",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "store::list_objects::test::list_path_gather_results_counts_common_prefixes_before_page_limit"
|
||||
},
|
||||
{
|
||||
"invariant": "quota-same-name-owner-and-disk-authority",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "bucket::metadata_sys::tests::quota_snapshot_reuses_same_name_owner_and_rereads_disk_authority"
|
||||
},
|
||||
{
|
||||
"invariant": "quota-snapshot-scope-and-owner",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "bucket::metadata_sys::tests::quota_snapshot_rejects_wrong_scope_incarnation_and_transaction_guard"
|
||||
},
|
||||
{
|
||||
"invariant": "quota-snapshot-lifecycle-and-incarnation",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "bucket::metadata_sys::tests::quota_snapshot_rejects_lifecycle_loss_during_read_and_changed_disk_incarnation"
|
||||
},
|
||||
{
|
||||
"invariant": "marker-purge-receipt-reopen",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::ops::object::marker_purge_receipt_tests::quorum_marker_purge_receipt_survives_reopen_with_stale_member"
|
||||
},
|
||||
{
|
||||
"invariant": "external-minio-shards",
|
||||
"suite": "rustfs-ecstore::pinned_erasure_fixtures_test",
|
||||
"name": "pinned_minio_shards_decode_and_reconstruct_plaintext"
|
||||
},
|
||||
{
|
||||
"invariant": "external-legacy-shards",
|
||||
"suite": "rustfs-ecstore::pinned_erasure_fixtures_test",
|
||||
"name": "pinned_legacy_shards_decode_and_reconstruct_plaintext"
|
||||
},
|
||||
{
|
||||
"invariant": "external-shard-bitrot",
|
||||
"suite": "rustfs-ecstore::pinned_erasure_fixtures_test",
|
||||
"name": "pinned_erasure_shards_reject_bitrot_corruption"
|
||||
},
|
||||
{
|
||||
"invariant": "external-shard-quorum-codec",
|
||||
"suite": "rustfs-ecstore::pinned_erasure_fixtures_test",
|
||||
"name": "pinned_erasure_shards_reject_insufficient_quorum_and_wrong_codec"
|
||||
}
|
||||
],
|
||||
"fixtures": [
|
||||
@@ -302,6 +372,21 @@
|
||||
"path": "crates/ecstore/tests/fixtures/list_namespace_keys.json",
|
||||
"sha256": "5afddb60565deba0938e617470255563f0e15a15fab699645c59bb7ccc84898e",
|
||||
"source": "Shared static LIST namespace corpus; ECStore differential tests and ODM list_through_static_namespace_boundary_matrix"
|
||||
},
|
||||
{
|
||||
"path": "crates/ecstore/tests/fixtures/erasure-shards/minio/manifest.json",
|
||||
"sha256": "c398f8dcb76903f6cf8eee07931c76592895ced201ff9d59ed4ff9b0ead2b13c",
|
||||
"source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/erasure-shards/README.md"
|
||||
},
|
||||
{
|
||||
"path": "crates/ecstore/tests/fixtures/erasure-shards/legacy/manifest.json",
|
||||
"sha256": "dce397d5b2f42748eb5091b4355bb9d133a50cbba003626fafac28095dbe540b",
|
||||
"source": "RustFS 1.0.0-alpha.37; crates/ecstore/tests/fixtures/erasure-shards/README.md"
|
||||
},
|
||||
{
|
||||
"path": "crates/ecstore/tests/fixtures/erasure-shards/plaintext.bin",
|
||||
"sha256": "b5a83332327961a31b65a8731c64559fd04ff8ac9f6124022201dbbdba5ffa01",
|
||||
"source": "Independent fixed SHA256 counter-stream oracle; crates/ecstore/tests/fixtures/erasure-shards/README.md"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
+14
-3
@@ -92,6 +92,12 @@ setup = 'ecstore-large-stack'
|
||||
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
# These handler/store fixtures and the relocated-pool GET compose deep storage
|
||||
# futures in libtest threads; on Linux they overflow the default 2 MiB stack.
|
||||
[[profile.default.scripts]]
|
||||
filter = 'package(rustfs) & (test(/^admin::handlers::table_catalog::tests::/) | test(/^table_catalog::store::tests::/) | test(=app::object::get::tests::execute_get_object_resumes_from_relocated_pool_without_splicing_body))'
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
[[profile.default.scripts]]
|
||||
filter = 'binary(lifecycle_integration_test) | (package(rustfs) & test(/^app::lifecycle_transition_api_test::/))'
|
||||
setup = 'lifecycle-large-stack'
|
||||
@@ -170,10 +176,11 @@ filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnost
|
||||
test-group = 'embedded-test-ports'
|
||||
|
||||
# Registration, inventory delivery, and real drive/object probes include
|
||||
# durable filesystem IO in short deadlines. Reserve capacity so sibling test
|
||||
# filesystem or HTTP IO in short deadlines. Reserve capacity so sibling test
|
||||
# processes cannot exhaust those budgets. Keep deadlines and assertions intact.
|
||||
# The short HTTP fixtures are tracked in rustfs/backlog#2714.
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_object) & test(=real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put)))'
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# This cancellation probe hashes the serving executable before acquiring its
|
||||
@@ -336,6 +343,10 @@ setup = 'ecstore-large-stack'
|
||||
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
[[profile.ci.scripts]]
|
||||
filter = 'package(rustfs) & (test(/^admin::handlers::table_catalog::tests::/) | test(/^table_catalog::store::tests::/) | test(=app::object::get::tests::execute_get_object_resumes_from_relocated_pool_without_splicing_body))'
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
[[profile.ci.scripts]]
|
||||
filter = 'binary(lifecycle_integration_test) | (package(rustfs) & test(/^app::lifecycle_transition_api_test::/))'
|
||||
setup = 'lifecycle-large-stack'
|
||||
@@ -428,7 +439,7 @@ filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnost
|
||||
test-group = 'embedded-test-ports'
|
||||
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_object) & test(=real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put)))'
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# Preserve the cancellation probe's startup budget and assertions in CI too.
|
||||
|
||||
@@ -98,6 +98,10 @@ runs:
|
||||
shell: bash
|
||||
run: make script-tests
|
||||
|
||||
- name: Check object generation protocol model
|
||||
shell: bash
|
||||
run: python3 ./scripts/test_object_generation_protocol_model.py
|
||||
|
||||
- name: Check test wiring
|
||||
shell: bash
|
||||
run: python3 ./scripts/check_test_wiring.py
|
||||
|
||||
@@ -88,7 +88,7 @@ jobs:
|
||||
# database lookup with a second ignore list to maintain.
|
||||
cargo-deny:
|
||||
name: Cargo Deny
|
||||
if: github.event_name != 'pull_request' || github.event.action != 'closed'
|
||||
if: github.repository == 'rustfs/rustfs' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
runs-on: sm-standard-2
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
@@ -131,7 +131,7 @@ jobs:
|
||||
|
||||
workflow-pin-report:
|
||||
name: Workflow Pin Report
|
||||
if: github.event_name != 'pull_request' || github.event.action != 'closed'
|
||||
if: github.repository == 'rustfs/rustfs' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
runs-on: sm-standard-2
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
@@ -203,7 +203,7 @@ jobs:
|
||||
# workflow produces, and until now it was only visible to whoever happened to
|
||||
# open the Actions tab. Same ci-8 mechanism coverage.yml and
|
||||
# e2e-replication-nightly.yml already use.
|
||||
if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
|
||||
if: github.repository == 'rustfs/rustfs' && (always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure'))
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
|
||||
@@ -440,6 +440,17 @@ jobs:
|
||||
fi
|
||||
done
|
||||
|
||||
- name: Verify macOS binary has no dynamic liblzma dependency
|
||||
if: matrix.platform == 'macos'
|
||||
shell: bash
|
||||
run: |
|
||||
binary="target/${{ matrix.target }}/release/rustfs"
|
||||
if otool -L "$binary" | grep -E 'liblzma.*\.dylib'; then
|
||||
echo "macOS release binary must not depend on a dynamic liblzma library" >&2
|
||||
exit 1
|
||||
fi
|
||||
"$binary" --version
|
||||
|
||||
- name: Generate the CPU symbol catalog
|
||||
if: endsWith(matrix.target, '-linux-gnu') && matrix.features == 'pyroscope'
|
||||
shell: bash
|
||||
|
||||
@@ -36,7 +36,9 @@ on:
|
||||
- "flake.lock"
|
||||
pull_request:
|
||||
types: [ opened, synchronize, reopened, closed ]
|
||||
branches: [ main ]
|
||||
# gateway/integration carries main's ruleset (rustfs/backlog#2734), so its
|
||||
# pull requests must produce the same required checks.
|
||||
branches: [ main, gateway/integration ]
|
||||
merge_group:
|
||||
types: [ checks_requested ]
|
||||
schedule:
|
||||
@@ -509,7 +511,7 @@ jobs:
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
strategy:
|
||||
# Preserve both independent results when one protocol fails, so diagnosis
|
||||
# Preserve independent results when one protocol fails, so diagnosis
|
||||
# and a focused fix do not require rebuilding an interrupted sibling.
|
||||
fail-fast: false
|
||||
matrix:
|
||||
@@ -518,6 +520,8 @@ jobs:
|
||||
flags: "--features swift"
|
||||
- name: sftp
|
||||
flags: "--features sftp"
|
||||
- name: http3
|
||||
flags: "--features http3,swift"
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
|
||||
@@ -78,6 +78,7 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
distributed:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Distributed 4-node 4-disk e2e
|
||||
# GitHub-hosted VM: loop and tmpfs mounts work here. sm-standard-4 is an
|
||||
# ARC pod and rejects both (`mount -o loop` ENOENT, tmpfs "read-only").
|
||||
@@ -167,11 +168,13 @@ jobs:
|
||||
FILTER: ${{ inputs.filter }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
rm -f target/nextest/e2e-distributed/junit.xml
|
||||
if [ -n "${FILTER}" ]; then
|
||||
python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-distributed -p e2e_test -E "${FILTER}"
|
||||
else
|
||||
python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-distributed -p e2e_test --no-tests=fail
|
||||
fi
|
||||
test -s target/nextest/e2e-distributed/junit.xml
|
||||
|
||||
- name: Upload distributed e2e diagnostics
|
||||
if: always()
|
||||
@@ -200,7 +203,7 @@ jobs:
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [distributed]
|
||||
if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
|
||||
if: github.repository == 'rustfs/rustfs' && (always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure'))
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
|
||||
@@ -53,6 +53,7 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
repl-nightly:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Replication e2e (nightly)
|
||||
# dual-node tests spawn two full rustfs servers each; the extra headroom
|
||||
# over the per-PR sm-standard-2 e2e job keeps the parallel process fan-out
|
||||
@@ -119,6 +120,7 @@ jobs:
|
||||
if-no-files-found: ignore
|
||||
|
||||
cluster-nightly:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Cluster fault e2e (nightly)
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
@@ -168,6 +170,7 @@ jobs:
|
||||
if-no-files-found: warn
|
||||
|
||||
protocols-nightly:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Protocol e2e (nightly)
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
@@ -229,7 +232,7 @@ jobs:
|
||||
# Only scheduled runs open/append the tracking issue (backlog#1149 ci-8);
|
||||
# manual workflow_dispatch runs stay quiet so a debugging run never files a
|
||||
# spurious alert.
|
||||
if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
|
||||
if: github.repository == 'rustfs/rustfs' && (always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure'))
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
|
||||
@@ -11,6 +11,7 @@ concurrency:
|
||||
cancel-in-progress: false
|
||||
jobs:
|
||||
collect:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
|
||||
@@ -72,9 +72,11 @@ jobs:
|
||||
fuzz-build:
|
||||
name: Build Fuzz Harness
|
||||
if: >
|
||||
(github.event_name == 'pull_request' && github.event.action != 'closed') ||
|
||||
github.event_name == 'schedule' ||
|
||||
github.event_name == 'workflow_dispatch'
|
||||
github.repository == 'rustfs/rustfs' && (
|
||||
(github.event_name == 'pull_request' && github.event.action != 'closed') ||
|
||||
github.event_name == 'schedule' ||
|
||||
github.event_name == 'workflow_dispatch'
|
||||
)
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
@@ -239,7 +241,7 @@ jobs:
|
||||
# `always()` is required: without it this job is skipped when a needed
|
||||
# job fails. Alerts only for scheduled (nightly) runs (backlog#1149
|
||||
# ci-8); PR and manual dispatch failures are already watched by a human.
|
||||
if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
|
||||
if: github.repository == 'rustfs/rustfs' && (always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure'))
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
|
||||
@@ -0,0 +1,197 @@
|
||||
# Copyright 2026 RustFS Team
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
name: Gateway Integration Sync
|
||||
|
||||
# Merges `main` into the long-lived `gateway/integration` branch every weekday
|
||||
# so the s3s -> gateway migration (rustfs/backlog#2734, task T0.1 in #2736)
|
||||
# never accumulates merge debt. A clean merge lands as a `--no-ff` merge commit
|
||||
# pushed straight to the branch. A conflicting merge is aborted and surfaced as
|
||||
# a pull request from `sync/<source>-<date>`, whose head is the un-merged
|
||||
# source commit, so the conflict is resolved the same day, in review, with no
|
||||
# other change riding along. The run itself stays green in that case: the pull
|
||||
# request is the signal. Any other failure (push rejected, pull request not
|
||||
# created) fails the run and, on the schedule, files the shared failure issue.
|
||||
#
|
||||
# Resolving a conflict pull request: merge `origin/gateway/integration` into
|
||||
# the sync branch, resolve, push, get the review. Land it with a fast-forward
|
||||
# push of the resolved merge commit to `gateway/integration`, not with the
|
||||
# squash button: a squash drops the merge ancestry and the same hunks conflict
|
||||
# again on the next run.
|
||||
#
|
||||
# Write scope: only `gateway/integration` and `sync/*` are ever pushed, and
|
||||
# both push targets below are literal refs. The pushes use a one-hour
|
||||
# installation token of the organisation's own GitHub App (contents: write)
|
||||
# because that app is the only non-admin bypass actor of the
|
||||
# `gateway/integration` ruleset; GITHUB_TOKEN cannot be a ruleset bypass actor,
|
||||
# so it cannot push through the required reviews and checks. The conflict pull
|
||||
# request is opened with FLAKE_UPDATE_TOKEN, the PAT that already opens the
|
||||
# weekly flake.lock pull requests, because this repository keeps "Allow GitHub
|
||||
# Actions to create and approve pull requests" switched off. GITHUB_TOKEN is
|
||||
# therefore read-only here.
|
||||
#
|
||||
# `schedule` fires only from the default branch, so the cron is live once this
|
||||
# file exists on `main`. `workflow_dispatch` runs from any ref that carries the
|
||||
# file and always acts on `gateway/integration`.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 2 * * 1-5" # Weekdays 02:00 UTC
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
source_ref:
|
||||
description: >-
|
||||
Branch to merge into gateway/integration. The schedule always merges
|
||||
main; only the conflict acceptance test (rustfs/backlog#2736) sets
|
||||
another branch.
|
||||
required: false
|
||||
default: main
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
TARGET_BRANCH: gateway/integration
|
||||
|
||||
jobs:
|
||||
sync:
|
||||
name: Merge main into gateway/integration
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Create the branch push token
|
||||
id: push-token
|
||||
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3
|
||||
with:
|
||||
app-id: ${{ vars.CLA_BOT_APP_ID }}
|
||||
private-key: ${{ secrets.CLA_BOT_APP_PRIVATE_KEY }}
|
||||
owner: ${{ github.repository_owner }}
|
||||
repositories: rustfs
|
||||
permission-contents: write
|
||||
|
||||
- name: Checkout gateway/integration
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
# persist-credentials-exempt: this checkout's token IS the push
|
||||
# credential — the merge commit and the sync branch are pushed with it
|
||||
# below. It is a one-hour installation token scoped to contents.
|
||||
ref: gateway/integration
|
||||
fetch-depth: 0
|
||||
token: ${{ steps.push-token.outputs.token }}
|
||||
|
||||
- name: Configure the bot identity
|
||||
run: |
|
||||
git config user.name "rustfs-cla-assistant[bot]"
|
||||
git config user.email "273121464+rustfs-cla-assistant[bot]@users.noreply.github.com"
|
||||
|
||||
- name: Merge the source branch
|
||||
id: merge
|
||||
env:
|
||||
SOURCE_REF: ${{ inputs.source_ref || 'main' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if ! [[ "$SOURCE_REF" =~ ^[A-Za-z0-9][A-Za-z0-9._/-]*$ ]] \
|
||||
|| [[ "$SOURCE_REF" == *..* ]] \
|
||||
|| [[ "$SOURCE_REF" == "$TARGET_BRANCH" ]]; then
|
||||
echo "::error::refusing source_ref '${SOURCE_REF}'"
|
||||
exit 1
|
||||
fi
|
||||
git fetch origin "refs/heads/${SOURCE_REF}:refs/remotes/origin/${SOURCE_REF}"
|
||||
source_sha=$(git rev-parse "origin/${SOURCE_REF}")
|
||||
target_sha=$(git rev-parse HEAD)
|
||||
{
|
||||
echo "source_ref=${SOURCE_REF}"
|
||||
echo "source_sha=${source_sha}"
|
||||
echo "target_sha=${target_sha}"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
if git merge --no-ff --no-edit "origin/${SOURCE_REF}"; then
|
||||
if [[ "$(git rev-parse HEAD)" == "$target_sha" ]]; then
|
||||
echo "result=up-to-date" >> "$GITHUB_OUTPUT"
|
||||
echo "\`${TARGET_BRANCH}\` already contains \`${SOURCE_REF}\` (${source_sha}); nothing to push." >> "$GITHUB_STEP_SUMMARY"
|
||||
else
|
||||
echo "result=merged" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
git diff --name-only --diff-filter=U > "$RUNNER_TEMP/conflicts.txt"
|
||||
git merge --abort
|
||||
if [[ ! -s "$RUNNER_TEMP/conflicts.txt" ]]; then
|
||||
echo "::error::git merge failed without reporting conflicting paths; this is not a conflict"
|
||||
exit 1
|
||||
fi
|
||||
echo "result=conflict" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Push the merge commit
|
||||
if: steps.merge.outputs.result == 'merged'
|
||||
env:
|
||||
SOURCE_REF: ${{ steps.merge.outputs.source_ref }}
|
||||
SOURCE_SHA: ${{ steps.merge.outputs.source_sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git push origin HEAD:refs/heads/gateway/integration
|
||||
echo "Merged \`${SOURCE_REF}\` (${SOURCE_SHA}) into \`${TARGET_BRANCH}\` as $(git rev-parse HEAD)." >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Open the conflict pull request
|
||||
if: steps.merge.outputs.result == 'conflict'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.FLAKE_UPDATE_TOKEN }}
|
||||
SOURCE_REF: ${{ steps.merge.outputs.source_ref }}
|
||||
SOURCE_SHA: ${{ steps.merge.outputs.source_sha }}
|
||||
TARGET_SHA: ${{ steps.merge.outputs.target_sha }}
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
date_utc=$(date -u +%Y-%m-%d)
|
||||
sync_branch="sync/${SOURCE_REF//\//-}-${date_utc}"
|
||||
git push origin "refs/remotes/origin/${SOURCE_REF}:refs/heads/${sync_branch}"
|
||||
{
|
||||
echo "The scheduled merge of \`${SOURCE_REF}\` into \`${TARGET_BRANCH}\` stopped on conflicts and was aborted; \`${TARGET_BRANCH}\` is unchanged."
|
||||
echo
|
||||
echo "This branch is the un-merged \`${SOURCE_REF}\` head. Merge \`origin/${TARGET_BRANCH}\` into it, resolve the files below, push, and get the review. Land the resolved merge commit with a fast-forward push to \`${TARGET_BRANCH}\`; a squash merge drops the merge ancestry and the same hunks conflict again on the next run. Do not include any change that is not conflict resolution."
|
||||
echo
|
||||
echo "- Run: ${RUN_URL}"
|
||||
echo "- Source: \`${SOURCE_REF}\` at ${SOURCE_SHA}"
|
||||
echo "- Target: \`${TARGET_BRANCH}\` at ${TARGET_SHA}"
|
||||
echo
|
||||
echo "## Conflicting files"
|
||||
echo
|
||||
echo "Output of \`git diff --name-only --diff-filter=U\` captured before \`git merge --abort\`:"
|
||||
echo
|
||||
sed 's/^/- /' "$RUNNER_TEMP/conflicts.txt"
|
||||
} > "$RUNNER_TEMP/pr-body.md"
|
||||
title="sync: merge ${SOURCE_REF} ${date_utc} (conflicts)"
|
||||
existing=$(gh pr list --repo "$GITHUB_REPOSITORY" --base "$TARGET_BRANCH" --head "$sync_branch" --state open --json number --jq '.[0].number // empty')
|
||||
if [[ -n "$existing" ]]; then
|
||||
gh pr edit "$existing" --repo "$GITHUB_REPOSITORY" --title "$title" --body-file "$RUNNER_TEMP/pr-body.md"
|
||||
pr_url=$(gh pr view "$existing" --repo "$GITHUB_REPOSITORY" --json url --jq .url)
|
||||
else
|
||||
pr_url=$(gh pr create --repo "$GITHUB_REPOSITORY" --base "$TARGET_BRANCH" --head "$sync_branch" --title "$title" --body-file "$RUNNER_TEMP/pr-body.md")
|
||||
fi
|
||||
echo "Conflicts in \`${SOURCE_REF}\` -> \`${TARGET_BRANCH}\`; pull request: ${pr_url}" >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [sync]
|
||||
# Same ci-8 mechanism audit.yml and coverage.yml use: a scheduled run that
|
||||
# could not push or could not open the conflict pull request must not stay
|
||||
# visible only in the Actions tab.
|
||||
if: github.repository == 'rustfs/rustfs' && (always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure'))
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Open or update failure-tracking issue
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}
|
||||
@@ -43,6 +43,7 @@ env:
|
||||
|
||||
jobs:
|
||||
resolve-source:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Resolve nightly source
|
||||
runs-on: sm-standard-2
|
||||
timeout-minutes: 10
|
||||
@@ -496,8 +497,10 @@ jobs:
|
||||
name: Alert on scheduled failure
|
||||
needs: [build, kms-vault-lane, kms-vault-ha-failover]
|
||||
if: >-
|
||||
always() && github.event_name == 'schedule' &&
|
||||
(contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
|
||||
github.repository == 'rustfs/rustfs' && (
|
||||
always() && github.event_name == 'schedule' &&
|
||||
(contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
|
||||
)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
|
||||
@@ -32,6 +32,7 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
update-flake:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Update flake.lock
|
||||
runs-on: sm-standard-2
|
||||
timeout-minutes: 90
|
||||
|
||||
@@ -49,6 +49,7 @@ env:
|
||||
|
||||
jobs:
|
||||
warp-ab:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Warp A/B budget gate
|
||||
runs-on: sm-standard-2
|
||||
# A normal nightly restores the last successful binary and builds only the
|
||||
@@ -364,8 +365,10 @@ jobs:
|
||||
# composite action already reports cancelled/timed-out jobs in the issue
|
||||
# body.
|
||||
if: >-
|
||||
always() && github.event_name == 'schedule' &&
|
||||
(contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
|
||||
github.repository == 'rustfs/rustfs' && (
|
||||
always() && github.event_name == 'schedule' &&
|
||||
(contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
|
||||
)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
|
||||
@@ -42,6 +42,7 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
check-ephemerality:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Check runner ephemerality
|
||||
runs-on: sm-standard-2
|
||||
timeout-minutes: 15
|
||||
@@ -65,7 +66,7 @@ jobs:
|
||||
# Same ci-8 mechanism as coverage.yml, audit.yml and the nightly lanes:
|
||||
# scheduled runs file a tracking issue, manual dispatch stays quiet so
|
||||
# debugging never produces a spurious alert.
|
||||
if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
|
||||
if: github.repository == 'rustfs/rustfs' && (always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure'))
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
|
||||
@@ -266,7 +266,10 @@ jobs:
|
||||
# the previous 45m step cap killed four consecutive runs mid-rebalance
|
||||
# with steps 1-7 green. 240m covers the observed pace (rebalance 1/4
|
||||
# in ~21m) plus one decommission pass; override per dispatch.
|
||||
timeout-minutes: ${{ inputs.pool_timeout_minutes || 240 }}
|
||||
# fromJSON coerces the string dispatch input to the number that
|
||||
# timeout-minutes requires; the bare || form yielded the string '240'
|
||||
# and the runner rejected the template (Unexpected value '240').
|
||||
timeout-minutes: ${{ fromJSON(inputs.pool_timeout_minutes || '240') }}
|
||||
# Case failures keep the run green: the report and the backlog
|
||||
# issue manager carry the product signal.
|
||||
continue-on-error: true
|
||||
@@ -583,25 +586,6 @@ jobs:
|
||||
echo "pool test script reported a failure (step FAIL verdict or die() abort)" >&2
|
||||
failed=1
|
||||
fi
|
||||
# A runner-level kill (step timeout) truncates the suite without a
|
||||
# FAIL verdict or an [ERROR] line, so require every expected
|
||||
# [POOL-STEP] marker to be present with PASS - missing markers mean
|
||||
# the run was truncated, not green. This also closes the gap for
|
||||
# standalone dispatches, where the chain driver's evidence
|
||||
# validation does not run.
|
||||
last_step=6
|
||||
if [ "${{ inputs.pools || '3' }}" = "3" ]; then
|
||||
last_step=8
|
||||
if [ "${{ inputs.run_decommission != 'false' }}" = "true" ]; then
|
||||
last_step=9
|
||||
fi
|
||||
fi
|
||||
for n in $(seq 1 "${last_step}"); do
|
||||
grep -q "^\[POOL-STEP\] ${n} .* PASS" "${POOL_ARTIFACT_DIR}/pool-test.log" 2>/dev/null || {
|
||||
echo "pool test log is missing [POOL-STEP] ${n} PASS (expected through step ${last_step}); the run was truncated" >&2
|
||||
failed=1
|
||||
}
|
||||
done
|
||||
[ "${failed}" -eq 0 ] || exit 1
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
|
||||
@@ -28,6 +28,7 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
check-freshness:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Check scheduled validation freshness
|
||||
# Stay on GitHub-hosted runners: this monitor must keep running when the
|
||||
# self-hosted sm-standard-* pool is down, which is exactly when the
|
||||
|
||||
@@ -39,6 +39,7 @@ jobs:
|
||||
alert-on-incomplete-run:
|
||||
name: Alert on incomplete scheduled run
|
||||
if: >-
|
||||
github.repository == 'rustfs/rustfs' &&
|
||||
github.event.workflow_run.event == 'schedule' &&
|
||||
github.event.workflow_run.conclusion != 'success' &&
|
||||
github.event.workflow_run.conclusion != 'failure'
|
||||
|
||||
@@ -26,6 +26,7 @@ on:
|
||||
|
||||
jobs:
|
||||
stale:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
runs-on: sm-standard-2
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
|
||||
@@ -14,6 +14,7 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
update:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
runs-on: sm-standard-2
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
|
||||
@@ -40,6 +40,7 @@ env:
|
||||
|
||||
jobs:
|
||||
targets-live:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: PostgreSQL, MySQL, AMQP, and NATS
|
||||
runs-on: sm-standard-2
|
||||
timeout-minutes: 90
|
||||
@@ -175,8 +176,10 @@ jobs:
|
||||
name: Alert on scheduled failure
|
||||
needs: [targets-live]
|
||||
if: >-
|
||||
always() && github.event_name == 'schedule' &&
|
||||
(contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
|
||||
github.repository == 'rustfs/rustfs' && (
|
||||
always() && github.event_name == 'schedule' &&
|
||||
(contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
|
||||
)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
|
||||
Generated
+376
-178
File diff suppressed because it is too large
Load Diff
+30
-19
@@ -18,6 +18,7 @@ members = [
|
||||
"crates/audit", # Audit target management system with multi-target fan-out
|
||||
"crates/checksums", # client checksums
|
||||
"crates/common", # Shared utilities and data structures
|
||||
"crates/capture-authority", # Durable capture authority storage
|
||||
"crates/data-usage", # Shared data usage models and algorithms
|
||||
"crates/config", # Configuration management
|
||||
"crates/connect-symbol-catalog", # Release-only CPU profile symbol catalog generator
|
||||
@@ -73,7 +74,7 @@ resolver = "3"
|
||||
edition = "2024"
|
||||
license = "Apache-2.0"
|
||||
repository = "https://github.com/rustfs/rustfs"
|
||||
rust-version = "1.98.1"
|
||||
rust-version = "1.99.0"
|
||||
version = "1.0.1"
|
||||
homepage = "https://rustfs.com"
|
||||
description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
|
||||
@@ -103,6 +104,7 @@ rustfs-config = { path = "./crates/config", version = "1.0.1" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.1" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.1" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.1" }
|
||||
rustfs-capture-authority = { path = "crates/capture-authority", version = "1.0.1" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.1" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.1" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.1" }
|
||||
@@ -157,7 +159,7 @@ futures-util = "0.3.34"
|
||||
pollster = "1.0.1"
|
||||
pulsar = { default-features = false, version = "6.9.0" }
|
||||
lapin = { default-features = false, version = "4.12.0" }
|
||||
hyper = { version = "1.11.1" }
|
||||
hyper = { version = "1.12.0" }
|
||||
hyper-rustls = { default-features = false, version = "0.27.10" }
|
||||
hyper-util = { version = "0.1.21" }
|
||||
http = "1.5.0"
|
||||
@@ -166,9 +168,9 @@ http-body-util = "0.1.5"
|
||||
memchr = "2.8.3"
|
||||
minlz = "1.2.3"
|
||||
reqwest = "0.13.5"
|
||||
rustfs-kafka-async = { version = "1.3.1" }
|
||||
rustfs-kafka-async = { version = "1.4.0" }
|
||||
socket2 = { version = "0.6.5" }
|
||||
tokio = { version = "1.53.1" }
|
||||
tokio = { version = "1.53.2" }
|
||||
tokio-rustls = { default-features = false, version = "0.26.6" }
|
||||
tokio-stream = { version = "0.1.19" }
|
||||
tokio-test = "0.4.6"
|
||||
@@ -178,6 +180,13 @@ tonic-prost = { version = "0.14.6" }
|
||||
tonic-prost-build = { version = "0.14.6" }
|
||||
tower = { version = "0.5.3" }
|
||||
tower-http = { version = "0.7.1" }
|
||||
h3 = "0.0.8"
|
||||
h3-quinn = "0.0.10"
|
||||
quinn = { version = "0.11.11", default-features = false, features = [
|
||||
"futures-io",
|
||||
"runtime-tokio",
|
||||
"rustls-aws-lc-rs",
|
||||
] }
|
||||
|
||||
# Serialization and Data Formats
|
||||
apache-avro = { version = "0.22.0", features = ["snappy", "zstandard"] }
|
||||
@@ -219,13 +228,13 @@ x509-parser = "0.18.1"
|
||||
sha1 = "0.11.0"
|
||||
sha2 = "0.11.0"
|
||||
subtle = "2.6"
|
||||
zeroize = { version = "1.9.0" }
|
||||
zeroize = { version = "1.9.1" }
|
||||
proptest = "1"
|
||||
|
||||
# Time and Date
|
||||
chrono = { version = "0.4.45" }
|
||||
humantime = "2.4.0"
|
||||
jiff = { version = "0.2.37" }
|
||||
jiff = { version = "0.2.38" }
|
||||
time = { version = "0.3.55" }
|
||||
|
||||
# Database
|
||||
@@ -244,9 +253,9 @@ atoi = "3.1.0"
|
||||
atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.12.0" }
|
||||
aws-credential-types = { version = "1.3.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.122.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.151.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.118.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.123.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.152.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.119.0" }
|
||||
aws-smithy-async = { version = "1.3.0" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.5.0" }
|
||||
aws-smithy-runtime-api = { version = "1.19.0" }
|
||||
@@ -279,7 +288,7 @@ highway = { version = "1.3.0" }
|
||||
hostname = "0.4.2"
|
||||
ipnetwork = { version = "0.21.1" }
|
||||
lazy_static = "1.5.1"
|
||||
libc = "0.2.189"
|
||||
libc = "0.2.190"
|
||||
libsystemd = "0.7.2"
|
||||
local-ip-address = "0.6.13"
|
||||
log = "0.4"
|
||||
@@ -299,7 +308,7 @@ percent-encoding = "2.3.2"
|
||||
# Server-side QR rendering for TOTP enrollment, so neither the console nor the
|
||||
# CLI needs its own QR encoder. No default features: the image/render backends
|
||||
# pull in an image stack this only needs SVG and text output from.
|
||||
qrcode-rs = { version = "2.1.1", default-features = false, features = ["std", "svg"] }
|
||||
qrcode-rs = { version = "2.2.0", default-features = false, features = ["std", "svg"] }
|
||||
pin-project-lite = "0.2.17"
|
||||
pretty_assertions = "1.4.1"
|
||||
rand = { version = "0.10.3" }
|
||||
@@ -307,6 +316,8 @@ ratelimit = "2.0.0"
|
||||
rayon = "1.12.0"
|
||||
rustfs-erasure-codec = { version = "9.0.0" }
|
||||
reed-solomon-simd = "3.1.0"
|
||||
openraft = { version = "=0.9.25", default-features = false, features = ["serde"] }
|
||||
redb = "=4.3.0"
|
||||
regex = { version = "1.13.1" }
|
||||
rumqttc = { package = "rumqttc-next", version = "0.34.0" }
|
||||
redis = { version = "1.7.1" }
|
||||
@@ -314,15 +325,15 @@ rustify = { version = "0.7", default-features = false }
|
||||
rustix = { version = "1.1.5" }
|
||||
rust-embed = { version = "8.12.0" }
|
||||
rustc-hash = { version = "2.1.3" }
|
||||
s3s = { version = "0.17.0", features = ["minio"] }
|
||||
s3s-sigv4 = "0.17.0"
|
||||
s3s = { git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fcac87e3b99fd5bd9e176b442c0", features = ["minio"] }
|
||||
s3s-sigv4 = { git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fcac87e3b99fd5bd9e176b442c0" }
|
||||
serial_test = "4.0.1"
|
||||
shadow-rs = { default-features = false, version = "2.0.0" }
|
||||
siphasher = "1.0.4"
|
||||
smallvec = { version = "1.16.2" }
|
||||
compact_str = "0.10.0"
|
||||
snap = "1.1.2"
|
||||
starshard = { version = "2.3.0" }
|
||||
starshard = { version = "2.4.0" }
|
||||
strum = { version = "0.28.0" }
|
||||
sysinfo = "0.39.6"
|
||||
temp-env = "0.3.6"
|
||||
@@ -338,7 +349,7 @@ tracing-subscriber = { version = "0.3.23" }
|
||||
transform-stream = "0.3.1"
|
||||
url = "2.5.8"
|
||||
urlencoding = "2.1.3"
|
||||
uuid = { version = "1.26.1" }
|
||||
uuid = { version = "1.27.0" }
|
||||
vaultrs = { version = "0.8.0" }
|
||||
tar = "0.4.46"
|
||||
walkdir = "2.5.0"
|
||||
@@ -352,7 +363,7 @@ zstd = "0.14.0"
|
||||
# Observability and Metrics
|
||||
metrics = "0.24.6"
|
||||
metrics-util = "0.20"
|
||||
dial9 = { version = "0.5.2", default-features = false }
|
||||
dial9 = { version = "0.5.3", default-features = false }
|
||||
opentelemetry = { version = "0.33.0" }
|
||||
opentelemetry-appender-tracing = { version = "0.33.0" }
|
||||
opentelemetry-otlp = { version = "0.33.0" }
|
||||
@@ -367,7 +378,7 @@ libunftp = { version = "0.23.1" }
|
||||
unftp-core = "0.1.0"
|
||||
suppaftp = { version = "12.1.1" }
|
||||
rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
russh = { version = "0.63.3" }
|
||||
russh = { version = "0.64.1" }
|
||||
russh-sftp = "3.0.1"
|
||||
|
||||
# WebDAV
|
||||
@@ -376,9 +387,9 @@ dav-server = "0.11.0"
|
||||
# Performance Analysis and Memory Profiling
|
||||
rustfs-mimalloc = { version = "0.5.6" }
|
||||
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
|
||||
hotpath = { version = "0.28.0", default-features = false }
|
||||
hotpath = { version = "0.28.5", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.48" }
|
||||
insta = { version = "1.49" }
|
||||
|
||||
[workspace.metadata.cargo-shear]
|
||||
ignored = ["hotpath", "rustfs"]
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
[package]
|
||||
name = "rustfs-capture-authority"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
repository.workspace = true
|
||||
description = "Durable local storage for capture authority"
|
||||
|
||||
[dependencies]
|
||||
openraft = { workspace = true, features = ["storage-v2"] }
|
||||
redb = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
thiserror = { workspace = true }
|
||||
tokio = { workspace = true, features = ["net", "io-util", "time", "sync", "rt"] }
|
||||
url = { workspace = true }
|
||||
tempfile = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
|
||||
[target.'cfg(any(target_os = "linux", target_os = "macos"))'.dependencies]
|
||||
rustix = { workspace = true, features = ["process"] }
|
||||
|
||||
[dev-dependencies]
|
||||
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,531 @@
|
||||
use super::*;
|
||||
use crate::application::{APPLY_RESULTS, EVENTS, OPERATIONS, VERSIONS};
|
||||
use crate::tests::CrashBackend;
|
||||
use redb::ReadableTableMetadata;
|
||||
use std::{
|
||||
io::{BufRead, BufReader, Write},
|
||||
process::{Command, Stdio},
|
||||
sync::mpsc,
|
||||
time::Duration,
|
||||
};
|
||||
fn identity() -> NodeIdentity {
|
||||
NodeIdentity {
|
||||
cluster_id: [7; 16],
|
||||
node_id: 0,
|
||||
}
|
||||
}
|
||||
fn binding() -> CaptureBinding {
|
||||
CaptureBinding {
|
||||
revision: 9,
|
||||
binding_id: [8; 16],
|
||||
target_id: [9; 16],
|
||||
}
|
||||
}
|
||||
fn command() -> DecideCreated {
|
||||
DecideCreated {
|
||||
operation_id: [1; 16],
|
||||
object: ObjectIdentity {
|
||||
bucket_incarnation: [2; 16],
|
||||
key: vec![0, 255, 3],
|
||||
},
|
||||
expected_head: None,
|
||||
prepared: PreparedIdentity {
|
||||
version_id: [3; 16],
|
||||
preparation_id: [4; 16],
|
||||
content_digest: [5; 32],
|
||||
content_length: 123,
|
||||
},
|
||||
binding_revision: 9,
|
||||
}
|
||||
}
|
||||
fn id(index: u64) -> LogId {
|
||||
LogId {
|
||||
term: 8,
|
||||
leader_node: 0,
|
||||
index,
|
||||
}
|
||||
}
|
||||
fn append(store: &mut Store, index: u64, c: &DecideCreated) {
|
||||
store
|
||||
.append(&[LogEntry {
|
||||
id: id(index),
|
||||
payload: c.encode().expect("encode command"),
|
||||
}])
|
||||
.expect("append command");
|
||||
}
|
||||
fn apply(store: &mut Store, index: u64, c: &DecideCreated) -> CreatedResult {
|
||||
append(store, index, c);
|
||||
store.save_committed(id(index)).expect("commit log");
|
||||
store.apply_next_created(id(index)).expect("apply command")
|
||||
}
|
||||
fn success(c: &DecideCreated, index: u64) -> CreatedResult {
|
||||
CreatedResult::Created {
|
||||
version_id: c.prepared.version_id,
|
||||
event_id: c.operation_id,
|
||||
decision_log_id: id(index),
|
||||
}
|
||||
}
|
||||
fn complete(store: &Store) {
|
||||
let c = command();
|
||||
assert_eq!(store.capture_binding().expect("binding"), Some(binding()));
|
||||
assert_eq!(
|
||||
store.created_head(&c.object).expect("head"),
|
||||
Some(c.prepared.version_id),
|
||||
"U11 acknowledged head lost"
|
||||
);
|
||||
assert_eq!(
|
||||
store.created_version(&c.object, c.prepared.version_id).expect("version"),
|
||||
Some(CreatedVersion {
|
||||
prepared: c.prepared.clone(),
|
||||
decision_log_id: id(0),
|
||||
binding_revision: 9
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
store.created_event(c.operation_id).expect("event"),
|
||||
Some(CreatedEvent {
|
||||
object: c.object.clone(),
|
||||
prepared: c.prepared.clone(),
|
||||
decision_log_id: id(0),
|
||||
binding: binding()
|
||||
}),
|
||||
"U11 complete event missing"
|
||||
);
|
||||
assert_eq!(
|
||||
store.created_operation(c.operation_id).expect("operation"),
|
||||
Some(CreatedOperation {
|
||||
canonical_command: c.encode().expect("encode"),
|
||||
result: success(&c, 0)
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
store.created_apply_result(0).expect("result"),
|
||||
Some(CreatedApplyResult {
|
||||
log_id: id(0),
|
||||
result: success(&c, 0)
|
||||
})
|
||||
);
|
||||
assert_eq!(store.last_applied().expect("last applied"), Some(id(0)));
|
||||
}
|
||||
fn fixture() -> (tempfile::TempDir, Store) {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let mut store = Store::initialize(dir.path().join("db"), identity()).expect("initialize");
|
||||
store.pin_capture_binding(binding()).expect("pin");
|
||||
(dir, store)
|
||||
}
|
||||
fn reopen(path: &std::path::Path) -> Store {
|
||||
let result = Store::open(path, identity());
|
||||
assert!(result.is_ok(), "U11 complete reopen: {:?}", result.as_ref().err());
|
||||
result.expect("asserted successful reopen")
|
||||
}
|
||||
#[test]
|
||||
fn created_codec_is_canonical_and_strict() {
|
||||
let c = command();
|
||||
let b = c.encode().expect("encode");
|
||||
let mut golden = b"CRTD\x01".to_vec();
|
||||
golden.extend([1; 16]);
|
||||
golden.extend([2; 16]);
|
||||
golden.extend(3u32.to_be_bytes());
|
||||
golden.extend([0, 255, 3]);
|
||||
golden.push(0);
|
||||
golden.extend([3; 16]);
|
||||
golden.extend([4; 16]);
|
||||
golden.extend([5; 32]);
|
||||
golden.extend(123u64.to_be_bytes());
|
||||
golden.extend(9u64.to_be_bytes());
|
||||
assert_eq!(b, golden);
|
||||
assert_eq!(DecideCreated::decode(&b).expect("decode"), c);
|
||||
let mut variants = Vec::new();
|
||||
let mut v = c.clone();
|
||||
v.operation_id = [6; 16];
|
||||
variants.push(v);
|
||||
let mut v = c.clone();
|
||||
v.object.bucket_incarnation = [6; 16];
|
||||
variants.push(v);
|
||||
let mut v = c.clone();
|
||||
v.object.key.push(4);
|
||||
variants.push(v);
|
||||
let mut v = c.clone();
|
||||
v.expected_head = Some([6; 16]);
|
||||
variants.push(v);
|
||||
let mut v = c.clone();
|
||||
v.prepared.version_id = [6; 16];
|
||||
variants.push(v);
|
||||
let mut v = c.clone();
|
||||
v.prepared.preparation_id = [6; 16];
|
||||
variants.push(v);
|
||||
let mut v = c.clone();
|
||||
v.prepared.content_digest = [6; 32];
|
||||
variants.push(v);
|
||||
let mut v = c.clone();
|
||||
v.prepared.content_length += 1;
|
||||
variants.push(v);
|
||||
let mut v = c.clone();
|
||||
v.binding_revision += 1;
|
||||
variants.push(v);
|
||||
for v in variants {
|
||||
assert_ne!(v.encode().expect("variant"), b)
|
||||
}
|
||||
for n in 0..b.len() {
|
||||
assert!(DecideCreated::decode(&b[..n]).is_err())
|
||||
}
|
||||
let mut bad = b.clone();
|
||||
bad[4] = 2;
|
||||
assert!(DecideCreated::decode(&bad).is_err());
|
||||
let mut bad = b.clone();
|
||||
bad[44] = 2;
|
||||
assert!(DecideCreated::decode(&bad).is_err());
|
||||
let mut bad = b.clone();
|
||||
bad.push(0);
|
||||
assert!(DecideCreated::decode(&bad).is_err());
|
||||
let mut bad = b;
|
||||
bad[37..41].copy_from_slice(&4097u32.to_be_bytes());
|
||||
assert!(DecideCreated::decode(&bad).is_err());
|
||||
let mut bad = c.clone();
|
||||
bad.operation_id = [0; 16];
|
||||
assert!(bad.encode().is_err());
|
||||
bad = c;
|
||||
bad.object.key = vec![0; MAX_OBJECT_KEY_LENGTH + 1];
|
||||
assert!(bad.encode().is_err());
|
||||
}
|
||||
#[test]
|
||||
fn created_committed_order_and_identity_are_enforced() {
|
||||
let (_dir, mut s) = fixture();
|
||||
let c = command();
|
||||
append(&mut s, 0, &c);
|
||||
append(&mut s, 1, &c);
|
||||
assert!(s.apply_next_created(id(0)).is_err());
|
||||
assert_eq!(s.last_applied().expect("no apply"), None);
|
||||
s.save_committed(id(1)).expect("committed");
|
||||
assert!(matches!(s.apply_next_created(id(1)), Err(StoreError::InvalidApplySequence)));
|
||||
let mut wrong = id(0);
|
||||
wrong.term += 1;
|
||||
assert!(s.apply_next_created(wrong).is_err());
|
||||
wrong = id(0);
|
||||
wrong.leader_node += 1;
|
||||
assert!(s.apply_next_created(wrong).is_err());
|
||||
assert_eq!(s.apply_next_created(id(0)).expect("first"), success(&c, 0));
|
||||
complete(&s);
|
||||
assert_eq!(s.apply_next_created(id(0)).expect("retry"), success(&c, 0));
|
||||
complete(&s);
|
||||
assert!(s.apply_next_created(wrong).is_err());
|
||||
assert_eq!(s.apply_next_created(id(1)).expect("duplicate log"), success(&c, 0));
|
||||
assert_eq!(s.last_applied().expect("last"), Some(id(1)));
|
||||
assert_eq!(
|
||||
s.created_apply_result(1).expect("result"),
|
||||
Some(CreatedApplyResult {
|
||||
log_id: id(1),
|
||||
result: success(&c, 0)
|
||||
})
|
||||
);
|
||||
let tx = s.database.begin_read().expect("read");
|
||||
assert_eq!(tx.open_table(EVENTS).expect("events").len().expect("len"), 1);
|
||||
}
|
||||
#[test]
|
||||
fn created_head_version_event_and_dedup_reopen_together() {
|
||||
let (dir, mut s) = fixture();
|
||||
let c = command();
|
||||
assert_eq!(apply(&mut s, 0, &c), success(&c, 0));
|
||||
drop(s);
|
||||
let mut s = reopen(&dir.path().join("db"));
|
||||
complete(&s);
|
||||
assert_eq!(apply(&mut s, 1, &c), success(&c, 0));
|
||||
let mut changed = c.clone();
|
||||
changed.prepared.content_digest = [6; 32];
|
||||
assert_eq!(
|
||||
apply(&mut s, 2, &changed),
|
||||
CreatedResult::OperationConflict,
|
||||
"U11 full canonical operation conflict"
|
||||
);
|
||||
assert_eq!(
|
||||
s.created_operation(c.operation_id)
|
||||
.expect("original")
|
||||
.expect("operation")
|
||||
.result,
|
||||
success(&c, 0)
|
||||
);
|
||||
let mut stale = c.clone();
|
||||
stale.operation_id = [6; 16];
|
||||
stale.prepared.version_id = [6; 16];
|
||||
assert_eq!(
|
||||
apply(&mut s, 3, &stale),
|
||||
CreatedResult::HeadMismatch {
|
||||
actual_head: Some(c.prepared.version_id)
|
||||
}
|
||||
);
|
||||
let mut reuse = c.clone();
|
||||
reuse.operation_id = [7; 16];
|
||||
reuse.expected_head = Some(c.prepared.version_id);
|
||||
assert_eq!(apply(&mut s, 4, &reuse), CreatedResult::VersionConflict);
|
||||
let mut replacement = stale.clone();
|
||||
replacement.operation_id = [10; 16];
|
||||
replacement.expected_head = Some(c.prepared.version_id);
|
||||
assert_eq!(apply(&mut s, 5, &replacement), success(&replacement, 5));
|
||||
assert_eq!(s.created_head(&c.object).expect("head"), Some(replacement.prepared.version_id));
|
||||
assert_eq!(
|
||||
s.created_version(&c.object, c.prepared.version_id)
|
||||
.expect("old version")
|
||||
.expect("version")
|
||||
.decision_log_id,
|
||||
id(0)
|
||||
);
|
||||
assert!(s.created_event(c.operation_id).expect("old event").is_some());
|
||||
assert!(s.created_event(replacement.operation_id).expect("new event").is_some());
|
||||
assert!(s.created_event(stale.operation_id).expect("rejection event").is_none());
|
||||
assert_eq!(
|
||||
apply(&mut s, 6, &stale),
|
||||
CreatedResult::HeadMismatch {
|
||||
actual_head: Some(c.prepared.version_id)
|
||||
}
|
||||
);
|
||||
drop(s);
|
||||
let s = reopen(&dir.path().join("db"));
|
||||
assert_eq!(s.last_applied().expect("last"), Some(id(6)));
|
||||
let tx = s.database.begin_read().expect("read");
|
||||
assert_eq!(tx.open_table(EVENTS).expect("events").len().expect("len"), 2);
|
||||
}
|
||||
#[test]
|
||||
fn created_binding_is_pinned_and_not_caller_selected() {
|
||||
let (dir, mut s) = fixture();
|
||||
let mut bad = command();
|
||||
bad.binding_revision = 10;
|
||||
assert_eq!(apply(&mut s, 0, &bad), CreatedResult::BindingMismatch);
|
||||
assert!(s.created_head(&bad.object).expect("head").is_none());
|
||||
assert!(s.created_event(bad.operation_id).expect("event").is_none());
|
||||
assert_eq!(
|
||||
s.created_operation(bad.operation_id)
|
||||
.expect("operation")
|
||||
.expect("saved")
|
||||
.result,
|
||||
CreatedResult::BindingMismatch
|
||||
);
|
||||
s.pin_capture_binding(binding()).expect("idempotent pin");
|
||||
let mut changed = binding();
|
||||
changed.target_id = [10; 16];
|
||||
assert!(matches!(s.pin_capture_binding(changed), Err(StoreError::BindingConflict)));
|
||||
drop(s);
|
||||
let s = reopen(&dir.path().join("db"));
|
||||
assert_eq!(s.capture_binding().expect("binding"), Some(binding()));
|
||||
}
|
||||
#[test]
|
||||
fn created_schema_compatibility_and_corruption_are_checked() {
|
||||
let dir = tempfile::tempdir().expect("dir");
|
||||
let path = dir.path().join("db");
|
||||
let s = Store::initialize(&path, identity()).expect("schema1");
|
||||
assert_eq!(s.capture_binding().expect("unmigrated"), None);
|
||||
drop(s);
|
||||
let mut s = reopen(&path);
|
||||
assert_eq!(s.last_applied().expect("no apply"), None);
|
||||
append(&mut s, 0, &command());
|
||||
s.save_committed(id(0)).expect("schema2");
|
||||
drop(s);
|
||||
let mut s = reopen(&path);
|
||||
assert_eq!(s.capture_binding().expect("schema2 unchanged"), None);
|
||||
s.pin_capture_binding(binding()).expect("explicit upgrade");
|
||||
drop(s);
|
||||
let mut s = reopen(&path);
|
||||
assert_eq!(s.capture_binding().expect("schema3"), Some(binding()));
|
||||
s.apply_next_created(id(0)).expect("apply");
|
||||
drop(s);
|
||||
let db = Database::open(&path).expect("raw db");
|
||||
let tx = db.begin_write().expect("write corruption");
|
||||
tx.open_table(EVENTS)
|
||||
.expect("events")
|
||||
.remove(command().operation_id.as_slice())
|
||||
.expect("remove reference");
|
||||
tx.commit().expect("commit corruption");
|
||||
drop(db);
|
||||
let result = Store::open(&path, identity());
|
||||
assert!(
|
||||
matches!(result, Err(StoreError::CorruptRecord)),
|
||||
"U11 missing event rejected before expect"
|
||||
);
|
||||
let (dir, mut s) = fixture();
|
||||
assert_eq!(s.read_committed().expect("empty committed"), None);
|
||||
append(&mut s, 0, &command());
|
||||
s.save_committed(id(0)).expect("committed schema3");
|
||||
assert_eq!(s.capture_binding().expect("no downgrade"), Some(binding()));
|
||||
drop(s);
|
||||
let path = dir.path().join("db");
|
||||
let db = Database::open(&path).expect("raw db");
|
||||
let tx = db.begin_write().expect("write");
|
||||
let mut marker = vec![1];
|
||||
marker.extend(encode_id(id(0)));
|
||||
tx.open_table(META)
|
||||
.expect("meta")
|
||||
.insert("last_applied", marker.as_slice())
|
||||
.expect("corrupt applied");
|
||||
tx.commit().expect("commit");
|
||||
drop(db);
|
||||
assert!(
|
||||
matches!(Store::open(&path, identity()), Err(StoreError::CorruptRecord)),
|
||||
"U11 last applied missing result rejected"
|
||||
);
|
||||
let (dir, s) = fixture();
|
||||
drop(s);
|
||||
let path = dir.path().join("db");
|
||||
let db = Database::open(&path).expect("raw pinned db");
|
||||
let tx = db.begin_write().expect("missing binding write");
|
||||
tx.open_table(META)
|
||||
.expect("meta")
|
||||
.remove("capture_binding")
|
||||
.expect("remove binding");
|
||||
tx.commit().expect("binding corruption commit");
|
||||
drop(db);
|
||||
assert!(
|
||||
matches!(Store::open(&path, identity()), Err(StoreError::CorruptRecord)),
|
||||
"U11 missing pinned binding rejected"
|
||||
);
|
||||
|
||||
let (dir, mut s) = fixture();
|
||||
let c = command();
|
||||
assert_eq!(apply(&mut s, 0, &c), success(&c, 0));
|
||||
assert_eq!(apply(&mut s, 1, &c), success(&c, 0));
|
||||
drop(s);
|
||||
let path = dir.path().join("db");
|
||||
let db = Database::open(&path).expect("raw duplicate db");
|
||||
let tx = db.begin_write().expect("retarget decision write");
|
||||
let mut key = created::object_bytes(&c.object).expect("object key");
|
||||
key.extend(c.prepared.version_id);
|
||||
let mut version = Vec::new();
|
||||
created::put_prepared(&mut version, &c.prepared);
|
||||
version.extend(encode_id(id(1)));
|
||||
version.extend(binding().revision.to_be_bytes());
|
||||
tx.open_table(VERSIONS)
|
||||
.expect("versions")
|
||||
.insert(key.as_slice(), version.as_slice())
|
||||
.expect("retarget version");
|
||||
let mut event = created::object_bytes(&c.object).expect("event object");
|
||||
created::put_prepared(&mut event, &c.prepared);
|
||||
event.extend(encode_id(id(1)));
|
||||
event.extend(created::binding_bytes(&binding()).expect("binding bytes"));
|
||||
tx.open_table(EVENTS)
|
||||
.expect("events")
|
||||
.insert(c.operation_id.as_slice(), event.as_slice())
|
||||
.expect("retarget event");
|
||||
let canonical = c.encode().expect("canonical");
|
||||
let mut operation = u64::try_from(canonical.len()).expect("length").to_be_bytes().to_vec();
|
||||
operation.extend(canonical);
|
||||
operation.extend(created::result_bytes(&success(&c, 1)));
|
||||
tx.open_table(OPERATIONS)
|
||||
.expect("operations")
|
||||
.insert(c.operation_id.as_slice(), operation.as_slice())
|
||||
.expect("retarget original response");
|
||||
for index in 0..=1 {
|
||||
let mut result = encode_id(id(index));
|
||||
result.extend(created::result_bytes(&success(&c, 1)));
|
||||
tx.open_table(APPLY_RESULTS)
|
||||
.expect("results")
|
||||
.insert(index, result.as_slice())
|
||||
.expect("retarget duplicate response");
|
||||
}
|
||||
tx.commit().expect("coherent retarget commit");
|
||||
drop(db);
|
||||
assert!(
|
||||
matches!(Store::open(&path, identity()), Err(StoreError::CorruptRecord)),
|
||||
"U11 original decision cannot retarget duplicate log"
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn created_process_child() {
|
||||
if let Some(path) = std::env::var_os("U11_CHILD_DB") {
|
||||
let mut s = Store::initialize(path, identity()).expect("child init");
|
||||
s.pin_capture_binding(binding()).expect("child pin");
|
||||
assert_eq!(apply(&mut s, 0, &command()), success(&command(), 0));
|
||||
complete(&s);
|
||||
println!("U11_ACK");
|
||||
std::io::stdout().flush().expect("flush ACK");
|
||||
loop {
|
||||
std::thread::park()
|
||||
}
|
||||
}
|
||||
let (_dir, mut s) = fixture();
|
||||
assert_eq!(apply(&mut s, 0, &command()), success(&command(), 0));
|
||||
complete(&s);
|
||||
}
|
||||
#[test]
|
||||
fn created_acknowledged_apply_survives_child_kill() {
|
||||
let dir = tempfile::tempdir().expect("dir");
|
||||
let path = dir.path().join("db");
|
||||
let mut child = Command::new(std::env::current_exe().expect("test executable"))
|
||||
.args(["--exact", "application_tests::created_process_child", "--nocapture"])
|
||||
.env("U11_CHILD_DB", &path)
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::inherit())
|
||||
.spawn()
|
||||
.expect("spawn");
|
||||
let stdout = child.stdout.take().expect("stdout");
|
||||
let (tx, rx) = mpsc::channel();
|
||||
let reader = std::thread::spawn(move || {
|
||||
for line in BufReader::new(stdout).lines() {
|
||||
let line = line.expect("child line");
|
||||
if line.ends_with("U11_ACK") {
|
||||
let _ = tx.send(());
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
let acknowledged = rx.recv_timeout(Duration::from_secs(20));
|
||||
child.kill().expect("kill child");
|
||||
child.wait().expect("wait child");
|
||||
reader.join().expect("reader cleanup");
|
||||
assert!(acknowledged.is_ok(), "U11 bounded child ACK missing");
|
||||
complete(&reopen(&path));
|
||||
}
|
||||
#[test]
|
||||
fn created_power_cut_before_and_after_apply_flush() {
|
||||
let (dir, mut s) = fixture();
|
||||
append(&mut s, 0, &command());
|
||||
s.save_committed(id(0)).expect("committed");
|
||||
drop(s);
|
||||
let path = dir.path().join("db");
|
||||
let backend = CrashBackend::new(&path);
|
||||
let mut s = Store::from_database(backend.database(), identity()).expect("backend");
|
||||
assert_eq!(s.apply_next_created(id(0)).expect("apply ACK"), success(&command(), 0));
|
||||
backend.cut();
|
||||
drop(s);
|
||||
complete(&reopen(&path));
|
||||
for sync in 1..=2 {
|
||||
for after in [false, true] {
|
||||
let (dir, mut s) = fixture();
|
||||
append(&mut s, 0, &command());
|
||||
s.save_committed(id(0)).expect("committed");
|
||||
drop(s);
|
||||
let path = dir.path().join("db");
|
||||
let backend = CrashBackend::new(&path);
|
||||
let mut s = Store::from_database(backend.database(), identity()).expect("backend");
|
||||
let (entered, resume) = backend.pause_sync(sync, after);
|
||||
let writer = std::thread::spawn(move || {
|
||||
let result = s.apply_next_created(id(0));
|
||||
(s, result)
|
||||
});
|
||||
entered.wait();
|
||||
backend.cut();
|
||||
resume.wait();
|
||||
let (s, result) = writer.join().expect("writer");
|
||||
drop(s);
|
||||
let reopened = Store::open(&path, identity());
|
||||
assert!(
|
||||
reopened.is_ok(),
|
||||
"U11 apply cut must reopen whole old/new state sync={sync} after={after}: {:?}",
|
||||
reopened.as_ref().err()
|
||||
);
|
||||
let s = reopened.expect("asserted reopen");
|
||||
if s.last_applied().expect("last").is_some() {
|
||||
complete(&s)
|
||||
} else {
|
||||
assert!(result.is_err());
|
||||
assert_eq!(s.created_head(&command().object).expect("old head"), None);
|
||||
assert_eq!(
|
||||
s.created_version(&command().object, command().prepared.version_id)
|
||||
.expect("old version"),
|
||||
None
|
||||
);
|
||||
assert_eq!(s.created_event(command().operation_id).expect("old event"), None);
|
||||
assert_eq!(s.created_operation(command().operation_id).expect("old op"), None);
|
||||
assert_eq!(s.created_apply_result(0).expect("old result"), None)
|
||||
}
|
||||
eprintln!("U11 cut boundary sync={sync} after={after}");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,579 @@
|
||||
//! Immutable checkpoint state and disk-backed validation of the retained suffix.
|
||||
use crate::{
|
||||
CaptureMembership, CreatedApplyResult, CreatedEvent, CreatedResult, CreatedVersion, LogId, StoreError, application,
|
||||
created::{DecideCreated, Decoder, object_bytes, result_bytes},
|
||||
native_checkpoint::{CheckpointLimits, recovery_scratch},
|
||||
snapshot,
|
||||
};
|
||||
use redb::{ReadTransaction, ReadableDatabase, ReadableTable, ReadableTableMetadata, TableDefinition};
|
||||
|
||||
pub(crate) const META_KEY: &str = "checkpoint_baseline";
|
||||
pub(crate) const PURGED_KEY: &str = "checkpoint_purged";
|
||||
pub(crate) const HEADS: TableDefinition<&[u8], &[u8]> = TableDefinition::new("checkpoint_heads_v1");
|
||||
pub(crate) const VERSIONS: TableDefinition<&[u8], &[u8]> = TableDefinition::new("checkpoint_versions_v1");
|
||||
pub(crate) const EVENTS: TableDefinition<&[u8], &[u8]> = TableDefinition::new("checkpoint_events_v1");
|
||||
pub(crate) const OPERATIONS: TableDefinition<&[u8], &[u8]> = TableDefinition::new("checkpoint_operations_v1");
|
||||
const FIRST_DECISIONS: TableDefinition<u64, &[u8]> = TableDefinition::new("checkpoint_validation_decisions");
|
||||
const VERSION_OPERATIONS: TableDefinition<&[u8], &[u8]> = TableDefinition::new("checkpoint_validation_versions");
|
||||
const METADATA_BYTES: usize = 1024 * 1024;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct BaselineMetadata {
|
||||
pub(crate) generation: u64,
|
||||
pub(crate) header: snapshot::Header,
|
||||
pub(crate) limits: CheckpointLimits,
|
||||
}
|
||||
|
||||
struct EmptySink;
|
||||
impl snapshot::StagingSink for EmptySink {
|
||||
fn row(&mut self, _: snapshot::Section, _: &[u8], _: &[u8]) -> std::io::Result<()> {
|
||||
Err(std::io::Error::other("checkpoint metadata contains rows"))
|
||||
}
|
||||
}
|
||||
fn metadata_limits() -> snapshot::Limits {
|
||||
snapshot::Limits {
|
||||
total_bytes: 1024 * 1024,
|
||||
membership_bytes: 512 * 1024,
|
||||
key_bytes: 0,
|
||||
value_bytes: 0,
|
||||
records: 0,
|
||||
}
|
||||
}
|
||||
pub(crate) fn encode_metadata(metadata: &BaselineMetadata) -> Result<Vec<u8>, StoreError> {
|
||||
metadata.limits.validate()?;
|
||||
if metadata.generation == 0 {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let mut bytes = b"CBAS\x01".to_vec();
|
||||
for value in [
|
||||
metadata.generation,
|
||||
metadata.limits.snapshot.total_bytes,
|
||||
metadata.limits.snapshot.membership_bytes,
|
||||
metadata.limits.snapshot.key_bytes,
|
||||
metadata.limits.snapshot.value_bytes,
|
||||
metadata.limits.snapshot.records,
|
||||
metadata.limits.staging_bytes,
|
||||
] {
|
||||
bytes.extend(value.to_be_bytes());
|
||||
}
|
||||
let frame = snapshot::Encoder::new(Vec::new(), &metadata.header, [0; 4], metadata_limits())
|
||||
.and_then(snapshot::Encoder::finish)
|
||||
.map_err(|_| StoreError::CorruptRecord)?;
|
||||
bytes.extend(frame);
|
||||
if bytes.len() > METADATA_BYTES {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
pub(crate) fn read_metadata(
|
||||
meta: &impl ReadableTable<&'static str, &'static [u8]>,
|
||||
) -> Result<Option<BaselineMetadata>, StoreError> {
|
||||
let schema = crate::number(required(meta.get("schema")?)?.value())?;
|
||||
let stored = meta.get(META_KEY)?;
|
||||
if schema != 7 {
|
||||
if stored.is_some() || meta.get(PURGED_KEY)?.is_some() {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
return Ok(None);
|
||||
}
|
||||
let value = required(stored)?;
|
||||
let bytes = value.value();
|
||||
if bytes.len() > METADATA_BYTES || bytes.get(..5) != Some(b"CBAS\x01") {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let mut numbers = [0u64; 7];
|
||||
for (index, number) in numbers.iter_mut().enumerate() {
|
||||
let start = 5 + index * 8;
|
||||
*number = crate::number(bytes.get(start..start + 8).ok_or(StoreError::CorruptRecord)?)?;
|
||||
}
|
||||
let limits = CheckpointLimits {
|
||||
snapshot: snapshot::Limits {
|
||||
total_bytes: numbers[1],
|
||||
membership_bytes: numbers[2],
|
||||
key_bytes: numbers[3],
|
||||
value_bytes: numbers[4],
|
||||
records: numbers[5],
|
||||
},
|
||||
staging_bytes: numbers[6],
|
||||
};
|
||||
limits.validate()?;
|
||||
let header = snapshot::decode(bytes.get(61..).ok_or(StoreError::CorruptRecord)?, &mut EmptySink, metadata_limits())
|
||||
.map_err(|_| StoreError::CorruptRecord)?;
|
||||
let metadata = BaselineMetadata {
|
||||
generation: numbers[0],
|
||||
header,
|
||||
limits,
|
||||
};
|
||||
if encode_metadata(&metadata)?.as_slice() != bytes {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
Ok(Some(metadata))
|
||||
}
|
||||
pub(crate) fn read_baseline_cut(meta: &impl ReadableTable<&'static str, &'static [u8]>) -> Result<Option<LogId>, StoreError> {
|
||||
Ok(read_metadata(meta)?.and_then(|metadata| metadata.header.applied))
|
||||
}
|
||||
pub(crate) fn encode_purged(id: LogId) -> Vec<u8> {
|
||||
let mut bytes = b"CPUR\x01".to_vec();
|
||||
bytes.extend(crate::encode_id(id));
|
||||
bytes
|
||||
}
|
||||
pub(crate) fn read_purged(meta: &impl ReadableTable<&'static str, &'static [u8]>) -> Result<Option<LogId>, StoreError> {
|
||||
let cut = read_baseline_cut(meta)?;
|
||||
let Some(value) = meta.get(PURGED_KEY)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
let bytes = value.value();
|
||||
if bytes.len() != 29 || bytes.get(..5) != Some(b"CPUR\x01") {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let id = crate::decode_id(&bytes[5..])?;
|
||||
at_or_before(id, required(cut)?)?;
|
||||
Ok(Some(id))
|
||||
}
|
||||
fn at_or_before(id: LogId, cut: LogId) -> Result<(), StoreError> {
|
||||
if id.index > cut.index || (id.index == cut.index && id != cut) {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
fn required<T>(value: Option<T>) -> Result<T, StoreError> {
|
||||
value.ok_or(StoreError::CorruptRecord)
|
||||
}
|
||||
struct Budget {
|
||||
limits: snapshot::Limits,
|
||||
bytes: u64,
|
||||
records: u64,
|
||||
}
|
||||
impl Budget {
|
||||
fn new(limits: snapshot::Limits) -> Self {
|
||||
Self {
|
||||
limits,
|
||||
bytes: 0,
|
||||
records: 0,
|
||||
}
|
||||
}
|
||||
fn row(&mut self, key: &[u8], value: &[u8]) -> Result<(), StoreError> {
|
||||
let key = u64::try_from(key.len()).map_err(|_| StoreError::CorruptRecord)?;
|
||||
let value = u64::try_from(value.len()).map_err(|_| StoreError::CorruptRecord)?;
|
||||
if key > 1024 * 1024 || value > 1024 * 1024 {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
self.bytes = self
|
||||
.bytes
|
||||
.checked_add(key)
|
||||
.and_then(|n| n.checked_add(value))
|
||||
.ok_or(StoreError::ResourceLimit)?;
|
||||
self.records = self.records.checked_add(1).ok_or(StoreError::ResourceLimit)?;
|
||||
if key > self.limits.key_bytes
|
||||
|| value > self.limits.value_bytes
|
||||
|| self.bytes > self.limits.total_bytes
|
||||
|| self.records > self.limits.records
|
||||
{
|
||||
return Err(StoreError::ResourceLimit);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn resource_error(error: StoreError) -> StoreError {
|
||||
use std::error::Error;
|
||||
let mut cause: Option<&(dyn Error + 'static)> = Some(&error);
|
||||
while let Some(current) = cause {
|
||||
if current
|
||||
.downcast_ref::<std::io::Error>()
|
||||
.is_some_and(|io| io.kind() == std::io::ErrorKind::StorageFull)
|
||||
{
|
||||
return StoreError::ResourceLimit;
|
||||
}
|
||||
cause = current.source();
|
||||
}
|
||||
error
|
||||
}
|
||||
|
||||
/// Validate relationships, preserving negative history rather than re-evaluating it.
|
||||
/// The caller separately establishes local capture or authenticated installation authority.
|
||||
pub(crate) fn validate_baseline_with_budget(
|
||||
tx: &ReadTransaction,
|
||||
metadata: &BaselineMetadata,
|
||||
index_max: u64,
|
||||
) -> Result<(), StoreError> {
|
||||
validate_baseline_inner(tx, metadata, index_max).map_err(resource_error)
|
||||
}
|
||||
|
||||
fn validate_baseline_inner(tx: &ReadTransaction, metadata: &BaselineMetadata, index_max: u64) -> Result<(), StoreError> {
|
||||
encode_metadata(metadata)?;
|
||||
let Some(cut) = metadata.header.applied else {
|
||||
if metadata.header.membership != CaptureMembership::new(None, vec![], std::collections::BTreeMap::new())? {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
for definition in [HEADS, VERSIONS, EVENTS, OPERATIONS] {
|
||||
if tx.open_table(definition)?.len()? != 0 {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
}
|
||||
return Ok(());
|
||||
};
|
||||
let heads = tx.open_table(HEADS)?;
|
||||
let versions = tx.open_table(VERSIONS)?;
|
||||
let events = tx.open_table(EVENTS)?;
|
||||
let operations = tx.open_table(OPERATIONS)?;
|
||||
let (_directory, index) = recovery_scratch(index_max)?;
|
||||
let mut index_write = index.begin_write()?;
|
||||
index_write.open_table(FIRST_DECISIONS)?;
|
||||
index_write.open_table(VERSION_OPERATIONS)?;
|
||||
let mut budget = Budget::new(metadata.limits.snapshot);
|
||||
let mut pending = 0;
|
||||
for row in operations.iter()? {
|
||||
let (key, value) = row?;
|
||||
budget.row(key.value(), value.value())?;
|
||||
let provenance = application::decode_provenance(value.value())?;
|
||||
let command = DecideCreated::decode(&provenance.operation.canonical_command)?;
|
||||
let canonical = command.encode()?;
|
||||
if key.value() != command.operation_id || canonical != provenance.operation.canonical_command {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
at_or_before(provenance.first_decision_log_id, cut)?;
|
||||
let mut original = u64::try_from(canonical.len())
|
||||
.map_err(|_| StoreError::CorruptRecord)?
|
||||
.to_be_bytes()
|
||||
.to_vec();
|
||||
original.extend(&canonical);
|
||||
original.extend(result_bytes(&provenance.operation.result));
|
||||
if application::encode_provenance(provenance.first_decision_log_id, &original)?.as_slice() != value.value() {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
if index_write
|
||||
.open_table(FIRST_DECISIONS)?
|
||||
.insert(provenance.first_decision_log_id.index, key.value())?
|
||||
.is_some()
|
||||
{
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
if provenance.operation.result == CreatedResult::OperationConflict {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
if let CreatedResult::Created {
|
||||
version_id,
|
||||
event_id,
|
||||
decision_log_id,
|
||||
} = provenance.operation.result
|
||||
{
|
||||
if version_id != command.prepared.version_id
|
||||
|| event_id != command.operation_id
|
||||
|| decision_log_id != provenance.first_decision_log_id
|
||||
|| command.binding_revision != metadata.header.binding.revision
|
||||
{
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let event = application::decode_event(required(events.get(key.value())?)?.value())?;
|
||||
if event
|
||||
!= (CreatedEvent {
|
||||
object: command.object.clone(),
|
||||
prepared: command.prepared.clone(),
|
||||
decision_log_id,
|
||||
binding: metadata.header.binding.clone(),
|
||||
})
|
||||
{
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let version_key = application::version_key(&command.object, version_id)?;
|
||||
let version = application::decode_version(required(versions.get(version_key.as_slice())?)?.value())?;
|
||||
if version
|
||||
!= (CreatedVersion {
|
||||
prepared: command.prepared.clone(),
|
||||
decision_log_id,
|
||||
binding_revision: metadata.header.binding.revision,
|
||||
})
|
||||
{
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
if index_write
|
||||
.open_table(VERSION_OPERATIONS)?
|
||||
.insert(version_key.as_slice(), key.value())?
|
||||
.is_some()
|
||||
{
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
} else if events.get(key.value())?.is_some() {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
pending += 1;
|
||||
if pending == 128 {
|
||||
index_write.commit().map_err(StoreError::StagingCommit)?;
|
||||
index_write = index.begin_write()?;
|
||||
pending = 0;
|
||||
}
|
||||
}
|
||||
index_write.commit().map_err(StoreError::StagingCommit)?;
|
||||
let index_read = index.begin_read()?;
|
||||
let version_operations = index_read.open_table(VERSION_OPERATIONS)?;
|
||||
if version_operations.len()? != versions.len()? {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
for row in versions.iter()? {
|
||||
let (key, value) = row?;
|
||||
budget.row(key.value(), value.value())?;
|
||||
required(version_operations.get(key.value())?)?;
|
||||
let mut decoder = Decoder::new(key.value());
|
||||
let object = decoder.object()?;
|
||||
let id = decoder.uuid()?;
|
||||
decoder.finish()?;
|
||||
let version = application::decode_version(value.value())?;
|
||||
let head_bytes = required(heads.get(object_bytes(&object)?.as_slice())?)?;
|
||||
let mut decoder = Decoder::new(head_bytes.value());
|
||||
let head = decoder.uuid()?;
|
||||
decoder.finish()?;
|
||||
let current =
|
||||
application::decode_version(required(versions.get(application::version_key(&object, head)?.as_slice())?)?.value())?;
|
||||
if id != version.prepared.version_id
|
||||
|| current.decision_log_id.index < version.decision_log_id.index
|
||||
|| (current.decision_log_id.index == version.decision_log_id.index
|
||||
&& current.decision_log_id != version.decision_log_id)
|
||||
{
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
}
|
||||
for row in events.iter()? {
|
||||
let (key, value) = row?;
|
||||
budget.row(key.value(), value.value())?;
|
||||
let event = application::decode_event(value.value())?;
|
||||
let operation = application::decode_provenance(required(operations.get(key.value())?)?.value())?;
|
||||
if operation.operation.result
|
||||
!= (CreatedResult::Created {
|
||||
version_id: event.prepared.version_id,
|
||||
event_id: key.value().try_into().map_err(|_| StoreError::CorruptRecord)?,
|
||||
decision_log_id: event.decision_log_id,
|
||||
})
|
||||
{
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
}
|
||||
for row in heads.iter()? {
|
||||
let (key, value) = row?;
|
||||
budget.row(key.value(), value.value())?;
|
||||
let mut decoder = Decoder::new(key.value());
|
||||
let object = decoder.object()?;
|
||||
decoder.finish()?;
|
||||
let mut decoder = Decoder::new(value.value());
|
||||
let id = decoder.uuid()?;
|
||||
decoder.finish()?;
|
||||
required(versions.get(application::version_key(&object, id)?.as_slice())?)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn validate_recovery(tx: &ReadTransaction) -> Result<(), StoreError> {
|
||||
validate_recovery_with_limits(tx, None)
|
||||
}
|
||||
|
||||
pub(crate) fn validate_recovery_with_limits(
|
||||
tx: &ReadTransaction,
|
||||
override_limits: Option<CheckpointLimits>,
|
||||
) -> Result<(), StoreError> {
|
||||
validate_recovery_inner(tx, override_limits).map_err(resource_error)
|
||||
}
|
||||
|
||||
fn validate_recovery_inner(tx: &ReadTransaction, override_limits: Option<CheckpointLimits>) -> Result<(), StoreError> {
|
||||
let meta = tx.open_table(crate::META)?;
|
||||
let metadata = required(read_metadata(&meta)?)?;
|
||||
crate::raft_state_machine::selected_meta(&meta, &metadata)?;
|
||||
let cut = metadata.header.applied;
|
||||
let recovery_limits = override_limits.unwrap_or(metadata.limits);
|
||||
recovery_limits.validate()?;
|
||||
let identity = crate::decode_identity(required(meta.get("identity")?)?.value())?;
|
||||
let binding = application::read_binding(&meta)?;
|
||||
if identity.cluster_id != metadata.header.cluster_id || binding != metadata.header.binding {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let purged = read_purged(&meta)?;
|
||||
validate_baseline_with_budget(tx, &metadata, recovery_limits.staging_bytes)?;
|
||||
let last = application::read_applied(&meta)?;
|
||||
optional_at_or_before(cut, last)?;
|
||||
let logs = tx.open_table(crate::LOGS)?;
|
||||
let committed = crate::committed::validate_committed(&meta, &logs)?;
|
||||
optional_at_or_before(last, committed)?;
|
||||
let declared_tail = meta.get("tail")?.map(|v| crate::decode_id(v.value())).transpose()?;
|
||||
optional_at_or_before(committed, declared_tail)?;
|
||||
let (_directory, scratch) = recovery_scratch(recovery_limits.staging_bytes)?;
|
||||
for (source, target) in [
|
||||
(HEADS, application::HEADS),
|
||||
(VERSIONS, application::VERSIONS),
|
||||
(EVENTS, application::EVENTS),
|
||||
(OPERATIONS, application::OPERATIONS),
|
||||
] {
|
||||
let table = tx.open_table(source)?;
|
||||
let mut write = scratch.begin_write()?;
|
||||
write.open_table(target)?;
|
||||
let mut pending = 0;
|
||||
for row in table.iter()? {
|
||||
let (key, value) = row?;
|
||||
write.open_table(target)?.insert(key.value(), value.value())?;
|
||||
pending += 1;
|
||||
if pending == 128 {
|
||||
write.commit().map_err(StoreError::StagingCommit)?;
|
||||
write = scratch.begin_write()?;
|
||||
pending = 0;
|
||||
}
|
||||
}
|
||||
write.commit().map_err(StoreError::StagingCommit)?;
|
||||
}
|
||||
let mut membership: CaptureMembership = metadata.header.membership.clone();
|
||||
if let Some(cut) = cut
|
||||
&& let Some(retained_cut) = logs.get(cut.index)?
|
||||
&& crate::decode_log(retained_cut.value())?.id != cut
|
||||
{
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let results = tx.open_table(application::APPLY_RESULTS)?;
|
||||
// Covered entries may remain until purge. Check their exact typed responses
|
||||
// without rebuilding the historical state or allocating a prefix-sized map.
|
||||
let operations = tx.open_table(OPERATIONS)?;
|
||||
let mut actual_tail = purged;
|
||||
let mut prefix_next = purged.map_or(Some(0), |id| id.index.checked_add(1));
|
||||
if let Some(cut) = cut {
|
||||
for row in logs.range(..=cut.index)? {
|
||||
let (key, bytes) = row?;
|
||||
if purged.is_some_and(|purged| key.value() <= purged.index) {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
if bytes.value().len() > 1024 * 1024 {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let entry = crate::decode_log(bytes.value())?;
|
||||
if entry.id.index != key.value() || prefix_next != Some(key.value()) {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
actual_tail = Some(entry.id);
|
||||
prefix_next = key.value().checked_add(1);
|
||||
let native = application::decode_native_entry(entry)?;
|
||||
let bytes = required(results.get(key.value())?)?;
|
||||
if bytes.value().len() > 1024 * 1024 {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let response = application::decode_response(bytes.value())?;
|
||||
application::validate_response_kind(&native, &response)?;
|
||||
if let openraft::EntryPayload::Normal(command) = native.payload {
|
||||
let operation = application::decode_provenance(
|
||||
required(operations.get(command.created().operation_id.as_slice())?)?.value(),
|
||||
)?;
|
||||
let canonical = command.created().encode()?;
|
||||
let expected = if operation.operation.canonical_command == canonical {
|
||||
operation.operation.result
|
||||
} else {
|
||||
CreatedResult::OperationConflict
|
||||
};
|
||||
if response.created_result() != Some(&expected) {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
if operation.first_decision_log_id.index == key.value()
|
||||
&& operation.first_decision_log_id != LogId::from(native.log_id)
|
||||
{
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
}
|
||||
}
|
||||
for row in results.range(..=cut.index)? {
|
||||
let (key, _) = row?;
|
||||
if purged.is_some_and(|purged| key.value() <= purged.index) || logs.get(key.value())?.is_none() {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
}
|
||||
if purged.is_none_or(|purged| purged.index < cut.index) && logs.get(cut.index)?.is_none() {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
}
|
||||
if actual_tail != cut {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
|
||||
let mut observed = cut;
|
||||
let mut next = cut.map_or(Some(0), |cut| cut.index.checked_add(1));
|
||||
let mut budget = Budget::new(recovery_limits.snapshot);
|
||||
if let Some(start) = next {
|
||||
for row in logs.range(start..)? {
|
||||
let (key, bytes) = row?;
|
||||
budget.row(&key.value().to_be_bytes(), bytes.value())?;
|
||||
let entry = crate::decode_log(bytes.value())?;
|
||||
if next != Some(key.value()) || entry.id.index != key.value() {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
next = key.value().checked_add(1);
|
||||
actual_tail = Some(entry.id);
|
||||
let native = application::decode_native_entry(entry.clone())?;
|
||||
if last.is_none_or(|last| key.value() > last.index) {
|
||||
continue;
|
||||
}
|
||||
let response_bytes = required(results.get(key.value())?)?;
|
||||
if response_bytes.value().len() > 1024 * 1024 {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
if u64::try_from(response_bytes.value().len()).map_err(|_| StoreError::CorruptRecord)?
|
||||
> recovery_limits.snapshot.value_bytes
|
||||
{
|
||||
return Err(StoreError::ResourceLimit);
|
||||
}
|
||||
let response = application::decode_response(response_bytes.value())?;
|
||||
application::validate_response_kind(&native, &response)?;
|
||||
let write = scratch.begin_write()?;
|
||||
let expected = match native.payload {
|
||||
openraft::EntryPayload::Blank => crate::CaptureResponse::blank(entry.id)?,
|
||||
openraft::EntryPayload::Membership(value) => {
|
||||
membership = CaptureMembership::try_from(&openraft::StoredMembership::new(Some(entry.id.into()), value))?;
|
||||
crate::CaptureResponse::membership(entry.id)?
|
||||
}
|
||||
openraft::EntryPayload::Normal(command) => {
|
||||
let canonical = command.created().encode()?;
|
||||
let result = application::apply_created(&write, entry.id, command.created(), &canonical, &binding, 6)?;
|
||||
crate::CaptureResponse::created(CreatedApplyResult {
|
||||
log_id: entry.id,
|
||||
result,
|
||||
})?
|
||||
}
|
||||
};
|
||||
if response != expected {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
write.commit().map_err(StoreError::StagingCommit)?;
|
||||
observed = Some(entry.id);
|
||||
}
|
||||
for row in results.range(start..)? {
|
||||
let (key, _) = row?;
|
||||
if last.is_none_or(|last| key.value() > last.index) || logs.get(key.value())?.is_none() {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
}
|
||||
}
|
||||
if actual_tail != declared_tail || observed != last || application::native_application_state(tx)?.1 != membership {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let recovered = scratch.begin_read()?;
|
||||
let mut live_budget = Budget::new(recovery_limits.snapshot);
|
||||
for definition in [
|
||||
application::HEADS,
|
||||
application::VERSIONS,
|
||||
application::EVENTS,
|
||||
application::OPERATIONS,
|
||||
] {
|
||||
let expected = recovered.open_table(definition)?;
|
||||
let actual = tx.open_table(definition)?;
|
||||
if expected.len()? != actual.len()? {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
for row in actual.iter()? {
|
||||
let (key, value) = row?;
|
||||
live_budget.row(key.value(), value.value())?;
|
||||
if required(expected.get(key.value())?)?.value() != value.value() {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn optional_at_or_before(id: Option<LogId>, cut: Option<LogId>) -> Result<(), StoreError> {
|
||||
match (id, cut) {
|
||||
(None, _) => Ok(()),
|
||||
(Some(id), Some(cut)) => at_or_before(id, cut),
|
||||
_ => Err(StoreError::CorruptRecord),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
use crate::{LOGS, LogId, META, Store, StoreError, decode_id, decode_log, encode_id, number};
|
||||
use redb::{ReadableDatabase, ReadableTable};
|
||||
|
||||
pub(crate) fn validate_committed(
|
||||
meta: &impl ReadableTable<&'static str, &'static [u8]>,
|
||||
logs: &impl ReadableTable<u64, &'static [u8]>,
|
||||
) -> Result<Option<LogId>, StoreError> {
|
||||
let schema = number(meta.get("schema")?.ok_or(StoreError::CorruptRecord)?.value())?;
|
||||
let marker = meta.get("committed")?;
|
||||
match (schema, marker) {
|
||||
(1 | 3 | 4 | 5 | 6 | 7, None) => {
|
||||
if matches!(schema, 3..=7) && meta.get("last_applied")?.is_none_or(|v| v.value() != [0]) {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
(2 | 3 | 4 | 5 | 6 | 7, Some(marker)) => {
|
||||
let id = decode_id(marker.value())?;
|
||||
match logs.get(id.index)? {
|
||||
Some(record) if decode_log(record.value())?.id == id => {}
|
||||
None if schema == 7 && crate::checkpoint_baseline::read_baseline_cut(meta)? == Some(id) => {}
|
||||
_ => return Err(StoreError::CorruptRecord),
|
||||
}
|
||||
Ok(Some(id))
|
||||
}
|
||||
(1 | 2, _) => Err(StoreError::CorruptRecord),
|
||||
_ => Err(StoreError::UnsupportedSchema),
|
||||
}
|
||||
}
|
||||
|
||||
impl Store {
|
||||
/// Read the exact committed boundary, validating it against the stored log.
|
||||
pub fn read_committed(&self) -> Result<Option<LogId>, StoreError> {
|
||||
let tx = self.database.begin_read()?;
|
||||
validate_committed(&tx.open_table(META)?, &tx.open_table(LOGS)?)
|
||||
}
|
||||
|
||||
/// Persist a monotonic committed boundary supplied by the authority owner.
|
||||
pub fn save_committed(&mut self, id: LogId) -> Result<(), StoreError> {
|
||||
let tx = self.write_transaction()?;
|
||||
{
|
||||
let mut meta = tx.open_table(META)?;
|
||||
let logs = tx.open_table(LOGS)?;
|
||||
let previous = validate_committed(&meta, &logs)?;
|
||||
if previous == Some(id) {
|
||||
return Ok(());
|
||||
}
|
||||
let record = logs.get(id.index)?.ok_or(StoreError::InvalidCommittedLog)?;
|
||||
if decode_log(record.value())?.id != id {
|
||||
return Err(StoreError::InvalidCommittedLog);
|
||||
}
|
||||
if let Some(previous) = previous {
|
||||
if id.index < previous.index {
|
||||
return Err(StoreError::CommittedRegression);
|
||||
}
|
||||
}
|
||||
meta.insert("committed", encode_id(id).as_slice())?;
|
||||
if previous.is_none() && !matches!(number(meta.get("schema")?.ok_or(StoreError::CorruptRecord)?.value())?, 3..=7) {
|
||||
meta.insert("schema", 2u64.to_be_bytes().as_slice())?;
|
||||
}
|
||||
}
|
||||
self.commit(tx)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,278 @@
|
||||
//! Reuse the existing actual Raft owner fixture, never a caller-constructed result proof.
|
||||
use super::*;
|
||||
|
||||
async fn decide(cluster: &Cluster, value: u8) -> (DecideCreated, LogId) {
|
||||
let input = command(value);
|
||||
let expected = input.created().clone();
|
||||
let response = cluster
|
||||
.node(0)
|
||||
.owner()
|
||||
.client_write(input)
|
||||
.await
|
||||
.expect("actual Raft Created decision");
|
||||
let Some(CreatedResult::Created { decision_log_id, .. }) = response.data.created_result() else {
|
||||
panic!("successful Created result required")
|
||||
};
|
||||
(expected, *decision_log_id)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn committed_created_authorization_exact() {
|
||||
let cluster = Cluster::start(&[0], 100).await;
|
||||
let (expected, first) = decide(&cluster, 1).await;
|
||||
let authorization = cluster
|
||||
.node(0)
|
||||
.owner()
|
||||
.authorize_created(expected.clone(), first)
|
||||
.await
|
||||
.expect("durable exact authorization");
|
||||
assert_eq!(authorization.identity(), identity(0));
|
||||
assert_eq!(authorization.operation_id(), expected.operation_id);
|
||||
assert_eq!(authorization.object(), &expected.object);
|
||||
assert_eq!(authorization.prepared(), &expected.prepared);
|
||||
assert_eq!(authorization.binding(), &binding());
|
||||
assert_eq!(authorization.decision_log_id(), first);
|
||||
assert_eq!(cluster.node(0).page().await.events[0].captured.decision_log_id, first);
|
||||
cluster.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn committed_created_authorization_retry_first_decision() {
|
||||
let cluster = Cluster::start(&[0], 100).await;
|
||||
let (expected, first) = decide(&cluster, 2).await;
|
||||
let before = cluster
|
||||
.node(0)
|
||||
.owner()
|
||||
.authorize_created(expected.clone(), first)
|
||||
.await
|
||||
.expect("first authorization");
|
||||
let response = cluster
|
||||
.node(0)
|
||||
.owner()
|
||||
.client_write(CaptureCommand::try_created(expected.clone()).expect("retry command"))
|
||||
.await
|
||||
.expect("real retry");
|
||||
assert_ne!(LogId::from(response.log_id), first);
|
||||
assert!(
|
||||
matches!(response.data.created_result(), Some(CreatedResult::Created{decision_log_id,..}) if *decision_log_id==first)
|
||||
);
|
||||
let after = cluster
|
||||
.node(0)
|
||||
.owner()
|
||||
.authorize_created(expected.clone(), first)
|
||||
.await
|
||||
.expect("first decision retained");
|
||||
assert_eq!(before, after);
|
||||
assert!(
|
||||
cluster
|
||||
.node(0)
|
||||
.owner()
|
||||
.authorize_created(expected, response.log_id.into())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(cluster.node(0).page().await.events.len(), 1);
|
||||
cluster.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn committed_created_authorization_rejects_mismatch() {
|
||||
let cluster = Cluster::start(&[0], 100).await;
|
||||
let (expected, first) = decide(&cluster, 3).await;
|
||||
let before = cluster.node(0).page().await;
|
||||
let durable_before = cluster
|
||||
.node(0)
|
||||
.writer
|
||||
.write(|store| {
|
||||
Ok((
|
||||
store.read_vote()?,
|
||||
store.read_committed()?,
|
||||
store.read_entries(0, u64::MAX)?,
|
||||
store.native_application_state()?,
|
||||
store.created_events_page(None, 256)?,
|
||||
))
|
||||
})
|
||||
.await
|
||||
.expect("durable state before rejected authorization");
|
||||
let mut variants = Vec::new();
|
||||
let mut altered = expected.clone();
|
||||
altered.operation_id = [4; 16];
|
||||
variants.push(altered);
|
||||
let mut altered = expected.clone();
|
||||
altered.object.key.push(9);
|
||||
variants.push(altered);
|
||||
let mut altered = expected.clone();
|
||||
altered.object.bucket_incarnation = [9; 16];
|
||||
variants.push(altered);
|
||||
let mut altered = expected.clone();
|
||||
altered.expected_head = Some([9; 16]);
|
||||
variants.push(altered);
|
||||
let mut altered = expected.clone();
|
||||
altered.prepared.preparation_id = [9; 16];
|
||||
variants.push(altered);
|
||||
let mut altered = expected.clone();
|
||||
altered.prepared.version_id = [9; 16];
|
||||
variants.push(altered);
|
||||
let mut altered = expected.clone();
|
||||
altered.prepared.content_digest = [9; 32];
|
||||
variants.push(altered);
|
||||
let mut altered = expected.clone();
|
||||
altered.prepared.content_length += 1;
|
||||
variants.push(altered);
|
||||
let mut altered = expected.clone();
|
||||
altered.binding_revision += 1;
|
||||
variants.push(altered);
|
||||
for altered in variants {
|
||||
assert!(
|
||||
cluster.node(0).owner().authorize_created(altered, first).await.is_err(),
|
||||
"AUTHORIZATION_REQUIRES_EXACT_PREPARED_IDENTITY_AND_COMMAND"
|
||||
);
|
||||
}
|
||||
for wrong in [
|
||||
LogId {
|
||||
term: first.term + 1,
|
||||
..first
|
||||
},
|
||||
LogId {
|
||||
leader_node: first.leader_node + 1,
|
||||
..first
|
||||
},
|
||||
LogId {
|
||||
index: first.index + 1,
|
||||
..first
|
||||
},
|
||||
] {
|
||||
assert!(
|
||||
cluster
|
||||
.node(0)
|
||||
.owner()
|
||||
.authorize_created(expected.clone(), wrong)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
let durable_after = cluster
|
||||
.node(0)
|
||||
.writer
|
||||
.write(|store| {
|
||||
Ok((
|
||||
store.read_vote()?,
|
||||
store.read_committed()?,
|
||||
store.read_entries(0, u64::MAX)?,
|
||||
store.native_application_state()?,
|
||||
store.created_events_page(None, 256)?,
|
||||
))
|
||||
})
|
||||
.await
|
||||
.expect("durable state after rejected authorization");
|
||||
assert_eq!(durable_before, durable_after, "REJECTED_AUTHORIZATION_DOES_NOT_WRITE");
|
||||
let mut failed = expected.clone();
|
||||
failed.operation_id = [5; 16];
|
||||
failed.prepared.version_id = [5; 16];
|
||||
let response = cluster
|
||||
.node(0)
|
||||
.owner()
|
||||
.client_write(CaptureCommand::try_created(failed.clone()).expect("failed command"))
|
||||
.await
|
||||
.expect("real head mismatch decision");
|
||||
assert!(matches!(response.data.created_result(), Some(CreatedResult::HeadMismatch { .. })));
|
||||
assert!(
|
||||
cluster
|
||||
.node(0)
|
||||
.owner()
|
||||
.authorize_created(failed, response.log_id.into())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(cluster.node(0).page().await, before);
|
||||
cluster.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn committed_created_authorization_restart() {
|
||||
let routes = Arc::new(Routes::default());
|
||||
let mut node = Node::start(0, routes.clone(), config(100)).await;
|
||||
node.owner().bootstrap(members(&[0])).await.expect("bootstrap actual owner");
|
||||
node.owner()
|
||||
.raft()
|
||||
.wait(Some(Duration::from_secs(15)))
|
||||
.state(ServerState::Leader, "leader")
|
||||
.await
|
||||
.expect("leader ready");
|
||||
let expected = command(6).created().clone();
|
||||
let response = node.owner().client_write(command(6)).await.expect("real first write");
|
||||
let Some(CreatedResult::Created {
|
||||
decision_log_id: first, ..
|
||||
}) = response.data.created_result()
|
||||
else {
|
||||
panic!("Created")
|
||||
};
|
||||
let first = *first;
|
||||
let before = node
|
||||
.owner()
|
||||
.authorize_created(expected.clone(), first)
|
||||
.await
|
||||
.expect("before restart");
|
||||
node.stop(&routes).await;
|
||||
let Node {
|
||||
directory,
|
||||
writer,
|
||||
owner: _,
|
||||
id,
|
||||
} = node;
|
||||
drop(writer);
|
||||
let writer = NativeLogWriter::new(Store::open(directory.path().join("db"), identity(id)).expect("retained redb"));
|
||||
// A retained commit-before-apply crash fixture is not a client-provided proof.
|
||||
// Only real owner initialization may replay it before authorization can succeed.
|
||||
use openraft::storage::{RaftLogStorage, RaftLogStorageExt};
|
||||
let pending = command(7).created().clone();
|
||||
let mut logs = NativeRaftLogStore::new(writer.clone());
|
||||
let last = logs.get_log_state().await.expect("retained tail").last_log_id.expect("tail");
|
||||
let pending_id = LogId {
|
||||
term: last.leader_id.term,
|
||||
leader_node: last.leader_id.node_id,
|
||||
index: last.index + 1,
|
||||
};
|
||||
logs.blocking_append(vec![CaptureRaftEntry {
|
||||
log_id: pending_id.into(),
|
||||
payload: openraft::EntryPayload::Normal(CaptureCommand::try_created(pending.clone()).expect("pending command")),
|
||||
}])
|
||||
.await
|
||||
.expect("retained crash suffix");
|
||||
logs.save_committed(Some(pending_id.into()))
|
||||
.await
|
||||
.expect("retained commit marker");
|
||||
assert!(
|
||||
writer.authorize_created(pending.clone(), pending_id).await.is_err(),
|
||||
"UNAPPLIED_COMMIT_IS_NOT_AUTHORIZATION"
|
||||
);
|
||||
drop(logs);
|
||||
let owner = NativeRaftOwner::start(
|
||||
writer.clone(),
|
||||
id,
|
||||
config(100),
|
||||
CheckpointLimits::default(),
|
||||
DirectFactory {
|
||||
source: id,
|
||||
routes: routes.clone(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("real recovery");
|
||||
routes.register(id, owner.raft().clone());
|
||||
let after = owner
|
||||
.authorize_created(expected, first)
|
||||
.await
|
||||
.expect("durable authorization after restart");
|
||||
assert_eq!(before, after);
|
||||
let replayed = owner
|
||||
.authorize_created(pending.clone(), pending_id)
|
||||
.await
|
||||
.expect("actual owner committed recovery authorizes");
|
||||
assert_eq!(replayed.prepared(), &pending.prepared);
|
||||
assert_eq!(replayed.decision_log_id(), pending_id);
|
||||
owner.shutdown().await.expect("drain recovered owner");
|
||||
routes.remove(id);
|
||||
drop(owner);
|
||||
drop(writer);
|
||||
}
|
||||
@@ -0,0 +1,297 @@
|
||||
use crate::{LogId, StoreError, decode_id, encode_id};
|
||||
|
||||
pub const MAX_OBJECT_KEY_LENGTH: usize = 4096;
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ObjectIdentity {
|
||||
pub bucket_incarnation: [u8; 16],
|
||||
pub key: Vec<u8>,
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct PreparedIdentity {
|
||||
pub version_id: [u8; 16],
|
||||
pub preparation_id: [u8; 16],
|
||||
pub content_digest: [u8; 32],
|
||||
pub content_length: u64,
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct DecideCreated {
|
||||
pub operation_id: [u8; 16],
|
||||
pub object: ObjectIdentity,
|
||||
pub expected_head: Option<[u8; 16]>,
|
||||
pub prepared: PreparedIdentity,
|
||||
pub binding_revision: u64,
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct CaptureBinding {
|
||||
pub revision: u64,
|
||||
pub binding_id: [u8; 16],
|
||||
pub target_id: [u8; 16],
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum CreatedResult {
|
||||
Created {
|
||||
version_id: [u8; 16],
|
||||
event_id: [u8; 16],
|
||||
decision_log_id: LogId,
|
||||
},
|
||||
HeadMismatch {
|
||||
actual_head: Option<[u8; 16]>,
|
||||
},
|
||||
OperationConflict,
|
||||
BindingMismatch,
|
||||
VersionConflict,
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CreatedVersion {
|
||||
pub prepared: PreparedIdentity,
|
||||
pub decision_log_id: LogId,
|
||||
pub binding_revision: u64,
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct CreatedEvent {
|
||||
pub object: ObjectIdentity,
|
||||
pub prepared: PreparedIdentity,
|
||||
pub decision_log_id: LogId,
|
||||
pub binding: CaptureBinding,
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CreatedOperation {
|
||||
pub canonical_command: Vec<u8>,
|
||||
pub result: CreatedResult,
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CreatedOperationProvenance {
|
||||
pub operation: CreatedOperation,
|
||||
pub first_decision_log_id: LogId,
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CreatedApplyResult {
|
||||
pub log_id: LogId,
|
||||
pub result: CreatedResult,
|
||||
}
|
||||
|
||||
/// Local publication authority re-read from the actual owner's durable Created decision.
|
||||
/// This is not a serializable remote proof and does not attest physical preparation durability.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CommittedCreatedAuthorization {
|
||||
identity: crate::NodeIdentity,
|
||||
command: DecideCreated,
|
||||
binding: CaptureBinding,
|
||||
decision_log_id: LogId,
|
||||
}
|
||||
impl CommittedCreatedAuthorization {
|
||||
pub(crate) fn verified(
|
||||
identity: crate::NodeIdentity,
|
||||
command: DecideCreated,
|
||||
binding: CaptureBinding,
|
||||
decision_log_id: LogId,
|
||||
) -> Self {
|
||||
Self {
|
||||
identity,
|
||||
command,
|
||||
binding,
|
||||
decision_log_id,
|
||||
}
|
||||
}
|
||||
pub fn identity(&self) -> crate::NodeIdentity {
|
||||
self.identity
|
||||
}
|
||||
pub fn operation_id(&self) -> [u8; 16] {
|
||||
self.command.operation_id
|
||||
}
|
||||
pub fn object(&self) -> &ObjectIdentity {
|
||||
&self.command.object
|
||||
}
|
||||
pub fn expected_head(&self) -> Option<[u8; 16]> {
|
||||
self.command.expected_head
|
||||
}
|
||||
pub fn prepared(&self) -> &PreparedIdentity {
|
||||
&self.command.prepared
|
||||
}
|
||||
pub fn binding(&self) -> &CaptureBinding {
|
||||
&self.binding
|
||||
}
|
||||
pub fn decision_log_id(&self) -> LogId {
|
||||
self.decision_log_id
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct Decoder<'a> {
|
||||
bytes: &'a [u8],
|
||||
}
|
||||
impl<'a> Decoder<'a> {
|
||||
pub(crate) fn new(bytes: &'a [u8]) -> Self {
|
||||
Self { bytes }
|
||||
}
|
||||
pub(crate) fn take<const N: usize>(&mut self) -> Result<[u8; N], StoreError> {
|
||||
let a = self
|
||||
.bytes
|
||||
.get(..N)
|
||||
.ok_or(StoreError::CorruptRecord)?
|
||||
.try_into()
|
||||
.map_err(|_| StoreError::CorruptRecord)?;
|
||||
self.bytes = &self.bytes[N..];
|
||||
Ok(a)
|
||||
}
|
||||
pub(crate) fn u64(&mut self) -> Result<u64, StoreError> {
|
||||
Ok(u64::from_be_bytes(self.take()?))
|
||||
}
|
||||
pub(crate) fn uuid(&mut self) -> Result<[u8; 16], StoreError> {
|
||||
let id = self.take()?;
|
||||
if id == [0; 16] {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
Ok(id)
|
||||
}
|
||||
pub(crate) fn option(&mut self) -> Result<Option<[u8; 16]>, StoreError> {
|
||||
match self.take::<1>()?[0] {
|
||||
0 => Ok(None),
|
||||
1 => Ok(Some(self.uuid()?)),
|
||||
_ => Err(StoreError::CorruptRecord),
|
||||
}
|
||||
}
|
||||
pub(crate) fn object(&mut self) -> Result<ObjectIdentity, StoreError> {
|
||||
let bucket_incarnation = self.uuid()?;
|
||||
let len = usize::try_from(u32::from_be_bytes(self.take()?)).map_err(|_| StoreError::CorruptRecord)?;
|
||||
if len > MAX_OBJECT_KEY_LENGTH {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let key = self.bytes.get(..len).ok_or(StoreError::CorruptRecord)?.to_vec();
|
||||
self.bytes = &self.bytes[len..];
|
||||
Ok(ObjectIdentity { bucket_incarnation, key })
|
||||
}
|
||||
pub(crate) fn prepared(&mut self) -> Result<PreparedIdentity, StoreError> {
|
||||
Ok(PreparedIdentity {
|
||||
version_id: self.uuid()?,
|
||||
preparation_id: self.uuid()?,
|
||||
content_digest: self.take()?,
|
||||
content_length: self.u64()?,
|
||||
})
|
||||
}
|
||||
pub(crate) fn id(&mut self) -> Result<LogId, StoreError> {
|
||||
decode_id(&self.take::<24>()?)
|
||||
}
|
||||
pub(crate) fn binding(&mut self) -> Result<CaptureBinding, StoreError> {
|
||||
Ok(CaptureBinding {
|
||||
revision: self.u64()?,
|
||||
binding_id: self.uuid()?,
|
||||
target_id: self.uuid()?,
|
||||
})
|
||||
}
|
||||
pub(crate) fn finish(self) -> Result<(), StoreError> {
|
||||
if self.bytes.is_empty() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(StoreError::CorruptRecord)
|
||||
}
|
||||
}
|
||||
pub(crate) fn result(&mut self) -> Result<CreatedResult, StoreError> {
|
||||
match self.take::<1>()?[0] {
|
||||
0 => Ok(CreatedResult::Created {
|
||||
version_id: self.uuid()?,
|
||||
event_id: self.uuid()?,
|
||||
decision_log_id: self.id()?,
|
||||
}),
|
||||
1 => Ok(CreatedResult::HeadMismatch {
|
||||
actual_head: self.option()?,
|
||||
}),
|
||||
2 => Ok(CreatedResult::OperationConflict),
|
||||
3 => Ok(CreatedResult::BindingMismatch),
|
||||
4 => Ok(CreatedResult::VersionConflict),
|
||||
_ => Err(StoreError::CorruptRecord),
|
||||
}
|
||||
}
|
||||
}
|
||||
pub(crate) fn put_option(b: &mut Vec<u8>, id: Option<[u8; 16]>) {
|
||||
b.push(u8::from(id.is_some()));
|
||||
if let Some(id) = id {
|
||||
b.extend(id)
|
||||
}
|
||||
}
|
||||
pub(crate) fn object_bytes(o: &ObjectIdentity) -> Result<Vec<u8>, StoreError> {
|
||||
if o.bucket_incarnation == [0; 16] || o.key.len() > MAX_OBJECT_KEY_LENGTH {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let capacity = 20usize.checked_add(o.key.len()).ok_or(StoreError::CorruptRecord)?;
|
||||
let mut b = Vec::with_capacity(capacity);
|
||||
b.extend(o.bucket_incarnation);
|
||||
b.extend(
|
||||
u32::try_from(o.key.len())
|
||||
.map_err(|_| StoreError::CorruptRecord)?
|
||||
.to_be_bytes(),
|
||||
);
|
||||
b.extend(&o.key);
|
||||
Ok(b)
|
||||
}
|
||||
pub(crate) fn put_prepared(b: &mut Vec<u8>, p: &PreparedIdentity) {
|
||||
b.extend(p.version_id);
|
||||
b.extend(p.preparation_id);
|
||||
b.extend(p.content_digest);
|
||||
b.extend(p.content_length.to_be_bytes())
|
||||
}
|
||||
pub(crate) fn binding_bytes(v: &CaptureBinding) -> Result<Vec<u8>, StoreError> {
|
||||
let mut b = v.revision.to_be_bytes().to_vec();
|
||||
b.extend(v.binding_id);
|
||||
b.extend(v.target_id);
|
||||
let mut d = Decoder::new(&b);
|
||||
d.binding()?;
|
||||
d.finish()?;
|
||||
Ok(b)
|
||||
}
|
||||
pub(crate) fn result_bytes(r: &CreatedResult) -> Vec<u8> {
|
||||
let mut b = Vec::new();
|
||||
match r {
|
||||
CreatedResult::Created {
|
||||
version_id,
|
||||
event_id,
|
||||
decision_log_id,
|
||||
} => {
|
||||
b.push(0);
|
||||
b.extend(version_id);
|
||||
b.extend(event_id);
|
||||
b.extend(encode_id(*decision_log_id))
|
||||
}
|
||||
CreatedResult::HeadMismatch { actual_head } => {
|
||||
b.push(1);
|
||||
put_option(&mut b, *actual_head)
|
||||
}
|
||||
CreatedResult::OperationConflict => b.push(2),
|
||||
CreatedResult::BindingMismatch => b.push(3),
|
||||
CreatedResult::VersionConflict => b.push(4),
|
||||
}
|
||||
b
|
||||
}
|
||||
impl DecideCreated {
|
||||
pub fn encode(&self) -> Result<Vec<u8>, StoreError> {
|
||||
let object = object_bytes(&self.object)?;
|
||||
let capacity = object.len().checked_add(134).ok_or(StoreError::CorruptRecord)?;
|
||||
let mut b = Vec::with_capacity(capacity);
|
||||
b.extend(b"CRTD\x01");
|
||||
b.extend(self.operation_id);
|
||||
b.extend(object);
|
||||
put_option(&mut b, self.expected_head);
|
||||
put_prepared(&mut b, &self.prepared);
|
||||
b.extend(self.binding_revision.to_be_bytes());
|
||||
Self::decode(&b)?;
|
||||
Ok(b)
|
||||
}
|
||||
pub fn decode(bytes: &[u8]) -> Result<Self, StoreError> {
|
||||
let mut d = Decoder::new(bytes);
|
||||
if d.take::<5>()? != *b"CRTD\x01" {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let c = Self {
|
||||
operation_id: d.uuid()?,
|
||||
object: d.object()?,
|
||||
expected_head: d.option()?,
|
||||
prepared: d.prepared()?,
|
||||
binding_revision: d.u64()?,
|
||||
};
|
||||
d.finish()?;
|
||||
Ok(c)
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,588 @@
|
||||
//! Local checkpoints are captured from this writer's owned transaction only.
|
||||
use crate::{
|
||||
LogId, NativeLogWriter, NativeSnapshotError, NativeSnapshotView, StoreError,
|
||||
checkpoint_baseline::{self, BaselineMetadata},
|
||||
snapshot,
|
||||
};
|
||||
use redb::{ReadableDatabase, ReadableTable, StorageBackend};
|
||||
use std::{
|
||||
fs::File,
|
||||
io::{self, Seek, Write},
|
||||
ops::Bound,
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
},
|
||||
};
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct CheckpointLimits {
|
||||
pub snapshot: snapshot::Limits,
|
||||
pub staging_bytes: u64,
|
||||
}
|
||||
|
||||
impl CheckpointLimits {
|
||||
pub(crate) fn validate(&self) -> Result<(), StoreError> {
|
||||
let rows = self.snapshot;
|
||||
if rows.total_bytes == 0
|
||||
|| rows.membership_bytes == 0
|
||||
|| rows.membership_bytes > 512 * 1024
|
||||
|| rows.key_bytes == 0
|
||||
|| rows.key_bytes > 1024 * 1024
|
||||
|| rows.value_bytes == 0
|
||||
|| rows.value_bytes > 1024 * 1024
|
||||
|| rows.records == 0
|
||||
|| self.staging_bytes < 4 * 1024 * 1024
|
||||
{
|
||||
return Err(StoreError::ResourceLimit);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for CheckpointLimits {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
snapshot: snapshot::Limits {
|
||||
total_bytes: 256 * 1024 * 1024,
|
||||
membership_bytes: 512 * 1024,
|
||||
key_bytes: 8192,
|
||||
value_bytes: 8192,
|
||||
records: 1_000_000,
|
||||
},
|
||||
staging_bytes: 1024 * 1024 * 1024,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct CheckpointReceipt {
|
||||
generation: u64,
|
||||
cut: LogId,
|
||||
}
|
||||
|
||||
impl CheckpointReceipt {
|
||||
pub fn generation(&self) -> u64 {
|
||||
self.generation
|
||||
}
|
||||
pub fn cut(&self) -> LogId {
|
||||
self.cut
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum CheckpointError {
|
||||
#[error(transparent)]
|
||||
Store(#[from] StoreError),
|
||||
#[error(transparent)]
|
||||
Capture(#[from] NativeSnapshotError),
|
||||
#[error(transparent)]
|
||||
Snapshot(#[from] snapshot::SnapshotError),
|
||||
#[error("checkpoint filesystem failure")]
|
||||
Io(#[from] io::Error),
|
||||
#[error("checkpoint worker failed")]
|
||||
Worker(#[source] tokio::task::JoinError),
|
||||
#[error("checkpoint store lock poisoned")]
|
||||
Poisoned,
|
||||
#[error("checkpoint conflict: {0}")]
|
||||
Conflict(&'static str),
|
||||
#[error("checkpoint cancelled before publication")]
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
/// Enforces the file-length budget before redb can allocate or write a page.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct BoundedBackend {
|
||||
inner: redb::backends::FileBackend,
|
||||
maximum: u64,
|
||||
}
|
||||
|
||||
impl BoundedBackend {
|
||||
fn new(file: File, maximum: u64) -> Result<Self, redb::DatabaseError> {
|
||||
Ok(Self {
|
||||
inner: redb::backends::FileBackend::new(file)?,
|
||||
maximum,
|
||||
})
|
||||
}
|
||||
|
||||
fn check(&self, end: u64) -> io::Result<()> {
|
||||
if end > self.maximum {
|
||||
return Err(io::Error::new(io::ErrorKind::StorageFull, "checkpoint staging budget"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl StorageBackend for BoundedBackend {
|
||||
fn len(&self) -> io::Result<u64> {
|
||||
self.inner.len()
|
||||
}
|
||||
fn read(&self, offset: u64, out: &mut [u8]) -> io::Result<()> {
|
||||
self.inner.read(offset, out)
|
||||
}
|
||||
fn set_len(&self, len: u64) -> io::Result<()> {
|
||||
self.check(len)?;
|
||||
self.inner.set_len(len)
|
||||
}
|
||||
fn sync_data(&self) -> io::Result<()> {
|
||||
self.inner.sync_data()
|
||||
}
|
||||
fn write(&self, offset: u64, data: &[u8]) -> io::Result<()> {
|
||||
let length = u64::try_from(data.len()).map_err(io::Error::other)?;
|
||||
self.check(
|
||||
offset
|
||||
.checked_add(length)
|
||||
.ok_or_else(|| io::Error::other("checkpoint write overflow"))?,
|
||||
)?;
|
||||
self.inner.write(offset, data)
|
||||
}
|
||||
fn close(&self) -> io::Result<()> {
|
||||
self.inner.close()
|
||||
}
|
||||
fn try_lock_range(&self, start: Bound<u64>, end: Bound<u64>) -> Result<bool, redb::BackendError> {
|
||||
self.inner.try_lock_range(start, end)
|
||||
}
|
||||
fn try_lock_shared_range(&self, start: Bound<u64>, end: Bound<u64>) -> Result<bool, redb::BackendError> {
|
||||
self.inner.try_lock_shared_range(start, end)
|
||||
}
|
||||
fn lock_range(&self, start: Bound<u64>, end: Bound<u64>) -> Result<(), redb::BackendError> {
|
||||
self.inner.lock_range(start, end)
|
||||
}
|
||||
fn lock_shared_range(&self, start: Bound<u64>, end: Bound<u64>) -> Result<(), redb::BackendError> {
|
||||
self.inner.lock_shared_range(start, end)
|
||||
}
|
||||
fn unlock_range(&self, start: Bound<u64>, end: Bound<u64>) -> Result<(), redb::BackendError> {
|
||||
self.inner.unlock_range(start, end)
|
||||
}
|
||||
fn query_lock_range(&self, start: Bound<u64>, end: Bound<u64>) -> Result<bool, redb::BackendError> {
|
||||
self.inner.query_lock_range(start, end)
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn recovery_scratch(maximum: u64) -> Result<(tempfile::TempDir, redb::Database), StoreError> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let database = stage_database(directory.path().join("stage.redb"), maximum)?;
|
||||
Ok((directory, database))
|
||||
}
|
||||
|
||||
pub(crate) fn stage_database(path: std::path::PathBuf, maximum: u64) -> Result<redb::Database, StoreError> {
|
||||
let file = File::options().read(true).write(true).create_new(true).open(path)?;
|
||||
let backend = BoundedBackend::new(file, maximum)?;
|
||||
Ok(redb::Database::builder()
|
||||
.set_cache_size(8 * 1024 * 1024)
|
||||
.create_with_backend(backend)?)
|
||||
}
|
||||
|
||||
struct PayloadFile {
|
||||
file: File,
|
||||
maximum: u64,
|
||||
written: u64,
|
||||
}
|
||||
|
||||
impl Write for PayloadFile {
|
||||
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
|
||||
let length = u64::try_from(bytes.len()).map_err(io::Error::other)?;
|
||||
let end = self
|
||||
.written
|
||||
.checked_add(length)
|
||||
.ok_or_else(|| io::Error::other("checkpoint payload overflow"))?;
|
||||
if end > self.maximum {
|
||||
return Err(io::Error::new(io::ErrorKind::StorageFull, "checkpoint payload budget"));
|
||||
}
|
||||
let count = self.file.write(bytes)?;
|
||||
self.written = self
|
||||
.written
|
||||
.checked_add(u64::try_from(count).map_err(io::Error::other)?)
|
||||
.ok_or_else(|| io::Error::other("checkpoint payload overflow"))?;
|
||||
Ok(count)
|
||||
}
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
self.file.flush()
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct StageSink<'a>(pub(crate) &'a redb::WriteTransaction);
|
||||
|
||||
impl snapshot::StagingSink for StageSink<'_> {
|
||||
fn row(&mut self, section: snapshot::Section, key: &[u8], value: &[u8]) -> io::Result<()> {
|
||||
let definition = match section {
|
||||
snapshot::Section::Heads => checkpoint_baseline::HEADS,
|
||||
snapshot::Section::Versions => checkpoint_baseline::VERSIONS,
|
||||
snapshot::Section::Events => checkpoint_baseline::EVENTS,
|
||||
snapshot::Section::Operations => checkpoint_baseline::OPERATIONS,
|
||||
};
|
||||
let mut table = self.0.open_table(definition).map_err(io::Error::other)?;
|
||||
if table.insert(key, value).map_err(io::Error::other)?.is_some() {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "duplicate checkpoint key"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// This type has no external constructors, deserialization, or path-based import.
|
||||
// Its source remains the owned local read transaction, not the decoded artifact.
|
||||
struct LocalStage {
|
||||
source: NativeSnapshotView,
|
||||
database: redb::Database,
|
||||
_directory: tempfile::TempDir,
|
||||
metadata: BaselineMetadata,
|
||||
}
|
||||
|
||||
fn capture_stage(source: NativeSnapshotView, limits: CheckpointLimits) -> Result<LocalStage, CheckpointError> {
|
||||
limits.validate()?;
|
||||
source
|
||||
.header()
|
||||
.applied
|
||||
.ok_or(CheckpointError::Conflict("empty applied cut"))?;
|
||||
let directory = tempfile::tempdir()?;
|
||||
let payload = File::options()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(directory.path().join("capture.csnp"))?;
|
||||
let artifact = source.stream_local_capability(
|
||||
PayloadFile {
|
||||
file: payload,
|
||||
maximum: limits.staging_bytes.min(limits.snapshot.total_bytes),
|
||||
written: 0,
|
||||
},
|
||||
limits.snapshot,
|
||||
)?;
|
||||
let remaining = limits
|
||||
.staging_bytes
|
||||
.checked_sub(artifact.payload.written)
|
||||
.ok_or(CheckpointError::Conflict("staging budget"))?;
|
||||
let mut payload = artifact.payload.file;
|
||||
payload.sync_all()?;
|
||||
payload.rewind()?;
|
||||
let database = stage_database(directory.path().join("stage.redb"), remaining)?;
|
||||
let transaction = database.begin_write().map_err(StoreError::from)?;
|
||||
for definition in [
|
||||
checkpoint_baseline::HEADS,
|
||||
checkpoint_baseline::VERSIONS,
|
||||
checkpoint_baseline::EVENTS,
|
||||
checkpoint_baseline::OPERATIONS,
|
||||
] {
|
||||
transaction.open_table(definition).map_err(StoreError::from)?;
|
||||
}
|
||||
let header = snapshot::decode_capability_artifact(
|
||||
snapshot::CapabilityArtifact {
|
||||
manifest: artifact.manifest,
|
||||
payload,
|
||||
},
|
||||
&mut StageSink(&transaction),
|
||||
limits.snapshot,
|
||||
)?;
|
||||
std::fs::remove_file(directory.path().join("capture.csnp"))?;
|
||||
if header != *source.header() {
|
||||
return Err(CheckpointError::Conflict("capture header"));
|
||||
}
|
||||
let metadata = BaselineMetadata {
|
||||
generation: 1,
|
||||
header,
|
||||
limits,
|
||||
};
|
||||
let encoded = checkpoint_baseline::encode_metadata(&metadata)?;
|
||||
{
|
||||
let mut meta = transaction.open_table(crate::META).map_err(StoreError::from)?;
|
||||
meta.insert("schema", 7u64.to_be_bytes().as_slice())
|
||||
.map_err(StoreError::from)?;
|
||||
meta.insert("checkpoint_baseline", encoded.as_slice())
|
||||
.map_err(StoreError::from)?;
|
||||
}
|
||||
transaction.commit().map_err(StoreError::StagingCommit)?;
|
||||
let stage = LocalStage {
|
||||
source,
|
||||
database,
|
||||
_directory: directory,
|
||||
metadata,
|
||||
};
|
||||
Ok(stage)
|
||||
}
|
||||
|
||||
fn verify_local_stage(stage: &LocalStage) -> Result<(), CheckpointError> {
|
||||
let transaction = stage.database.begin_read().map_err(StoreError::from)?;
|
||||
for (source_definition, stage_definition) in [
|
||||
(crate::application::HEADS, checkpoint_baseline::HEADS),
|
||||
(crate::application::VERSIONS, checkpoint_baseline::VERSIONS),
|
||||
(crate::application::EVENTS, checkpoint_baseline::EVENTS),
|
||||
(crate::application::OPERATIONS, checkpoint_baseline::OPERATIONS),
|
||||
] {
|
||||
let source = stage
|
||||
.source
|
||||
.transaction()
|
||||
.open_table(source_definition)
|
||||
.map_err(StoreError::from)?;
|
||||
let staged = transaction.open_table(stage_definition).map_err(StoreError::from)?;
|
||||
let mut source_rows = source.iter().map_err(StoreError::from)?;
|
||||
let mut staged_rows = staged.iter().map_err(StoreError::from)?;
|
||||
loop {
|
||||
match (source_rows.next(), staged_rows.next()) {
|
||||
(None, None) => break,
|
||||
(Some(source), Some(staged)) => {
|
||||
let (source_key, source_value) = source.map_err(StoreError::from)?;
|
||||
let (stage_key, stage_value) = staged.map_err(StoreError::from)?;
|
||||
if source_key.value() != stage_key.value() || source_value.value() != stage_value.value() {
|
||||
return Err(CheckpointError::Conflict("staging differs from owned local capture"));
|
||||
}
|
||||
}
|
||||
_ => return Err(CheckpointError::Conflict("staging row count")),
|
||||
}
|
||||
}
|
||||
}
|
||||
let staged_meta = transaction.open_table(crate::META).map_err(StoreError::from)?;
|
||||
let staged_metadata =
|
||||
checkpoint_baseline::read_metadata(&staged_meta)?.ok_or(CheckpointError::Conflict("missing staging metadata"))?;
|
||||
if checkpoint_baseline::encode_metadata(&staged_metadata)? != checkpoint_baseline::encode_metadata(&stage.metadata)? {
|
||||
return Err(CheckpointError::Conflict("staging metadata differs from local capture"));
|
||||
}
|
||||
let stage_bytes = std::fs::metadata(stage._directory.path().join("stage.redb"))?.len();
|
||||
let index_budget = stage
|
||||
.metadata
|
||||
.limits
|
||||
.staging_bytes
|
||||
.checked_sub(stage_bytes)
|
||||
.ok_or(CheckpointError::Conflict("staging disk budget"))?;
|
||||
checkpoint_baseline::validate_baseline_with_budget(&transaction, &stage.metadata, index_budget)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn resource_error(error: CheckpointError) -> CheckpointError {
|
||||
use std::error::Error;
|
||||
if matches!(&error, CheckpointError::Store(StoreError::CommitIndeterminate(_))) {
|
||||
return error;
|
||||
}
|
||||
let mut source: Option<&(dyn Error + 'static)> = Some(&error);
|
||||
while let Some(current) = source {
|
||||
if current
|
||||
.downcast_ref::<io::Error>()
|
||||
.is_some_and(|error| error.kind() == io::ErrorKind::StorageFull)
|
||||
{
|
||||
return CheckpointError::Store(StoreError::ResourceLimit);
|
||||
}
|
||||
source = current.source();
|
||||
}
|
||||
error
|
||||
}
|
||||
|
||||
struct CancelOnDrop(Arc<AtomicBool>);
|
||||
impl Drop for CancelOnDrop {
|
||||
fn drop(&mut self) {
|
||||
self.0.store(true, Ordering::Release);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
type CaptureHook = Box<dyn FnOnce() + Send>;
|
||||
#[cfg(test)]
|
||||
type StageHook = Box<dyn FnOnce(&redb::Database) + Send>;
|
||||
#[cfg(test)]
|
||||
static CAPTURE_HOOKS: std::sync::Mutex<std::collections::BTreeMap<usize, CaptureHook>> =
|
||||
std::sync::Mutex::new(std::collections::BTreeMap::new());
|
||||
#[cfg(test)]
|
||||
static STAGE_HOOKS: std::sync::Mutex<std::collections::BTreeMap<usize, StageHook>> =
|
||||
std::sync::Mutex::new(std::collections::BTreeMap::new());
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) struct CheckpointHook {
|
||||
owner: usize,
|
||||
stage: bool,
|
||||
}
|
||||
#[cfg(test)]
|
||||
impl Drop for CheckpointHook {
|
||||
fn drop(&mut self) {
|
||||
if self.stage {
|
||||
STAGE_HOOKS.lock().expect("stage hooks").remove(&self.owner);
|
||||
} else {
|
||||
CAPTURE_HOOKS.lock().expect("capture hooks").remove(&self.owner);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn after_local_capture(writer: &NativeLogWriter, hook: impl FnOnce() + Send + 'static) -> CheckpointHook {
|
||||
let owner = Arc::as_ptr(&writer.store).addr();
|
||||
assert!(
|
||||
CAPTURE_HOOKS
|
||||
.lock()
|
||||
.expect("capture hooks")
|
||||
.insert(owner, Box::new(hook))
|
||||
.is_none()
|
||||
);
|
||||
CheckpointHook { owner, stage: false }
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn after_local_stage(writer: &NativeLogWriter, hook: impl FnOnce(&redb::Database) + Send + 'static) -> CheckpointHook {
|
||||
let owner = Arc::as_ptr(&writer.store).addr();
|
||||
assert!(
|
||||
STAGE_HOOKS
|
||||
.lock()
|
||||
.expect("stage hooks")
|
||||
.insert(owner, Box::new(hook))
|
||||
.is_none()
|
||||
);
|
||||
CheckpointHook { owner, stage: true }
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn run_capture_hook(owner: &Arc<std::sync::Mutex<crate::Store>>) {
|
||||
let hook = CAPTURE_HOOKS
|
||||
.lock()
|
||||
.expect("capture hooks")
|
||||
.remove(&Arc::as_ptr(owner).addr());
|
||||
if let Some(hook) = hook {
|
||||
hook();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn run_stage_hook(owner: &Arc<std::sync::Mutex<crate::Store>>, database: &redb::Database) {
|
||||
let hook = STAGE_HOOKS.lock().expect("stage hooks").remove(&Arc::as_ptr(owner).addr());
|
||||
if let Some(hook) = hook {
|
||||
hook(database);
|
||||
}
|
||||
}
|
||||
|
||||
impl NativeLogWriter {
|
||||
pub async fn checkpoint_local(&self, limits: CheckpointLimits) -> Result<CheckpointReceipt, CheckpointError> {
|
||||
self.checkpoint_local_inner(limits).await.map_err(resource_error)
|
||||
}
|
||||
|
||||
async fn checkpoint_local_inner(&self, limits: CheckpointLimits) -> Result<CheckpointReceipt, CheckpointError> {
|
||||
limits.validate()?;
|
||||
let admission = self.checkpoint_admission.clone().lock_owned().await;
|
||||
let owner = self.store.clone();
|
||||
let stage = tokio::task::spawn_blocking(move || {
|
||||
let source = NativeSnapshotView::acquire(owner.clone())?;
|
||||
#[cfg(test)]
|
||||
run_capture_hook(&owner);
|
||||
let stage = capture_stage(source, limits)?;
|
||||
#[cfg(test)]
|
||||
run_stage_hook(&owner, &stage.database);
|
||||
verify_local_stage(&stage)?;
|
||||
Ok::<_, CheckpointError>((stage, admission))
|
||||
})
|
||||
.await
|
||||
.map_err(CheckpointError::Worker)??;
|
||||
let ordering = self.ordering.clone().lock_owned().await;
|
||||
let owner = self.store.clone();
|
||||
let cancellation = CancelOnDrop(Arc::new(AtomicBool::new(false)));
|
||||
let cancelled = cancellation.0.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
// The blocking owner retains both guards until publication finishes,
|
||||
// even if the async caller is dropped while redb is committing.
|
||||
let _ordering = ordering;
|
||||
let (stage, _admission) = stage;
|
||||
if cancelled.load(Ordering::Acquire) {
|
||||
return Err(CheckpointError::Cancelled);
|
||||
}
|
||||
if !Arc::ptr_eq(&owner, stage.source.owner()) {
|
||||
return Err(CheckpointError::Conflict("local capture owner"));
|
||||
}
|
||||
let mut store = owner.lock().map_err(|_| CheckpointError::Poisoned)?;
|
||||
let transaction = store.write_transaction()?;
|
||||
let mut meta = transaction.open_table(crate::META).map_err(StoreError::from)?;
|
||||
let schema = crate::number(
|
||||
meta.get("schema")
|
||||
.map_err(StoreError::from)?
|
||||
.ok_or(StoreError::CorruptRecord)?
|
||||
.value(),
|
||||
)?;
|
||||
if !matches!(schema, 6 | 7) {
|
||||
return Err(StoreError::UnsupportedSchema.into());
|
||||
}
|
||||
let previous = checkpoint_baseline::read_metadata(&meta)?;
|
||||
if crate::application::read_binding(&meta)? != stage.metadata.header.binding {
|
||||
return Err(CheckpointError::Conflict("live capture binding changed"));
|
||||
}
|
||||
let identity = crate::decode_identity(
|
||||
meta.get("identity")
|
||||
.map_err(StoreError::from)?
|
||||
.ok_or(StoreError::CorruptRecord)?
|
||||
.value(),
|
||||
)?;
|
||||
if identity.cluster_id != stage.metadata.header.cluster_id {
|
||||
return Err(CheckpointError::Conflict("live capture cluster changed"));
|
||||
}
|
||||
let cut = stage
|
||||
.metadata
|
||||
.header
|
||||
.applied
|
||||
.ok_or(CheckpointError::Conflict("empty applied cut"))?;
|
||||
let applied = crate::application::read_applied(&meta)?.ok_or(StoreError::CorruptRecord)?;
|
||||
let logs = transaction.open_table(crate::LOGS).map_err(StoreError::from)?;
|
||||
let committed = crate::committed::validate_committed(&meta, &logs)?.ok_or(StoreError::CorruptRecord)?;
|
||||
if cut.index > applied.index
|
||||
|| (cut.index == applied.index && cut != applied)
|
||||
|| applied.index > committed.index
|
||||
|| (applied.index == committed.index && applied != committed)
|
||||
{
|
||||
return Err(CheckpointError::Conflict("live applied and committed bounds"));
|
||||
}
|
||||
drop(logs);
|
||||
if let Some(old) = previous.as_ref().and_then(|previous| previous.header.applied)
|
||||
&& (old.index > cut.index || (old.index == cut.index && old != cut))
|
||||
{
|
||||
return Err(CheckpointError::Conflict("published checkpoint cut"));
|
||||
}
|
||||
let mut metadata = stage.metadata;
|
||||
metadata.generation = previous.map_or(Ok(1), |old| {
|
||||
old.generation
|
||||
.checked_add(1)
|
||||
.ok_or(CheckpointError::Conflict("checkpoint generation exhausted"))
|
||||
})?;
|
||||
let stage_transaction = stage.database.begin_read().map_err(StoreError::from)?;
|
||||
for (source_definition, definition) in [
|
||||
(crate::application::HEADS, checkpoint_baseline::HEADS),
|
||||
(crate::application::VERSIONS, checkpoint_baseline::VERSIONS),
|
||||
(crate::application::EVENTS, checkpoint_baseline::EVENTS),
|
||||
(crate::application::OPERATIONS, checkpoint_baseline::OPERATIONS),
|
||||
] {
|
||||
let captured = stage
|
||||
.source
|
||||
.transaction()
|
||||
.open_table(source_definition)
|
||||
.map_err(StoreError::from)?;
|
||||
let staged = stage_transaction.open_table(definition).map_err(StoreError::from)?;
|
||||
let mut source_rows = captured.iter().map_err(StoreError::from)?;
|
||||
let mut stage_rows = staged.iter().map_err(StoreError::from)?;
|
||||
let mut target = transaction.open_table(definition).map_err(StoreError::from)?;
|
||||
target.retain(|_, _| false).map_err(StoreError::from)?;
|
||||
// Recheck disk bytes while copying; the earlier relationship
|
||||
// validation remains outside the writer ordering interval.
|
||||
loop {
|
||||
match (source_rows.next(), stage_rows.next()) {
|
||||
(None, None) => break,
|
||||
(Some(source), Some(staged)) => {
|
||||
let (source_key, source_value) = source.map_err(StoreError::from)?;
|
||||
let (key, value) = staged.map_err(StoreError::from)?;
|
||||
if source_key.value() != key.value() || source_value.value() != value.value() {
|
||||
return Err(CheckpointError::Conflict("publication differs from local capture"));
|
||||
}
|
||||
target.insert(key.value(), value.value()).map_err(StoreError::from)?;
|
||||
}
|
||||
_ => return Err(CheckpointError::Conflict("publication row count")),
|
||||
}
|
||||
}
|
||||
}
|
||||
let encoded = checkpoint_baseline::encode_metadata(&metadata)?;
|
||||
meta.insert("checkpoint_baseline", encoded.as_slice())
|
||||
.map_err(StoreError::from)?;
|
||||
meta.insert("schema", 7u64.to_be_bytes().as_slice())
|
||||
.map_err(StoreError::from)?;
|
||||
crate::raft_state_machine::publish_local_meta(&mut meta, &metadata)?;
|
||||
drop(meta);
|
||||
if cancelled.load(Ordering::Acquire) {
|
||||
return Err(CheckpointError::Cancelled);
|
||||
}
|
||||
store.commit(transaction)?;
|
||||
Ok(CheckpointReceipt {
|
||||
generation: metadata.generation,
|
||||
cut,
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(CheckpointError::Worker)?
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,607 @@
|
||||
use super::*;
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
fn identity() -> NodeIdentity {
|
||||
NodeIdentity {
|
||||
cluster_id: [7; 16],
|
||||
node_id: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn binding() -> CaptureBinding {
|
||||
CaptureBinding {
|
||||
revision: 9,
|
||||
binding_id: [8; 16],
|
||||
target_id: [9; 16],
|
||||
}
|
||||
}
|
||||
|
||||
fn id(index: u64) -> LogId {
|
||||
LogId {
|
||||
term: 8,
|
||||
leader_node: 0,
|
||||
index,
|
||||
}
|
||||
}
|
||||
|
||||
fn command(operation: u8) -> DecideCreated {
|
||||
DecideCreated {
|
||||
operation_id: [operation; 16],
|
||||
object: ObjectIdentity {
|
||||
bucket_incarnation: [2; 16],
|
||||
key: b"checkpoint-object".to_vec(),
|
||||
},
|
||||
expected_head: None,
|
||||
prepared: PreparedIdentity {
|
||||
version_id: [3; 16],
|
||||
preparation_id: [4; 16],
|
||||
content_digest: [5; 32],
|
||||
content_length: 4097,
|
||||
},
|
||||
binding_revision: 9,
|
||||
}
|
||||
}
|
||||
|
||||
fn apply(store: &mut Store, index: u64, command: DecideCreated) -> CaptureResponse {
|
||||
let entry = CaptureRaftEntry {
|
||||
log_id: id(index).into(),
|
||||
payload: openraft::EntryPayload::Normal(CaptureCommand::try_created(command).expect("canonical command")),
|
||||
};
|
||||
store.append_native(&[entry]).expect("actual durable append");
|
||||
store.save_committed(id(index)).expect("actual durable commit");
|
||||
store.apply_next_native(id(index)).expect("actual native apply")
|
||||
}
|
||||
|
||||
fn fixture() -> (tempfile::TempDir, Store, DecideCreated, CreatedOperationProvenance) {
|
||||
let dir = tempfile::tempdir().expect("actual native database directory");
|
||||
let mut store = Store::initialize(dir.path().join("db"), identity()).expect("initialize actual redb");
|
||||
store.pin_capture_binding(binding()).expect("bind actual store");
|
||||
store.enable_native_empty().expect("native schema4");
|
||||
store.promote_created_provenance().expect("native schema6 provenance");
|
||||
let membership = CaptureMembership::new(
|
||||
Some(id(0)),
|
||||
vec![BTreeSet::from([0])],
|
||||
BTreeMap::from([(0, CaptureNode::new("http://127.0.0.1:7000".into()).expect("native member"))]),
|
||||
)
|
||||
.expect("full native membership");
|
||||
let stored = openraft::StoredMembership::try_from(&membership).expect("OpenRaft membership");
|
||||
store
|
||||
.append_native(&[CaptureRaftEntry {
|
||||
log_id: id(0).into(),
|
||||
payload: openraft::EntryPayload::Membership(stored.membership().clone()),
|
||||
}])
|
||||
.expect("append actual membership");
|
||||
store.save_committed(id(0)).expect("commit membership");
|
||||
store.apply_next_native(id(0)).expect("apply membership");
|
||||
let negative = command(11);
|
||||
let mut original = negative.clone();
|
||||
original.expected_head = Some([12; 16]);
|
||||
apply(&mut store, 1, original.clone());
|
||||
let first = store
|
||||
.created_operation_provenance(original.operation_id)
|
||||
.expect("read first negative")
|
||||
.expect("canonical negative operation");
|
||||
assert_eq!(first.first_decision_log_id, id(1), "CHECKPOINT_FIRST_NEGATIVE_BOUND");
|
||||
assert!(
|
||||
matches!(first.operation.result, CreatedResult::HeadMismatch { .. }),
|
||||
"CHECKPOINT_FIRST_NEGATIVE_BOUND"
|
||||
);
|
||||
apply(&mut store, 2, command(1));
|
||||
(dir, store, original, first)
|
||||
}
|
||||
|
||||
fn runtime() -> tokio::runtime::Runtime {
|
||||
tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("checkpoint runtime")
|
||||
}
|
||||
|
||||
fn schema(store: &Store) -> u64 {
|
||||
let tx = store.database.begin_read().expect("read actual schema");
|
||||
let table = tx.open_table(META).expect("schema table");
|
||||
number(table.get("schema").expect("schema read").expect("schema exists").value()).expect("schema number")
|
||||
}
|
||||
|
||||
fn assert_negative(store: &Store, original: &DecideCreated, first: &CreatedOperationProvenance) {
|
||||
assert_eq!(
|
||||
store
|
||||
.created_operation_provenance(original.operation_id)
|
||||
.expect("canonical read"),
|
||||
Some(first.clone()),
|
||||
"CHECKPOINT_FIRST_NEGATIVE_BOUND: preserve original bytes/result/full first ID"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_checkpoint_negative_retry_and_purge_reopen() {
|
||||
let (dir, store, original, first) = fixture();
|
||||
assert_eq!(schema(&store), 6);
|
||||
let writer = NativeLogWriter::new(store);
|
||||
let receipt = runtime()
|
||||
.block_on(writer.checkpoint_local(CheckpointLimits::default()))
|
||||
.expect("publish local checkpoint");
|
||||
assert_eq!(receipt.cut(), id(2), "CHECKPOINT_BASELINE_BOUND");
|
||||
assert_eq!(receipt.generation(), 1);
|
||||
{
|
||||
let mut store = writer.store.lock().expect("actual writer store");
|
||||
assert_eq!(schema(&store), 7);
|
||||
assert_negative(&store, &original, &first);
|
||||
apply(&mut store, 3, original.clone());
|
||||
assert_negative(&store, &original, &first);
|
||||
let mut conflict = original.clone();
|
||||
conflict.binding_revision += 1;
|
||||
apply(&mut store, 4, conflict);
|
||||
assert_eq!(
|
||||
store
|
||||
.created_apply_result(4)
|
||||
.expect("conflicting response")
|
||||
.expect("response")
|
||||
.result,
|
||||
CreatedResult::OperationConflict,
|
||||
"CHECKPOINT_RETRY_BOUND"
|
||||
);
|
||||
assert_negative(&store, &original, &first);
|
||||
}
|
||||
runtime()
|
||||
.block_on(writer.purge_checkpoint(&receipt))
|
||||
.expect("purge exact durable baseline");
|
||||
drop(writer);
|
||||
let reopened = Store::open(dir.path().join("db"), identity()).expect("reopen purged baseline plus suffix");
|
||||
assert_negative(&reopened, &original, &first);
|
||||
assert_eq!(reopened.last_applied().expect("suffix applied"), Some(id(4)), "CHECKPOINT_SUFFIX_BOUND");
|
||||
assert!(reopened.read_entry(2).expect("purged cut").is_none());
|
||||
assert!(reopened.read_entry(3).expect("retained suffix").is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_checkpoint_capture_keeps_concurrent_suffix() {
|
||||
let (dir, store, original, first) = fixture();
|
||||
let writer = NativeLogWriter::new(store);
|
||||
let (entered_tx, entered_rx) = std::sync::mpsc::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel();
|
||||
let _hook = native_checkpoint::after_local_capture(&writer, move || {
|
||||
entered_tx.send(()).expect("capture reached");
|
||||
release_rx
|
||||
.recv_timeout(std::time::Duration::from_secs(30))
|
||||
.expect("release owned capture");
|
||||
});
|
||||
let worker = writer.clone();
|
||||
let thread = std::thread::spawn(move || runtime().block_on(worker.checkpoint_local(CheckpointLimits::default())));
|
||||
entered_rx
|
||||
.recv_timeout(std::time::Duration::from_secs(30))
|
||||
.expect("actual capture boundary");
|
||||
{
|
||||
let mut store = writer.store.lock().expect("capture releases live write owner");
|
||||
apply(&mut store, 3, original.clone());
|
||||
}
|
||||
release_tx.send(()).expect("release capture");
|
||||
let receipt = thread.join().expect("checkpoint worker").expect("concurrent checkpoint");
|
||||
assert_eq!(receipt.cut(), id(2), "CHECKPOINT_SAME_CUT_BOUND");
|
||||
runtime()
|
||||
.block_on(writer.purge_checkpoint(&receipt))
|
||||
.expect("purge captured prefix");
|
||||
drop(writer);
|
||||
let reopened = Store::open(dir.path().join("db"), identity()).expect("reopen concurrent suffix");
|
||||
assert_eq!(reopened.last_applied().expect("actual suffix"), Some(id(3)), "CHECKPOINT_SUFFIX_BOUND");
|
||||
assert_negative(&reopened, &original, &first);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_checkpoint_staging_corruption_preserves_live_generation() {
|
||||
let (dir, store, original, first) = fixture();
|
||||
let writer = NativeLogWriter::new(store);
|
||||
let before = runtime()
|
||||
.block_on(writer.checkpoint_local(CheckpointLimits::default()))
|
||||
.expect("initial baseline");
|
||||
let _hook = native_checkpoint::after_local_stage(&writer, |database| {
|
||||
let tx = database.begin_write().expect("actual isolated stage mutation");
|
||||
tx.open_table(checkpoint_baseline::OPERATIONS)
|
||||
.expect("stage operations")
|
||||
.retain(|_, _| false)
|
||||
.expect("damage cross-record relationships");
|
||||
tx.commit().expect("persist actual damaged stage");
|
||||
});
|
||||
assert!(
|
||||
runtime()
|
||||
.block_on(writer.checkpoint_local(CheckpointLimits::default()))
|
||||
.is_err(),
|
||||
"CHECKPOINT_FAILURE_ATOMIC"
|
||||
);
|
||||
let store = writer.store.lock().expect("unchanged live store");
|
||||
let tx = store.database.begin_read().expect("published generation");
|
||||
let meta = tx.open_table(META).expect("published meta");
|
||||
assert_eq!(
|
||||
checkpoint_baseline::read_metadata(&meta)
|
||||
.expect("valid old metadata")
|
||||
.expect("old baseline")
|
||||
.generation,
|
||||
before.generation(),
|
||||
"CHECKPOINT_FAILURE_ATOMIC"
|
||||
);
|
||||
assert_negative(&store, &original, &first);
|
||||
drop(meta);
|
||||
drop(tx);
|
||||
drop(store);
|
||||
drop(writer);
|
||||
let reopened = Store::open(dir.path().join("db"), identity()).expect("reopen after failed staging validation");
|
||||
assert_negative(&reopened, &original, &first);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_checkpoint_budget_failure_preserves_schema_and_history() {
|
||||
for staging in [false, true] {
|
||||
let (dir, mut store, original, first) = fixture();
|
||||
if staging {
|
||||
for index in 0u64..256 {
|
||||
let mut large = command(31);
|
||||
large.operation_id[..8].copy_from_slice(&index.to_be_bytes());
|
||||
large.prepared.version_id[..8].copy_from_slice(&index.to_be_bytes());
|
||||
large.object.key = vec![b'x'; MAX_OBJECT_KEY_LENGTH];
|
||||
large.object.key[..8].copy_from_slice(&index.to_be_bytes());
|
||||
apply(&mut store, index + 3, large);
|
||||
}
|
||||
}
|
||||
let writer = NativeLogWriter::new(store);
|
||||
let mut limits = CheckpointLimits::default();
|
||||
if staging {
|
||||
limits.staging_bytes = 4 * 1024 * 1024;
|
||||
} else {
|
||||
limits.snapshot.total_bytes = 64;
|
||||
}
|
||||
let result = runtime().block_on(writer.checkpoint_local(limits));
|
||||
if staging {
|
||||
assert!(
|
||||
matches!(&result, Err(CheckpointError::Store(StoreError::ResourceLimit))),
|
||||
"CHECKPOINT_RESOURCE_BOUND: actual 4MiB physical staging limit: {result:?}"
|
||||
);
|
||||
} else {
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"CHECKPOINT_RESOURCE_BOUND: actual bounded output must refuse: {result:?}"
|
||||
);
|
||||
}
|
||||
{
|
||||
let store = writer.store.lock().expect("live after exhausted budget");
|
||||
assert_eq!(schema(&store), 6, "CHECKPOINT_FAILURE_ATOMIC");
|
||||
assert_negative(&store, &original, &first);
|
||||
assert!(store.read_entry(0).expect("prefix intact").is_some());
|
||||
}
|
||||
drop(writer);
|
||||
assert_negative(
|
||||
&Store::open(dir.path().join("db"), identity()).expect("reopen budget failure"),
|
||||
&original,
|
||||
&first,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_checkpoint_cancelled_capture_cannot_publish_or_purge() {
|
||||
let (dir, store, original, first) = fixture();
|
||||
let writer = NativeLogWriter::new(store);
|
||||
let (entered_tx, entered_rx) = std::sync::mpsc::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel();
|
||||
let _hook = native_checkpoint::after_local_capture(&writer, move || {
|
||||
entered_tx.send(()).expect("capture cancellation boundary");
|
||||
release_rx
|
||||
.recv_timeout(std::time::Duration::from_secs(30))
|
||||
.expect("release pending actual capture");
|
||||
});
|
||||
let rt = runtime();
|
||||
let worker = writer.clone();
|
||||
rt.block_on(async {
|
||||
let task = tokio::spawn(async move { worker.checkpoint_local(CheckpointLimits::default()).await });
|
||||
tokio::task::spawn_blocking(move || entered_rx.recv_timeout(std::time::Duration::from_secs(30)))
|
||||
.await
|
||||
.expect("capture observer")
|
||||
.expect("actual capture owned transaction");
|
||||
task.abort();
|
||||
assert!(task.await.expect_err("cancelled waiter").is_cancelled(), "CHECKPOINT_CANCEL_BOUND");
|
||||
release_tx.send(()).expect("release actual worker after waiter cancellation");
|
||||
// A fresh checkpoint waits for the same production admission; completion
|
||||
// establishes that the cancelled capture worker released its real resources.
|
||||
let receipt = writer
|
||||
.checkpoint_local(CheckpointLimits::default())
|
||||
.await
|
||||
.expect("capture after cancellation");
|
||||
assert_eq!(receipt.generation(), 1, "CHECKPOINT_CANCEL_BOUND: abandoned capture never published");
|
||||
});
|
||||
drop(rt);
|
||||
drop(writer);
|
||||
let reopened = Store::open(dir.path().join("db"), identity()).expect("reopen after cancellation and successful next capture");
|
||||
assert_negative(&reopened, &original, &first);
|
||||
assert!(
|
||||
reopened.read_entry(0).expect("cancellation cannot purge").is_some(),
|
||||
"CHECKPOINT_CANCEL_BOUND"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_checkpoint_stale_receipt_cannot_remove_history() {
|
||||
let (_dir, store, original, first) = fixture();
|
||||
let writer = NativeLogWriter::new(store);
|
||||
let rt = runtime();
|
||||
let old = rt
|
||||
.block_on(writer.checkpoint_local(CheckpointLimits::default()))
|
||||
.expect("first generation");
|
||||
let new = rt
|
||||
.block_on(writer.checkpoint_local(CheckpointLimits::default()))
|
||||
.expect("next generation");
|
||||
assert!(new.generation() > old.generation());
|
||||
assert!(
|
||||
matches!(
|
||||
rt.block_on(writer.purge_checkpoint(&old)),
|
||||
Err(NativeWriteError::Store(StoreError::StaleCheckpoint))
|
||||
),
|
||||
"CHECKPOINT_PURGE_BOUND"
|
||||
);
|
||||
let store = writer.store.lock().expect("history after stale purge");
|
||||
assert_negative(&store, &original, &first);
|
||||
assert!(store.read_entry(1).expect("first decision retained").is_some());
|
||||
assert!(store.created_apply_result(1).expect("typed result retained").is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_checkpoint_reopen_rejects_conflicting_cut_and_suffix_gap() {
|
||||
for corruption in 0..4 {
|
||||
let gap = corruption == 1;
|
||||
let (dir, store, original, _) = fixture();
|
||||
let writer = NativeLogWriter::new(store);
|
||||
let rt = runtime();
|
||||
let receipt = rt
|
||||
.block_on(writer.checkpoint_local(CheckpointLimits::default()))
|
||||
.expect("valid checkpoint");
|
||||
if gap {
|
||||
{
|
||||
let mut store = writer.store.lock().expect("suffix owner");
|
||||
apply(&mut store, 3, original.clone());
|
||||
apply(&mut store, 4, original.clone());
|
||||
}
|
||||
rt.block_on(writer.purge_checkpoint(&receipt))
|
||||
.expect("purge before gap corruption");
|
||||
{
|
||||
let store = writer.store.lock().expect("actual purged metadata owner");
|
||||
let tx = store.database.begin_write().expect("actual CPUR mutation");
|
||||
let mut meta = tx.open_table(META).expect("actual CPUR table");
|
||||
let mut conflict = receipt.cut();
|
||||
conflict.term += 1;
|
||||
conflict.leader_node += 1;
|
||||
let encoded = checkpoint_baseline::encode_purged(conflict);
|
||||
meta.insert(checkpoint_baseline::PURGED_KEY, encoded.as_slice())
|
||||
.expect("persist conflicting full purge ID");
|
||||
drop(meta);
|
||||
tx.commit().expect("durable conflicting CPUR");
|
||||
let tx = store.database.begin_read().expect("actual conflicting CPUR read");
|
||||
let meta = tx.open_table(META).expect("actual persisted CPUR");
|
||||
assert!(
|
||||
matches!(checkpoint_baseline::read_purged(&meta), Err(StoreError::CorruptRecord)),
|
||||
"CHECKPOINT_FULL_CUT_BOUND: same index with different term/leader is corruption"
|
||||
);
|
||||
drop(meta);
|
||||
drop(tx);
|
||||
let tx = store
|
||||
.database
|
||||
.begin_write()
|
||||
.expect("restore valid purge boundary for gap scenario");
|
||||
let encoded = checkpoint_baseline::encode_purged(receipt.cut());
|
||||
tx.open_table(META)
|
||||
.expect("actual purge meta")
|
||||
.insert(checkpoint_baseline::PURGED_KEY, encoded.as_slice())
|
||||
.expect("restore exact cut");
|
||||
tx.commit().expect("persist exact purge boundary");
|
||||
}
|
||||
}
|
||||
if corruption >= 2 {
|
||||
let mut store = writer.store.lock().expect("actual suffix before tail corruption");
|
||||
apply(&mut store, 3, original.clone());
|
||||
}
|
||||
{
|
||||
let store = writer.store.lock().expect("actual corruption owner");
|
||||
let tx = store.database.begin_write().expect("actual redb mutation");
|
||||
let mut logs = tx.open_table(LOGS).expect("actual log table");
|
||||
if gap {
|
||||
logs.remove(3).expect("remove required suffix entry");
|
||||
} else if corruption >= 2 {
|
||||
let mut tail = id(3);
|
||||
if corruption == 2 {
|
||||
tail.index += 1;
|
||||
} else {
|
||||
tail.term += 1;
|
||||
tail.leader_node += 1;
|
||||
}
|
||||
let encoded = encode_id(tail);
|
||||
tx.open_table(META)
|
||||
.expect("actual metadata")
|
||||
.insert("tail", encoded.as_slice())
|
||||
.expect("corrupt durable tail");
|
||||
} else {
|
||||
let mut entry = store.read_entry(2).expect("retained cut").expect("cut entry");
|
||||
entry.id.term += 1;
|
||||
entry.id.leader_node += 1;
|
||||
let encoded = encode_log(&entry).expect("valid conflicting full ID encoding");
|
||||
logs.insert(2, encoded.as_slice()).expect("replace exact cut identity");
|
||||
}
|
||||
drop(logs);
|
||||
tx.commit().expect("persist actual inconsistent recovery input");
|
||||
}
|
||||
assert!(
|
||||
rt.block_on(writer.purge_checkpoint(&receipt)).is_err(),
|
||||
"CHECKPOINT_PURGE_BOUND: invalid recovery cannot authorize cleanup"
|
||||
);
|
||||
if !gap {
|
||||
let store = writer.store.lock().expect("refused purge retains original evidence");
|
||||
assert!(store.read_entry(1).expect("original log retained").is_some(), "CHECKPOINT_PURGE_BOUND");
|
||||
assert!(
|
||||
store.created_apply_result(1).expect("original response retained").is_some(),
|
||||
"CHECKPOINT_PURGE_BOUND"
|
||||
);
|
||||
}
|
||||
drop(writer);
|
||||
drop(rt);
|
||||
assert!(Store::open(dir.path().join("db"), identity()).is_err(), "CHECKPOINT_RECOVERY_BOUND");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_checkpoint_publication_fault_selects_complete_generation() {
|
||||
for (sync, after) in [(1, false), (1, true), (2, false), (2, true)] {
|
||||
let (dir, store, original, first) = fixture();
|
||||
let vote = VoteRecord {
|
||||
term: 9,
|
||||
voted_node: 0,
|
||||
committed: true,
|
||||
};
|
||||
let mut store = store;
|
||||
store.save_vote(vote).expect("local hard state before checkpoint");
|
||||
drop(store);
|
||||
let path = dir.path().join("db");
|
||||
let backend = tests::CrashBackend::new(&path);
|
||||
let store = Store::from_database(backend.database(), identity()).expect("existing simulated durability backend");
|
||||
let writer = NativeLogWriter::new(store);
|
||||
let (entered, release) = backend.pause_sync(sync, after);
|
||||
let worker = writer.clone();
|
||||
let task = std::thread::spawn(move || runtime().block_on(worker.checkpoint_local(CheckpointLimits::default())));
|
||||
entered.wait();
|
||||
backend.cut();
|
||||
release.wait();
|
||||
let _unknown = task.join().expect("interrupted publication worker");
|
||||
drop(writer);
|
||||
let reopened = Store::open(&path, identity()).expect("recover complete generation after simulated sync cut");
|
||||
assert!(matches!(schema(&reopened), 6 | 7), "CHECKPOINT_CRASH_ATOMIC");
|
||||
assert_negative(&reopened, &original, &first);
|
||||
assert_eq!(
|
||||
reopened.read_vote().expect("preserved local vote"),
|
||||
Some(vote),
|
||||
"CHECKPOINT_LOCAL_STATE_BOUND"
|
||||
);
|
||||
assert_eq!(reopened.last_applied().expect("complete applied position"), Some(id(2)));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_checkpoint_recovery_budget_can_be_raised_without_changing_baseline() {
|
||||
use redb::ReadableTableMetadata;
|
||||
let (dir, store, original, first) = fixture();
|
||||
let mut limits = CheckpointLimits::default();
|
||||
let tx = store.database.begin_read().expect("measure real baseline rows");
|
||||
limits.snapshot.records = [
|
||||
application::HEADS,
|
||||
application::VERSIONS,
|
||||
application::EVENTS,
|
||||
application::OPERATIONS,
|
||||
]
|
||||
.into_iter()
|
||||
.map(|definition| {
|
||||
tx.open_table(definition)
|
||||
.expect("actual baseline table")
|
||||
.len()
|
||||
.expect("actual row count")
|
||||
})
|
||||
.sum();
|
||||
drop(tx);
|
||||
let writer = NativeLogWriter::new(store);
|
||||
let receipt = runtime()
|
||||
.block_on(writer.checkpoint_local(limits))
|
||||
.expect("capture exact bounded baseline");
|
||||
{
|
||||
let mut store = writer.store.lock().expect("actual suffix writer");
|
||||
let mut suffix = command(21);
|
||||
suffix.object.key = b"suffix-new-key".to_vec();
|
||||
suffix.prepared.version_id = [22; 16];
|
||||
apply(&mut store, 3, suffix);
|
||||
}
|
||||
drop(writer);
|
||||
let path = dir.path().join("db");
|
||||
assert!(
|
||||
matches!(Store::open(&path, identity()), Err(StoreError::ResourceLimit)),
|
||||
"CHECKPOINT_RESOURCE_BOUND"
|
||||
);
|
||||
let mut raised = limits;
|
||||
raised.snapshot.records += 32;
|
||||
let reopened =
|
||||
Store::open_with_checkpoint_limits(&path, identity(), raised).expect("explicitly raise bounded recovery budget");
|
||||
assert_negative(&reopened, &original, &first);
|
||||
assert_eq!(reopened.last_applied().expect("retained suffix position"), Some(id(3)));
|
||||
let tx = reopened.database.begin_read().expect("unchanged baseline metadata");
|
||||
let meta = tx.open_table(META).expect("actual metadata");
|
||||
let baseline = checkpoint_baseline::read_metadata(&meta)
|
||||
.expect("baseline decode")
|
||||
.expect("baseline selected");
|
||||
assert_eq!(baseline.generation, receipt.generation());
|
||||
assert_eq!(
|
||||
baseline.limits.snapshot.records, limits.snapshot.records,
|
||||
"CHECKPOINT_RESOURCE_BOUND: override never rewrites evidence"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_checkpoint_interrupted_purge_reopens_and_retries_idempotently() {
|
||||
for (sync, after) in [(1, false), (1, true), (2, false), (2, true)] {
|
||||
let (dir, store, original, first) = fixture();
|
||||
let writer = NativeLogWriter::new(store);
|
||||
let receipt = runtime()
|
||||
.block_on(writer.checkpoint_local(CheckpointLimits::default()))
|
||||
.expect("durable baseline before purge");
|
||||
drop(writer);
|
||||
let path = dir.path().join("db");
|
||||
let backend = tests::CrashBackend::new(&path);
|
||||
let store = Store::from_database(backend.database(), identity()).expect("simulated purge durability backend");
|
||||
let writer = NativeLogWriter::new(store);
|
||||
let (entered, release) = backend.pause_sync(sync, after);
|
||||
let worker = writer.clone();
|
||||
let task = std::thread::spawn(move || runtime().block_on(worker.purge_checkpoint(&receipt)));
|
||||
entered.wait();
|
||||
backend.cut();
|
||||
release.wait();
|
||||
let _unknown = task.join().expect("interrupted purge worker");
|
||||
drop(writer);
|
||||
let reopened = Store::open(&path, identity()).expect("baseline survives simulated interrupted purge");
|
||||
assert_negative(&reopened, &original, &first);
|
||||
let retained = (0..=2)
|
||||
.map(|index| reopened.read_entry(index).expect("atomic purge log state").is_some())
|
||||
.collect::<Vec<_>>();
|
||||
assert!(
|
||||
retained.iter().all(|present| *present) || retained.iter().all(|present| !*present),
|
||||
"CHECKPOINT_PURGE_ATOMIC: complete old prefix or complete new purge"
|
||||
);
|
||||
let writer = NativeLogWriter::new(reopened);
|
||||
let rt = runtime();
|
||||
rt.block_on(writer.purge_checkpoint(&receipt)).expect("resume exact purge");
|
||||
rt.block_on(writer.purge_checkpoint(&receipt))
|
||||
.expect("repeat exact purge idempotently");
|
||||
drop(writer);
|
||||
let reopened = Store::open(&path, identity()).expect("reopen resumed purge");
|
||||
assert_negative(&reopened, &original, &first);
|
||||
assert!(reopened.read_entry(2).expect("purged cut absent").is_none(), "CHECKPOINT_PURGE_ATOMIC");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_checkpoint_ack_and_purge_ack_survive_simulated_power_cut() {
|
||||
let (dir, store, original, first) = fixture();
|
||||
drop(store);
|
||||
let path = dir.path().join("db");
|
||||
let backend = tests::CrashBackend::new(&path);
|
||||
let writer =
|
||||
NativeLogWriter::new(Store::from_database(backend.database(), identity()).expect("actual checkpoint fault backend"));
|
||||
let receipt = runtime()
|
||||
.block_on(writer.checkpoint_local(CheckpointLimits::default()))
|
||||
.expect("successful checkpoint ACK");
|
||||
backend.cut();
|
||||
drop(writer);
|
||||
let reopened = Store::open(&path, identity()).expect("ACK checkpoint survives simulated power cut");
|
||||
assert_eq!(schema(&reopened), 7, "CHECKPOINT_ACK_DURABLE");
|
||||
assert_negative(&reopened, &original, &first);
|
||||
let backend = tests::CrashBackend::new(&path);
|
||||
drop(reopened);
|
||||
let writer = NativeLogWriter::new(Store::from_database(backend.database(), identity()).expect("actual purge fault backend"));
|
||||
runtime()
|
||||
.block_on(writer.purge_checkpoint(&receipt))
|
||||
.expect("successful purge ACK");
|
||||
backend.cut();
|
||||
drop(writer);
|
||||
let reopened = Store::open(&path, identity()).expect("ACK purge survives simulated power cut");
|
||||
assert_negative(&reopened, &original, &first);
|
||||
assert!(reopened.read_entry(2).expect("ACK purge cut absent").is_none(), "CHECKPOINT_ACK_DURABLE");
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
//! Owned consistent native application snapshot view.
|
||||
use crate::{Store, StoreError, application, snapshot};
|
||||
use redb::{ReadTransaction, ReadableDatabase, ReadableTable, ReadableTableMetadata};
|
||||
use std::{
|
||||
io::Write,
|
||||
sync::{Arc, Mutex},
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
std::thread_local! {
|
||||
static AFTER_CAPTURE: std::cell::RefCell<Option<Box<dyn FnOnce()>>> = const { std::cell::RefCell::new(None) };
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) struct CaptureHook;
|
||||
#[cfg(test)]
|
||||
impl Drop for CaptureHook {
|
||||
fn drop(&mut self) {
|
||||
AFTER_CAPTURE.with(|hook| {
|
||||
hook.borrow_mut().take();
|
||||
});
|
||||
}
|
||||
}
|
||||
#[cfg(test)]
|
||||
pub(crate) fn after_capture(hook: impl FnOnce() + 'static) -> CaptureHook {
|
||||
AFTER_CAPTURE.with(|slot| {
|
||||
assert!(slot.borrow().is_none(), "one capture hook per test thread");
|
||||
*slot.borrow_mut() = Some(Box::new(hook));
|
||||
});
|
||||
CaptureHook
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum NativeSnapshotError {
|
||||
#[error(transparent)]
|
||||
Store(#[from] StoreError),
|
||||
#[error(transparent)]
|
||||
Snapshot(#[from] snapshot::SnapshotError),
|
||||
#[error("native snapshot Store lock poisoned")]
|
||||
Poisoned,
|
||||
#[error("native snapshot worker failed")]
|
||||
Worker(#[source] tokio::task::JoinError),
|
||||
}
|
||||
|
||||
pub struct NativeSnapshotView {
|
||||
// Drop the transaction before the owner so the Database stays open throughout the view.
|
||||
transaction: ReadTransaction,
|
||||
header: snapshot::Header,
|
||||
counts: [u64; 4],
|
||||
schema: u64,
|
||||
_owner: Arc<Mutex<Store>>,
|
||||
}
|
||||
impl NativeSnapshotView {
|
||||
pub(crate) fn acquire(owner: Arc<Mutex<Store>>) -> Result<Self, NativeSnapshotError> {
|
||||
let transaction = {
|
||||
let store = owner.lock().map_err(|_| NativeSnapshotError::Poisoned)?;
|
||||
if !store.writable {
|
||||
return Err(StoreError::WriteDisabled.into());
|
||||
}
|
||||
store.database.begin_read().map_err(StoreError::from)?
|
||||
};
|
||||
#[cfg(test)]
|
||||
if let Some(hook) = AFTER_CAPTURE.with(|slot| slot.borrow_mut().take()) {
|
||||
hook();
|
||||
}
|
||||
let (applied, membership) = application::native_application_state(&transaction)?;
|
||||
let (header, schema) = {
|
||||
let meta = transaction.open_table(crate::META).map_err(StoreError::from)?;
|
||||
let logs = transaction.open_table(crate::LOGS).map_err(StoreError::from)?;
|
||||
let committed = crate::committed::validate_committed(&meta, &logs)?;
|
||||
if let Some(applied) = applied {
|
||||
let committed = committed.ok_or(StoreError::CorruptRecord)?;
|
||||
if applied.index > committed.index || (applied.index == committed.index && applied != committed) {
|
||||
return Err(StoreError::CorruptRecord.into());
|
||||
}
|
||||
}
|
||||
let identity = meta
|
||||
.get("identity")
|
||||
.map_err(StoreError::from)?
|
||||
.ok_or(StoreError::CorruptRecord)?;
|
||||
let schema = crate::number(
|
||||
meta.get("schema")
|
||||
.map_err(StoreError::from)?
|
||||
.ok_or(StoreError::CorruptRecord)?
|
||||
.value(),
|
||||
)?;
|
||||
(
|
||||
snapshot::Header {
|
||||
cluster_id: crate::decode_identity(identity.value())?.cluster_id,
|
||||
applied,
|
||||
membership,
|
||||
binding: application::read_binding(&meta)?,
|
||||
},
|
||||
schema,
|
||||
)
|
||||
};
|
||||
let mut counts = [0; 4];
|
||||
for (index, definition) in [
|
||||
application::HEADS,
|
||||
application::VERSIONS,
|
||||
application::EVENTS,
|
||||
application::OPERATIONS,
|
||||
]
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
{
|
||||
counts[index] = transaction
|
||||
.open_table(definition)
|
||||
.map_err(StoreError::from)?
|
||||
.len()
|
||||
.map_err(StoreError::from)?;
|
||||
}
|
||||
Ok(Self {
|
||||
transaction,
|
||||
header,
|
||||
counts,
|
||||
schema,
|
||||
_owner: owner,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn stream_capability_artifact<W: Write>(
|
||||
self,
|
||||
writer: W,
|
||||
limits: snapshot::Limits,
|
||||
) -> Result<snapshot::CapabilityArtifact<W>, NativeSnapshotError> {
|
||||
if !matches!(self.schema, 6 | 7) {
|
||||
return Err(StoreError::UnsupportedSchema.into());
|
||||
}
|
||||
let output = snapshot::CapabilityOutput::new(writer, limits.total_bytes);
|
||||
Ok(self.stream(output, limits)?.finish())
|
||||
}
|
||||
|
||||
pub(crate) fn transaction(&self) -> &ReadTransaction {
|
||||
&self.transaction
|
||||
}
|
||||
|
||||
pub(crate) fn header(&self) -> &snapshot::Header {
|
||||
&self.header
|
||||
}
|
||||
|
||||
pub(crate) fn owner(&self) -> &Arc<Mutex<Store>> {
|
||||
&self._owner
|
||||
}
|
||||
|
||||
pub(crate) fn stream_local_capability<W: Write>(
|
||||
&self,
|
||||
writer: W,
|
||||
limits: snapshot::Limits,
|
||||
) -> Result<snapshot::CapabilityArtifact<W>, NativeSnapshotError> {
|
||||
if !matches!(self.schema, 6 | 7) {
|
||||
return Err(StoreError::UnsupportedSchema.into());
|
||||
}
|
||||
let output = snapshot::CapabilityOutput::new(writer, limits.total_bytes);
|
||||
Ok(self.stream_borrowed(output, limits)?.finish())
|
||||
}
|
||||
|
||||
pub fn stream<W: Write>(self, writer: W, limits: snapshot::Limits) -> Result<W, NativeSnapshotError> {
|
||||
self.stream_borrowed(writer, limits)
|
||||
}
|
||||
|
||||
fn stream_borrowed<W: Write>(&self, writer: W, limits: snapshot::Limits) -> Result<W, NativeSnapshotError> {
|
||||
let mut encoder = snapshot::Encoder::new(writer, &self.header, self.counts, limits)?;
|
||||
for (definition, section) in [
|
||||
(application::HEADS, snapshot::Section::Heads),
|
||||
(application::VERSIONS, snapshot::Section::Versions),
|
||||
(application::EVENTS, snapshot::Section::Events),
|
||||
(application::OPERATIONS, snapshot::Section::Operations),
|
||||
] {
|
||||
let table = self.transaction.open_table(definition).map_err(StoreError::from)?;
|
||||
for row in table.iter().map_err(StoreError::from)? {
|
||||
let (key, value) = row.map_err(StoreError::from)?;
|
||||
encoder.row(snapshot::Row {
|
||||
section,
|
||||
key: key.value(),
|
||||
value: value.value(),
|
||||
})?;
|
||||
}
|
||||
}
|
||||
Ok(encoder.finish()?)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
//! Bounded snapshot transport bytes; receive ownership follows the buffer into install.
|
||||
use std::{
|
||||
io::{self, Read, Seek, SeekFrom, Write},
|
||||
pin::Pin,
|
||||
task::{Context, Poll},
|
||||
};
|
||||
use tokio::io::{AsyncRead, AsyncSeek, AsyncWrite, ReadBuf};
|
||||
|
||||
pub struct NativeSnapshotData {
|
||||
bytes: Vec<u8>,
|
||||
position: u64,
|
||||
limit: u64,
|
||||
pub(crate) receive_guard: Option<crate::raft_state_machine::ReceiveGuard>,
|
||||
}
|
||||
|
||||
fn boundary() -> io::Error {
|
||||
io::Error::new(io::ErrorKind::InvalidInput, "native snapshot byte limit")
|
||||
}
|
||||
|
||||
impl NativeSnapshotData {
|
||||
pub fn new(max_bytes: u64) -> io::Result<Self> {
|
||||
if max_bytes == 0 || usize::try_from(max_bytes).is_err() {
|
||||
return Err(boundary());
|
||||
}
|
||||
Ok(Self {
|
||||
bytes: Vec::new(),
|
||||
position: 0,
|
||||
limit: max_bytes,
|
||||
receive_guard: None,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn as_slice(&self) -> &[u8] {
|
||||
&self.bytes
|
||||
}
|
||||
|
||||
pub(crate) fn attach_receive_guard(&mut self, guard: crate::raft_state_machine::ReceiveGuard) {
|
||||
self.receive_guard = Some(guard);
|
||||
}
|
||||
}
|
||||
|
||||
impl Read for NativeSnapshotData {
|
||||
fn read(&mut self, output: &mut [u8]) -> io::Result<usize> {
|
||||
let position = usize::try_from(self.position).map_err(|_| boundary())?;
|
||||
let available = self.bytes.get(position..).unwrap_or_default();
|
||||
let count = output.len().min(available.len());
|
||||
output[..count].copy_from_slice(&available[..count]);
|
||||
self.position = self
|
||||
.position
|
||||
.checked_add(u64::try_from(count).map_err(|_| boundary())?)
|
||||
.ok_or_else(boundary)?;
|
||||
Ok(count)
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for NativeSnapshotData {
|
||||
fn write(&mut self, input: &[u8]) -> io::Result<usize> {
|
||||
if input.is_empty() {
|
||||
return Ok(0);
|
||||
}
|
||||
let count = u64::try_from(input.len()).map_err(|_| boundary())?;
|
||||
let end = self
|
||||
.position
|
||||
.checked_add(count)
|
||||
.filter(|end| *end <= self.limit)
|
||||
.ok_or_else(boundary)?;
|
||||
let position = usize::try_from(self.position).map_err(|_| boundary())?;
|
||||
let length = usize::try_from(end).map_err(|_| boundary())?;
|
||||
if length > self.bytes.len() {
|
||||
self.bytes
|
||||
.try_reserve_exact(length - self.bytes.len())
|
||||
.map_err(|_| io::Error::new(io::ErrorKind::OutOfMemory, "native snapshot allocation"))?;
|
||||
self.bytes.resize(length, 0);
|
||||
}
|
||||
self.bytes[position..length].copy_from_slice(input);
|
||||
self.position = end;
|
||||
Ok(input.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Seek for NativeSnapshotData {
|
||||
fn seek(&mut self, from: SeekFrom) -> io::Result<u64> {
|
||||
let position = match from {
|
||||
SeekFrom::Start(position) => Some(position),
|
||||
SeekFrom::Current(offset) => self.position.checked_add_signed(offset),
|
||||
SeekFrom::End(offset) => u64::try_from(self.bytes.len())
|
||||
.map_err(|_| boundary())?
|
||||
.checked_add_signed(offset),
|
||||
}
|
||||
.filter(|position| *position <= self.limit)
|
||||
.ok_or_else(boundary)?;
|
||||
self.position = position;
|
||||
Ok(position)
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncRead for NativeSnapshotData {
|
||||
fn poll_read(self: Pin<&mut Self>, _cx: &mut Context<'_>, output: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
|
||||
let this = self.get_mut();
|
||||
let position = match usize::try_from(this.position) {
|
||||
Ok(position) => position,
|
||||
Err(_) => return Poll::Ready(Err(boundary())),
|
||||
};
|
||||
let available = this.bytes.get(position..).unwrap_or_default();
|
||||
let count = output.remaining().min(available.len());
|
||||
let end = this
|
||||
.position
|
||||
.checked_add(u64::try_from(count).map_err(|_| boundary())?)
|
||||
.ok_or_else(boundary)?;
|
||||
output.put_slice(&available[..count]);
|
||||
this.position = end;
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncWrite for NativeSnapshotData {
|
||||
fn poll_write(self: Pin<&mut Self>, _cx: &mut Context<'_>, input: &[u8]) -> Poll<io::Result<usize>> {
|
||||
Poll::Ready(Write::write(self.get_mut(), input))
|
||||
}
|
||||
|
||||
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
|
||||
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncSeek for NativeSnapshotData {
|
||||
fn start_seek(self: Pin<&mut Self>, from: SeekFrom) -> io::Result<()> {
|
||||
Seek::seek(self.get_mut(), from).map(|_| ())
|
||||
}
|
||||
|
||||
fn poll_complete(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<u64>> {
|
||||
Poll::Ready(Ok(self.position))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn native_snapshot_data_rejects_growth_before_mutation() {
|
||||
let mut data = NativeSnapshotData::new(4).expect("bounded snapshot");
|
||||
Write::write_all(&mut data, b"ab").expect("initial bytes");
|
||||
assert!(Write::write_all(&mut data, b"cde").is_err());
|
||||
assert_eq!(data.as_slice(), b"ab", "rejected write leaves bytes intact");
|
||||
assert_eq!(Seek::stream_position(&mut data).expect("position"), 2);
|
||||
Seek::seek(&mut data, SeekFrom::Start(4)).expect("seek exactly to limit");
|
||||
Write::write(&mut data, b"").expect("empty write");
|
||||
assert_eq!(data.as_slice(), b"ab", "empty write never allocates a gap");
|
||||
assert!(Write::write(&mut data, b"x").is_err());
|
||||
assert!(Seek::seek(&mut data, SeekFrom::Current(1)).is_err());
|
||||
assert!(Seek::seek(&mut data, SeekFrom::Current(i64::MIN)).is_err());
|
||||
assert_eq!(Seek::stream_position(&mut data).expect("unchanged position"), 4);
|
||||
Seek::seek(&mut data, SeekFrom::Start(3)).expect("bounded sparse position");
|
||||
Write::write_all(&mut data, b"z").expect("write ending exactly at limit");
|
||||
assert_eq!(data.as_slice(), b"ab\0z");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_snapshot_data_checks_length_overflow() {
|
||||
assert!(NativeSnapshotData::new(0).is_err());
|
||||
if usize::BITS == 64 {
|
||||
let mut data = NativeSnapshotData::new(u64::MAX).expect("representable byte limit");
|
||||
Seek::seek(&mut data, SeekFrom::Start(u64::MAX)).expect("seek does not allocate");
|
||||
assert!(Write::write(&mut data, b"x").is_err());
|
||||
assert!(Seek::seek(&mut data, SeekFrom::Current(1)).is_err());
|
||||
assert!(data.as_slice().is_empty(), "overflow never allocates");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn native_snapshot_data_async_io_uses_the_same_bounds() {
|
||||
use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt};
|
||||
let mut data = NativeSnapshotData::new(4).expect("bounded async snapshot");
|
||||
AsyncWriteExt::write_all(&mut data, b"abcd").await.expect("write exact limit");
|
||||
assert!(AsyncWriteExt::write_all(&mut data, b"e").await.is_err());
|
||||
assert!(AsyncSeekExt::seek(&mut data, SeekFrom::Start(5)).await.is_err());
|
||||
AsyncSeekExt::seek(&mut data, SeekFrom::End(-2))
|
||||
.await
|
||||
.expect("rewind within snapshot");
|
||||
let mut output = Vec::new();
|
||||
AsyncReadExt::read_to_end(&mut data, &mut output)
|
||||
.await
|
||||
.expect("read snapshot suffix");
|
||||
assert_eq!(output, b"cd");
|
||||
AsyncWriteExt::shutdown(&mut data).await.expect("transport shutdown");
|
||||
assert_eq!(data.as_slice(), b"abcd");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,217 @@
|
||||
//! Validate isolated received bytes before one durable shared-owner publication.
|
||||
use crate::{
|
||||
CaptureMembership, CheckpointLimits, NativeLogWriter, NativeSnapshotData, StoreError, application, checkpoint_baseline,
|
||||
native_checkpoint, snapshot,
|
||||
};
|
||||
use redb::{ReadableDatabase, ReadableTable};
|
||||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
};
|
||||
|
||||
struct Cancel(Arc<AtomicBool>);
|
||||
impl Drop for Cancel {
|
||||
fn drop(&mut self) {
|
||||
self.0.store(true, Ordering::Release);
|
||||
}
|
||||
}
|
||||
struct Stage {
|
||||
database: redb::Database,
|
||||
_directory: tempfile::TempDir,
|
||||
baseline: checkpoint_baseline::BaselineMetadata,
|
||||
}
|
||||
fn stage(data: &NativeSnapshotData, limits: CheckpointLimits) -> Result<Stage, StoreError> {
|
||||
let bytes = data.as_slice();
|
||||
if u64::try_from(bytes.len()).map_err(|_| StoreError::ResourceLimit)?
|
||||
> limits.snapshot.total_bytes.checked_add(56).ok_or(StoreError::ResourceLimit)?
|
||||
{
|
||||
return Err(StoreError::ResourceLimit);
|
||||
}
|
||||
let manifest = snapshot::CapabilityManifest::decode(bytes.get(..56).ok_or(StoreError::CorruptRecord)?)
|
||||
.map_err(|_| StoreError::CorruptRecord)?;
|
||||
let directory = tempfile::tempdir()?;
|
||||
let database = native_checkpoint::stage_database(directory.path().join("received.redb"), limits.staging_bytes)?;
|
||||
let tx = database.begin_write()?;
|
||||
for definition in [
|
||||
checkpoint_baseline::HEADS,
|
||||
checkpoint_baseline::VERSIONS,
|
||||
checkpoint_baseline::EVENTS,
|
||||
checkpoint_baseline::OPERATIONS,
|
||||
] {
|
||||
tx.open_table(definition)?;
|
||||
}
|
||||
let header = snapshot::decode_capability_artifact(
|
||||
snapshot::CapabilityArtifact {
|
||||
manifest,
|
||||
payload: &bytes[56..],
|
||||
},
|
||||
&mut native_checkpoint::StageSink(&tx),
|
||||
limits.snapshot,
|
||||
)
|
||||
.map_err(|_| StoreError::CorruptRecord)?;
|
||||
let baseline = checkpoint_baseline::BaselineMetadata {
|
||||
generation: 1,
|
||||
header,
|
||||
limits,
|
||||
};
|
||||
let encoded = checkpoint_baseline::encode_metadata(&baseline)?;
|
||||
{
|
||||
let mut meta = tx.open_table(crate::META)?;
|
||||
meta.insert("schema", 7u64.to_be_bytes().as_slice())?;
|
||||
meta.insert(checkpoint_baseline::META_KEY, encoded.as_slice())?;
|
||||
}
|
||||
tx.commit().map_err(StoreError::StagingCommit)?;
|
||||
checkpoint_baseline::validate_baseline_with_budget(&database.begin_read()?, &baseline, limits.staging_bytes)?;
|
||||
Ok(Stage {
|
||||
database,
|
||||
_directory: directory,
|
||||
baseline,
|
||||
})
|
||||
}
|
||||
pub(crate) async fn install(
|
||||
writer: NativeLogWriter,
|
||||
limits: CheckpointLimits,
|
||||
meta: openraft::SnapshotMeta<u64, crate::CaptureNode>,
|
||||
data: Box<NativeSnapshotData>,
|
||||
) -> Result<(), StoreError> {
|
||||
if data
|
||||
.receive_guard
|
||||
.as_ref()
|
||||
.is_none_or(|guard| !guard.belongs_to(&writer.snapshot_receives))
|
||||
{
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let (mut stage, data) = tokio::task::spawn_blocking(move || Ok::<_, StoreError>((stage(&data, limits)?, data)))
|
||||
.await
|
||||
.map_err(|_| StoreError::CorruptRecord)??;
|
||||
if meta.last_log_id.map(crate::LogId::from) != stage.baseline.header.applied {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
if CaptureMembership::try_from(&meta.last_membership)? != stage.baseline.header.membership {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let ordering = writer.ordering.clone().lock_owned().await;
|
||||
let cancel = Cancel(Arc::new(AtomicBool::new(false)));
|
||||
let cancelled = cancel.0.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let _ordering = ordering;
|
||||
let _data = data;
|
||||
if cancelled.load(Ordering::Acquire) {
|
||||
return Err(StoreError::WriteDisabled);
|
||||
}
|
||||
let mut store = writer.store.lock().map_err(|_| StoreError::CorruptRecord)?;
|
||||
let read = store.database.begin_read()?;
|
||||
let old_meta = read.open_table(crate::META)?;
|
||||
let identity = crate::decode_identity(old_meta.get("identity")?.ok_or(StoreError::CorruptRecord)?.value())?;
|
||||
if identity.cluster_id != stage.baseline.header.cluster_id
|
||||
|| application::read_binding(&old_meta)? != stage.baseline.header.binding
|
||||
{
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let old_applied = application::read_applied(&old_meta)?;
|
||||
let old_logs = read.open_table(crate::LOGS)?;
|
||||
let committed = crate::committed::validate_committed(&old_meta, &old_logs)?;
|
||||
let cut = stage.baseline.header.applied;
|
||||
if old_applied.is_some_and(|id| cut.is_none_or(|cut| id.index > cut.index || (id.index == cut.index && id != cut))) {
|
||||
return Err(StoreError::CommittedRegression);
|
||||
}
|
||||
let compatible = match cut {
|
||||
None => true,
|
||||
Some(cut) => {
|
||||
let at_cut = old_logs
|
||||
.get(cut.index)?
|
||||
.map(|v| crate::decode_log(v.value()).map(|v| v.id))
|
||||
.transpose()?
|
||||
.or(checkpoint_baseline::read_baseline_cut(&old_meta)?.filter(|id| id.index == cut.index));
|
||||
if committed
|
||||
.is_some_and(|id| id.index >= cut.index && (at_cut != Some(cut) || (id.index == cut.index && id != cut)))
|
||||
{
|
||||
return Err(StoreError::CommittedRegression);
|
||||
}
|
||||
at_cut == Some(cut)
|
||||
}
|
||||
};
|
||||
let previous = checkpoint_baseline::read_metadata(&old_meta)?;
|
||||
stage.baseline.generation =
|
||||
previous.map_or(Ok(1), |old| old.generation.checked_add(1).ok_or(StoreError::ResourceLimit))?;
|
||||
drop(old_logs);
|
||||
drop(old_meta);
|
||||
drop(read);
|
||||
let source = stage.database.begin_read()?;
|
||||
let tx = store.write_transaction()?;
|
||||
for (baseline, live) in [
|
||||
(checkpoint_baseline::HEADS, application::HEADS),
|
||||
(checkpoint_baseline::VERSIONS, application::VERSIONS),
|
||||
(checkpoint_baseline::EVENTS, application::EVENTS),
|
||||
(checkpoint_baseline::OPERATIONS, application::OPERATIONS),
|
||||
] {
|
||||
let table = source.open_table(baseline)?;
|
||||
for target in [baseline, live] {
|
||||
let mut target = tx.open_table(target)?;
|
||||
target.retain(|_, _| false)?;
|
||||
for row in table.iter()? {
|
||||
let (key, value) = row?;
|
||||
target.insert(key.value(), value.value())?;
|
||||
}
|
||||
}
|
||||
}
|
||||
{
|
||||
let mut index = tx.open_table(crate::source_page::EVENT_SEQUENCE)?;
|
||||
index.retain(|_, _| false)?;
|
||||
}
|
||||
for row in source.open_table(checkpoint_baseline::EVENTS)?.iter()? {
|
||||
let (key, value) = row?;
|
||||
let event = application::decode_event(value.value())?;
|
||||
crate::source_page::insert_event_index(
|
||||
&tx,
|
||||
event.decision_log_id,
|
||||
key.value().try_into().map_err(|_| StoreError::CorruptRecord)?,
|
||||
)?;
|
||||
}
|
||||
let tail = {
|
||||
let mut logs = tx.open_table(crate::LOGS)?;
|
||||
logs.retain(|index, _| compatible && cut.is_none_or(|cut| index > cut.index))?;
|
||||
logs.last()?
|
||||
.map(|(_, v)| crate::decode_log(v.value()).map(|v| v.id))
|
||||
.transpose()?
|
||||
.or(cut)
|
||||
};
|
||||
tx.open_table(application::APPLY_RESULTS)?.retain(|_, _| false)?;
|
||||
{
|
||||
let mut target = tx.open_table(crate::META)?;
|
||||
target.insert("schema", 7u64.to_be_bytes().as_slice())?;
|
||||
target.insert(
|
||||
checkpoint_baseline::META_KEY,
|
||||
checkpoint_baseline::encode_metadata(&stage.baseline)?.as_slice(),
|
||||
)?;
|
||||
let mut applied = vec![0];
|
||||
if let Some(cut) = cut {
|
||||
target.insert(checkpoint_baseline::PURGED_KEY, checkpoint_baseline::encode_purged(cut).as_slice())?;
|
||||
applied = vec![1];
|
||||
applied.extend(crate::encode_id(cut));
|
||||
} else {
|
||||
target.remove(checkpoint_baseline::PURGED_KEY)?;
|
||||
}
|
||||
target.insert("last_applied", applied.as_slice())?;
|
||||
target.insert("native_membership", stage.baseline.header.membership.encode()?.as_slice())?;
|
||||
if let Some(tail) = tail {
|
||||
target.insert("tail", crate::encode_id(tail).as_slice())?;
|
||||
} else {
|
||||
target.remove("tail")?;
|
||||
}
|
||||
let committed = committed.filter(|id| cut.is_none_or(|cut| id.index > cut.index)).or(cut);
|
||||
if let Some(committed) = committed {
|
||||
target.insert("committed", crate::encode_id(committed).as_slice())?;
|
||||
} else {
|
||||
target.remove("committed")?;
|
||||
}
|
||||
crate::raft_state_machine::publish_meta(&mut target, &stage.baseline, meta)?;
|
||||
}
|
||||
if cancelled.load(Ordering::Acquire) {
|
||||
return Err(StoreError::WriteDisabled);
|
||||
}
|
||||
store.commit(tx)
|
||||
})
|
||||
.await
|
||||
.map_err(|_| StoreError::CorruptRecord)?
|
||||
}
|
||||
@@ -0,0 +1,536 @@
|
||||
//! Atomic installation failures over real native stores and disk fault backend.
|
||||
use super::raft_state_machine_tests::{Fixture, entries, entry, id, install, receive, snapshot_bytes};
|
||||
use super::*;
|
||||
use openraft::{
|
||||
RaftLogReader, RaftSnapshotBuilder,
|
||||
storage::{RaftLogStorage, RaftLogStorageExt, RaftStateMachine},
|
||||
};
|
||||
|
||||
async fn media_target() -> (Fixture, tests::CrashBackend) {
|
||||
let mut f = Fixture::new(1);
|
||||
f.apply(entries(1)).await;
|
||||
f.logs
|
||||
.save_vote(&openraft::Vote::new_committed(9, 1))
|
||||
.await
|
||||
.expect("preserved target vote");
|
||||
let Fixture {
|
||||
directory,
|
||||
writer,
|
||||
logs,
|
||||
sm,
|
||||
identity,
|
||||
} = f;
|
||||
drop(sm);
|
||||
drop(logs);
|
||||
drop(writer);
|
||||
let backend = tests::CrashBackend::new(&directory.path().join("db"));
|
||||
let writer = NativeLogWriter::new(Store::from_database(backend.database(), identity).expect("actual redb fault target"));
|
||||
let logs = NativeRaftLogStore::new(writer.clone());
|
||||
let sm = NativeRaftStateMachine::new(writer.clone(), CheckpointLimits::default()).expect("fault target state machine");
|
||||
(
|
||||
Fixture {
|
||||
directory,
|
||||
writer,
|
||||
logs,
|
||||
sm,
|
||||
identity,
|
||||
},
|
||||
backend,
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn native_snapshot_install_corruption_preserves_state() {
|
||||
let mut source = Fixture::new(0);
|
||||
source.apply(entries(2)).await;
|
||||
let mut snapshot = source.build().await;
|
||||
let bytes = snapshot_bytes(&mut snapshot).await;
|
||||
let mut target = Fixture::new(1);
|
||||
target.apply(entries(1)).await;
|
||||
let before = target.page().await;
|
||||
let state = target.sm.applied_state().await.expect("old applied state");
|
||||
let mut corrupted = bytes.clone();
|
||||
*corrupted.last_mut().expect("nonempty snapshot") ^= 1;
|
||||
for payload in [corrupted, bytes[..bytes.len() - 1].to_vec()] {
|
||||
let stable = std::fs::read(target.directory.path().join("db")).expect("old stable bytes");
|
||||
let data = receive(&mut target.sm, &payload).await;
|
||||
assert!(
|
||||
target.sm.install_snapshot(&snapshot.meta, data).await.is_err(),
|
||||
"SNAPSHOT_INSTALL_CORRUPT_FRAME_REJECTED"
|
||||
);
|
||||
assert_eq!(target.sm.applied_state().await.expect("unchanged applied state"), state);
|
||||
assert_eq!(target.page().await, before, "SNAPSHOT_INSTALL_CORRUPTION_NO_PARTIAL_TABLES");
|
||||
assert_eq!(std::fs::read(target.directory.path().join("db")).expect("unchanged stable bytes"), stable);
|
||||
assert!(
|
||||
target
|
||||
.sm
|
||||
.get_current_snapshot()
|
||||
.await
|
||||
.expect("unpublished corrupt snapshot")
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
install(&mut target.sm, &mut snapshot).await;
|
||||
assert_eq!(target.page().await, source.page().await, "SNAPSHOT_INSTALL_RETRY_AFTER_CORRUPT_RECEIVE");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn native_snapshot_install_stale_cut_rejected() {
|
||||
let mut source = Fixture::new(0);
|
||||
source.apply(entries(1)).await;
|
||||
let mut snapshot = source.build().await;
|
||||
let bytes = snapshot_bytes(&mut snapshot).await;
|
||||
let mut target = Fixture::new(1);
|
||||
target.apply(entries(3)).await;
|
||||
let current = target.build().await.meta;
|
||||
let events = target.page().await;
|
||||
let data = receive(&mut target.sm, &bytes).await;
|
||||
assert!(
|
||||
target.sm.install_snapshot(&snapshot.meta, data).await.is_err(),
|
||||
"SNAPSHOT_INSTALL_NEVER_ROLLS_BACK_APPLIED_STATE"
|
||||
);
|
||||
assert_eq!(target.sm.applied_state().await.expect("non-regressed applied").0, Some(id(3).into()));
|
||||
assert_eq!(
|
||||
target
|
||||
.sm
|
||||
.get_current_snapshot()
|
||||
.await
|
||||
.expect("old current snapshot")
|
||||
.expect("present")
|
||||
.meta,
|
||||
current
|
||||
);
|
||||
assert_eq!(target.page().await, events);
|
||||
let mut source = Fixture::new(0);
|
||||
source.apply(entries(2)).await;
|
||||
let mut snapshot = source.build().await;
|
||||
let bytes = snapshot_bytes(&mut snapshot).await;
|
||||
let mut target = Fixture::new(1);
|
||||
let mut batch = entries(3);
|
||||
for entry in &mut batch[2..] {
|
||||
entry.log_id = LogId {
|
||||
term: 9,
|
||||
..id(entry.log_id.index)
|
||||
}
|
||||
.into();
|
||||
}
|
||||
target
|
||||
.logs
|
||||
.blocking_append(batch)
|
||||
.await
|
||||
.expect("conflicting committed suffix logs");
|
||||
target.sm.apply(entries(1)).await.expect("existing applied prefix");
|
||||
target
|
||||
.logs
|
||||
.save_committed(Some(LogId { term: 9, ..id(3) }.into()))
|
||||
.await
|
||||
.expect("protect conflicting committed suffix");
|
||||
let before = target
|
||||
.logs
|
||||
.get_log_reader()
|
||||
.await
|
||||
.try_get_log_entries(..)
|
||||
.await
|
||||
.expect("protected logs");
|
||||
let data = receive(&mut target.sm, &bytes).await;
|
||||
assert!(
|
||||
target.sm.install_snapshot(&snapshot.meta, data).await.is_err(),
|
||||
"SNAPSHOT_INSTALL_NEVER_DISCARDS_COMMITTED_CONFLICT"
|
||||
);
|
||||
assert_eq!(
|
||||
target
|
||||
.logs
|
||||
.get_log_reader()
|
||||
.await
|
||||
.try_get_log_entries(..)
|
||||
.await
|
||||
.expect("unchanged protected logs"),
|
||||
before
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn native_snapshot_install_cancelled_owner() {
|
||||
let mut source = Fixture::new(0);
|
||||
source.apply(entries(2)).await;
|
||||
let mut snapshot = source.build().await;
|
||||
let bytes = snapshot_bytes(&mut snapshot).await;
|
||||
let (mut target, backend) = media_target().await;
|
||||
let data = receive(&mut target.sm, &bytes).await;
|
||||
let (entered, resume) = backend.pause_sync(1, false);
|
||||
let mut installing = target.sm.clone();
|
||||
let meta = snapshot.meta.clone();
|
||||
let task = tokio::spawn(async move { installing.install_snapshot(&meta, data).await });
|
||||
entered.wait();
|
||||
assert!(!task.is_finished(), "SNAPSHOT_INSTALL_NOT_ACKED_BEFORE_SYNC");
|
||||
task.abort();
|
||||
assert!(task.await.expect_err("cancelled install waiter").is_cancelled());
|
||||
assert!(
|
||||
target.writer.ordering.try_lock().is_err(),
|
||||
"SNAPSHOT_INSTALL_CANCELLED_WAITER_RETAINS_OWNER"
|
||||
);
|
||||
resume.wait();
|
||||
// Acquiring the production owner waits for the abandoned blocking commit.
|
||||
drop(target.writer.ordering.lock().await);
|
||||
assert_eq!(
|
||||
target.sm.applied_state().await.expect("completed background install").0,
|
||||
Some(id(2).into()),
|
||||
"SNAPSHOT_INSTALL_COMMIT_NOT_INTERRUPTED_BY_WAITER_CANCEL"
|
||||
);
|
||||
let vote = target.logs.read_vote().await.expect("preserved vote");
|
||||
assert_eq!(vote, Some(openraft::Vote::new_committed(9, 1)));
|
||||
backend.cut();
|
||||
let Fixture {
|
||||
directory,
|
||||
writer,
|
||||
logs,
|
||||
sm,
|
||||
identity,
|
||||
} = target;
|
||||
drop(sm);
|
||||
drop(logs);
|
||||
drop(writer);
|
||||
let writer = NativeLogWriter::new(Store::open(directory.path().join("db"), identity).expect("reopen cancelled installation"));
|
||||
let mut sm =
|
||||
NativeRaftStateMachine::new(writer.clone(), CheckpointLimits::default()).expect("reopen installed state machine");
|
||||
assert_eq!(
|
||||
sm.get_current_snapshot()
|
||||
.await
|
||||
.expect("completed durable snapshot")
|
||||
.expect("present")
|
||||
.meta,
|
||||
snapshot.meta
|
||||
);
|
||||
assert_eq!(
|
||||
writer.created_events_page(None, 256).await.expect("installed events"),
|
||||
source.page().await
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn native_snapshot_install_power_cut_atomic() {
|
||||
let mut source = Fixture::new(0);
|
||||
source.apply(entries(2)).await;
|
||||
let mut snapshot = source.build().await;
|
||||
let bytes = snapshot_bytes(&mut snapshot).await;
|
||||
let new_events = source.page().await;
|
||||
for after_sync in [false, true] {
|
||||
let (mut target, backend) = media_target().await;
|
||||
let old_events = target.page().await;
|
||||
let data = receive(&mut target.sm, &bytes).await;
|
||||
let (entered, resume) = backend.pause_sync(1, after_sync);
|
||||
let meta = snapshot.meta.clone();
|
||||
let mut installing = target.sm.clone();
|
||||
let task = tokio::spawn(async move { installing.install_snapshot(&meta, data).await });
|
||||
entered.wait();
|
||||
backend.cut();
|
||||
resume.wait();
|
||||
assert!(
|
||||
task.await.expect("faulted install task").is_err(),
|
||||
"SNAPSHOT_INSTALL_POWER_CUT_NO_FALSE_ACK"
|
||||
);
|
||||
assert!(
|
||||
target.sm.get_current_snapshot().await.is_err(),
|
||||
"SNAPSHOT_INSTALL_FENCED_HANDLE_CANNOT_EXPORT"
|
||||
);
|
||||
let Fixture {
|
||||
directory,
|
||||
writer,
|
||||
logs,
|
||||
sm,
|
||||
identity,
|
||||
} = target;
|
||||
drop(sm);
|
||||
drop(logs);
|
||||
drop(writer);
|
||||
let writer =
|
||||
NativeLogWriter::new(Store::open(directory.path().join("db"), identity).expect("atomic installation recovery"));
|
||||
let mut logs = NativeRaftLogStore::new(writer.clone());
|
||||
let mut sm = NativeRaftStateMachine::new(writer.clone(), CheckpointLimits::default()).expect("recovered SM");
|
||||
let events = writer
|
||||
.created_events_page(None, 256)
|
||||
.await
|
||||
.expect("complete recovered tables");
|
||||
let cut = sm.applied_state().await.expect("recovered applied marker").0;
|
||||
if events == old_events {
|
||||
assert_eq!(cut, Some(id(1).into()), "SNAPSHOT_INSTALL_OLD_TABLES_OLD_MARKER");
|
||||
assert!(sm.get_current_snapshot().await.expect("old snapshot selection").is_none());
|
||||
} else {
|
||||
assert_eq!(events, new_events, "SNAPSHOT_INSTALL_ALL_TABLES_OLD_OR_NEW");
|
||||
assert_eq!(cut, Some(id(2).into()), "SNAPSHOT_INSTALL_NEW_TABLES_NEW_MARKER");
|
||||
let mut current = sm
|
||||
.get_current_snapshot()
|
||||
.await
|
||||
.expect("new snapshot selection")
|
||||
.expect("new snapshot published");
|
||||
assert_eq!(current.meta, snapshot.meta);
|
||||
assert_eq!(snapshot_bytes(&mut current).await, bytes);
|
||||
}
|
||||
assert_eq!(writer.durable_identity().await.expect("recovered local identity"), identity);
|
||||
assert_eq!(
|
||||
logs.read_vote().await.expect("recovered local vote"),
|
||||
Some(openraft::Vote::new_committed(9, 1))
|
||||
);
|
||||
let state = logs.get_log_state().await.expect("coherent recovered log state");
|
||||
assert!(
|
||||
state
|
||||
.last_log_id
|
||||
.is_some_and(|last| cut.is_some_and(|applied| last.index >= applied.index)),
|
||||
"SNAPSHOT_INSTALL_LOG_STATE_COVERS_APPLIED_CUT"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn native_snapshot_install_ack_survives_power_cut() {
|
||||
let mut source = Fixture::new(0);
|
||||
source.apply(entries(2)).await;
|
||||
let mut snapshot = source.build().await;
|
||||
let bytes = snapshot_bytes(&mut snapshot).await;
|
||||
let (mut target, backend) = media_target().await;
|
||||
install(&mut target.sm, &mut snapshot).await;
|
||||
backend.cut();
|
||||
let Fixture {
|
||||
directory,
|
||||
writer,
|
||||
logs,
|
||||
sm,
|
||||
identity,
|
||||
} = target;
|
||||
drop(sm);
|
||||
drop(logs);
|
||||
drop(writer);
|
||||
let writer = NativeLogWriter::new(
|
||||
Store::open(directory.path().join("db"), identity).expect("acknowledged installation survives power cut"),
|
||||
);
|
||||
let mut logs = NativeRaftLogStore::new(writer.clone());
|
||||
let mut sm = NativeRaftStateMachine::new(writer.clone(), CheckpointLimits::default()).expect("reopened durable SM");
|
||||
let mut current = sm
|
||||
.get_current_snapshot()
|
||||
.await
|
||||
.expect("durable current snapshot")
|
||||
.expect("present");
|
||||
assert_eq!(current.meta, snapshot.meta, "SNAPSHOT_INSTALL_ACK_COMPLETE_META_DURABLE");
|
||||
assert_eq!(snapshot_bytes(&mut current).await, bytes);
|
||||
assert_eq!(
|
||||
writer.created_events_page(None, 256).await.expect("durable installed events"),
|
||||
source.page().await
|
||||
);
|
||||
logs.blocking_append(vec![entry(3)]).await.expect("append after ACK recovery");
|
||||
sm.apply(vec![entry(3)]).await.expect("apply after ACK recovery");
|
||||
assert_eq!(sm.applied_state().await.expect("continued state").0, Some(id(3).into()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn native_snapshot_install_purge_without_pending_rejected() {
|
||||
let mut target = Fixture::new(1);
|
||||
target.apply(entries(1)).await;
|
||||
let before = target.page().await;
|
||||
let bytes = std::fs::read(target.directory.path().join("db")).expect("old unauthorized purge bytes");
|
||||
let result = tokio::time::timeout(std::time::Duration::from_secs(2), target.logs.purge(id(3).into()))
|
||||
.await
|
||||
.expect("no pending receiver means purge cannot wait for hypothetical installation");
|
||||
assert!(result.is_err(), "SNAPSHOT_PURGE_WITHOUT_PENDING_FAILS_IMMEDIATELY");
|
||||
assert_eq!(target.page().await, before);
|
||||
assert_eq!(
|
||||
std::fs::read(target.directory.path().join("db")).expect("rejected purge preserves bytes"),
|
||||
bytes
|
||||
);
|
||||
}
|
||||
|
||||
async fn wait_for_pending_purge(writer: &NativeLogWriter) {
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
while writer.snapshot_receives.waiters.load(std::sync::atomic::Ordering::Acquire) == 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("actual trait purge reached pending-receive wait");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn native_snapshot_install_pending_purge_waits_for_install() {
|
||||
let mut source = Fixture::new(0);
|
||||
source.apply(entries(3)).await;
|
||||
let mut snapshot = source.build().await;
|
||||
let bytes = snapshot_bytes(&mut snapshot).await;
|
||||
let mut target = Fixture::new(1);
|
||||
target.apply(entries(1)).await;
|
||||
let data = receive(&mut target.sm, &bytes).await;
|
||||
let mut purging = target.logs.clone();
|
||||
let purge = tokio::spawn(async move { purging.purge(id(3).into()).await });
|
||||
wait_for_pending_purge(&target.writer).await;
|
||||
assert!(!purge.is_finished(), "SNAPSHOT_PURGE_WAITS_FOR_INCOMING_CUT_BEYOND_LOCAL_TAIL");
|
||||
target
|
||||
.sm
|
||||
.install_snapshot(&snapshot.meta, data)
|
||||
.await
|
||||
.expect("actual install while OpenRaft-style purge waits");
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), purge)
|
||||
.await
|
||||
.expect("pending purge wakes after install")
|
||||
.expect("purge task")
|
||||
.expect("installed checkpoint now authorizes exact purge");
|
||||
assert_eq!(
|
||||
target
|
||||
.logs
|
||||
.get_log_state()
|
||||
.await
|
||||
.expect("installed log state")
|
||||
.last_purged_log_id,
|
||||
Some(id(3).into()),
|
||||
"SNAPSHOT_PENDING_PURGE_USES_DURABLE_INSTALLED_CUT"
|
||||
);
|
||||
assert_eq!(target.page().await, source.page().await);
|
||||
assert_eq!(
|
||||
target
|
||||
.writer
|
||||
.snapshot_receives
|
||||
.waiters
|
||||
.load(std::sync::atomic::Ordering::Acquire),
|
||||
0,
|
||||
"SNAPSHOT_PURGE_WAIT_OWNER_RELEASED"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn native_snapshot_install_cancelled_receive_wakes_purge() {
|
||||
let mut target = Fixture::new(1);
|
||||
target.apply(entries(1)).await;
|
||||
let page = target.page().await;
|
||||
let before = std::fs::read(target.directory.path().join("db")).expect("stable target before cancelled receive");
|
||||
let data = target.sm.begin_receiving_snapshot().await.expect("actual pending receiver");
|
||||
let mut purging = target.logs.clone();
|
||||
let purge = tokio::spawn(async move { purging.purge(id(3).into()).await });
|
||||
wait_for_pending_purge(&target.writer).await;
|
||||
assert!(!purge.is_finished(), "SNAPSHOT_CANCELLED_RECEIVE_TEST_HAS_REAL_WAITING_PURGE");
|
||||
drop(data);
|
||||
assert!(
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), purge)
|
||||
.await
|
||||
.expect("cancelled receive wakes pending purge")
|
||||
.expect("purge task")
|
||||
.is_err(),
|
||||
"SNAPSHOT_CANCELLED_RECEIVE_CANNOT_AUTHORIZE_PURGE"
|
||||
);
|
||||
assert_eq!(target.page().await, page);
|
||||
assert_eq!(
|
||||
std::fs::read(target.directory.path().join("db")).expect("retained cancelled-receive state"),
|
||||
before
|
||||
);
|
||||
assert_eq!(
|
||||
target
|
||||
.writer
|
||||
.snapshot_receives
|
||||
.waiters
|
||||
.load(std::sync::atomic::Ordering::Acquire),
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn native_snapshot_install_failed_receive_wakes_purge() {
|
||||
let mut source = Fixture::new(0);
|
||||
source.apply(entries(3)).await;
|
||||
let mut snapshot = source.build().await;
|
||||
let bytes = snapshot_bytes(&mut snapshot).await;
|
||||
let mut target = Fixture::new(1);
|
||||
target.apply(entries(1)).await;
|
||||
let page = target.page().await;
|
||||
let before = std::fs::read(target.directory.path().join("db")).expect("stable target before failed install");
|
||||
let data = receive(&mut target.sm, &bytes).await;
|
||||
let mut purging = target.logs.clone();
|
||||
let purge = tokio::spawn(async move { purging.purge(id(3).into()).await });
|
||||
wait_for_pending_purge(&target.writer).await;
|
||||
assert!(!purge.is_finished(), "SNAPSHOT_FAILED_INSTALL_TEST_HAS_REAL_WAITING_PURGE");
|
||||
let mut wrong = snapshot.meta.clone();
|
||||
wrong.last_log_id = Some(LogId { term: 9, ..id(3) }.into());
|
||||
assert!(
|
||||
target.sm.install_snapshot(&wrong, data).await.is_err(),
|
||||
"SNAPSHOT_INVALID_META_INSTALL_REJECTED"
|
||||
);
|
||||
assert!(
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), purge)
|
||||
.await
|
||||
.expect("failed install wakes pending purge")
|
||||
.expect("purge task")
|
||||
.is_err(),
|
||||
"SNAPSHOT_FAILED_INSTALL_CANNOT_AUTHORIZE_PURGE"
|
||||
);
|
||||
assert_eq!(target.page().await, page);
|
||||
assert_eq!(
|
||||
std::fs::read(target.directory.path().join("db")).expect("retained failed-install state"),
|
||||
before
|
||||
);
|
||||
assert_eq!(
|
||||
target
|
||||
.writer
|
||||
.snapshot_receives
|
||||
.waiters
|
||||
.load(std::sync::atomic::Ordering::Acquire),
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn native_snapshot_install_completed_between_purge_query_and_wait() {
|
||||
let mut source = Fixture::new(0);
|
||||
source.apply(entries(2)).await;
|
||||
let mut snapshot = source.build().await;
|
||||
let data_bytes = snapshot_bytes(&mut snapshot).await;
|
||||
let mut target = Fixture::new(1);
|
||||
let data = receive(&mut target.sm, &data_bytes).await;
|
||||
let entered = std::sync::Arc::new(tokio::sync::Notify::new());
|
||||
let resume = std::sync::Arc::new(tokio::sync::Notify::new());
|
||||
*target.writer.snapshot_receives.purge_recheck.lock().expect("test pause") = Some((entered.clone(), resume.clone()));
|
||||
let mut logs = target.logs.clone();
|
||||
let purge = tokio::spawn(async move { logs.purge(id(2).into()).await });
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), entered.notified())
|
||||
.await
|
||||
.expect("purge failed coverage under owner before continuation");
|
||||
target
|
||||
.sm
|
||||
.install_snapshot(&snapshot.meta, data)
|
||||
.await
|
||||
.expect("install completes before purge continuation");
|
||||
resume.notify_one();
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), purge)
|
||||
.await
|
||||
.expect("completed install wake retained")
|
||||
.expect("purge task")
|
||||
.expect("old StaleCheckpoint rechecked after install");
|
||||
assert_eq!(
|
||||
target.page().await,
|
||||
source.page().await,
|
||||
"SNAPSHOT_COMPLETED_INSTALL_RECHECKS_STALE_PURGE"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn native_snapshot_install_empty_baseline_pending_purge() {
|
||||
let mut source = Fixture::new(0);
|
||||
source.apply(entries(2)).await;
|
||||
let mut snapshot = source.build().await;
|
||||
let bytes = snapshot_bytes(&mut snapshot).await;
|
||||
let mut target = Fixture::new(1);
|
||||
target.sm.build_snapshot().await.expect("real local empty baseline");
|
||||
let data = receive(&mut target.sm, &bytes).await;
|
||||
let mut logs = target.logs.clone();
|
||||
let purge = tokio::spawn(async move { logs.purge(id(2).into()).await });
|
||||
wait_for_pending_purge(&target.writer).await;
|
||||
target
|
||||
.sm
|
||||
.install_snapshot(&snapshot.meta, data)
|
||||
.await
|
||||
.expect("install supersedes empty baseline");
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), purge)
|
||||
.await
|
||||
.expect("empty coverage wake")
|
||||
.expect("purge task")
|
||||
.expect("new covered purge");
|
||||
assert_eq!(
|
||||
target.page().await,
|
||||
source.page().await,
|
||||
"SNAPSHOT_EMPTY_BASELINE_IS_UNCOVERED_NOT_CORRUPT"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,371 @@
|
||||
//! Validated capture peer metadata and its independent CNOD v1 codec.
|
||||
use serde::{Deserialize, Deserializer, Serialize, Serializer, ser::SerializeStruct};
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct CaptureNode {
|
||||
rpc_uri: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum CaptureNodeError {
|
||||
#[error("invalid RPC URI")]
|
||||
InvalidRpcUri,
|
||||
#[error("invalid RPC URI")]
|
||||
InvalidRpcUriParse(#[source] url::ParseError),
|
||||
#[error("RPC URI too long")]
|
||||
RpcUriTooLong,
|
||||
#[error("invalid node magic")]
|
||||
InvalidNodeMagic,
|
||||
#[error("unsupported node version")]
|
||||
UnsupportedNodeVersion,
|
||||
#[error("invalid node length")]
|
||||
InvalidNodeLength,
|
||||
#[error("truncated node record")]
|
||||
TruncatedNodeRecord,
|
||||
#[error("invalid node UTF-8")]
|
||||
InvalidNodeUtf8(#[source] std::str::Utf8Error),
|
||||
#[error("trailing node bytes")]
|
||||
TrailingNodeBytes,
|
||||
}
|
||||
impl CaptureNodeError {
|
||||
pub fn code(&self) -> &'static str {
|
||||
match self {
|
||||
Self::InvalidRpcUri | Self::InvalidRpcUriParse(_) => "invalid_rpc_uri",
|
||||
Self::RpcUriTooLong => "rpc_uri_too_long",
|
||||
Self::InvalidNodeMagic => "invalid_node_magic",
|
||||
Self::UnsupportedNodeVersion => "unsupported_node_version",
|
||||
Self::InvalidNodeLength => "invalid_node_length",
|
||||
Self::TruncatedNodeRecord => "truncated_node_record",
|
||||
Self::InvalidNodeUtf8(_) => "invalid_node_utf8",
|
||||
Self::TrailingNodeBytes => "trailing_node_bytes",
|
||||
}
|
||||
}
|
||||
}
|
||||
impl CaptureNode {
|
||||
pub fn new(rpc_uri: String) -> Result<Self, CaptureNodeError> {
|
||||
let node = Self { rpc_uri };
|
||||
node.validate()?;
|
||||
Ok(node)
|
||||
}
|
||||
|
||||
pub fn rpc_uri(&self) -> &str {
|
||||
&self.rpc_uri
|
||||
}
|
||||
|
||||
pub fn validate(&self) -> Result<(), CaptureNodeError> {
|
||||
let uri = &self.rpc_uri;
|
||||
if uri.len() > 4096 {
|
||||
return Err(CaptureNodeError::RpcUriTooLong);
|
||||
}
|
||||
if uri.is_empty() || uri.chars().any(|c| c.is_whitespace() || c.is_control() || c == '\\') {
|
||||
return Err(CaptureNodeError::InvalidRpcUri);
|
||||
}
|
||||
let rest = uri
|
||||
.strip_prefix("http://")
|
||||
.or_else(|| uri.strip_prefix("https://"))
|
||||
.ok_or(CaptureNodeError::InvalidRpcUri)?;
|
||||
if uri.contains('?') || uri.contains('#') {
|
||||
return Err(CaptureNodeError::InvalidRpcUri);
|
||||
}
|
||||
let (authority, path) = rest.split_once('/').map_or((rest, ""), |(a, p)| (a, p));
|
||||
// An empty suffix after the first slash is the only allowed path.
|
||||
if !path.is_empty() || authority.contains('@') {
|
||||
return Err(CaptureNodeError::InvalidRpcUri);
|
||||
}
|
||||
let (host, port) = if authority.starts_with('[') {
|
||||
let end = authority.find(']').ok_or(CaptureNodeError::InvalidRpcUri)?;
|
||||
let port = authority
|
||||
.get(end + 1..)
|
||||
.and_then(|suffix| suffix.strip_prefix(':'))
|
||||
.ok_or(CaptureNodeError::InvalidRpcUri)?;
|
||||
(&authority[..=end], port)
|
||||
} else {
|
||||
authority.split_once(':').ok_or(CaptureNodeError::InvalidRpcUri)?
|
||||
};
|
||||
if host.is_empty() || port.is_empty() || !port.bytes().all(|b| b.is_ascii_digit()) {
|
||||
return Err(CaptureNodeError::InvalidRpcUri);
|
||||
}
|
||||
let port = port.parse::<u16>().map_err(|_| CaptureNodeError::InvalidRpcUri)?;
|
||||
if port == 0 {
|
||||
return Err(CaptureNodeError::InvalidRpcUri);
|
||||
}
|
||||
let parsed_url = url::Url::parse(uri).map_err(CaptureNodeError::InvalidRpcUriParse)?;
|
||||
if parsed_url.host().is_none() {
|
||||
return Err(CaptureNodeError::InvalidRpcUri);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn encode(&self) -> Result<Vec<u8>, CaptureNodeError> {
|
||||
self.validate()?;
|
||||
let length = u32::try_from(self.rpc_uri.len()).map_err(|_| CaptureNodeError::InvalidNodeLength)?;
|
||||
let mut bytes = b"CNOD\x01".to_vec();
|
||||
bytes.extend_from_slice(&length.to_be_bytes());
|
||||
bytes.extend_from_slice(self.rpc_uri.as_bytes());
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
pub fn decode(bytes: &[u8]) -> Result<Self, CaptureNodeError> {
|
||||
if bytes.len() < 9 {
|
||||
return Err(CaptureNodeError::TruncatedNodeRecord);
|
||||
}
|
||||
if &bytes[..4] != b"CNOD" {
|
||||
return Err(CaptureNodeError::InvalidNodeMagic);
|
||||
}
|
||||
if bytes[4] != 1 {
|
||||
return Err(CaptureNodeError::UnsupportedNodeVersion);
|
||||
}
|
||||
let length = u32::from_be_bytes(bytes[5..9].try_into().map_err(|_| CaptureNodeError::InvalidNodeLength)?);
|
||||
if !(1..=4096).contains(&length) {
|
||||
return Err(CaptureNodeError::InvalidNodeLength);
|
||||
}
|
||||
let length = usize::try_from(length).map_err(|_| CaptureNodeError::InvalidNodeLength)?;
|
||||
let end = 9_usize.checked_add(length).ok_or(CaptureNodeError::InvalidNodeLength)?;
|
||||
if bytes.len() < end {
|
||||
return Err(CaptureNodeError::TruncatedNodeRecord);
|
||||
}
|
||||
if bytes.len() != end {
|
||||
return Err(CaptureNodeError::TrailingNodeBytes);
|
||||
}
|
||||
let uri = std::str::from_utf8(&bytes[9..end]).map_err(CaptureNodeError::InvalidNodeUtf8)?;
|
||||
Self::new(uri.to_owned())
|
||||
}
|
||||
}
|
||||
impl Serialize for CaptureNode {
|
||||
fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
|
||||
self.validate().map_err(serde::ser::Error::custom)?;
|
||||
let mut state = serializer.serialize_struct("CaptureNode", 1)?;
|
||||
state.serialize_field("rpc_uri", &self.rpc_uri)?;
|
||||
state.end()
|
||||
}
|
||||
}
|
||||
impl<'de> Deserialize<'de> for CaptureNode {
|
||||
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct Fields {
|
||||
rpc_uri: String,
|
||||
}
|
||||
let fields = Fields::deserialize(deserializer).map_err(|_| serde::de::Error::custom("invalid capture node fields"))?;
|
||||
Self::new(fields.rpc_uri).map_err(serde::de::Error::custom)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::error::Error;
|
||||
|
||||
fn record(payload: &[u8]) -> Vec<u8> {
|
||||
let mut bytes = b"CNOD\x01".to_vec();
|
||||
bytes.extend_from_slice(&u32::try_from(payload.len()).expect("test payload length").to_be_bytes());
|
||||
bytes.extend_from_slice(payload);
|
||||
bytes
|
||||
}
|
||||
fn reject_uri(uri: &str) {
|
||||
let result = CaptureNode::new(uri.to_owned());
|
||||
assert!(result.is_err(), "URI should reject: {uri:?}");
|
||||
assert_eq!(result.expect_err("rejected URI").code(), "invalid_rpc_uri");
|
||||
}
|
||||
fn reject_record(bytes: &[u8], code: &str) {
|
||||
let result = CaptureNode::decode(bytes);
|
||||
assert!(result.is_err(), "record should reject with {code}");
|
||||
assert_eq!(result.expect_err("rejected record").code(), code);
|
||||
}
|
||||
#[test]
|
||||
fn actual_openraft_node_bound() {
|
||||
fn require_node<T: openraft::Node>() {}
|
||||
require_node::<CaptureNode>();
|
||||
assert_eq!(CaptureNode::default().rpc_uri(), "");
|
||||
}
|
||||
#[test]
|
||||
fn default_and_empty_rejected_at_every_boundary() {
|
||||
let node = CaptureNode::default();
|
||||
assert_eq!(node.rpc_uri(), "");
|
||||
reject_uri("");
|
||||
assert_eq!(node.validate().expect_err("empty validation").code(), "invalid_rpc_uri");
|
||||
assert!(serde_json::to_string(&node).is_err());
|
||||
assert!(serde_json::from_str::<CaptureNode>(r#"{"rpc_uri":""}"#).is_err());
|
||||
assert_eq!(node.encode().expect_err("empty encoding").code(), "invalid_rpc_uri");
|
||||
reject_record(&record(b""), "invalid_node_length");
|
||||
}
|
||||
#[test]
|
||||
fn accepted_uris_preserve_exact_bytes() {
|
||||
for uri in [
|
||||
"http://node:80",
|
||||
"https://node:443",
|
||||
"http://node:1",
|
||||
"https://node:65535/",
|
||||
"http://127.0.0.1:9000",
|
||||
"https://[::1]:443/",
|
||||
"http://[2001:db8::1]:1",
|
||||
"http://NoDe:00080/",
|
||||
] {
|
||||
let result = CaptureNode::new(uri.to_owned());
|
||||
assert!(result.is_ok(), "valid URI construction: {uri}");
|
||||
let node = result.expect("valid URI");
|
||||
assert_eq!(node.rpc_uri(), uri, "preserve explicit default port and raw URI");
|
||||
assert_eq!(node.encode().expect("encode URI"), record(uri.as_bytes()));
|
||||
assert_eq!(CaptureNode::decode(&record(uri.as_bytes())).expect("decode URI"), node);
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn golden_cnod_bytes() {
|
||||
let node = CaptureNode {
|
||||
rpc_uri: "http://node:80".to_owned(),
|
||||
};
|
||||
let golden = b"CNOD\x01\x00\x00\x00\x0ehttp://node:80";
|
||||
assert_eq!(node.encode().ok().as_deref(), Some(golden.as_slice()), "golden CNOD bytes");
|
||||
}
|
||||
#[test]
|
||||
fn valid_cnod_decode() {
|
||||
let golden = b"CNOD\x01\x00\x00\x00\x0ehttp://node:80";
|
||||
let decoded = CaptureNode::decode(golden);
|
||||
assert!(decoded.is_ok(), "golden decode succeeds");
|
||||
assert_eq!(decoded.expect("golden decoded node").rpc_uri(), "http://node:80");
|
||||
}
|
||||
#[test]
|
||||
fn serde_shape_roundtrip_and_strict_fields() {
|
||||
let node = CaptureNode::new("https://node:443/".to_owned()).expect("serde node");
|
||||
let json = r#"{"rpc_uri":"https://node:443/"}"#;
|
||||
assert_eq!(serde_json::to_string(&node).expect("serialize node"), json);
|
||||
assert_eq!(serde_json::from_str::<CaptureNode>(json).expect("deserialize node"), node);
|
||||
for json in [
|
||||
r#"{}"#,
|
||||
r#"{"rpc_uri":1}"#,
|
||||
r#"{"rpc_uri":null}"#,
|
||||
r#"{"rpc_uri":"http://node:1","extra":0}"#,
|
||||
r#"{"rpc_uri":"http://node:1","rpc_uri":"http://node:2"}"#,
|
||||
] {
|
||||
assert!(serde_json::from_str::<CaptureNode>(json).is_err(), "strict fields: {json}");
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn credentials_and_raw_uri_shapes_reject() {
|
||||
for uri in [
|
||||
"http://user@node:1",
|
||||
"http://user:password@node:1",
|
||||
"http://@node:1",
|
||||
"HTTP://node:1",
|
||||
"Https://node:1",
|
||||
"http://node",
|
||||
"http://node:",
|
||||
"http://node:0",
|
||||
"http://node:-1",
|
||||
"http://node:+1",
|
||||
"http://node:65536",
|
||||
"http://node:999999999999999999999",
|
||||
"http://[::1:1",
|
||||
"http://[::1]extra:1",
|
||||
"http://[invalid]:1",
|
||||
"http://::1:1",
|
||||
"http://:1",
|
||||
"/disk/path",
|
||||
"file:///disk/path",
|
||||
"http://node:1/./",
|
||||
"http://node:1/../",
|
||||
"http://node:1/%2e/",
|
||||
"http://node:1/%2E%2e/",
|
||||
"http://node:1/path",
|
||||
"http://node:1\\",
|
||||
" http://node:1",
|
||||
"http://node:1\n",
|
||||
"http://no\tde:1",
|
||||
"http://node:1\u{00a0}",
|
||||
"http://node:1\u{0085}",
|
||||
"http://node:1\0",
|
||||
] {
|
||||
reject_uri(uri);
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn query_and_fragment_markers_reject() {
|
||||
for uri in [
|
||||
"http://node:1?",
|
||||
"http://node:1?x=y",
|
||||
"http://node:1/?",
|
||||
"http://node:1#",
|
||||
"http://node:1#fragment",
|
||||
"http://node:1/#",
|
||||
] {
|
||||
reject_uri(uri);
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn utf8_byte_length_boundaries() {
|
||||
for length in [4095, 4096] {
|
||||
let prefix = "http://node:";
|
||||
let uri = format!("{prefix}{}1", "0".repeat(length - prefix.len() - 1));
|
||||
assert_eq!(uri.len(), length);
|
||||
assert!(url::Url::parse(&uri).is_ok(), "boundary fixture accepted by actual URL parser");
|
||||
let node = CaptureNode::new(uri.clone()).expect("boundary node");
|
||||
assert_eq!(node.rpc_uri(), uri);
|
||||
assert_eq!(
|
||||
CaptureNode::decode(&node.encode().expect("boundary encoding")).expect("boundary decoding"),
|
||||
node
|
||||
);
|
||||
}
|
||||
let uri = format!("http://node:{}1", "0".repeat(4097 - "http://node:".len() - 1));
|
||||
assert_eq!(CaptureNode::new(uri).expect_err("oversize URI").code(), "rpc_uri_too_long");
|
||||
let unicode = format!("http://{}:1", "é".repeat(2044));
|
||||
assert!(unicode.chars().count() < 4096);
|
||||
assert!(unicode.len() > 4096);
|
||||
assert_eq!(CaptureNode::new(unicode).expect_err("UTF-8 byte limit").code(), "rpc_uri_too_long");
|
||||
}
|
||||
#[test]
|
||||
fn cnod_headers_and_lengths_reject_exact_codes() {
|
||||
let valid = record(b"http://node:80");
|
||||
for end in 0..9 {
|
||||
reject_record(&valid[..end], "truncated_node_record");
|
||||
}
|
||||
let mut bad = valid.clone();
|
||||
bad[0] = b'X';
|
||||
reject_record(&bad, "invalid_node_magic");
|
||||
bad = valid.clone();
|
||||
bad[4] = 2;
|
||||
reject_record(&bad, "unsupported_node_version");
|
||||
for length in [0_u32, 4097, u32::MAX] {
|
||||
bad = b"CNOD\x01".to_vec();
|
||||
bad.extend_from_slice(&length.to_be_bytes());
|
||||
reject_record(&bad, "invalid_node_length");
|
||||
}
|
||||
for end in 9..valid.len() {
|
||||
reject_record(&valid[..end], "truncated_node_record");
|
||||
}
|
||||
reject_record(&record(&[0xff]), "invalid_node_utf8");
|
||||
reject_record(&record(&[0xc3, 0x28]), "invalid_node_utf8");
|
||||
reject_record(&record(b"invalid URI"), "invalid_rpc_uri");
|
||||
}
|
||||
#[test]
|
||||
fn trailing_bytes_reject() {
|
||||
for tail in [&b"x"[..], &b"xyz"[..]] {
|
||||
let mut bytes = record(b"http://node:80");
|
||||
bytes.extend_from_slice(tail);
|
||||
reject_record(&bytes, "trailing_node_bytes");
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn errors_and_sources_redact_input() {
|
||||
let sentinel = "http://SECRET_USER:SECRET_PASSWORD@node:1";
|
||||
let error = CaptureNode::new(sentinel.to_owned()).expect_err("credentials reject");
|
||||
let mut current: &dyn Error = &error;
|
||||
loop {
|
||||
let rendered = format!("{current} {current:?}");
|
||||
assert!(!rendered.contains("SECRET"));
|
||||
match current.source() {
|
||||
Some(source) => current = source,
|
||||
None => break,
|
||||
}
|
||||
}
|
||||
let json = format!(r#"{{"rpc_uri":"{sentinel}"}}"#);
|
||||
let error = serde_json::from_str::<CaptureNode>(&json).expect_err("serde credentials reject");
|
||||
assert!(!format!("{error} {error:?}").contains("SECRET"));
|
||||
let json = format!(r#"{{"{sentinel}":0}}"#);
|
||||
let error = serde_json::from_str::<CaptureNode>(&json).expect_err("unknown fields reject");
|
||||
assert!(!format!("{error} {error:?}").contains("SECRET"));
|
||||
let error = CaptureNode::new("http://[bad]:1".to_owned()).expect_err("parser rejection");
|
||||
assert!(error.source().is_some(), "retain URL parser source");
|
||||
let error = CaptureNode::decode(&record(&[0xff])).expect_err("UTF-8 rejection");
|
||||
assert!(error.source().is_some(), "retain UTF-8 source");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
//! OpenRaft log storage backed by the existing native writer and local durable baseline.
|
||||
use crate::{CaptureRaftEntry, CaptureTypeConfig, NativeLogReader, NativeLogWriter, NativeWriteError, StoreError};
|
||||
use openraft::{
|
||||
LogId, OptionalSend, RaftLogReader, StorageError, StorageIOError, Vote,
|
||||
storage::{LogFlushed, RaftLogStorage},
|
||||
};
|
||||
use redb::{ReadableDatabase, ReadableTable};
|
||||
|
||||
/// Clones share one native Store, write ordering and checkpoint authority.
|
||||
#[derive(Clone)]
|
||||
pub struct NativeRaftLogStore {
|
||||
writer: NativeLogWriter,
|
||||
// Last: owner drain is signaled only after this clone releases its writer.
|
||||
pub(crate) owner_lease: Option<std::sync::Arc<crate::raft_owner::OwnerLease>>,
|
||||
}
|
||||
impl NativeRaftLogStore {
|
||||
pub fn new(writer: NativeLogWriter) -> Self {
|
||||
Self {
|
||||
writer,
|
||||
owner_lease: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
fn error(error: NativeWriteError, operation: fn(openraft::AnyError) -> StorageIOError<u64>) -> StorageError<u64> {
|
||||
StorageError::IO {
|
||||
source: operation(openraft::AnyError::new(&error)),
|
||||
}
|
||||
}
|
||||
impl RaftLogReader<CaptureTypeConfig> for NativeRaftLogStore {
|
||||
async fn try_get_log_entries<RB: std::ops::RangeBounds<u64> + Clone + std::fmt::Debug + OptionalSend>(
|
||||
&mut self,
|
||||
range: RB,
|
||||
) -> Result<Vec<CaptureRaftEntry>, StorageError<u64>> {
|
||||
let mut reader = self.writer.reader();
|
||||
reader.owner_lease = self.owner_lease.clone();
|
||||
reader.try_get_log_entries(range).await
|
||||
}
|
||||
}
|
||||
impl RaftLogStorage<CaptureTypeConfig> for NativeRaftLogStore {
|
||||
type LogReader = NativeLogReader;
|
||||
async fn get_log_state(&mut self) -> Result<openraft::LogState<CaptureTypeConfig>, StorageError<u64>> {
|
||||
self.writer
|
||||
.write(|store| {
|
||||
if !store.writable {
|
||||
return Err(StoreError::WriteDisabled);
|
||||
}
|
||||
let read = store.database.begin_read()?;
|
||||
let meta = read.open_table(crate::META)?;
|
||||
let purged = crate::checkpoint_baseline::read_purged(&meta)?;
|
||||
let logs = read.open_table(crate::LOGS)?;
|
||||
let last = logs
|
||||
.last()?
|
||||
.map(|(index, value)| {
|
||||
let id = crate::decode_log(value.value())?.id;
|
||||
if id.index != index.value() {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
Ok(id)
|
||||
})
|
||||
.transpose()?
|
||||
.or(purged);
|
||||
let tail = meta.get("tail")?.map(|value| crate::decode_id(value.value())).transpose()?;
|
||||
if tail != last {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
Ok(openraft::LogState {
|
||||
last_purged_log_id: purged.map(Into::into),
|
||||
last_log_id: last.map(Into::into),
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(|e| error(e, StorageIOError::read_logs))
|
||||
}
|
||||
async fn get_log_reader(&mut self) -> Self::LogReader {
|
||||
let mut reader = self.writer.reader();
|
||||
reader.owner_lease = self.owner_lease.clone();
|
||||
reader
|
||||
}
|
||||
async fn save_vote(&mut self, vote: &Vote<u64>) -> Result<(), StorageError<u64>> {
|
||||
self.writer
|
||||
.save_vote(*vote)
|
||||
.await
|
||||
.map_err(|e| error(e, StorageIOError::write_vote))
|
||||
}
|
||||
async fn read_vote(&mut self) -> Result<Option<Vote<u64>>, StorageError<u64>> {
|
||||
self.writer.read_vote().await.map_err(|e| error(e, StorageIOError::read_vote))
|
||||
}
|
||||
async fn save_committed(&mut self, committed: Option<LogId<u64>>) -> Result<(), StorageError<u64>> {
|
||||
self.writer
|
||||
.write(move |store| match committed {
|
||||
Some(id) => store.save_committed(id.into()),
|
||||
None if store.read_committed()?.is_none() => Ok(()),
|
||||
None => Err(StoreError::CommittedRegression),
|
||||
})
|
||||
.await
|
||||
.map_err(|e| error(e, StorageIOError::write_logs))
|
||||
}
|
||||
async fn read_committed(&mut self) -> Result<Option<LogId<u64>>, StorageError<u64>> {
|
||||
self.writer
|
||||
.write(|store| store.read_committed().map(|id| id.map(Into::into)))
|
||||
.await
|
||||
.map_err(|e| error(e, StorageIOError::read_logs))
|
||||
}
|
||||
async fn append<I>(&mut self, entries: I, callback: LogFlushed<CaptureTypeConfig>) -> Result<(), StorageError<u64>>
|
||||
where
|
||||
I: IntoIterator<Item = CaptureRaftEntry> + OptionalSend,
|
||||
I::IntoIter: OptionalSend,
|
||||
{
|
||||
let entries = entries.into_iter().collect::<Vec<_>>();
|
||||
// The original blocking worker owns ordering and callback through durable commit,
|
||||
// even if OpenRaft cancels its awaiting append future.
|
||||
self.writer
|
||||
.write(move |store| {
|
||||
let result = store.append_native(&entries);
|
||||
callback.log_io_completed(
|
||||
result
|
||||
.as_ref()
|
||||
.map(|_| ())
|
||||
.map_err(|error| std::io::Error::other(error.code())),
|
||||
);
|
||||
result
|
||||
})
|
||||
.await
|
||||
.map_err(|e| error(e, StorageIOError::write_logs))
|
||||
}
|
||||
async fn truncate(&mut self, log_id: LogId<u64>) -> Result<(), StorageError<u64>> {
|
||||
self.writer
|
||||
.truncate_suffix(log_id.index)
|
||||
.await
|
||||
.map_err(|e| error(e, StorageIOError::write_logs))
|
||||
}
|
||||
async fn purge(&mut self, log_id: LogId<u64>) -> Result<(), StorageError<u64>> {
|
||||
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(30);
|
||||
loop {
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return Err(error(NativeWriteError::Store(StoreError::StaleCheckpoint), StorageIOError::write_logs));
|
||||
}
|
||||
let notified = self.writer.snapshot_receives.changed.notified();
|
||||
tokio::pin!(notified);
|
||||
notified.as_mut().enable();
|
||||
let receives = self.writer.snapshot_receives.clone();
|
||||
// Observe coverage and receive ownership while install is excluded by ordering.
|
||||
let (result, pending) = self
|
||||
.writer
|
||||
.write(move |store| Ok((store.purge_native_prefix(log_id.into(), None), receives.pending())))
|
||||
.await
|
||||
.map_err(|e| error(e, StorageIOError::write_logs))?;
|
||||
if matches!(&result, Err(StoreError::StaleCheckpoint)) {
|
||||
#[cfg(test)]
|
||||
{
|
||||
let hook = self
|
||||
.writer
|
||||
.snapshot_receives
|
||||
.purge_recheck
|
||||
.lock()
|
||||
.expect("test hook lock")
|
||||
.take();
|
||||
if let Some((entered, resume)) = hook {
|
||||
entered.notify_one();
|
||||
resume.notified().await;
|
||||
}
|
||||
}
|
||||
if pending {
|
||||
#[cfg(test)]
|
||||
let _waiter = crate::raft_state_machine::Waiter::new(self.writer.snapshot_receives.clone());
|
||||
if tokio::time::timeout_at(deadline, notified).await.is_err() {
|
||||
return Err(error(NativeWriteError::Store(StoreError::StaleCheckpoint), StorageIOError::write_logs));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
return result.map_err(|e| error(e.into(), StorageIOError::write_logs));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,439 @@
|
||||
//! Exercises actual OpenRaft storage traits, not a substitute consensus engine.
|
||||
use super::*;
|
||||
use openraft::{
|
||||
RaftLogReader,
|
||||
storage::{RaftLogStorage, RaftLogStorageExt},
|
||||
};
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
fn identity() -> NodeIdentity {
|
||||
NodeIdentity {
|
||||
cluster_id: [7; 16],
|
||||
node_id: 0,
|
||||
}
|
||||
}
|
||||
fn binding() -> CaptureBinding {
|
||||
CaptureBinding {
|
||||
revision: 9,
|
||||
binding_id: [8; 16],
|
||||
target_id: [9; 16],
|
||||
}
|
||||
}
|
||||
fn id(index: u64) -> LogId {
|
||||
LogId {
|
||||
term: 8,
|
||||
leader_node: 0,
|
||||
index,
|
||||
}
|
||||
}
|
||||
fn entry(index: u64) -> CaptureRaftEntry {
|
||||
let payload = if index == 0 {
|
||||
let membership = CaptureMembership::new(
|
||||
Some(id(0)),
|
||||
vec![BTreeSet::from([0])],
|
||||
BTreeMap::from([(0, CaptureNode::new("http://127.0.0.1:7000".into()).expect("test member"))]),
|
||||
)
|
||||
.expect("native membership");
|
||||
let membership = openraft::StoredMembership::try_from(&membership).expect("stored native membership");
|
||||
openraft::EntryPayload::Membership(membership.membership().clone())
|
||||
} else {
|
||||
let byte = u8::try_from(index).expect("small test event index");
|
||||
let command = DecideCreated {
|
||||
operation_id: [byte; 16],
|
||||
object: ObjectIdentity {
|
||||
bucket_incarnation: [2; 16],
|
||||
key: vec![byte],
|
||||
},
|
||||
expected_head: None,
|
||||
prepared: PreparedIdentity {
|
||||
version_id: [byte; 16],
|
||||
preparation_id: [4; 16],
|
||||
content_digest: [5; 32],
|
||||
content_length: 123,
|
||||
},
|
||||
binding_revision: binding().revision,
|
||||
};
|
||||
openraft::EntryPayload::Normal(CaptureCommand::try_created(command).expect("native test command"))
|
||||
};
|
||||
CaptureRaftEntry {
|
||||
log_id: id(index).into(),
|
||||
payload,
|
||||
}
|
||||
}
|
||||
fn entries(end: u64) -> Vec<CaptureRaftEntry> {
|
||||
(0..=end).map(entry).collect()
|
||||
}
|
||||
fn fixture() -> (tempfile::TempDir, NativeLogWriter, NativeRaftLogStore) {
|
||||
let directory = tempfile::tempdir().expect("actual native database directory");
|
||||
let mut store = Store::initialize(directory.path().join("db"), identity()).expect("actual redb initialization");
|
||||
store.pin_capture_binding(binding()).expect("capture binding");
|
||||
store.enable_native_empty().expect("native mode");
|
||||
store.promote_created_provenance().expect("native provenance");
|
||||
let writer = NativeLogWriter::new(store);
|
||||
let adapter = NativeRaftLogStore::new(writer.clone());
|
||||
(directory, writer, adapter)
|
||||
}
|
||||
fn media_fixture() -> (tempfile::TempDir, tests::CrashBackend, NativeLogWriter, NativeRaftLogStore) {
|
||||
let (directory, writer, adapter) = fixture();
|
||||
drop(adapter);
|
||||
drop(writer);
|
||||
let backend = tests::CrashBackend::new(&directory.path().join("db"));
|
||||
let store = Store::from_database(backend.database(), identity()).expect("existing real disk fault backend");
|
||||
let writer = NativeLogWriter::new(store);
|
||||
let adapter = NativeRaftLogStore::new(writer.clone());
|
||||
(directory, backend, writer, adapter)
|
||||
}
|
||||
async fn logs(adapter: &mut NativeRaftLogStore) -> Vec<CaptureRaftEntry> {
|
||||
adapter
|
||||
.get_log_reader()
|
||||
.await
|
||||
.try_get_log_entries(..)
|
||||
.await
|
||||
.expect("actual trait reader")
|
||||
}
|
||||
async fn checkpoint(writer: &NativeLogWriter, adapter: &mut NativeRaftLogStore) -> CheckpointReceipt {
|
||||
let batch = entries(2);
|
||||
adapter
|
||||
.blocking_append(batch.clone())
|
||||
.await
|
||||
.expect("trait append before checkpoint");
|
||||
// Prepares genuine applied/checkpoint evidence; this does not claim Raft consensus.
|
||||
writer
|
||||
.apply_committed(batch)
|
||||
.await
|
||||
.expect("trusted test owner applies preparation entries");
|
||||
writer
|
||||
.checkpoint_local(CheckpointLimits::default())
|
||||
.await
|
||||
.expect("real durable baseline")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raft_log_storage_flush_restart() {
|
||||
let (directory, backend, writer, mut adapter) = media_fixture();
|
||||
let batch = entries(2);
|
||||
// blocking_append constructs the real library-private LogFlushed callback.
|
||||
adapter
|
||||
.blocking_append(batch.clone())
|
||||
.await
|
||||
.expect("real durable flush callback");
|
||||
let state = adapter.get_log_state().await.expect("trait log state");
|
||||
assert_eq!(state.last_log_id, Some(id(2).into()), "RAFT_LOG_TAIL_NOT_APPLIED_STATE");
|
||||
assert_eq!(state.last_purged_log_id, None);
|
||||
assert_eq!(writer.applied_state().await.expect("application state").0, None);
|
||||
assert_eq!(logs(&mut adapter).await, batch);
|
||||
backend.cut();
|
||||
drop(adapter);
|
||||
drop(writer);
|
||||
let reopened = Store::open(directory.path().join("db"), identity()).expect("reopen after successful callback and power cut");
|
||||
let mut adapter = NativeRaftLogStore::new(NativeLogWriter::new(reopened));
|
||||
assert_eq!(logs(&mut adapter).await, batch, "RAFT_LOG_FLUSH_CALLBACK_DURABLE");
|
||||
assert_eq!(
|
||||
adapter.get_log_state().await.expect("reopened trait state").last_log_id,
|
||||
Some(id(2).into())
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raft_log_storage_vote_restart() {
|
||||
let (directory, backend, writer, mut adapter) = media_fixture();
|
||||
let vote = openraft::Vote::new_committed(9, 0);
|
||||
adapter.save_vote(&vote).await.expect("trait durable vote");
|
||||
assert_eq!(adapter.read_vote().await.expect("trait vote read"), Some(vote));
|
||||
adapter.blocking_append(entries(2)).await.expect("trait append");
|
||||
adapter
|
||||
.save_committed(Some(id(1).into()))
|
||||
.await
|
||||
.expect("trait durable commit marker");
|
||||
assert!(adapter.save_committed(None).await.is_err(), "RAFT_LOG_COMMITTED_NONE_CANNOT_RESET");
|
||||
assert_eq!(adapter.read_committed().await.expect("exact marker"), Some(id(1).into()));
|
||||
assert!(
|
||||
adapter.save_committed(Some(id(0).into())).await.is_err(),
|
||||
"RAFT_LOG_COMMITTED_NO_REGRESSION"
|
||||
);
|
||||
let mut wrong = id(2);
|
||||
wrong.term += 1;
|
||||
assert!(
|
||||
adapter.save_committed(Some(wrong.into())).await.is_err(),
|
||||
"RAFT_LOG_COMMITTED_EXACT_IDENTITY"
|
||||
);
|
||||
backend.cut();
|
||||
drop(adapter);
|
||||
drop(writer);
|
||||
let mut reopened = NativeRaftLogStore::new(NativeLogWriter::new(
|
||||
Store::open(directory.path().join("db"), identity()).expect("durable vote reopen"),
|
||||
));
|
||||
assert_eq!(
|
||||
reopened.read_vote().await.expect("reopened vote"),
|
||||
Some(vote),
|
||||
"RAFT_LOG_VOTE_ACK_DURABLE"
|
||||
);
|
||||
assert_eq!(reopened.read_committed().await.expect("reopened committed marker"), Some(id(1).into()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raft_log_storage_partial_purge_restart() {
|
||||
let (directory, writer, mut adapter) = fixture();
|
||||
let receipt = checkpoint(&writer, &mut adapter).await;
|
||||
adapter
|
||||
.blocking_append(vec![entry(3), entry(4)])
|
||||
.await
|
||||
.expect("uncovered suffix append");
|
||||
let events = writer
|
||||
.created_events_page(None, 256)
|
||||
.await
|
||||
.expect("pre-purge authoritative events");
|
||||
adapter
|
||||
.purge(id(1).into())
|
||||
.await
|
||||
.expect("checkpoint authorizes strict prefix");
|
||||
assert_eq!(
|
||||
logs(&mut adapter).await,
|
||||
vec![entry(2), entry(3), entry(4)],
|
||||
"RAFT_LOG_PARTIAL_PURGE_RETAINS_SUFFIX"
|
||||
);
|
||||
let state = adapter.get_log_state().await.expect("partially purged state");
|
||||
assert_eq!(state.last_purged_log_id, Some(id(1).into()));
|
||||
assert_eq!(state.last_log_id, Some(id(4).into()));
|
||||
adapter.purge(id(1).into()).await.expect("repeat exact purge is idempotent");
|
||||
assert!(adapter.purge(id(0).into()).await.is_err(), "RAFT_LOG_PURGE_NO_REGRESSION");
|
||||
{
|
||||
let store = writer.store.lock().expect("inspect task-owned store");
|
||||
let tx = store.database.begin_read().expect("real persisted prefix inspection");
|
||||
let results = tx.open_table(application::APPLY_RESULTS).expect("apply receipts");
|
||||
assert!(results.get(0).expect("membership receipt").is_none());
|
||||
assert!(results.get(1).expect("first receipt").is_none());
|
||||
assert!(
|
||||
results.get(2).expect("retained receipt").is_some(),
|
||||
"RAFT_LOG_PARTIAL_PURGE_RECEIPT_SUFFIX"
|
||||
);
|
||||
}
|
||||
drop(adapter);
|
||||
drop(writer);
|
||||
let writer = NativeLogWriter::new(Store::open(directory.path().join("db"), identity()).expect("restart partial purge"));
|
||||
let mut adapter = NativeRaftLogStore::new(writer.clone());
|
||||
assert_eq!(
|
||||
logs(&mut adapter).await,
|
||||
vec![entry(2), entry(3), entry(4)],
|
||||
"RAFT_LOG_PARTIAL_PURGE_REOPEN"
|
||||
);
|
||||
adapter
|
||||
.blocking_append(vec![entry(5)])
|
||||
.await
|
||||
.expect("continue append after partial purge");
|
||||
writer
|
||||
.apply_committed(vec![entry(3), entry(4), entry(5)])
|
||||
.await
|
||||
.expect("continue application from true applied state");
|
||||
assert_eq!(
|
||||
writer
|
||||
.created_events_page(None, 256)
|
||||
.await
|
||||
.expect("retained plus new events")
|
||||
.events
|
||||
.len(),
|
||||
5,
|
||||
"RAFT_LOG_PURGE_NEVER_DELETES_EVENTS"
|
||||
);
|
||||
writer
|
||||
.purge_checkpoint(&receipt)
|
||||
.await
|
||||
.expect("existing exact receipt API remains compatible after partial purge");
|
||||
assert_eq!(
|
||||
adapter
|
||||
.get_log_state()
|
||||
.await
|
||||
.expect("exact compatibility state")
|
||||
.last_purged_log_id,
|
||||
Some(id(2).into())
|
||||
);
|
||||
assert_eq!(logs(&mut adapter).await, vec![entry(3), entry(4), entry(5)]);
|
||||
assert_eq!(
|
||||
writer
|
||||
.created_events_page(None, 2)
|
||||
.await
|
||||
.expect("original events retained")
|
||||
.events,
|
||||
events.events
|
||||
);
|
||||
drop(adapter);
|
||||
drop(writer);
|
||||
let mut reopened = NativeRaftLogStore::new(NativeLogWriter::new(
|
||||
Store::open(directory.path().join("db"), identity()).expect("reopen continued apply and exact purge"),
|
||||
));
|
||||
assert_eq!(logs(&mut reopened).await, vec![entry(3), entry(4), entry(5)]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raft_log_storage_purge_rejects_uncovered() {
|
||||
let (directory, writer, mut adapter) = fixture();
|
||||
adapter
|
||||
.blocking_append(entries(2))
|
||||
.await
|
||||
.expect("trait append without snapshot authority");
|
||||
writer
|
||||
.apply_committed(entries(2))
|
||||
.await
|
||||
.expect("applied preparation without checkpoint");
|
||||
let before = std::fs::read(directory.path().join("db")).expect("stable pre-rejection bytes");
|
||||
assert!(
|
||||
adapter.purge(id(1).into()).await.is_err(),
|
||||
"PURGE_REQUIRES_CHECKPOINT_COVERAGE: application progress is not a durable checkpoint"
|
||||
);
|
||||
assert_eq!(
|
||||
std::fs::read(directory.path().join("db")).expect("rejected purge bytes"),
|
||||
before,
|
||||
"RAFT_LOG_UNAUTHORIZED_PURGE_NO_MUTATION"
|
||||
);
|
||||
writer
|
||||
.checkpoint_local(CheckpointLimits::default())
|
||||
.await
|
||||
.expect("checkpoint at applied cut 2");
|
||||
adapter
|
||||
.blocking_append(vec![entry(3)])
|
||||
.await
|
||||
.expect("log beyond checkpoint cut");
|
||||
let before = std::fs::read(directory.path().join("db")).expect("post-checkpoint bytes");
|
||||
assert!(
|
||||
adapter.purge(id(3).into()).await.is_err(),
|
||||
"PURGE_REQUIRES_CHECKPOINT_COVERAGE: retained target exceeds durable cut"
|
||||
);
|
||||
assert_eq!(
|
||||
std::fs::read(directory.path().join("db")).expect("uncovered target rejection bytes"),
|
||||
before,
|
||||
"RAFT_LOG_UNCOVERED_PURGE_NO_MUTATION"
|
||||
);
|
||||
assert_eq!(logs(&mut adapter).await, entries(3));
|
||||
assert_eq!(adapter.get_log_state().await.expect("unchanged marker").last_purged_log_id, None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raft_log_storage_purge_identity() {
|
||||
let (directory, writer, mut adapter) = fixture();
|
||||
checkpoint(&writer, &mut adapter).await;
|
||||
for target in [
|
||||
LogId { term: 9, ..id(1) },
|
||||
LogId { leader_node: 1, ..id(1) },
|
||||
LogId { term: 9, ..id(2) },
|
||||
] {
|
||||
let before = std::fs::read(directory.path().join("db")).expect("before wrong complete LogId");
|
||||
assert!(adapter.purge(target.into()).await.is_err(), "RAFT_LOG_PURGE_EXACT_LOG_ID");
|
||||
assert_eq!(std::fs::read(directory.path().join("db")).expect("after identity rejection"), before);
|
||||
}
|
||||
adapter.purge(id(1).into()).await.expect("valid identity");
|
||||
let mut wrong = id(1);
|
||||
wrong.term += 1;
|
||||
assert!(adapter.purge(wrong.into()).await.is_err(), "RAFT_LOG_PURGED_REPEAT_IDENTITY_BOUND");
|
||||
assert_eq!(
|
||||
adapter
|
||||
.get_log_state()
|
||||
.await
|
||||
.expect("unchanged identity marker")
|
||||
.last_purged_log_id,
|
||||
Some(id(1).into())
|
||||
);
|
||||
adapter.purge(id(2).into()).await.expect("final exact covered target");
|
||||
let state = adapter.get_log_state().await.expect("no remaining logs");
|
||||
assert_eq!(state.last_purged_log_id, Some(id(2).into()));
|
||||
assert_eq!(state.last_log_id, Some(id(2).into()), "RAFT_LOG_EMPTY_TAIL_USES_PURGED_LOG_ID");
|
||||
assert!(logs(&mut adapter).await.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn raft_log_storage_cancelled_append() {
|
||||
let (directory, backend, writer, adapter) = media_fixture();
|
||||
let (entered, resume) = backend.pause_sync(1, false);
|
||||
let mut first = adapter.clone();
|
||||
let task = tokio::spawn(async move { first.blocking_append(entries(2)).await });
|
||||
entered.wait(); // Actual redb commit has begun but its first sync is paused.
|
||||
assert!(!task.is_finished(), "RAFT_LOG_FLUSH_NOT_BEFORE_DURABILITY");
|
||||
task.abort();
|
||||
assert!(task.await.expect_err("cancelled blocking append waiter").is_cancelled());
|
||||
assert!(writer.ordering.try_lock().is_err(), "RAFT_LOG_CANCELLED_APPEND_RETAINS_ORDERING_OWNER");
|
||||
let mut following = adapter.clone();
|
||||
let next = tokio::spawn(async move { following.blocking_append(vec![entry(3)]).await });
|
||||
assert!(writer.ordering.try_lock().is_err(), "RAFT_LOG_NEXT_APPEND_CANNOT_BYPASS_PENDING_SYNC");
|
||||
resume.wait();
|
||||
next.await.expect("next trait append task").expect("ordered follow-up append");
|
||||
let mut adapter = adapter;
|
||||
assert_eq!(
|
||||
logs(&mut adapter).await,
|
||||
entries(3),
|
||||
"RAFT_LOG_CANCELLED_WAITER_BACKGROUND_COMMIT_COMPLETE"
|
||||
);
|
||||
backend.cut();
|
||||
drop(adapter);
|
||||
drop(writer);
|
||||
let mut reopened = NativeRaftLogStore::new(NativeLogWriter::new(
|
||||
Store::open(directory.path().join("db"), identity()).expect("reopen cancelled waiter background commit"),
|
||||
));
|
||||
assert_eq!(logs(&mut reopened).await, entries(3));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn raft_log_storage_failed_append() {
|
||||
let (directory, backend, writer, adapter) = media_fixture();
|
||||
let (entered, resume) = backend.pause_sync(1, false);
|
||||
let mut appending = adapter.clone();
|
||||
let task = tokio::spawn(async move { appending.blocking_append(entries(2)).await });
|
||||
entered.wait();
|
||||
assert!(!task.is_finished(), "RAFT_LOG_FAILED_FLUSH_NOT_EARLY_SUCCESS");
|
||||
backend.cut();
|
||||
resume.wait();
|
||||
assert!(
|
||||
task.await.expect("failed actual trait append task").is_err(),
|
||||
"RAFT_LOG_FAILED_APPEND_NEVER_REPORTS_CALLBACK_SUCCESS"
|
||||
);
|
||||
let mut adapter = adapter;
|
||||
assert!(
|
||||
adapter.blocking_append(entries(2)).await.is_err(),
|
||||
"RAFT_LOG_INDETERMINATE_STORE_REJECTS_LATER_APPEND"
|
||||
);
|
||||
assert!(
|
||||
adapter.save_vote(&openraft::Vote::new_committed(9, 0)).await.is_err(),
|
||||
"RAFT_LOG_INDETERMINATE_STORE_REJECTS_LATER_VOTE"
|
||||
);
|
||||
drop(adapter);
|
||||
drop(writer);
|
||||
let mut reopened = NativeRaftLogStore::new(NativeLogWriter::new(
|
||||
Store::open(directory.path().join("db"), identity()).expect("reopen failed-before-first-sync append"),
|
||||
));
|
||||
assert!(logs(&mut reopened).await.is_empty(), "RAFT_LOG_FAILURE_NO_PARTIAL_BATCH");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raft_log_storage_truncate_uncommitted_suffix() {
|
||||
let (directory, writer, mut adapter) = fixture();
|
||||
adapter.blocking_append(entries(4)).await.expect("trait append full log");
|
||||
adapter
|
||||
.save_committed(Some(id(2).into()))
|
||||
.await
|
||||
.expect("protect committed prefix");
|
||||
assert!(
|
||||
adapter.truncate(id(2).into()).await.is_err(),
|
||||
"RAFT_LOG_TRUNCATE_PROTECTS_COMMITTED_PREFIX"
|
||||
);
|
||||
adapter
|
||||
.truncate(id(3).into())
|
||||
.await
|
||||
.expect("truncate includes uncommitted target");
|
||||
assert_eq!(logs(&mut adapter).await, entries(2), "RAFT_LOG_TRUNCATE_INCLUSIVE");
|
||||
let mut replacement = entry(3);
|
||||
replacement.log_id = LogId { term: 9, ..id(3) }.into();
|
||||
adapter
|
||||
.blocking_append(vec![replacement.clone()])
|
||||
.await
|
||||
.expect("trait append replacement suffix");
|
||||
assert_eq!(
|
||||
adapter.get_log_state().await.expect("replacement tail").last_log_id,
|
||||
Some(replacement.log_id)
|
||||
);
|
||||
drop(adapter);
|
||||
drop(writer);
|
||||
let mut reopened = NativeRaftLogStore::new(NativeLogWriter::new(
|
||||
Store::open(directory.path().join("db"), identity()).expect("reopen truncate and replacement"),
|
||||
));
|
||||
let mut expected = entries(2);
|
||||
expected.push(replacement);
|
||||
assert_eq!(logs(&mut reopened).await, expected);
|
||||
}
|
||||
@@ -0,0 +1,251 @@
|
||||
//! Explicit lifecycle for real OpenRaft over one durable native capture owner.
|
||||
use crate::{
|
||||
CaptureCommand, CaptureNode, CaptureNodeError, CaptureTypeConfig, CheckpointLimits, NativeLogWriter, NativeRaftLogStore,
|
||||
NativeRaftStateMachine, NativeSnapshotError, NativeWriteError, StoreError,
|
||||
};
|
||||
use openraft::{
|
||||
Config, ConfigError, Raft, RaftMetrics, RaftNetworkFactory,
|
||||
error::{ClientWriteError, Fatal, InitializeError, RaftError},
|
||||
raft::ClientWriteResponse,
|
||||
};
|
||||
use redb::ReadableDatabase;
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
},
|
||||
};
|
||||
use tokio::sync::watch;
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum NativeRaftOwnerError {
|
||||
#[error(transparent)]
|
||||
Config(#[from] Box<ConfigError>),
|
||||
#[error(transparent)]
|
||||
Store(#[from] NativeWriteError),
|
||||
#[error(transparent)]
|
||||
Snapshot(#[from] NativeSnapshotError),
|
||||
#[error("runtime node ID {runtime} differs from durable node ID {durable}")]
|
||||
IdentityMismatch { runtime: u64, durable: u64 },
|
||||
#[error("bootstrap must contain the local node")]
|
||||
MissingLocalMember,
|
||||
#[error(transparent)]
|
||||
Node(#[from] CaptureNodeError),
|
||||
#[error("native Raft owner is stopping or stopped")]
|
||||
Stopped,
|
||||
#[error(transparent)]
|
||||
Fatal(Box<Fatal<u64>>),
|
||||
#[error(transparent)]
|
||||
Initialize(Box<RaftError<u64, InitializeError<u64, CaptureNode>>>),
|
||||
#[error(transparent)]
|
||||
Write(Box<RaftError<u64, ClientWriteError<u64, CaptureNode>>>),
|
||||
#[error("native Raft shutdown task failed")]
|
||||
Shutdown(#[source] tokio::task::JoinError),
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
#[error(transparent)]
|
||||
Reader(#[from] crate::source_reader::ReaderError),
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct DrainState {
|
||||
complete: AtomicBool,
|
||||
changed: tokio::sync::Notify,
|
||||
#[cfg(test)]
|
||||
waiters: Arc<std::sync::atomic::AtomicUsize>,
|
||||
}
|
||||
impl DrainState {
|
||||
async fn wait(&self) {
|
||||
#[cfg(test)]
|
||||
let _waiter = DrainWaiter::new(self.waiters.clone());
|
||||
loop {
|
||||
let notified = self.changed.notified();
|
||||
tokio::pin!(notified);
|
||||
notified.as_mut().enable();
|
||||
if self.complete.load(Ordering::Acquire) {
|
||||
return;
|
||||
}
|
||||
notified.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
#[cfg(test)]
|
||||
struct DrainWaiter(Arc<std::sync::atomic::AtomicUsize>);
|
||||
#[cfg(test)]
|
||||
impl DrainWaiter {
|
||||
fn new(waiters: Arc<std::sync::atomic::AtomicUsize>) -> Self {
|
||||
waiters.fetch_add(1, Ordering::AcqRel);
|
||||
Self(waiters)
|
||||
}
|
||||
}
|
||||
#[cfg(test)]
|
||||
impl Drop for DrainWaiter {
|
||||
fn drop(&mut self) {
|
||||
self.0.fetch_sub(1, Ordering::AcqRel);
|
||||
}
|
||||
}
|
||||
|
||||
// The real worker, snapshot builders and replication readers share this one lease.
|
||||
// Its final drop runs after their NativeLogWriter fields have been released.
|
||||
pub(crate) struct OwnerLease {
|
||||
drain: Arc<DrainState>,
|
||||
}
|
||||
impl Drop for OwnerLease {
|
||||
fn drop(&mut self) {
|
||||
self.drain.complete.store(true, Ordering::Release);
|
||||
self.drain.changed.notify_waiters();
|
||||
}
|
||||
}
|
||||
|
||||
/// A lifecycle owner, not a transport. All storage adapters share its original writer.
|
||||
pub struct NativeRaftOwner {
|
||||
raft: Raft<CaptureTypeConfig>,
|
||||
writer: NativeLogWriter,
|
||||
identity: crate::NodeIdentity,
|
||||
accepting: AtomicBool,
|
||||
drain: Arc<DrainState>,
|
||||
#[cfg(test)]
|
||||
pub(crate) snapshot_build_pause: Arc<crate::raft_state_machine::TestPause>,
|
||||
}
|
||||
impl NativeRaftOwner {
|
||||
pub async fn start<N>(
|
||||
writer: NativeLogWriter,
|
||||
node_id: u64,
|
||||
config: Arc<Config>,
|
||||
limits: CheckpointLimits,
|
||||
network: N,
|
||||
) -> Result<Self, NativeRaftOwnerError>
|
||||
where
|
||||
N: RaftNetworkFactory<CaptureTypeConfig>,
|
||||
{
|
||||
// All operator-input checks precede Raft::new and its background workers.
|
||||
let config = Arc::new(
|
||||
(*config)
|
||||
.clone()
|
||||
.validate()
|
||||
.map_err(|error| NativeRaftOwnerError::Config(Box::new(error)))?,
|
||||
);
|
||||
let mut state_machine = NativeRaftStateMachine::new(writer.clone(), limits)?;
|
||||
let identity = writer
|
||||
.write(|store| {
|
||||
if !store.writable {
|
||||
return Err(StoreError::WriteDisabled);
|
||||
}
|
||||
let transaction = store.database.begin_read()?;
|
||||
let meta = transaction.open_table(crate::META)?;
|
||||
crate::application::native_application_state(&transaction)?;
|
||||
crate::application::read_binding(&meta)?;
|
||||
crate::decode_identity(meta.get("identity")?.ok_or(StoreError::CorruptRecord)?.value())
|
||||
})
|
||||
.await?;
|
||||
if identity.node_id != node_id {
|
||||
return Err(NativeRaftOwnerError::IdentityMismatch {
|
||||
runtime: node_id,
|
||||
durable: identity.node_id,
|
||||
});
|
||||
}
|
||||
let drain = Arc::new(DrainState::default());
|
||||
let lease = Arc::new(OwnerLease { drain: drain.clone() });
|
||||
state_machine.owner_lease = Some(lease.clone());
|
||||
let mut logs = NativeRaftLogStore::new(writer.clone());
|
||||
logs.owner_lease = Some(lease);
|
||||
#[cfg(test)]
|
||||
let snapshot_build_pause = state_machine.build_before_ordering.clone();
|
||||
let raft = Raft::new(node_id, config, network, logs, state_machine)
|
||||
.await
|
||||
.map_err(|error| NativeRaftOwnerError::Fatal(Box::new(error)))?;
|
||||
Ok(Self {
|
||||
raft,
|
||||
writer,
|
||||
identity,
|
||||
accepting: AtomicBool::new(true),
|
||||
drain,
|
||||
#[cfg(test)]
|
||||
snapshot_build_pause,
|
||||
})
|
||||
}
|
||||
|
||||
/// Identity read from durable META and checked before starting this owner.
|
||||
pub fn identity(&self) -> crate::NodeIdentity {
|
||||
self.identity
|
||||
}
|
||||
|
||||
fn require_running(&self) -> Result<(), NativeRaftOwnerError> {
|
||||
if self.accepting.load(Ordering::Acquire) {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(NativeRaftOwnerError::Stopped)
|
||||
}
|
||||
}
|
||||
|
||||
/// Only explicit first-cluster formation calls initialize; start never does.
|
||||
pub async fn bootstrap(&self, members: BTreeMap<u64, CaptureNode>) -> Result<(), NativeRaftOwnerError> {
|
||||
self.require_running()?;
|
||||
if !members.contains_key(&self.identity.node_id) {
|
||||
return Err(NativeRaftOwnerError::MissingLocalMember);
|
||||
}
|
||||
for node in members.values() {
|
||||
node.validate()?;
|
||||
}
|
||||
self.raft
|
||||
.initialize(members)
|
||||
.await
|
||||
.map_err(|error| NativeRaftOwnerError::Initialize(Box::new(error)))
|
||||
}
|
||||
|
||||
/// The returned LogId and application response come from actual consensus apply.
|
||||
pub async fn client_write(
|
||||
&self,
|
||||
command: CaptureCommand,
|
||||
) -> Result<ClientWriteResponse<CaptureTypeConfig>, NativeRaftOwnerError> {
|
||||
self.require_running()?;
|
||||
self.raft
|
||||
.client_write(command)
|
||||
.await
|
||||
.map_err(|error| NativeRaftOwnerError::Write(Box::new(error)))
|
||||
}
|
||||
|
||||
/// Re-read an exact successful first decision from this owner's durable view.
|
||||
/// Use the Created result's decision LogId, not an idempotent retry entry's outer LogId.
|
||||
pub async fn authorize_created(
|
||||
&self,
|
||||
expected: crate::DecideCreated,
|
||||
expected_decision: crate::LogId,
|
||||
) -> Result<crate::CommittedCreatedAuthorization, NativeRaftOwnerError> {
|
||||
self.require_running()?;
|
||||
Ok(self.writer.authorize_created(expected, expected_decision).await?)
|
||||
}
|
||||
|
||||
pub fn metrics(&self) -> watch::Receiver<RaftMetrics<u64, CaptureNode>> {
|
||||
self.raft.metrics()
|
||||
}
|
||||
|
||||
/// Transport registration only; raw clones remain trusted caller-owned handles.
|
||||
pub fn raft(&self) -> &Raft<CaptureTypeConfig> {
|
||||
&self.raft
|
||||
}
|
||||
|
||||
/// The caller owns serve/shutdown and must drain it before releasing the Store.
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
pub fn bind_reader(
|
||||
&self,
|
||||
path: impl AsRef<std::path::Path>,
|
||||
) -> Result<(crate::source_reader::SourceReaderServer, crate::source_reader::Shutdown), NativeRaftOwnerError> {
|
||||
self.require_running()?;
|
||||
Ok(crate::source_reader::SourceReaderServer::bind(path, self.writer.clone())?)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn shutdown_waiters(&self) -> Arc<std::sync::atomic::AtomicUsize> {
|
||||
self.drain.waiters.clone()
|
||||
}
|
||||
|
||||
/// Stop admission before signaling Raft; await its real core/tick shutdown.
|
||||
pub async fn shutdown(&self) -> Result<(), NativeRaftOwnerError> {
|
||||
self.accepting.store(false, Ordering::Release);
|
||||
self.raft.shutdown().await.map_err(NativeRaftOwnerError::Shutdown)?;
|
||||
// OpenRaft joins core/tick but detaches the SM worker and snapshot builders.
|
||||
self.drain.wait().await;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,885 @@
|
||||
//! Real Raft engines and real redb; direct RPC routing exists only inside these tests.
|
||||
use super::*;
|
||||
use openraft::{
|
||||
Raft, RaftNetwork, RaftNetworkFactory, ServerState,
|
||||
error::{InstallSnapshotError, RPCError, RaftError, RemoteError, Unreachable},
|
||||
network::{Backoff, RPCOption},
|
||||
raft::{
|
||||
AppendEntriesRequest, AppendEntriesResponse, InstallSnapshotRequest, InstallSnapshotResponse, VoteRequest, VoteResponse,
|
||||
},
|
||||
};
|
||||
use std::{
|
||||
collections::{BTreeMap, BTreeSet},
|
||||
sync::{
|
||||
Arc, Mutex,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
type Engine = Raft<CaptureTypeConfig>;
|
||||
type RpcError = RPCError<u64, CaptureNode, RaftError<u64>>;
|
||||
type SnapshotRpcError = RPCError<u64, CaptureNode, RaftError<u64, InstallSnapshotError>>;
|
||||
type PausedLinks = Mutex<BTreeMap<(u64, u64), (Arc<tokio::sync::Notify>, Arc<tokio::sync::Notify>)>>;
|
||||
#[derive(Default)]
|
||||
struct Routes {
|
||||
nodes: Mutex<BTreeMap<u64, Engine>>,
|
||||
blocked: Mutex<BTreeSet<(u64, u64)>>,
|
||||
snapshot_installs: AtomicUsize,
|
||||
pause_append: PausedLinks,
|
||||
}
|
||||
impl Routes {
|
||||
fn register(&self, id: u64, raft: Engine) {
|
||||
self.nodes.lock().expect("routes").insert(id, raft);
|
||||
}
|
||||
fn remove(&self, id: u64) {
|
||||
self.nodes.lock().expect("routes").remove(&id);
|
||||
}
|
||||
fn isolate(&self, id: u64, members: &[u64]) {
|
||||
let mut blocked = self.blocked.lock().expect("partition routes");
|
||||
for peer in members {
|
||||
if *peer != id {
|
||||
blocked.insert((id, *peer));
|
||||
blocked.insert((*peer, id));
|
||||
}
|
||||
}
|
||||
}
|
||||
fn heal(&self) {
|
||||
self.blocked.lock().expect("restore routes").clear();
|
||||
}
|
||||
fn endpoint(&self, source: u64, target: u64) -> Result<Engine, Unreachable> {
|
||||
if self.blocked.lock().expect("partition check").contains(&(source, target)) {
|
||||
return Err(Unreachable::new(&std::io::Error::other("test partition")));
|
||||
}
|
||||
self.nodes
|
||||
.lock()
|
||||
.expect("endpoint registry")
|
||||
.get(&target)
|
||||
.cloned()
|
||||
.ok_or_else(|| Unreachable::new(&std::io::Error::other("test endpoint unavailable")))
|
||||
}
|
||||
}
|
||||
#[derive(Clone)]
|
||||
struct DirectFactory {
|
||||
source: u64,
|
||||
routes: Arc<Routes>,
|
||||
}
|
||||
struct DirectClient {
|
||||
source: u64,
|
||||
target: u64,
|
||||
node: CaptureNode,
|
||||
routes: Arc<Routes>,
|
||||
}
|
||||
impl RaftNetworkFactory<CaptureTypeConfig> for DirectFactory {
|
||||
type Network = DirectClient;
|
||||
async fn new_client(&mut self, target: u64, node: &CaptureNode) -> DirectClient {
|
||||
DirectClient {
|
||||
source: self.source,
|
||||
target,
|
||||
node: node.clone(),
|
||||
routes: self.routes.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
impl RaftNetwork<CaptureTypeConfig> for DirectClient {
|
||||
async fn append_entries(
|
||||
&mut self,
|
||||
request: AppendEntriesRequest<CaptureTypeConfig>,
|
||||
_option: RPCOption,
|
||||
) -> Result<AppendEntriesResponse<u64>, RpcError> {
|
||||
let pause = self
|
||||
.routes
|
||||
.pause_append
|
||||
.lock()
|
||||
.expect("actual RPC pause hook")
|
||||
.remove(&(self.source, self.target));
|
||||
if let Some((entered, resume)) = pause {
|
||||
entered.notify_one();
|
||||
resume.notified().await;
|
||||
}
|
||||
let raft = self.routes.endpoint(self.source, self.target)?;
|
||||
raft.append_entries(request)
|
||||
.await
|
||||
.map_err(|error| RemoteError::new_with_node(self.target, self.node.clone(), error).into())
|
||||
}
|
||||
async fn vote(&mut self, request: VoteRequest<u64>, _option: RPCOption) -> Result<VoteResponse<u64>, RpcError> {
|
||||
let raft = self.routes.endpoint(self.source, self.target)?;
|
||||
raft.vote(request)
|
||||
.await
|
||||
.map_err(|error| RemoteError::new_with_node(self.target, self.node.clone(), error).into())
|
||||
}
|
||||
async fn install_snapshot(
|
||||
&mut self,
|
||||
request: InstallSnapshotRequest<CaptureTypeConfig>,
|
||||
_option: RPCOption,
|
||||
) -> Result<InstallSnapshotResponse<u64>, SnapshotRpcError> {
|
||||
let raft = self.routes.endpoint(self.source, self.target)?;
|
||||
let done = request.done;
|
||||
let result = raft
|
||||
.install_snapshot(request)
|
||||
.await
|
||||
.map_err(|error| RemoteError::new_with_node(self.target, self.node.clone(), error).into());
|
||||
if done && result.is_ok() {
|
||||
self.routes.snapshot_installs.fetch_add(1, Ordering::AcqRel);
|
||||
}
|
||||
result
|
||||
}
|
||||
fn backoff(&self) -> Backoff {
|
||||
Backoff::new(std::iter::repeat(Duration::from_millis(25)))
|
||||
}
|
||||
}
|
||||
fn identity(node_id: u64) -> NodeIdentity {
|
||||
NodeIdentity {
|
||||
cluster_id: [7; 16],
|
||||
node_id,
|
||||
}
|
||||
}
|
||||
fn binding() -> CaptureBinding {
|
||||
CaptureBinding {
|
||||
revision: 9,
|
||||
binding_id: [8; 16],
|
||||
target_id: [9; 16],
|
||||
}
|
||||
}
|
||||
fn command(value: u8) -> CaptureCommand {
|
||||
CaptureCommand::try_created(DecideCreated {
|
||||
operation_id: [value; 16],
|
||||
object: ObjectIdentity {
|
||||
bucket_incarnation: [2; 16],
|
||||
key: vec![value],
|
||||
},
|
||||
expected_head: None,
|
||||
prepared: PreparedIdentity {
|
||||
version_id: [value; 16],
|
||||
preparation_id: [4; 16],
|
||||
content_digest: [5; 32],
|
||||
content_length: 123,
|
||||
},
|
||||
binding_revision: 9,
|
||||
})
|
||||
.expect("native client command")
|
||||
}
|
||||
fn config(snapshot_every: u64) -> Arc<openraft::Config> {
|
||||
Arc::new(
|
||||
openraft::Config {
|
||||
cluster_name: "native-owner-tests".into(),
|
||||
heartbeat_interval: 100,
|
||||
election_timeout_min: 500,
|
||||
election_timeout_max: 1000,
|
||||
snapshot_policy: openraft::SnapshotPolicy::LogsSinceLast(snapshot_every),
|
||||
max_in_snapshot_log_to_keep: 0,
|
||||
purge_batch_size: 1,
|
||||
..openraft::Config::default()
|
||||
}
|
||||
.validate()
|
||||
.expect("valid real Raft test configuration"),
|
||||
)
|
||||
}
|
||||
fn members(ids: &[u64]) -> BTreeMap<u64, CaptureNode> {
|
||||
ids.iter()
|
||||
.map(|id| {
|
||||
(
|
||||
*id,
|
||||
CaptureNode::new(format!("http://127.0.0.1:{}", 7000 + id)).expect("bounded test endpoint"),
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
struct Node {
|
||||
directory: tempfile::TempDir,
|
||||
writer: NativeLogWriter,
|
||||
owner: Option<NativeRaftOwner>,
|
||||
id: u64,
|
||||
}
|
||||
impl Node {
|
||||
async fn start(id: u64, routes: Arc<Routes>, config: Arc<openraft::Config>) -> Self {
|
||||
let directory = tempfile::tempdir().expect("real Raft owner directory");
|
||||
let mut store = Store::initialize(directory.path().join("db"), identity(id)).expect("real owner redb");
|
||||
store.pin_capture_binding(binding()).expect("captured binding");
|
||||
store.enable_native_empty().expect("native storage mode");
|
||||
store.promote_created_provenance().expect("native provenance");
|
||||
let writer = NativeLogWriter::new(store);
|
||||
let owner = NativeRaftOwner::start(
|
||||
writer.clone(),
|
||||
id,
|
||||
config,
|
||||
CheckpointLimits::default(),
|
||||
DirectFactory {
|
||||
source: id,
|
||||
routes: routes.clone(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("actual owner start");
|
||||
routes.register(id, owner.raft().clone());
|
||||
Self {
|
||||
directory,
|
||||
writer,
|
||||
owner: Some(owner),
|
||||
id,
|
||||
}
|
||||
}
|
||||
fn owner(&self) -> &NativeRaftOwner {
|
||||
self.owner.as_ref().expect("live owner")
|
||||
}
|
||||
async fn page(&self) -> EventPage {
|
||||
self.writer
|
||||
.created_events_page(None, 256)
|
||||
.await
|
||||
.expect("actual committed events")
|
||||
}
|
||||
async fn stop(&mut self, routes: &Routes) {
|
||||
self.owner
|
||||
.take()
|
||||
.expect("running owner")
|
||||
.shutdown()
|
||||
.await
|
||||
.expect("real Raft shutdown");
|
||||
routes.remove(self.id);
|
||||
}
|
||||
}
|
||||
struct Cluster {
|
||||
nodes: Vec<Node>,
|
||||
routes: Arc<Routes>,
|
||||
}
|
||||
impl Cluster {
|
||||
async fn start(ids: &[u64], snapshot_every: u64) -> Self {
|
||||
let routes = Arc::new(Routes::default());
|
||||
let mut nodes = Vec::new();
|
||||
for id in ids {
|
||||
nodes.push(Node::start(*id, routes.clone(), config(snapshot_every)).await);
|
||||
}
|
||||
nodes[0]
|
||||
.owner()
|
||||
.bootstrap(members(ids))
|
||||
.await
|
||||
.expect("explicit initial cluster bootstrap");
|
||||
let cluster = Self { nodes, routes };
|
||||
cluster.leader(ids).await;
|
||||
cluster
|
||||
}
|
||||
fn node(&self, id: u64) -> &Node {
|
||||
self.nodes.iter().find(|node| node.id == id).expect("cluster node")
|
||||
}
|
||||
async fn leader(&self, eligible: &[u64]) -> u64 {
|
||||
tokio::time::timeout(Duration::from_secs(15), async {
|
||||
loop {
|
||||
for id in eligible {
|
||||
let node = self.node(*id);
|
||||
let Some(owner) = &node.owner else {
|
||||
continue;
|
||||
};
|
||||
let metrics = owner.metrics().borrow().clone();
|
||||
assert!(metrics.running_state.is_ok(), "real Raft owner fatal state: {:?}", metrics.running_state);
|
||||
if metrics.state == ServerState::Leader && metrics.current_leader == Some(*id) {
|
||||
return *id;
|
||||
}
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("real leader election readiness")
|
||||
}
|
||||
async fn applied(&self, ids: &[u64], index: u64) {
|
||||
for id in ids {
|
||||
self.node(*id)
|
||||
.owner()
|
||||
.raft()
|
||||
.wait(Some(Duration::from_secs(15)))
|
||||
.applied_index_at_least(Some(index), "real replicated apply readiness")
|
||||
.await
|
||||
.expect("committed log applied on real node");
|
||||
}
|
||||
}
|
||||
async fn stop(mut self) {
|
||||
for node in &mut self.nodes {
|
||||
if node.owner.is_some() {
|
||||
node.stop(&self.routes).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raft_owner_single_node_bootstrap() {
|
||||
let cluster = Cluster::start(&[0], 100).await;
|
||||
let response = cluster
|
||||
.node(0)
|
||||
.owner()
|
||||
.client_write(command(1))
|
||||
.await
|
||||
.expect("real client_write");
|
||||
assert!(
|
||||
matches!(response.data.created_result(),Some(CreatedResult::Created { event_id,.. }) if *event_id==[1;16]),
|
||||
"OWNER_TRUE_RAFT_CREATED_RESPONSE"
|
||||
);
|
||||
let page = cluster.node(0).page().await;
|
||||
assert_eq!(page.events.len(), 1);
|
||||
assert_eq!(
|
||||
page.events[0].captured.decision_log_id,
|
||||
LogId::from(response.log_id),
|
||||
"OWNER_COMMITTED_EVENT_LOG_ID"
|
||||
);
|
||||
assert!(
|
||||
cluster.node(0).owner().bootstrap(members(&[0])).await.is_err(),
|
||||
"OWNER_BOOTSTRAP_NOT_REINITIALIZATION"
|
||||
);
|
||||
cluster.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raft_owner_single_node_restart() {
|
||||
let routes = Arc::new(Routes::default());
|
||||
let mut node = Node::start(0, routes.clone(), config(100)).await;
|
||||
node.owner().bootstrap(members(&[0])).await.expect("first bootstrap only");
|
||||
node.owner()
|
||||
.raft()
|
||||
.wait(Some(Duration::from_secs(15)))
|
||||
.state(ServerState::Leader, "single leader")
|
||||
.await
|
||||
.expect("leader ready");
|
||||
node.owner().client_write(command(1)).await.expect("first genuine write");
|
||||
let before = node.page().await;
|
||||
node.stop(&routes).await;
|
||||
let Node {
|
||||
directory,
|
||||
writer,
|
||||
owner: _,
|
||||
id,
|
||||
} = node;
|
||||
let vote = writer.read_vote().await.expect("durable vote before reopen");
|
||||
drop(writer);
|
||||
let store = Store::open(directory.path().join("db"), identity(id)).expect("restart same durable identity");
|
||||
assert_eq!(store.read_vote().expect("persisted vote"), vote.map(Into::into));
|
||||
let writer = NativeLogWriter::new(store);
|
||||
let owner = NativeRaftOwner::start(
|
||||
writer.clone(),
|
||||
id,
|
||||
config(100),
|
||||
CheckpointLimits::default(),
|
||||
DirectFactory {
|
||||
source: id,
|
||||
routes: routes.clone(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("restart existing Raft");
|
||||
routes.register(id, owner.raft().clone());
|
||||
assert!(
|
||||
owner.raft().is_initialized().await.expect("restored cluster membership"),
|
||||
"OWNER_RESTART_DOES_NOT_REQUIRE_BOOTSTRAP"
|
||||
);
|
||||
owner
|
||||
.raft()
|
||||
.wait(Some(Duration::from_secs(15)))
|
||||
.state(ServerState::Leader, "restored leader")
|
||||
.await
|
||||
.expect("restarted leader ready");
|
||||
owner
|
||||
.client_write(command(2))
|
||||
.await
|
||||
.expect("continue after restart without initialize");
|
||||
let after = writer.created_events_page(None, 256).await.expect("restart continued events");
|
||||
assert_eq!(after.events.len(), 2);
|
||||
assert_eq!(after.events[0], before.events[0], "OWNER_RESTART_PRESERVES_FIRST_DECISION");
|
||||
owner.shutdown().await.expect("restart owner shutdown");
|
||||
routes.remove(id);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raft_owner_idempotent_retry() {
|
||||
let cluster = Cluster::start(&[0], 100).await;
|
||||
let owner = cluster.node(0).owner();
|
||||
let first = owner.client_write(command(1)).await.expect("first write");
|
||||
let replay = owner.client_write(command(1)).await.expect("same operation retry");
|
||||
assert_eq!(
|
||||
replay.data.created_result(),
|
||||
first.data.created_result(),
|
||||
"OWNER_RETRY_PRESERVES_FIRST_DECISION_RESULT"
|
||||
);
|
||||
assert!(replay.log_id.index > first.log_id.index, "real retry can append another Raft log");
|
||||
assert_eq!(cluster.node(0).page().await.events.len(), 1, "OWNER_RETRY_NO_DUPLICATE_EVENT");
|
||||
cluster.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raft_owner_identity_binding() {
|
||||
let routes = Arc::new(Routes::default());
|
||||
let mut node = Node::start(0, routes.clone(), config(100)).await;
|
||||
node.owner().bootstrap(members(&[0])).await.expect("real durable membership");
|
||||
node.owner()
|
||||
.raft()
|
||||
.wait(Some(Duration::from_secs(15)))
|
||||
.state(ServerState::Leader, "single leader")
|
||||
.await
|
||||
.expect("leader");
|
||||
node.owner().client_write(command(1)).await.expect("nonempty durable store");
|
||||
node.stop(&routes).await;
|
||||
let before = std::fs::read(node.directory.path().join("db")).expect("stable identity-bound store");
|
||||
let result = NativeRaftOwner::start(
|
||||
node.writer.clone(),
|
||||
1,
|
||||
config(100),
|
||||
CheckpointLimits::default(),
|
||||
DirectFactory {
|
||||
source: 1,
|
||||
routes: routes.clone(),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
matches!(result, Err(NativeRaftOwnerError::IdentityMismatch { runtime: 1, durable: 0 })),
|
||||
"OWNER_REQUIRES_DURABLE_NODE_ID: runtime node cannot reinterpret stored membership/identity"
|
||||
);
|
||||
assert_eq!(
|
||||
std::fs::read(node.directory.path().join("db")).expect("unchanged identity-bound store"),
|
||||
before
|
||||
);
|
||||
let invalid_config = Arc::new(openraft::Config {
|
||||
election_timeout_min: 1000,
|
||||
election_timeout_max: 500,
|
||||
..openraft::Config::default()
|
||||
});
|
||||
let invalid = NativeRaftOwner::start(
|
||||
node.writer.clone(),
|
||||
0,
|
||||
invalid_config,
|
||||
CheckpointLimits::default(),
|
||||
DirectFactory {
|
||||
source: 0,
|
||||
routes: routes.clone(),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
matches!(invalid, Err(NativeRaftOwnerError::Config(_))),
|
||||
"OWNER_REJECTS_INVALID_CONFIG_BEFORE_START"
|
||||
);
|
||||
let mut limits = CheckpointLimits::default();
|
||||
limits.snapshot.total_bytes = 0;
|
||||
let invalid = NativeRaftOwner::start(node.writer.clone(), 0, config(100), limits, DirectFactory { source: 0, routes }).await;
|
||||
assert!(
|
||||
matches!(invalid, Err(NativeRaftOwnerError::Snapshot(_))),
|
||||
"OWNER_REJECTS_INVALID_LIMITS_BEFORE_START"
|
||||
);
|
||||
assert_eq!(
|
||||
std::fs::read(node.directory.path().join("db")).expect("invalid start leaves stable bytes"),
|
||||
before
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
#[tokio::test]
|
||||
async fn raft_owner_shared_reader() {
|
||||
use source_reader::SourceReaderClient;
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let cluster = Cluster::start(&[0], 100).await;
|
||||
let node = cluster.node(0);
|
||||
std::fs::set_permissions(node.directory.path(), std::fs::Permissions::from_mode(0o700)).expect("private reader parent");
|
||||
let socket = node.directory.path().join("reader.sock");
|
||||
let (server, shutdown) = node
|
||||
.owner()
|
||||
.bind_reader(&socket)
|
||||
.expect("owner shares live writer with reader");
|
||||
let task = tokio::spawn(server.serve());
|
||||
let client = SourceReaderClient::new(&socket).expect("live reader");
|
||||
assert!(
|
||||
client
|
||||
.page(identity(0).cluster_id, None, 1)
|
||||
.await
|
||||
.expect("initial reader page")
|
||||
.page
|
||||
.events
|
||||
.is_empty()
|
||||
);
|
||||
let response = node
|
||||
.owner()
|
||||
.client_write(command(1))
|
||||
.await
|
||||
.expect("real write while reader runs");
|
||||
let page = client
|
||||
.page(identity(0).cluster_id, None, 1)
|
||||
.await
|
||||
.expect("live committed event");
|
||||
assert_eq!(page.identity, identity(0));
|
||||
assert_eq!(
|
||||
page.page.events[0].captured.decision_log_id,
|
||||
LogId::from(response.log_id),
|
||||
"OWNER_READER_USES_REAL_COMMITTED_WRITER"
|
||||
);
|
||||
shutdown.shutdown();
|
||||
task.await.expect("reader serve task").expect("reader drains");
|
||||
cluster.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raft_owner_shutdown() {
|
||||
let cluster = Cluster::start(&[0], 100).await;
|
||||
let owner = cluster.node(0).owner();
|
||||
owner.client_write(command(1)).await.expect("accepted before shutdown");
|
||||
owner.shutdown().await.expect("actual owner shutdown");
|
||||
assert!(owner.client_write(command(2)).await.is_err(), "OWNER_SHUTDOWN_REJECTS_NEW_WRITES");
|
||||
assert_eq!(cluster.node(0).page().await.events.len(), 1);
|
||||
cluster.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn raft_owner_three_node_majority() {
|
||||
let cluster = Cluster::start(&[0, 1, 2], 100).await;
|
||||
let leader = cluster.leader(&[0, 1, 2]).await;
|
||||
let first = cluster
|
||||
.node(leader)
|
||||
.owner()
|
||||
.client_write(command(1))
|
||||
.await
|
||||
.expect("real quorum write");
|
||||
cluster.applied(&[0, 1, 2], first.log_id.index).await;
|
||||
let follower = [0, 1, 2].into_iter().find(|id| *id != leader).expect("nonleader");
|
||||
let rejected = cluster.node(follower).owner().client_write(command(254)).await;
|
||||
match rejected {
|
||||
Err(NativeRaftOwnerError::Write(error)) => {
|
||||
assert!(
|
||||
matches!(error.api_error(), Some(openraft::error::ClientWriteError::ForwardToLeader(_))),
|
||||
"OWNER_PRESERVES_TYPED_FORWARD_TO_LEADER"
|
||||
);
|
||||
}
|
||||
other => panic!("OWNER_NONLEADER_WRITE_MUST_NOT_SUCCEED: {other:?}"),
|
||||
}
|
||||
cluster.routes.isolate(leader, &[0, 1, 2]);
|
||||
let uncertain =
|
||||
tokio::time::timeout(Duration::from_millis(1500), cluster.node(leader).owner().client_write(command(2))).await;
|
||||
assert!(!matches!(uncertain, Ok(Ok(_))), "OWNER_MAJORITY_REQUIRED_FOR_SUCCESS");
|
||||
let others: Vec<_> = [0, 1, 2].into_iter().filter(|id| *id != leader).collect();
|
||||
let next = cluster.leader(&others).await;
|
||||
let response = cluster
|
||||
.node(next)
|
||||
.owner()
|
||||
.client_write(command(2))
|
||||
.await
|
||||
.expect("majority-owned retry of exact same operation");
|
||||
cluster.routes.heal();
|
||||
cluster.applied(&[0, 1, 2], response.log_id.index).await;
|
||||
for node in &cluster.nodes {
|
||||
assert_eq!(node.page().await.events.len(), 2, "OWNER_QUORUM_RETRY_NO_DUPLICATE_EVENT");
|
||||
}
|
||||
cluster.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn raft_owner_leader_failover() {
|
||||
let mut cluster = Cluster::start(&[0, 1, 2], 100).await;
|
||||
let leader = cluster.leader(&[0, 1, 2]).await;
|
||||
let first = cluster
|
||||
.node(leader)
|
||||
.owner()
|
||||
.client_write(command(1))
|
||||
.await
|
||||
.expect("first confirmed write");
|
||||
cluster.applied(&[0, 1, 2], first.log_id.index).await;
|
||||
let before = cluster.node(leader).page().await.events[0].clone();
|
||||
cluster
|
||||
.nodes
|
||||
.iter_mut()
|
||||
.find(|node| node.id == leader)
|
||||
.expect("old leader")
|
||||
.stop(&cluster.routes)
|
||||
.await;
|
||||
let survivors: Vec<_> = [0, 1, 2].into_iter().filter(|id| *id != leader).collect();
|
||||
let next = cluster.leader(&survivors).await;
|
||||
let response = cluster
|
||||
.node(next)
|
||||
.owner()
|
||||
.client_write(command(2))
|
||||
.await
|
||||
.expect("new leader actual quorum write");
|
||||
cluster.applied(&survivors, response.log_id.index).await;
|
||||
for id in survivors {
|
||||
let page = cluster.node(id).page().await;
|
||||
assert_eq!(page.events.len(), 2);
|
||||
assert_eq!(page.events[0], before, "OWNER_FAILOVER_PRESERVES_CONFIRMED_DECISION");
|
||||
}
|
||||
cluster.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn raft_owner_snapshot_catchup() {
|
||||
let cluster = Cluster::start(&[0, 1, 2], 1000).await;
|
||||
let leader = cluster.leader(&[0, 1, 2]).await;
|
||||
let follower = [0, 1, 2].into_iter().find(|id| *id != leader).expect("lagging follower");
|
||||
let first = cluster
|
||||
.node(leader)
|
||||
.owner()
|
||||
.client_write(command(1))
|
||||
.await
|
||||
.expect("initial confirmed event");
|
||||
cluster.applied(&[0, 1, 2], first.log_id.index).await;
|
||||
// Keep the catch-up test focused on replication rather than an isolated follower's election.
|
||||
cluster.node(follower).owner().raft().runtime_config().elect(false);
|
||||
cluster.routes.isolate(follower, &[0, 1, 2]);
|
||||
let mut last = first;
|
||||
for value in 2..=12 {
|
||||
last = cluster
|
||||
.node(leader)
|
||||
.owner()
|
||||
.client_write(command(value))
|
||||
.await
|
||||
.expect("majority writes while follower offline");
|
||||
}
|
||||
cluster
|
||||
.node(leader)
|
||||
.owner()
|
||||
.raft()
|
||||
.trigger()
|
||||
.snapshot()
|
||||
.await
|
||||
.expect("true Raft snapshot trigger");
|
||||
cluster
|
||||
.node(leader)
|
||||
.owner()
|
||||
.raft()
|
||||
.wait(Some(Duration::from_secs(15)))
|
||||
.metrics(
|
||||
|metrics| metrics.purged.is_some_and(|cut| cut.index >= last.log_id.index),
|
||||
"snapshot durably purged lagging log prefix",
|
||||
)
|
||||
.await
|
||||
.expect("real snapshot and purge");
|
||||
let before = cluster.routes.snapshot_installs.load(Ordering::Acquire);
|
||||
cluster.routes.heal();
|
||||
cluster.applied(&[0, 1, 2], last.log_id.index).await;
|
||||
cluster
|
||||
.node(follower)
|
||||
.owner()
|
||||
.raft()
|
||||
.wait(Some(Duration::from_secs(15)))
|
||||
.metrics(
|
||||
|metrics| metrics.snapshot.is_some_and(|cut| cut.index >= last.log_id.index),
|
||||
"follower installed actual transferred snapshot",
|
||||
)
|
||||
.await
|
||||
.expect("real snapshot catch-up");
|
||||
tokio::time::timeout(Duration::from_secs(15), async {
|
||||
while cluster.routes.snapshot_installs.load(Ordering::Acquire) <= before {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("snapshot readiness includes actual successful done-RPC return");
|
||||
assert!(
|
||||
cluster.routes.snapshot_installs.load(Ordering::Acquire) > before,
|
||||
"OWNER_CATCHUP_USED_ACTUAL_SNAPSHOT_RPC"
|
||||
);
|
||||
let expected = cluster.node(leader).page().await;
|
||||
for node in &cluster.nodes {
|
||||
assert_eq!(node.page().await, expected, "OWNER_SNAPSHOT_CATCHUP_FULL_SOURCE_EVENTS");
|
||||
}
|
||||
cluster.node(follower).owner().raft().runtime_config().elect(true);
|
||||
let current = cluster.leader(&[0, 1, 2]).await;
|
||||
let next = cluster
|
||||
.node(current)
|
||||
.owner()
|
||||
.client_write(command(13))
|
||||
.await
|
||||
.expect("continue after real snapshot install");
|
||||
cluster.applied(&[0, 1, 2], next.log_id.index).await;
|
||||
for node in &cluster.nodes {
|
||||
assert_eq!(node.page().await.events.len(), 13);
|
||||
}
|
||||
cluster.stop().await;
|
||||
}
|
||||
|
||||
async fn wait_for_shutdown_drain(waiters: &AtomicUsize) {
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while waiters.load(Ordering::Acquire) == 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("OWNER_SHUTDOWN_DRAINS_INFLIGHT_WORK: owner reached real worker drain wait");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raft_owner_shutdown_inflight_snapshot() {
|
||||
let routes = Arc::new(Routes::default());
|
||||
let mut node = Node::start(0, routes.clone(), config(1000)).await;
|
||||
node.owner()
|
||||
.bootstrap(members(&[0]))
|
||||
.await
|
||||
.expect("real single-node bootstrap");
|
||||
node.owner()
|
||||
.raft()
|
||||
.wait(Some(Duration::from_secs(15)))
|
||||
.state(ServerState::Leader, "leader before snapshot")
|
||||
.await
|
||||
.expect("leader readiness");
|
||||
node.owner()
|
||||
.client_write(command(1))
|
||||
.await
|
||||
.expect("real committed snapshot input");
|
||||
let entered = Arc::new(tokio::sync::Notify::new());
|
||||
let resume = Arc::new(tokio::sync::Notify::new());
|
||||
*node.owner().snapshot_build_pause.lock().expect("real builder hook") = Some((entered.clone(), resume.clone()));
|
||||
node.owner()
|
||||
.raft()
|
||||
.trigger()
|
||||
.snapshot()
|
||||
.await
|
||||
.expect("real Raft snapshot request");
|
||||
tokio::time::timeout(Duration::from_secs(5), entered.notified())
|
||||
.await
|
||||
.expect("real detached snapshot builder paused");
|
||||
let owner = node.owner.take().expect("running owner");
|
||||
let waiters = owner.shutdown_waiters();
|
||||
let observer = owner.raft().clone();
|
||||
let shutdown = tokio::spawn(async move {
|
||||
let result = owner.shutdown().await;
|
||||
(owner, result)
|
||||
});
|
||||
wait_for_shutdown_drain(&waiters).await;
|
||||
assert!(
|
||||
observer.metrics().borrow().running_state.is_err(),
|
||||
"OWNER_CORE_STOPPED_WHILE_SNAPSHOT_DRAIN_PENDING"
|
||||
);
|
||||
assert!(!shutdown.is_finished(), "OWNER_SHUTDOWN_DRAINS_INFLIGHT_SNAPSHOT");
|
||||
resume.notify_one();
|
||||
let (owner, result) = tokio::time::timeout(Duration::from_secs(10), shutdown)
|
||||
.await
|
||||
.expect("released snapshot lets shutdown finish")
|
||||
.expect("shutdown task");
|
||||
result.expect("snapshot work drained");
|
||||
routes.remove(0);
|
||||
drop(observer);
|
||||
drop(owner);
|
||||
drop(waiters);
|
||||
let Node {
|
||||
directory,
|
||||
writer,
|
||||
owner: _,
|
||||
id,
|
||||
} = node;
|
||||
drop(writer);
|
||||
let store = Store::open(directory.path().join("db"), identity(id)).expect("OWNER_SHUTDOWN_SNAPSHOT_RELEASES_LIVE_DATABASE");
|
||||
assert_eq!(
|
||||
store
|
||||
.created_events_page(None, 256)
|
||||
.expect("retained committed source events")
|
||||
.events
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn raft_owner_shutdown_inflight_replication() {
|
||||
// OpenRaft times append_entries by heartbeat_interval. Keep the real deadline
|
||||
// longer than readiness checks so only explicit RPC release ends this pause.
|
||||
let slow = Arc::new(
|
||||
openraft::Config {
|
||||
heartbeat_interval: 10_000,
|
||||
election_timeout_min: 20_000,
|
||||
election_timeout_max: 30_000,
|
||||
snapshot_policy: openraft::SnapshotPolicy::LogsSinceLast(1000),
|
||||
..openraft::Config::default()
|
||||
}
|
||||
.validate()
|
||||
.expect("long real RPC deadline"),
|
||||
);
|
||||
let routes = Arc::new(Routes::default());
|
||||
let mut nodes = Vec::new();
|
||||
for id in [0, 1, 2] {
|
||||
nodes.push(Node::start(id, routes.clone(), slow.clone()).await);
|
||||
}
|
||||
nodes[0]
|
||||
.owner()
|
||||
.bootstrap(members(&[0, 1, 2]))
|
||||
.await
|
||||
.expect("real three-node bootstrap");
|
||||
nodes[0]
|
||||
.owner()
|
||||
.raft()
|
||||
.trigger()
|
||||
.elect()
|
||||
.await
|
||||
.expect("explicit test election readiness");
|
||||
let mut cluster = Cluster { nodes, routes };
|
||||
let leader = cluster.leader(&[0, 1, 2]).await;
|
||||
let response = cluster
|
||||
.node(leader)
|
||||
.owner()
|
||||
.client_write(command(1))
|
||||
.await
|
||||
.expect("real replicated write before pause");
|
||||
cluster.applied(&[0, 1, 2], response.log_id.index).await;
|
||||
let follower = [0, 1, 2].into_iter().find(|id| *id != leader).expect("replication target");
|
||||
let entered = Arc::new(tokio::sync::Notify::new());
|
||||
let resume = Arc::new(tokio::sync::Notify::new());
|
||||
cluster
|
||||
.routes
|
||||
.pause_append
|
||||
.lock()
|
||||
.expect("actual request pause")
|
||||
.insert((leader, follower), (entered.clone(), resume.clone()));
|
||||
cluster
|
||||
.node(leader)
|
||||
.owner()
|
||||
.raft()
|
||||
.trigger()
|
||||
.heartbeat()
|
||||
.await
|
||||
.expect("real replication RPC");
|
||||
tokio::time::timeout(Duration::from_secs(5), entered.notified())
|
||||
.await
|
||||
.expect("actual AppendEntries in flight");
|
||||
let node = cluster.nodes.iter_mut().find(|node| node.id == leader).expect("leader node");
|
||||
let owner = node.owner.take().expect("leader owner");
|
||||
let waiters = owner.shutdown_waiters();
|
||||
let observer = owner.raft().clone();
|
||||
let shutdown = tokio::spawn(async move {
|
||||
let result = owner.shutdown().await;
|
||||
(owner, result)
|
||||
});
|
||||
wait_for_shutdown_drain(&waiters).await;
|
||||
assert!(
|
||||
observer.metrics().borrow().running_state.is_err(),
|
||||
"OWNER_CORE_STOPPED_WHILE_REPLICATION_DRAIN_PENDING"
|
||||
);
|
||||
assert!(!shutdown.is_finished(), "OWNER_SHUTDOWN_DRAINS_INFLIGHT_REPLICATION");
|
||||
resume.notify_one();
|
||||
let (owner, result) = tokio::time::timeout(Duration::from_secs(10), shutdown)
|
||||
.await
|
||||
.expect("released real RPC lets shutdown finish")
|
||||
.expect("replication shutdown task");
|
||||
result.expect("actual replication reader drained");
|
||||
cluster.routes.remove(leader);
|
||||
drop(observer);
|
||||
drop(owner);
|
||||
drop(waiters);
|
||||
let position = cluster
|
||||
.nodes
|
||||
.iter()
|
||||
.position(|node| node.id == leader)
|
||||
.expect("stopped leader");
|
||||
let Node {
|
||||
directory,
|
||||
writer,
|
||||
owner: _,
|
||||
id,
|
||||
} = cluster.nodes.remove(position);
|
||||
drop(writer);
|
||||
let store =
|
||||
Store::open(directory.path().join("db"), identity(id)).expect("OWNER_SHUTDOWN_REPLICATION_RELEASES_LIVE_DATABASE");
|
||||
assert_eq!(
|
||||
store
|
||||
.created_events_page(None, 256)
|
||||
.expect("real quorum event remains")
|
||||
.events
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
drop(store);
|
||||
cluster.stop().await;
|
||||
}
|
||||
|
||||
#[path = "committed_created_authorization_tests.rs"]
|
||||
mod authorization;
|
||||
@@ -0,0 +1,327 @@
|
||||
//! Durable OpenRaft application state and selected baseline snapshots.
|
||||
use crate::{
|
||||
CaptureRaftEntry, CaptureResponse, CaptureTypeConfig, CheckpointLimits, NativeLogWriter, NativeSnapshotData,
|
||||
NativeSnapshotError, StoreError, checkpoint_baseline, snapshot,
|
||||
};
|
||||
use openraft::{
|
||||
OptionalSend, RaftSnapshotBuilder, Snapshot, SnapshotMeta, StorageError, StorageIOError, StoredMembership,
|
||||
storage::RaftStateMachine,
|
||||
};
|
||||
use redb::{ReadableDatabase, ReadableTable, ReadableTableMetadata};
|
||||
use std::{
|
||||
io::{Seek, Write},
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
},
|
||||
};
|
||||
|
||||
pub(crate) const SNAPSHOT_META: &str = "native_snapshot_meta_v1";
|
||||
type Meta = SnapshotMeta<u64, crate::CaptureNode>;
|
||||
#[derive(serde::Serialize, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct SelectedMeta {
|
||||
generation: u64,
|
||||
meta: Meta,
|
||||
}
|
||||
|
||||
pub(crate) fn publish_meta(
|
||||
meta: &mut redb::Table<&str, &[u8]>,
|
||||
baseline: &checkpoint_baseline::BaselineMetadata,
|
||||
value: Meta,
|
||||
) -> Result<(), StoreError> {
|
||||
let bytes = serde_json::to_vec(&SelectedMeta {
|
||||
generation: baseline.generation,
|
||||
meta: value,
|
||||
})
|
||||
.map_err(|_| StoreError::CorruptRecord)?;
|
||||
if bytes.len() > 1024 * 1024 {
|
||||
return Err(StoreError::ResourceLimit);
|
||||
}
|
||||
meta.insert(SNAPSHOT_META, bytes.as_slice())?;
|
||||
meta.insert("native_snapshot_meta_required", &[1u8][..])?;
|
||||
Ok(())
|
||||
}
|
||||
pub(crate) fn local_meta(
|
||||
baseline: &checkpoint_baseline::BaselineMetadata,
|
||||
identity: crate::NodeIdentity,
|
||||
) -> Result<Meta, StoreError> {
|
||||
Ok(Meta {
|
||||
last_log_id: baseline.header.applied.map(Into::into),
|
||||
last_membership: StoredMembership::try_from(&baseline.header.membership)?,
|
||||
snapshot_id: format!("native-local-{:x?}-{}-{}", identity.cluster_id, identity.node_id, baseline.generation),
|
||||
})
|
||||
}
|
||||
pub(crate) fn publish_local_meta(
|
||||
meta: &mut redb::Table<&str, &[u8]>,
|
||||
baseline: &checkpoint_baseline::BaselineMetadata,
|
||||
) -> Result<(), StoreError> {
|
||||
let identity = crate::decode_identity(meta.get("identity")?.ok_or(StoreError::CorruptRecord)?.value())?;
|
||||
publish_meta(meta, baseline, local_meta(baseline, identity)?)
|
||||
}
|
||||
pub(crate) fn selected_meta(
|
||||
meta: &impl ReadableTable<&'static str, &'static [u8]>,
|
||||
baseline: &checkpoint_baseline::BaselineMetadata,
|
||||
) -> Result<Meta, StoreError> {
|
||||
let Some(bytes) = meta.get(SNAPSHOT_META)? else {
|
||||
if meta.get("native_snapshot_meta_required")?.is_some() {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
return local_meta(
|
||||
baseline,
|
||||
crate::decode_identity(meta.get("identity")?.ok_or(StoreError::CorruptRecord)?.value())?,
|
||||
);
|
||||
};
|
||||
if bytes.value().len() > 1024 * 1024 {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let selected: SelectedMeta = serde_json::from_slice(bytes.value()).map_err(|_| StoreError::CorruptRecord)?;
|
||||
if selected.generation != baseline.generation
|
||||
|| selected.meta.last_log_id.map(crate::LogId::from) != baseline.header.applied
|
||||
|| crate::CaptureMembership::try_from(&selected.meta.last_membership)? != baseline.header.membership
|
||||
{
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
Ok(selected.meta)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) type TestPause = std::sync::Mutex<Option<(Arc<tokio::sync::Notify>, Arc<tokio::sync::Notify>)>>;
|
||||
|
||||
#[derive(Default)]
|
||||
pub(crate) struct ReceiveState {
|
||||
count: AtomicUsize,
|
||||
#[cfg(test)]
|
||||
pub(crate) purge_recheck: TestPause,
|
||||
pub(crate) changed: tokio::sync::Notify,
|
||||
#[cfg(test)]
|
||||
pub(crate) waiters: AtomicUsize,
|
||||
}
|
||||
pub(crate) struct ReceiveGuard {
|
||||
state: Arc<ReceiveState>,
|
||||
}
|
||||
impl ReceiveGuard {
|
||||
pub(crate) fn belongs_to(&self, state: &Arc<ReceiveState>) -> bool {
|
||||
Arc::ptr_eq(&self.state, state)
|
||||
}
|
||||
fn new(state: Arc<ReceiveState>) -> Self {
|
||||
state.count.fetch_add(1, Ordering::AcqRel);
|
||||
Self { state }
|
||||
}
|
||||
}
|
||||
impl Drop for ReceiveGuard {
|
||||
fn drop(&mut self) {
|
||||
self.state.count.fetch_sub(1, Ordering::AcqRel);
|
||||
self.state.changed.notify_waiters();
|
||||
}
|
||||
}
|
||||
impl ReceiveState {
|
||||
pub(crate) fn pending(&self) -> bool {
|
||||
self.count.load(Ordering::Acquire) > 0
|
||||
}
|
||||
}
|
||||
|
||||
/// Both adapters must use clones of the same writer. Limits apply while receiving.
|
||||
#[derive(Clone)]
|
||||
pub struct NativeRaftStateMachine {
|
||||
pub(crate) writer: NativeLogWriter,
|
||||
pub(crate) limits: CheckpointLimits,
|
||||
#[cfg(test)]
|
||||
pub(crate) build_before_ordering: Arc<TestPause>,
|
||||
// Declared last: every clone releases its writer before the final owner lease.
|
||||
pub(crate) owner_lease: Option<Arc<crate::raft_owner::OwnerLease>>,
|
||||
}
|
||||
impl NativeRaftStateMachine {
|
||||
pub fn new(writer: NativeLogWriter, limits: CheckpointLimits) -> Result<Self, NativeSnapshotError> {
|
||||
limits.validate()?;
|
||||
limits.snapshot.total_bytes.checked_add(56).ok_or(StoreError::ResourceLimit)?;
|
||||
Ok(Self {
|
||||
writer,
|
||||
limits,
|
||||
#[cfg(test)]
|
||||
build_before_ordering: Arc::new(TestPause::default()),
|
||||
owner_lease: None,
|
||||
})
|
||||
}
|
||||
fn maximum(&self) -> u64 {
|
||||
self.limits.snapshot.total_bytes + 56
|
||||
}
|
||||
async fn current(&self) -> Result<Option<Snapshot<CaptureTypeConfig>>, NativeSnapshotError> {
|
||||
let owner = self.writer.store.clone();
|
||||
let limits = self.limits;
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let store = owner.lock().map_err(|_| StoreError::CorruptRecord)?;
|
||||
if !store.writable {
|
||||
return Err(StoreError::WriteDisabled);
|
||||
}
|
||||
let tx = store.database.begin_read()?;
|
||||
drop(store);
|
||||
let table = tx.open_table(crate::META)?;
|
||||
let Some(baseline) = checkpoint_baseline::read_metadata(&table)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
let meta = selected_meta(&table, &baseline)?;
|
||||
let definitions = [
|
||||
checkpoint_baseline::HEADS,
|
||||
checkpoint_baseline::VERSIONS,
|
||||
checkpoint_baseline::EVENTS,
|
||||
checkpoint_baseline::OPERATIONS,
|
||||
];
|
||||
let mut counts = [0; 4];
|
||||
for (i, definition) in definitions.iter().enumerate() {
|
||||
counts[i] = tx.open_table(*definition)?.len()?;
|
||||
}
|
||||
let out = snapshot::CapabilityOutput::new(Vec::new(), limits.snapshot.total_bytes);
|
||||
let mut encoder =
|
||||
snapshot::Encoder::new(out, &baseline.header, counts, limits.snapshot).map_err(|_| StoreError::CorruptRecord)?;
|
||||
for (definition, section) in definitions.into_iter().zip([
|
||||
snapshot::Section::Heads,
|
||||
snapshot::Section::Versions,
|
||||
snapshot::Section::Events,
|
||||
snapshot::Section::Operations,
|
||||
]) {
|
||||
for row in tx.open_table(definition)?.iter()? {
|
||||
let (key, value) = row?;
|
||||
encoder
|
||||
.row(snapshot::Row {
|
||||
section,
|
||||
key: key.value(),
|
||||
value: value.value(),
|
||||
})
|
||||
.map_err(|_| StoreError::CorruptRecord)?;
|
||||
}
|
||||
}
|
||||
let artifact = encoder.finish().map_err(|_| StoreError::CorruptRecord)?.finish();
|
||||
let mut data = NativeSnapshotData::new(limits.snapshot.total_bytes + 56).map_err(|_| StoreError::ResourceLimit)?;
|
||||
data.write_all(&artifact.manifest.bytes)
|
||||
.map_err(|_| StoreError::ResourceLimit)?;
|
||||
data.write_all(&artifact.payload).map_err(|_| StoreError::ResourceLimit)?;
|
||||
data.rewind().map_err(|_| StoreError::CorruptRecord)?;
|
||||
Ok::<_, StoreError>(Some(Snapshot {
|
||||
meta,
|
||||
snapshot: Box::new(data),
|
||||
}))
|
||||
})
|
||||
.await
|
||||
.map_err(NativeSnapshotError::Worker)?
|
||||
.map_err(NativeSnapshotError::Store)
|
||||
}
|
||||
}
|
||||
fn failure(error: impl std::error::Error + 'static) -> StorageError<u64> {
|
||||
StorageIOError::read_state_machine(openraft::AnyError::new(&error)).into()
|
||||
}
|
||||
impl RaftStateMachine<CaptureTypeConfig> for NativeRaftStateMachine {
|
||||
type SnapshotBuilder = Self;
|
||||
async fn applied_state(
|
||||
&mut self,
|
||||
) -> Result<(Option<openraft::LogId<u64>>, StoredMembership<u64, crate::CaptureNode>), StorageError<u64>> {
|
||||
let (id, membership) = self.writer.applied_state().await.map_err(failure)?;
|
||||
Ok((id.map(Into::into), StoredMembership::try_from(&membership).map_err(failure)?))
|
||||
}
|
||||
async fn apply<I>(&mut self, entries: I) -> Result<Vec<CaptureResponse>, StorageError<u64>>
|
||||
where
|
||||
I: IntoIterator<Item = CaptureRaftEntry> + OptionalSend,
|
||||
I::IntoIter: OptionalSend,
|
||||
{
|
||||
self.writer
|
||||
.apply_committed(entries.into_iter().collect())
|
||||
.await
|
||||
.map_err(failure)
|
||||
}
|
||||
async fn get_snapshot_builder(&mut self) -> Self {
|
||||
self.clone()
|
||||
}
|
||||
async fn begin_receiving_snapshot(&mut self) -> Result<Box<NativeSnapshotData>, StorageError<u64>> {
|
||||
let mut data = NativeSnapshotData::new(self.maximum()).map_err(failure)?;
|
||||
data.attach_receive_guard(ReceiveGuard::new(self.writer.snapshot_receives.clone()));
|
||||
Ok(Box::new(data))
|
||||
}
|
||||
async fn install_snapshot(&mut self, meta: &Meta, data: Box<NativeSnapshotData>) -> Result<(), StorageError<u64>> {
|
||||
crate::native_snapshot_install::install(self.writer.clone(), self.limits, meta.clone(), data)
|
||||
.await
|
||||
.map_err(failure)
|
||||
}
|
||||
async fn get_current_snapshot(&mut self) -> Result<Option<Snapshot<CaptureTypeConfig>>, StorageError<u64>> {
|
||||
self.current().await.map_err(failure)
|
||||
}
|
||||
}
|
||||
impl RaftSnapshotBuilder<CaptureTypeConfig> for NativeRaftStateMachine {
|
||||
async fn build_snapshot(&mut self) -> Result<Snapshot<CaptureTypeConfig>, StorageError<u64>> {
|
||||
#[cfg(test)]
|
||||
{
|
||||
let hook = self.build_before_ordering.lock().expect("build test hook").take();
|
||||
if let Some((entered, resume)) = hook {
|
||||
entered.notify_one();
|
||||
resume.notified().await;
|
||||
}
|
||||
}
|
||||
let limits = self.limits;
|
||||
let published_empty = self
|
||||
.writer
|
||||
.write(move |store| {
|
||||
let tx = store.write_transaction()?;
|
||||
let mut meta = tx.open_table(crate::META)?;
|
||||
// Decide the empty/nonempty path under the original writer ordering.
|
||||
if crate::application::read_applied(&meta)?.is_some() {
|
||||
return Ok(false);
|
||||
}
|
||||
let identity = crate::decode_identity(meta.get("identity")?.ok_or(StoreError::CorruptRecord)?.value())?;
|
||||
let membership =
|
||||
crate::CaptureMembership::decode(meta.get("native_membership")?.ok_or(StoreError::CorruptRecord)?.value())?;
|
||||
let previous = checkpoint_baseline::read_metadata(&meta)?;
|
||||
let generation = previous.map_or(Ok(1), |v| v.generation.checked_add(1).ok_or(StoreError::ResourceLimit))?;
|
||||
let baseline = checkpoint_baseline::BaselineMetadata {
|
||||
generation,
|
||||
header: snapshot::Header {
|
||||
cluster_id: identity.cluster_id,
|
||||
applied: None,
|
||||
membership,
|
||||
binding: crate::application::read_binding(&meta)?,
|
||||
},
|
||||
limits,
|
||||
};
|
||||
for (live, base) in [
|
||||
(crate::application::HEADS, checkpoint_baseline::HEADS),
|
||||
(crate::application::VERSIONS, checkpoint_baseline::VERSIONS),
|
||||
(crate::application::EVENTS, checkpoint_baseline::EVENTS),
|
||||
(crate::application::OPERATIONS, checkpoint_baseline::OPERATIONS),
|
||||
] {
|
||||
if tx.open_table(live)?.len()? != 0 {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
tx.open_table(base)?.retain(|_, _| false)?;
|
||||
}
|
||||
meta.insert("schema", 7u64.to_be_bytes().as_slice())?;
|
||||
meta.insert(checkpoint_baseline::META_KEY, checkpoint_baseline::encode_metadata(&baseline)?.as_slice())?;
|
||||
publish_local_meta(&mut meta, &baseline)?;
|
||||
drop(meta);
|
||||
store.commit(tx)?;
|
||||
Ok(true)
|
||||
})
|
||||
.await
|
||||
.map_err(failure)?;
|
||||
if !published_empty {
|
||||
self.writer.checkpoint_local(self.limits).await.map_err(failure)?;
|
||||
}
|
||||
self.current()
|
||||
.await
|
||||
.map_err(failure)?
|
||||
.ok_or_else(|| failure(StoreError::CorruptRecord))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) struct Waiter(Arc<ReceiveState>);
|
||||
#[cfg(test)]
|
||||
impl Waiter {
|
||||
pub(crate) fn new(state: Arc<ReceiveState>) -> Self {
|
||||
state.waiters.fetch_add(1, Ordering::AcqRel);
|
||||
Self(state)
|
||||
}
|
||||
}
|
||||
#[cfg(test)]
|
||||
impl Drop for Waiter {
|
||||
fn drop(&mut self) {
|
||||
self.0.waiters.fetch_sub(1, Ordering::AcqRel);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,583 @@
|
||||
//! Actual OpenRaft storage traits over real redb; no simulated state machine.
|
||||
use super::*;
|
||||
use openraft::{
|
||||
RaftLogReader, RaftSnapshotBuilder,
|
||||
storage::{RaftLogStorage, RaftLogStorageExt, RaftStateMachine},
|
||||
};
|
||||
use std::{
|
||||
collections::{BTreeMap, BTreeSet},
|
||||
io::SeekFrom,
|
||||
};
|
||||
use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt};
|
||||
|
||||
pub(super) type Snapshot = openraft::Snapshot<CaptureTypeConfig>;
|
||||
pub(super) fn identity(node_id: u64) -> NodeIdentity {
|
||||
NodeIdentity {
|
||||
cluster_id: [7; 16],
|
||||
node_id,
|
||||
}
|
||||
}
|
||||
pub(super) fn binding() -> CaptureBinding {
|
||||
CaptureBinding {
|
||||
revision: 9,
|
||||
binding_id: [8; 16],
|
||||
target_id: [9; 16],
|
||||
}
|
||||
}
|
||||
pub(super) fn id(index: u64) -> LogId {
|
||||
LogId {
|
||||
term: 8,
|
||||
leader_node: 0,
|
||||
index,
|
||||
}
|
||||
}
|
||||
pub(super) fn entry(index: u64) -> CaptureRaftEntry {
|
||||
let payload = if index == 0 {
|
||||
let membership = CaptureMembership::new(
|
||||
Some(id(0)),
|
||||
vec![BTreeSet::from([0])],
|
||||
BTreeMap::from([(0, CaptureNode::new("http://127.0.0.1:7000".into()).expect("member"))]),
|
||||
)
|
||||
.expect("native membership");
|
||||
let stored = openraft::StoredMembership::try_from(&membership).expect("stored membership");
|
||||
openraft::EntryPayload::Membership(stored.membership().clone())
|
||||
} else {
|
||||
let byte = u8::try_from(index).expect("bounded test event");
|
||||
openraft::EntryPayload::Normal(
|
||||
CaptureCommand::try_created(DecideCreated {
|
||||
operation_id: [byte; 16],
|
||||
object: ObjectIdentity {
|
||||
bucket_incarnation: [2; 16],
|
||||
key: vec![byte],
|
||||
},
|
||||
expected_head: None,
|
||||
prepared: PreparedIdentity {
|
||||
version_id: [byte; 16],
|
||||
preparation_id: [4; 16],
|
||||
content_digest: [5; 32],
|
||||
content_length: 123,
|
||||
},
|
||||
binding_revision: binding().revision,
|
||||
})
|
||||
.expect("native command"),
|
||||
)
|
||||
};
|
||||
CaptureRaftEntry {
|
||||
log_id: id(index).into(),
|
||||
payload,
|
||||
}
|
||||
}
|
||||
pub(super) fn entries(end: u64) -> Vec<CaptureRaftEntry> {
|
||||
(0..=end).map(entry).collect()
|
||||
}
|
||||
pub(super) struct Fixture {
|
||||
pub directory: tempfile::TempDir,
|
||||
pub writer: NativeLogWriter,
|
||||
pub logs: NativeRaftLogStore,
|
||||
pub sm: NativeRaftStateMachine,
|
||||
pub identity: NodeIdentity,
|
||||
}
|
||||
impl Fixture {
|
||||
pub fn new(node_id: u64) -> Self {
|
||||
Self::with_identity_binding(identity(node_id), binding())
|
||||
}
|
||||
pub fn with_identity_binding(identity: NodeIdentity, binding: CaptureBinding) -> Self {
|
||||
let directory = tempfile::tempdir().expect("real state machine directory");
|
||||
let mut store = Store::initialize(directory.path().join("db"), identity).expect("real redb initialization");
|
||||
store.pin_capture_binding(binding).expect("native binding");
|
||||
store.enable_native_empty().expect("native mode");
|
||||
store.promote_created_provenance().expect("provenance");
|
||||
let writer = NativeLogWriter::new(store);
|
||||
let logs = NativeRaftLogStore::new(writer.clone());
|
||||
let sm = NativeRaftStateMachine::new(writer.clone(), CheckpointLimits::default()).expect("real state machine adapter");
|
||||
Self {
|
||||
directory,
|
||||
writer,
|
||||
logs,
|
||||
sm,
|
||||
identity,
|
||||
}
|
||||
}
|
||||
pub async fn apply(&mut self, batch: Vec<CaptureRaftEntry>) -> Vec<CaptureResponse> {
|
||||
self.logs
|
||||
.blocking_append(batch.clone())
|
||||
.await
|
||||
.expect("actual trait append and library flush callback");
|
||||
self.sm.apply(batch).await.expect("actual state machine trait apply")
|
||||
}
|
||||
pub async fn build(&mut self) -> Snapshot {
|
||||
self.sm
|
||||
.get_snapshot_builder()
|
||||
.await
|
||||
.build_snapshot()
|
||||
.await
|
||||
.expect("actual durable snapshot builder")
|
||||
}
|
||||
pub async fn page(&self) -> EventPage {
|
||||
self.writer
|
||||
.created_events_page(None, 256)
|
||||
.await
|
||||
.expect("authoritative native events")
|
||||
}
|
||||
}
|
||||
pub(super) async fn snapshot_bytes(snapshot: &mut Snapshot) -> Vec<u8> {
|
||||
snapshot.snapshot.seek(SeekFrom::Start(0)).await.expect("snapshot rewind");
|
||||
let mut bytes = Vec::new();
|
||||
snapshot
|
||||
.snapshot
|
||||
.read_to_end(&mut bytes)
|
||||
.await
|
||||
.expect("actual snapshot bytes");
|
||||
bytes
|
||||
}
|
||||
pub(super) async fn receive(
|
||||
sm: &mut NativeRaftStateMachine,
|
||||
bytes: &[u8],
|
||||
) -> Box<<CaptureTypeConfig as openraft::RaftTypeConfig>::SnapshotData> {
|
||||
let mut data = sm.begin_receiving_snapshot().await.expect("actual bounded receive handle");
|
||||
data.write_all(bytes).await.expect("receive real snapshot bytes");
|
||||
data.seek(SeekFrom::Start(0)).await.expect("received snapshot rewind");
|
||||
data
|
||||
}
|
||||
pub(super) async fn install(sm: &mut NativeRaftStateMachine, snapshot: &mut Snapshot) {
|
||||
let bytes = snapshot_bytes(snapshot).await;
|
||||
let data = receive(sm, &bytes).await;
|
||||
sm.install_snapshot(&snapshot.meta, data)
|
||||
.await
|
||||
.expect("actual state machine trait install");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raft_state_machine_apply_restart() {
|
||||
let mut f = Fixture::new(0);
|
||||
assert_eq!(f.sm.applied_state().await.expect("empty applied state").0, None);
|
||||
assert!(f.sm.get_current_snapshot().await.expect("no published snapshot").is_none());
|
||||
let batch = entries(2);
|
||||
let responses = f.apply(batch.clone()).await;
|
||||
assert_eq!(responses.len(), 3, "RAFT_SM_REAL_RESPONSE_COUNT");
|
||||
assert_eq!(responses[0].kind(), CaptureResponseKind::Membership);
|
||||
assert!(matches!(responses[1].created_result(), Some(CreatedResult::Created { .. })));
|
||||
let state = f.sm.applied_state().await.expect("durable applied state");
|
||||
assert_eq!(state.0, Some(id(2).into()));
|
||||
assert_eq!(state.1.log_id(), &Some(id(0).into()));
|
||||
let before = f.page().await;
|
||||
let Fixture {
|
||||
directory,
|
||||
writer,
|
||||
logs,
|
||||
sm,
|
||||
identity,
|
||||
} = f;
|
||||
drop(sm);
|
||||
drop(logs);
|
||||
drop(writer);
|
||||
let writer = NativeLogWriter::new(Store::open(directory.path().join("db"), identity).expect("restart real store"));
|
||||
let mut sm = NativeRaftStateMachine::new(writer.clone(), CheckpointLimits::default()).expect("reopened state machine");
|
||||
assert_eq!(
|
||||
sm.applied_state().await.expect("reopened applied state"),
|
||||
state,
|
||||
"RAFT_SM_APPLY_ACK_DURABLE"
|
||||
);
|
||||
assert_eq!(writer.created_events_page(None, 256).await.expect("reopened events"), before);
|
||||
assert_eq!(
|
||||
sm.apply(batch).await.expect("exact replay responses"),
|
||||
responses,
|
||||
"RAFT_SM_REPLAY_SAME_RESPONSES"
|
||||
);
|
||||
assert_eq!(
|
||||
writer
|
||||
.created_events_page(None, 256)
|
||||
.await
|
||||
.expect("no duplicate replay events"),
|
||||
before
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raft_state_machine_snapshot_build_get_restart() {
|
||||
let mut f = Fixture::new(0);
|
||||
f.apply(entries(2)).await;
|
||||
let mut built = f.build().await;
|
||||
let meta = built.meta.clone();
|
||||
let bytes = snapshot_bytes(&mut built).await;
|
||||
assert_eq!(meta.last_log_id, Some(id(2).into()), "RAFT_SM_SNAPSHOT_META_CUT");
|
||||
assert_eq!(meta.last_membership, f.sm.applied_state().await.expect("membership").1);
|
||||
f.apply(vec![entry(3)]).await;
|
||||
let mut current =
|
||||
f.sm.get_current_snapshot()
|
||||
.await
|
||||
.expect("persisted snapshot")
|
||||
.expect("snapshot exists");
|
||||
assert_eq!(current.meta, meta, "RAFT_SM_LIVE_APPLY_DOES_NOT_REPIN_SNAPSHOT");
|
||||
assert_eq!(snapshot_bytes(&mut current).await, bytes, "RAFT_SM_SNAPSHOT_FROZEN_BYTES");
|
||||
let Fixture {
|
||||
directory,
|
||||
writer,
|
||||
logs,
|
||||
sm,
|
||||
identity,
|
||||
} = f;
|
||||
drop(current);
|
||||
drop(built);
|
||||
drop(sm);
|
||||
drop(logs);
|
||||
drop(writer);
|
||||
let writer = NativeLogWriter::new(Store::open(directory.path().join("db"), identity).expect("restart snapshot owner"));
|
||||
let mut sm = NativeRaftStateMachine::new(writer, CheckpointLimits::default()).expect("reopen adapter");
|
||||
let mut current = sm
|
||||
.get_current_snapshot()
|
||||
.await
|
||||
.expect("restarted current snapshot")
|
||||
.expect("durable snapshot exists");
|
||||
assert_eq!(current.meta, meta, "RAFT_SM_COMPLETE_META_RECONSTRUCTED");
|
||||
assert_eq!(snapshot_bytes(&mut current).await, bytes, "RAFT_SM_RESTART_SNAPSHOT_BYTES");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raft_state_machine_receive_bounds() {
|
||||
let f = Fixture::new(0);
|
||||
let mut limits = CheckpointLimits::default();
|
||||
limits.snapshot.total_bytes = 1024;
|
||||
let mut sm = NativeRaftStateMachine::new(f.writer.clone(), limits).expect("bounded state machine");
|
||||
let mut data = sm.begin_receiving_snapshot().await.expect("bounded receive");
|
||||
data.write_all(&[1; 16]).await.expect("small write");
|
||||
assert!(data.seek(SeekFrom::Start(u64::MAX)).await.is_err(), "RAFT_SM_RECEIVE_SEEK_BOUND");
|
||||
data.seek(SeekFrom::Start(0)).await.expect("restore seek");
|
||||
let mut bytes = [0; 16];
|
||||
data.read_exact(&mut bytes).await.expect("small received bytes");
|
||||
assert_eq!(bytes, [1; 16], "RAFT_SM_BAD_SEEK_PRESERVES_BYTES");
|
||||
assert!(data.write_all(&[2; 2048]).await.is_err(), "RAFT_SM_RECEIVE_WRITE_BOUND_BEFORE_INSTALL");
|
||||
assert_eq!(sm.applied_state().await.expect("receive is not install").0, None);
|
||||
assert!(sm.get_current_snapshot().await.expect("no received publication").is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raft_state_machine_install_rejects_meta_identity() {
|
||||
let mut source = Fixture::new(0);
|
||||
source.apply(entries(2)).await;
|
||||
let mut snapshot = source.build().await;
|
||||
let bytes = snapshot_bytes(&mut snapshot).await;
|
||||
let mut target = Fixture::new(1);
|
||||
target.apply(entries(1)).await;
|
||||
let old_page = target.page().await;
|
||||
let stable = std::fs::read(target.directory.path().join("db")).expect("nonempty target bytes");
|
||||
let mut wrong = snapshot.meta.clone();
|
||||
wrong.last_log_id = Some(LogId { term: 9, ..id(2) }.into());
|
||||
let data = receive(&mut target.sm, &bytes).await;
|
||||
assert!(
|
||||
target.sm.install_snapshot(&wrong, data).await.is_err(),
|
||||
"SNAPSHOT_META_BOUND_TO_FRAME: full LogId term cannot differ from snapshot header"
|
||||
);
|
||||
assert_eq!(target.sm.applied_state().await.expect("rejection state").0, Some(id(1).into()));
|
||||
assert_eq!(target.page().await, old_page, "RAFT_SM_META_REJECTION_RETAINS_EVENTS");
|
||||
assert_eq!(
|
||||
std::fs::read(target.directory.path().join("db")).expect("unchanged nonempty target"),
|
||||
stable,
|
||||
"RAFT_SM_META_REJECTION_NO_MUTATION"
|
||||
);
|
||||
assert!(target.sm.get_current_snapshot().await.expect("rejection snapshot").is_none());
|
||||
let mut wrong = snapshot.meta.clone();
|
||||
wrong.last_membership = openraft::StoredMembership::default();
|
||||
let data = receive(&mut target.sm, &bytes).await;
|
||||
assert!(
|
||||
target.sm.install_snapshot(&wrong, data).await.is_err(),
|
||||
"RAFT_SM_MEMBERSHIP_BOUND_TO_FRAME"
|
||||
);
|
||||
let mut foreign = Fixture::with_identity_binding(
|
||||
NodeIdentity {
|
||||
cluster_id: [6; 16],
|
||||
..identity(1)
|
||||
},
|
||||
binding(),
|
||||
);
|
||||
let data = receive(&mut foreign.sm, &bytes).await;
|
||||
assert!(
|
||||
foreign.sm.install_snapshot(&snapshot.meta, data).await.is_err(),
|
||||
"RAFT_SM_CLUSTER_BOUND_TO_DURABLE_OWNER"
|
||||
);
|
||||
let mut foreign = Fixture::with_identity_binding(
|
||||
identity(1),
|
||||
CaptureBinding {
|
||||
target_id: [5; 16],
|
||||
..binding()
|
||||
},
|
||||
);
|
||||
let data = receive(&mut foreign.sm, &bytes).await;
|
||||
assert!(
|
||||
foreign.sm.install_snapshot(&snapshot.meta, data).await.is_err(),
|
||||
"RAFT_SM_CAPTURE_BINDING_BOUND"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raft_state_machine_install_preserves_local_identity_vote() {
|
||||
let mut source = Fixture::new(0);
|
||||
source.apply(entries(2)).await;
|
||||
let mut snapshot = source.build().await;
|
||||
snapshot.meta.snapshot_id = "foreign-snapshot-id-preserved-verbatim".into();
|
||||
let mut target = Fixture::new(1);
|
||||
let vote = openraft::Vote::new_committed(10, 1);
|
||||
target.logs.save_vote(&vote).await.expect("local election vote");
|
||||
install(&mut target.sm, &mut snapshot).await;
|
||||
assert_eq!(
|
||||
target.writer.durable_identity().await.expect("local identity"),
|
||||
identity(1),
|
||||
"RAFT_SM_INSTALL_PRESERVES_LOCAL_NODE"
|
||||
);
|
||||
assert_eq!(
|
||||
target.logs.read_vote().await.expect("local vote"),
|
||||
Some(vote),
|
||||
"RAFT_SM_INSTALL_PRESERVES_LOCAL_VOTE"
|
||||
);
|
||||
let mut current = target
|
||||
.sm
|
||||
.get_current_snapshot()
|
||||
.await
|
||||
.expect("installed snapshot")
|
||||
.expect("present");
|
||||
assert_eq!(current.meta, snapshot.meta, "RAFT_SM_INSTALL_PRESERVES_COMPLETE_INCOMING_META");
|
||||
assert_eq!(snapshot_bytes(&mut current).await, snapshot_bytes(&mut snapshot).await);
|
||||
let Fixture {
|
||||
directory,
|
||||
writer,
|
||||
logs,
|
||||
sm,
|
||||
identity,
|
||||
} = target;
|
||||
drop(current);
|
||||
drop(sm);
|
||||
drop(logs);
|
||||
drop(writer);
|
||||
let writer = NativeLogWriter::new(Store::open(directory.path().join("db"), identity).expect("restart installed target"));
|
||||
let mut logs = NativeRaftLogStore::new(writer.clone());
|
||||
let mut sm = NativeRaftStateMachine::new(writer.clone(), CheckpointLimits::default()).expect("reopened installed SM");
|
||||
assert_eq!(writer.durable_identity().await.expect("restarted local identity"), identity);
|
||||
assert_eq!(logs.read_vote().await.expect("restarted vote"), Some(vote));
|
||||
assert_eq!(
|
||||
sm.get_current_snapshot()
|
||||
.await
|
||||
.expect("restarted snapshot")
|
||||
.expect("present")
|
||||
.meta,
|
||||
snapshot.meta,
|
||||
"RAFT_SM_INCOMING_SNAPSHOT_ID_DURABLE"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raft_state_machine_install_suffix() {
|
||||
let mut source = Fixture::new(0);
|
||||
source.apply(entries(2)).await;
|
||||
let mut snapshot = source.build().await;
|
||||
let mut target = Fixture::new(1);
|
||||
target
|
||||
.logs
|
||||
.blocking_append(entries(4))
|
||||
.await
|
||||
.expect("retained local log suffix");
|
||||
target.sm.apply(entries(1)).await.expect("old local applied prefix");
|
||||
target
|
||||
.logs
|
||||
.save_committed(Some(id(4).into()))
|
||||
.await
|
||||
.expect("known committed compatible suffix");
|
||||
install(&mut target.sm, &mut snapshot).await;
|
||||
assert_eq!(
|
||||
target
|
||||
.logs
|
||||
.get_log_reader()
|
||||
.await
|
||||
.try_get_log_entries(3..)
|
||||
.await
|
||||
.expect("compatible suffix"),
|
||||
vec![entry(3), entry(4)],
|
||||
"RAFT_SM_INSTALL_PRESERVES_COMPATIBLE_COMMITTED_SUFFIX"
|
||||
);
|
||||
assert_eq!(
|
||||
target.logs.read_committed().await.expect("preserved committed marker"),
|
||||
Some(id(4).into())
|
||||
);
|
||||
target
|
||||
.sm
|
||||
.apply(vec![entry(3), entry(4)])
|
||||
.await
|
||||
.expect("apply retained committed suffix");
|
||||
assert_eq!(target.page().await.events.len(), 4);
|
||||
let mut conflict = Fixture::new(2);
|
||||
let mut batch = entries(4);
|
||||
for entry in &mut batch[2..] {
|
||||
entry.log_id = LogId {
|
||||
term: 9,
|
||||
..id(entry.log_id.index)
|
||||
}
|
||||
.into();
|
||||
}
|
||||
conflict
|
||||
.logs
|
||||
.blocking_append(batch)
|
||||
.await
|
||||
.expect("conflicting uncommitted suffix");
|
||||
conflict.sm.apply(entries(1)).await.expect("protected applied prefix");
|
||||
install(&mut conflict.sm, &mut snapshot).await;
|
||||
assert!(
|
||||
conflict
|
||||
.logs
|
||||
.get_log_reader()
|
||||
.await
|
||||
.try_get_log_entries(3..)
|
||||
.await
|
||||
.expect("discarded conflict")
|
||||
.is_empty(),
|
||||
"RAFT_SM_INSTALL_DROPS_ONLY_UNCOMMITTED_CONFLICT"
|
||||
);
|
||||
conflict.apply(vec![entry(3)]).await;
|
||||
assert_eq!(conflict.sm.applied_state().await.expect("new suffix applied").0, Some(id(3).into()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raft_state_machine_install_restart_continue() {
|
||||
let mut source = Fixture::new(0);
|
||||
source.apply(entries(2)).await;
|
||||
let expected = source.page().await;
|
||||
let mut snapshot = source.build().await;
|
||||
let mut target = Fixture::new(1);
|
||||
install(&mut target.sm, &mut snapshot).await;
|
||||
assert_eq!(target.page().await, expected, "RAFT_SM_INSTALL_REBUILDS_SOURCE_EVENTS");
|
||||
let Fixture {
|
||||
directory,
|
||||
writer,
|
||||
logs,
|
||||
sm,
|
||||
identity,
|
||||
} = target;
|
||||
drop(sm);
|
||||
drop(logs);
|
||||
drop(writer);
|
||||
let writer = NativeLogWriter::new(Store::open(directory.path().join("db"), identity).expect("restart installed redb"));
|
||||
let mut logs = NativeRaftLogStore::new(writer.clone());
|
||||
let mut sm = NativeRaftStateMachine::new(writer.clone(), CheckpointLimits::default()).expect("restart installed adapter");
|
||||
assert_eq!(sm.applied_state().await.expect("snapshot applied state").0, Some(id(2).into()));
|
||||
logs.blocking_append(vec![entry(3)])
|
||||
.await
|
||||
.expect("append after install restart");
|
||||
sm.apply(vec![entry(3)]).await.expect("apply after install restart");
|
||||
let all = writer.created_events_page(None, 256).await.expect("retained plus new event");
|
||||
assert_eq!(all.events.len(), 3);
|
||||
assert_eq!(all.events[..2], expected.events);
|
||||
let next = writer
|
||||
.created_events_page(expected.next, 1)
|
||||
.await
|
||||
.expect("snapshot cursor resumes");
|
||||
assert_eq!(next.events[0].event_id, [3; 16], "RAFT_SM_INSTALL_CURSOR_CONTINUITY");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raft_state_machine_empty_snapshot_restart_continue() {
|
||||
let mut source = Fixture::new(0);
|
||||
let mut target = Fixture::new(42);
|
||||
let mut snapshot = source.sm.build_snapshot().await.expect("durable empty baseline");
|
||||
assert_eq!(snapshot.meta.last_log_id, None);
|
||||
let empty_meta = snapshot.meta.clone();
|
||||
let received = receive(&mut target.sm, &snapshot_bytes(&mut snapshot).await).await;
|
||||
target
|
||||
.sm
|
||||
.install_snapshot(&empty_meta, received)
|
||||
.await
|
||||
.expect("install empty frame");
|
||||
let mut target = target.reopen();
|
||||
let current = target
|
||||
.sm
|
||||
.get_current_snapshot()
|
||||
.await
|
||||
.expect("empty after reopen")
|
||||
.expect("durable selection");
|
||||
assert_eq!(current.meta, empty_meta);
|
||||
target
|
||||
.logs
|
||||
.blocking_append(entries(1))
|
||||
.await
|
||||
.expect("real append after empty install");
|
||||
target.sm.apply(entries(1)).await.expect("real apply after empty install");
|
||||
let mut target = target.reopen();
|
||||
assert_eq!(
|
||||
target.sm.applied_state().await.expect("replay empty baseline + suffix").0,
|
||||
Some(id(1).into())
|
||||
);
|
||||
assert_eq!(
|
||||
target
|
||||
.writer
|
||||
.created_events_page(None, 16)
|
||||
.await
|
||||
.expect("event after empty baseline")
|
||||
.events
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
let refreshed = target
|
||||
.sm
|
||||
.build_snapshot()
|
||||
.await
|
||||
.expect("nonempty checkpoint supersedes empty");
|
||||
assert_eq!(refreshed.meta.last_log_id, Some(id(1).into()));
|
||||
}
|
||||
|
||||
impl Fixture {
|
||||
fn reopen(self) -> Self {
|
||||
let Self {
|
||||
directory,
|
||||
writer,
|
||||
logs,
|
||||
sm,
|
||||
identity,
|
||||
} = self;
|
||||
drop(sm);
|
||||
drop(logs);
|
||||
drop(writer);
|
||||
let writer =
|
||||
NativeLogWriter::new(Store::open(directory.path().join("db"), identity).expect("real empty baseline restart"));
|
||||
let logs = NativeRaftLogStore::new(writer.clone());
|
||||
let sm =
|
||||
NativeRaftStateMachine::new(writer.clone(), CheckpointLimits::default()).expect("reopened empty baseline adapter");
|
||||
Self {
|
||||
directory,
|
||||
writer,
|
||||
logs,
|
||||
sm,
|
||||
identity,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raft_state_machine_build_competes_with_initial_apply() {
|
||||
let mut target = Fixture::new(0);
|
||||
target
|
||||
.logs
|
||||
.blocking_append(entries(1))
|
||||
.await
|
||||
.expect("real pending initial batch");
|
||||
let entered = std::sync::Arc::new(tokio::sync::Notify::new());
|
||||
let resume = std::sync::Arc::new(tokio::sync::Notify::new());
|
||||
*target.sm.build_before_ordering.lock().expect("build pause") = Some((entered.clone(), resume.clone()));
|
||||
let mut builder = target.sm.get_snapshot_builder().await;
|
||||
let task = tokio::spawn(async move { builder.build_snapshot().await });
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), entered.notified())
|
||||
.await
|
||||
.expect("builder began before initial apply");
|
||||
target
|
||||
.sm
|
||||
.apply(entries(1))
|
||||
.await
|
||||
.expect("actual initial apply while builder pending");
|
||||
resume.notify_one();
|
||||
let snapshot = tokio::time::timeout(std::time::Duration::from_secs(5), task)
|
||||
.await
|
||||
.expect("build continues")
|
||||
.expect("builder task")
|
||||
.expect("initial apply does not make snapshot build a fatal storage error");
|
||||
assert_eq!(
|
||||
snapshot.meta.last_log_id,
|
||||
Some(id(1).into()),
|
||||
"SNAPSHOT_BUILD_INITIAL_APPLY_RACE_USES_NONEMPTY_CHECKPOINT"
|
||||
);
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,559 @@
|
||||
//! Disposable event ordering index. EVENTS remains the sole event authority.
|
||||
use crate::{
|
||||
CaptureBinding, CreatedEvent, CreatedResult, LogId, NativeLogWriter, NativeWriteError, Store, StoreError, application,
|
||||
decode_id, encode_id,
|
||||
};
|
||||
use redb::{ReadableDatabase, ReadableTable, ReadableTableMetadata, TableDefinition, WriteTransaction};
|
||||
|
||||
pub const MAX_EVENT_PAGE: usize = 256;
|
||||
pub(crate) const EVENT_SEQUENCE: TableDefinition<u64, &[u8]> = TableDefinition::new("event_sequence_cache_v1");
|
||||
|
||||
/// Persist only after every event preceding this cursor has a durable consumer ACK.
|
||||
/// Deserialization does not grant authority: each use checks the stored event.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct EventCursor {
|
||||
cluster: [u8; 16],
|
||||
term: u64,
|
||||
leader: u64,
|
||||
index: u64,
|
||||
event: [u8; 16],
|
||||
binding_id: [u8; 16],
|
||||
target_id: [u8; 16],
|
||||
revision: u64,
|
||||
}
|
||||
impl EventCursor {
|
||||
pub(crate) fn new(cluster: [u8; 16], event: [u8; 16], value: &CreatedEvent) -> Self {
|
||||
Self {
|
||||
cluster,
|
||||
term: value.decision_log_id.term,
|
||||
leader: value.decision_log_id.leader_node,
|
||||
index: value.decision_log_id.index,
|
||||
event,
|
||||
binding_id: value.binding.binding_id,
|
||||
target_id: value.binding.target_id,
|
||||
revision: value.binding.revision,
|
||||
}
|
||||
}
|
||||
fn matches(&self, cluster: [u8; 16], event: [u8; 16], value: &CreatedEvent) -> bool {
|
||||
self == &Self::new(cluster, event, value)
|
||||
}
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct SourceEvent {
|
||||
pub event_id: [u8; 16],
|
||||
pub captured: CreatedEvent,
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct EventPage {
|
||||
pub events: Vec<SourceEvent>,
|
||||
pub next: Option<EventCursor>,
|
||||
}
|
||||
fn encoded_index(id: LogId, event: [u8; 16]) -> Vec<u8> {
|
||||
let mut bytes = encode_id(id).to_vec();
|
||||
bytes.extend(event);
|
||||
bytes
|
||||
}
|
||||
fn decode_index(bytes: &[u8]) -> Result<(LogId, [u8; 16]), StoreError> {
|
||||
if bytes.len() != 40 {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let id = decode_id(&bytes[..24])?;
|
||||
let event: [u8; 16] = bytes[24..].try_into().map_err(|_| StoreError::CorruptRecord)?;
|
||||
if event == [0; 16] {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
Ok((id, event))
|
||||
}
|
||||
pub(crate) fn insert_event_index(tx: &WriteTransaction, id: LogId, event: [u8; 16]) -> Result<(), StoreError> {
|
||||
let bytes = encoded_index(id, event);
|
||||
let mut index = tx.open_table(EVENT_SEQUENCE)?;
|
||||
if index.get(id.index)?.is_some() {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
index.insert(id.index, bytes.as_slice())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Check the event semantics required to derive ordering from checkpoint authority.
|
||||
/// This deliberately does not grant snapshot installation or source authority.
|
||||
fn validated_event(
|
||||
key: &[u8],
|
||||
bytes: &[u8],
|
||||
operations: &impl ReadableTable<&'static [u8], &'static [u8]>,
|
||||
versions: &impl ReadableTable<&'static [u8], &'static [u8]>,
|
||||
schema: u64,
|
||||
binding: &CaptureBinding,
|
||||
applied: Option<LogId>,
|
||||
) -> Result<([u8; 16], CreatedEvent), StoreError> {
|
||||
let event_id: [u8; 16] = key.try_into().map_err(|_| StoreError::CorruptRecord)?;
|
||||
let event = application::decode_event(bytes)?;
|
||||
let cut = applied.ok_or(StoreError::CorruptRecord)?;
|
||||
let operation = operations.get(key)?.ok_or(StoreError::CorruptRecord)?;
|
||||
let first_decision = if matches!(schema, 5 | 6 | 7) {
|
||||
Some(application::decode_provenance(operation.value())?.first_decision_log_id)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let operation = application::decode_stored_operation(operation.value(), schema)?;
|
||||
let command = crate::DecideCreated::decode(&operation.canonical_command)?;
|
||||
let version_key = application::version_key(&event.object, event.prepared.version_id)?;
|
||||
let version = versions.get(version_key.as_slice())?.ok_or(StoreError::CorruptRecord)?;
|
||||
let version = application::decode_version(version.value())?;
|
||||
if first_decision.is_some_and(|first| first != event.decision_log_id)
|
||||
|| event_id == [0; 16]
|
||||
|| event.decision_log_id.index > cut.index
|
||||
|| (event.decision_log_id.index == cut.index && event.decision_log_id != cut)
|
||||
|| event.binding != *binding
|
||||
|| command.operation_id != event_id
|
||||
|| command.object != event.object
|
||||
|| command.prepared != event.prepared
|
||||
|| command.binding_revision != event.binding.revision
|
||||
|| operation.result
|
||||
!= (CreatedResult::Created {
|
||||
version_id: event.prepared.version_id,
|
||||
event_id,
|
||||
decision_log_id: event.decision_log_id,
|
||||
})
|
||||
|| version
|
||||
!= (crate::CreatedVersion {
|
||||
prepared: event.prepared.clone(),
|
||||
decision_log_id: event.decision_log_id,
|
||||
binding_revision: event.binding.revision,
|
||||
})
|
||||
{
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
Ok((event_id, event))
|
||||
}
|
||||
|
||||
/// Rebuild from the same authority rows emitted by the existing four-section checkpoint.
|
||||
/// Callers must independently establish authority and validate all other state before promotion.
|
||||
pub(crate) fn rebuild_from_authority(
|
||||
tx: &WriteTransaction,
|
||||
schema: u64,
|
||||
binding: &CaptureBinding,
|
||||
applied: Option<LogId>,
|
||||
) -> Result<(), StoreError> {
|
||||
let events = tx.open_table(application::EVENTS)?;
|
||||
let operations = tx.open_table(application::OPERATIONS)?;
|
||||
let versions = tx.open_table(application::VERSIONS)?;
|
||||
{
|
||||
let mut index = tx.open_table(EVENT_SEQUENCE)?;
|
||||
index.retain(|_, _| false)?;
|
||||
}
|
||||
for row in events.iter()? {
|
||||
let (key, bytes) = row?;
|
||||
let (event, value) = validated_event(key.value(), bytes.value(), &operations, &versions, schema, binding, applied)?;
|
||||
insert_event_index(tx, value.decision_log_id, event)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
impl Store {
|
||||
pub(crate) fn ensure_event_index(&mut self) -> Result<(), StoreError> {
|
||||
let read = self.database.begin_read()?;
|
||||
let meta = read.open_table(crate::META)?;
|
||||
let schema = crate::number(meta.get("schema")?.ok_or(StoreError::CorruptRecord)?.value())?;
|
||||
if !matches!(schema, 3..=7) {
|
||||
return Ok(());
|
||||
}
|
||||
let binding = application::read_binding(&meta)?;
|
||||
let applied = application::read_applied(&meta)?;
|
||||
let events = read.open_table(application::EVENTS)?;
|
||||
let complete = match read.open_table(EVENT_SEQUENCE) {
|
||||
Ok(index) => {
|
||||
let mut complete = index.len()? == events.len()?;
|
||||
if complete {
|
||||
for row in events.iter()? {
|
||||
let (key, bytes) = row?;
|
||||
let value = application::decode_event(bytes.value())?;
|
||||
let expected =
|
||||
encoded_index(value.decision_log_id, key.value().try_into().map_err(|_| StoreError::CorruptRecord)?);
|
||||
if index
|
||||
.get(value.decision_log_id.index)?
|
||||
.is_none_or(|stored| stored.value() != expected.as_slice())
|
||||
{
|
||||
complete = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
complete
|
||||
}
|
||||
Err(redb::TableError::TableDoesNotExist(_)) => false,
|
||||
Err(error) => return Err(error.into()),
|
||||
};
|
||||
drop(events);
|
||||
drop(meta);
|
||||
drop(read);
|
||||
if !complete {
|
||||
let tx = self.write_transaction()?;
|
||||
rebuild_from_authority(&tx, schema, &binding, applied)?;
|
||||
self.commit(tx)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
/// Read a bounded page without changing any delivery ACK or cursor state.
|
||||
pub fn created_events_page(&self, cursor: Option<&EventCursor>, limit: usize) -> Result<EventPage, StoreError> {
|
||||
if !self.writable {
|
||||
return Err(StoreError::WriteDisabled);
|
||||
}
|
||||
if limit == 0 || limit > MAX_EVENT_PAGE {
|
||||
return Err(StoreError::InvalidEventPage);
|
||||
}
|
||||
let tx = self.database.begin_read()?;
|
||||
let meta = tx.open_table(crate::META)?;
|
||||
let schema = crate::number(meta.get("schema")?.ok_or(StoreError::CorruptRecord)?.value())?;
|
||||
if !matches!(schema, 3..=7) {
|
||||
return Err(StoreError::UnsupportedSchema);
|
||||
}
|
||||
let identity = crate::decode_identity(meta.get("identity")?.ok_or(StoreError::CorruptRecord)?.value())?;
|
||||
let events = tx.open_table(application::EVENTS)?;
|
||||
let index = tx.open_table(EVENT_SEQUENCE)?;
|
||||
if index.len()? != events.len()? {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let read_event = |id: u64, bytes: &[u8]| -> Result<SourceEvent, StoreError> {
|
||||
let (log, event_id) = decode_index(bytes)?;
|
||||
let value = events.get(event_id.as_slice())?.ok_or(StoreError::CorruptRecord)?;
|
||||
let captured = application::decode_event(value.value())?;
|
||||
if log.index != id || captured.decision_log_id != log {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
Ok(SourceEvent { event_id, captured })
|
||||
};
|
||||
if let Some(cursor) = cursor {
|
||||
let stored = index.get(cursor.index)?.ok_or(StoreError::IdentityMismatch)?;
|
||||
let event = read_event(cursor.index, stored.value())?;
|
||||
if !cursor.matches(identity.cluster_id, event.event_id, &event.captured) {
|
||||
return Err(StoreError::IdentityMismatch);
|
||||
}
|
||||
}
|
||||
let start = cursor.map_or(0, |c| c.index);
|
||||
let range = if cursor.is_some() {
|
||||
(std::ops::Bound::Excluded(start), std::ops::Bound::Unbounded)
|
||||
} else {
|
||||
(std::ops::Bound::Included(start), std::ops::Bound::Unbounded)
|
||||
};
|
||||
let mut page = EventPage {
|
||||
events: Vec::with_capacity(limit),
|
||||
next: cursor.cloned(),
|
||||
};
|
||||
for row in index.range(range)?.take(limit) {
|
||||
let (id, bytes) = row?;
|
||||
let event = read_event(id.value(), bytes.value())?;
|
||||
page.next = Some(EventCursor::new(identity.cluster_id, event.event_id, &event.captured));
|
||||
page.events.push(event);
|
||||
}
|
||||
Ok(page)
|
||||
}
|
||||
}
|
||||
impl NativeLogWriter {
|
||||
pub async fn created_events_page(&self, cursor: Option<EventCursor>, limit: usize) -> Result<EventPage, NativeWriteError> {
|
||||
let owner = self.store.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
owner
|
||||
.lock()
|
||||
.map_err(|_| NativeWriteError::Poisoned)?
|
||||
.created_events_page(cursor.as_ref(), limit)
|
||||
.map_err(NativeWriteError::Store)
|
||||
})
|
||||
.await
|
||||
.map_err(NativeWriteError::Worker)?
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::{CaptureCommand, CaptureRaftEntry, DecideCreated, NodeIdentity, ObjectIdentity, PreparedIdentity, snapshot};
|
||||
use std::io;
|
||||
fn identity() -> NodeIdentity {
|
||||
NodeIdentity {
|
||||
cluster_id: [7; 16],
|
||||
node_id: 0,
|
||||
}
|
||||
}
|
||||
fn binding() -> CaptureBinding {
|
||||
CaptureBinding {
|
||||
revision: 9,
|
||||
binding_id: [8; 16],
|
||||
target_id: [9; 16],
|
||||
}
|
||||
}
|
||||
fn id(index: u64) -> LogId {
|
||||
LogId {
|
||||
term: 8,
|
||||
leader_node: 0,
|
||||
index,
|
||||
}
|
||||
}
|
||||
fn command(event: u8) -> DecideCreated {
|
||||
DecideCreated {
|
||||
operation_id: [event; 16],
|
||||
object: ObjectIdentity {
|
||||
bucket_incarnation: [2; 16],
|
||||
key: vec![event],
|
||||
},
|
||||
expected_head: None,
|
||||
prepared: PreparedIdentity {
|
||||
version_id: [event; 16],
|
||||
preparation_id: [4; 16],
|
||||
content_digest: [5; 32],
|
||||
content_length: 123,
|
||||
},
|
||||
binding_revision: 9,
|
||||
}
|
||||
}
|
||||
fn apply(store: &mut Store, index: u64, command: DecideCreated) -> CreatedResult {
|
||||
store
|
||||
.append_native(&[CaptureRaftEntry {
|
||||
log_id: id(index).into(),
|
||||
payload: openraft::EntryPayload::Normal(CaptureCommand::try_created(command).expect("command")),
|
||||
}])
|
||||
.expect("append native");
|
||||
store.save_committed(id(index)).expect("commit native");
|
||||
store
|
||||
.apply_next_native(id(index))
|
||||
.expect("apply native")
|
||||
.created_result()
|
||||
.expect("created response")
|
||||
.clone()
|
||||
}
|
||||
fn fixture() -> (tempfile::TempDir, Store) {
|
||||
let dir = tempfile::tempdir().expect("directory");
|
||||
let mut store = Store::initialize(dir.path().join("source.redb"), identity()).expect("initialize");
|
||||
store.pin_capture_binding(binding()).expect("pin");
|
||||
store.enable_native_empty().expect("native");
|
||||
apply(&mut store, 0, command(9));
|
||||
apply(&mut store, 1, command(7));
|
||||
(dir, store)
|
||||
}
|
||||
#[test]
|
||||
fn committed_events_page_in_decision_order() {
|
||||
let (_dir, store) = fixture();
|
||||
let first = store.created_events_page(None, 1).expect("first");
|
||||
assert_eq!(
|
||||
first.events.iter().map(|e| e.event_id).collect::<Vec<_>>(),
|
||||
vec![[9; 16]],
|
||||
"SOURCE_PAGE_ORDER_BOUND"
|
||||
);
|
||||
let next = store.created_events_page(first.next.as_ref(), 1).expect("second");
|
||||
assert_eq!(next.events[0].event_id, [7; 16]);
|
||||
assert_eq!(next.events[0].captured.binding, binding());
|
||||
assert!(
|
||||
store
|
||||
.created_events_page(next.next.as_ref(), 1)
|
||||
.expect("end")
|
||||
.events
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn replay_and_rejections_do_not_duplicate_events() {
|
||||
let (_dir, mut store) = fixture();
|
||||
apply(&mut store, 2, command(9));
|
||||
let mut rejected = command(3);
|
||||
rejected.object = command(9).object;
|
||||
assert!(matches!(apply(&mut store, 3, rejected), CreatedResult::HeadMismatch { .. }));
|
||||
let page = store.created_events_page(None, 256).expect("page");
|
||||
assert_eq!(page.events.len(), 2, "SOURCE_PAGE_REPLAY_BOUND");
|
||||
}
|
||||
#[test]
|
||||
fn reopen_rebuilds_missing_or_stale_index() {
|
||||
let (dir, mut store) = fixture();
|
||||
apply(&mut store, 2, command(1));
|
||||
let expected = store.created_events_page(None, 256).expect("before");
|
||||
let tx = store.write_transaction().expect("remove cache");
|
||||
tx.delete_table(EVENT_SEQUENCE).expect("delete cache");
|
||||
store.commit(tx).expect("cache deletion");
|
||||
drop(store);
|
||||
let mut store = Store::open(dir.path().join("source.redb"), identity()).expect("rebuild missing");
|
||||
assert_eq!(store.created_events_page(None, 256).expect("rebuilt"), expected);
|
||||
let tx = store.write_transaction().expect("stale cache");
|
||||
{
|
||||
let mut index = tx.open_table(EVENT_SEQUENCE).expect("cache");
|
||||
index.remove(1).expect("omit middle item");
|
||||
index.insert(200, b"bad".as_slice()).expect("same-count stale item");
|
||||
}
|
||||
store.commit(tx).expect("cache damage");
|
||||
drop(store);
|
||||
let store = Store::open(dir.path().join("source.redb"), identity()).expect("rebuild stale");
|
||||
assert_eq!(store.created_events_page(None, 256).expect("rebuilt stale"), expected);
|
||||
}
|
||||
#[test]
|
||||
fn cursor_binds_cluster_term_leader_and_event() {
|
||||
let (_dir, store) = fixture();
|
||||
let cursor = store.created_events_page(None, 1).expect("page").next.expect("cursor");
|
||||
let mut bad = Vec::new();
|
||||
let mut c = cursor.clone();
|
||||
c.cluster = [6; 16];
|
||||
bad.push(c);
|
||||
let mut c = cursor.clone();
|
||||
c.term += 1;
|
||||
bad.push(c);
|
||||
let mut c = cursor.clone();
|
||||
c.leader += 1;
|
||||
bad.push(c);
|
||||
let mut c = cursor.clone();
|
||||
c.event = [7; 16];
|
||||
bad.push(c);
|
||||
let mut c = cursor.clone();
|
||||
c.revision += 1;
|
||||
bad.push(c);
|
||||
let mut c = cursor.clone();
|
||||
c.binding_id = [6; 16];
|
||||
bad.push(c);
|
||||
let mut c = cursor.clone();
|
||||
c.target_id = [6; 16];
|
||||
bad.push(c);
|
||||
for cursor in bad {
|
||||
assert_eq!(
|
||||
store.created_events_page(Some(&cursor), 1).err().map(|e| e.code()),
|
||||
Some("identity_mismatch"),
|
||||
"SOURCE_PAGE_CURSOR_BOUND"
|
||||
);
|
||||
}
|
||||
let encoded = serde_json::to_vec(&cursor).expect("cursor bytes");
|
||||
let decoded: EventCursor = serde_json::from_slice(&encoded).expect("cursor decode");
|
||||
assert_eq!(store.created_events_page(Some(&decoded), 1).expect("resume").events[0].event_id, [7; 16]);
|
||||
}
|
||||
#[test]
|
||||
fn page_is_bounded_and_new_lower_uuid_is_not_lost() {
|
||||
let (_dir, mut store) = fixture();
|
||||
let page = store.created_events_page(None, 1).expect("one");
|
||||
assert_eq!(page.events.len(), 1);
|
||||
apply(&mut store, 2, command(1));
|
||||
let next = store.created_events_page(page.next.as_ref(), 256).expect("later");
|
||||
assert_eq!(next.events.iter().map(|e| e.event_id).collect::<Vec<_>>(), vec![[7; 16], [1; 16]]);
|
||||
for limit in [0, MAX_EVENT_PAGE + 1, usize::MAX] {
|
||||
assert_eq!(store.created_events_page(None, limit).err().map(|e| e.code()), Some("invalid_event_page"));
|
||||
}
|
||||
store.writable = false;
|
||||
assert_eq!(store.created_events_page(None, 1).err().map(|e| e.code()), Some("write_disabled"));
|
||||
store.writable = true;
|
||||
let mut max = page.next.expect("cursor");
|
||||
max.index = u64::MAX;
|
||||
assert_eq!(
|
||||
store.created_events_page(Some(&max), 1).err().map(|e| e.code()),
|
||||
Some("identity_mismatch")
|
||||
);
|
||||
}
|
||||
struct Staging<'a>(&'a WriteTransaction);
|
||||
impl snapshot::StagingSink for Staging<'_> {
|
||||
fn row(&mut self, section: snapshot::Section, key: &[u8], value: &[u8]) -> io::Result<()> {
|
||||
let table = match section {
|
||||
snapshot::Section::Heads => application::HEADS,
|
||||
snapshot::Section::Versions => application::VERSIONS,
|
||||
snapshot::Section::Events => application::EVENTS,
|
||||
snapshot::Section::Operations => application::OPERATIONS,
|
||||
};
|
||||
self.0
|
||||
.open_table(table)
|
||||
.map_err(io::Error::other)?
|
||||
.insert(key, value)
|
||||
.map_err(io::Error::other)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
fn limits() -> snapshot::Limits {
|
||||
snapshot::Limits {
|
||||
total_bytes: 1_048_576,
|
||||
membership_bytes: 4096,
|
||||
key_bytes: 4096,
|
||||
value_bytes: 8192,
|
||||
records: 1024,
|
||||
}
|
||||
}
|
||||
async fn artifact(store: Store) -> (snapshot::CapabilityArtifact<Vec<u8>>, EventPage) {
|
||||
let writer = NativeLogWriter::new(store);
|
||||
let before = writer.created_events_page(None, 256).await.expect("async page");
|
||||
let artifact = writer
|
||||
.snapshot_view()
|
||||
.await
|
||||
.expect("snapshot view")
|
||||
.stream_capability_artifact(Vec::new(), limits())
|
||||
.expect("capability export");
|
||||
(artifact, before)
|
||||
}
|
||||
fn stage(artifact: snapshot::CapabilityArtifact<Vec<u8>>, damage: bool) -> Result<(tempfile::TempDir, Store), StoreError> {
|
||||
let dir = tempfile::tempdir().expect("stage directory");
|
||||
let database = redb::Database::create(dir.path().join("stage.redb")).expect("isolated database");
|
||||
let tx = database.begin_write()?;
|
||||
let header = snapshot::decode_capability_artifact(
|
||||
snapshot::CapabilityArtifact {
|
||||
manifest: artifact.manifest,
|
||||
payload: std::io::Cursor::new(artifact.payload),
|
||||
},
|
||||
&mut Staging(&tx),
|
||||
limits(),
|
||||
)
|
||||
.expect("real capability decoder");
|
||||
if damage {
|
||||
let mut events = tx.open_table(application::EVENTS)?;
|
||||
let value = events.get([9; 16].as_slice())?.expect("event").value().to_vec();
|
||||
events.insert([6; 16].as_slice(), value.as_slice())?;
|
||||
events.remove([9; 16].as_slice())?;
|
||||
}
|
||||
rebuild_from_authority(&tx, 6, &header.binding, header.applied)?;
|
||||
{
|
||||
let mut meta = tx.open_table(crate::META)?;
|
||||
meta.insert("schema", 6u64.to_be_bytes().as_slice())?;
|
||||
meta.insert(
|
||||
"identity",
|
||||
crate::encode_identity(NodeIdentity {
|
||||
cluster_id: header.cluster_id,
|
||||
..identity()
|
||||
})
|
||||
.as_slice(),
|
||||
)?;
|
||||
meta.insert("capture_binding", crate::created::binding_bytes(&header.binding)?.as_slice())?;
|
||||
let mut applied = vec![1];
|
||||
applied.extend(encode_id(header.applied.expect("cut")));
|
||||
meta.insert("last_applied", applied.as_slice())?;
|
||||
}
|
||||
tx.commit().expect("stage commit");
|
||||
// A test-only staging handle, not a production installer or authority grant.
|
||||
Ok((
|
||||
dir,
|
||||
Store {
|
||||
database,
|
||||
writable: true,
|
||||
},
|
||||
))
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn snapshot_authority_rebuild_matches_source() {
|
||||
let (_dir, mut store) = fixture();
|
||||
apply(&mut store, 2, command(9));
|
||||
let mut rejected = command(3);
|
||||
rejected.object = command(9).object;
|
||||
apply(&mut store, 3, rejected);
|
||||
store.promote_created_provenance().expect("schema6");
|
||||
let (artifact, before) = artifact(store).await;
|
||||
let (_stage, restored) = stage(artifact, false).expect("staged authority index");
|
||||
assert_eq!(
|
||||
restored.created_events_page(None, 256).expect("restored page"),
|
||||
before,
|
||||
"SOURCE_PAGE_SNAPSHOT_BOUND"
|
||||
);
|
||||
let first = restored.created_events_page(None, 1).expect("first");
|
||||
assert_eq!(
|
||||
restored.created_events_page(first.next.as_ref(), 1).expect("resume").events[0],
|
||||
before.events[1]
|
||||
);
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn snapshot_event_operation_mismatch_rejected() {
|
||||
let (_dir, mut store) = fixture();
|
||||
store.promote_created_provenance().expect("schema6");
|
||||
let (artifact, _) = artifact(store).await;
|
||||
assert_eq!(
|
||||
stage(artifact, true).err().map(|e| e.code()),
|
||||
Some("corrupt_record"),
|
||||
"SOURCE_PAGE_SNAPSHOT_BOUND"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,451 @@
|
||||
//! Same-UID local reads from an owner's live native store. No ACK or cursor writes.
|
||||
use crate::{EventCursor, EventPage, NativeLogWriter, NativeWriteError, NodeIdentity, StoreError};
|
||||
use redb::ReadableDatabase;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{
|
||||
fs, io,
|
||||
os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt},
|
||||
path::{Path, PathBuf},
|
||||
time::Duration,
|
||||
};
|
||||
use tokio::{
|
||||
io::{AsyncReadExt, AsyncWriteExt},
|
||||
net::{UnixListener, UnixStream},
|
||||
sync::{OwnedSemaphorePermit, watch},
|
||||
task::JoinSet,
|
||||
};
|
||||
|
||||
pub const WIRE_VERSION: u16 = 1;
|
||||
pub const MAX_REQUEST_BYTES: usize = 4096;
|
||||
pub const MAX_RESPONSE_BYTES: usize = 8 * 1024 * 1024;
|
||||
pub const MAX_CONNECTIONS: usize = 4;
|
||||
pub const IO_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ReadPage {
|
||||
pub version: u16,
|
||||
pub expected_cluster: [u8; 16],
|
||||
pub cursor: Option<EventCursor>,
|
||||
pub limit: usize,
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ReadPageResponse {
|
||||
pub version: u16,
|
||||
pub identity: NodeIdentity,
|
||||
pub page: EventPage,
|
||||
}
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub enum Rejection {
|
||||
InvalidRequest,
|
||||
WrongCluster,
|
||||
InvalidCursor,
|
||||
StoreUnavailable,
|
||||
ResponseLimit,
|
||||
}
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
#[serde(tag = "status", deny_unknown_fields)]
|
||||
pub enum ReadPageReply {
|
||||
Page { response: ReadPageResponse },
|
||||
Rejected { version: u16, error: Rejection },
|
||||
}
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum ReaderError {
|
||||
#[error("SOURCE_READER_IO")]
|
||||
Io(#[from] io::Error),
|
||||
#[error("SOURCE_READER_PERMISSION")]
|
||||
Permission,
|
||||
#[error("SOURCE_READER_PATH_OCCUPIED")]
|
||||
Occupied,
|
||||
#[error("SOURCE_READER_PATH_CHANGED")]
|
||||
PathChanged,
|
||||
#[error("SOURCE_READER_TIMEOUT")]
|
||||
Timeout,
|
||||
#[error("SOURCE_READER_PROTOCOL")]
|
||||
Protocol,
|
||||
#[error("SOURCE_READER_REJECTED: {0:?}")]
|
||||
Rejected(Rejection),
|
||||
#[error("SOURCE_READER_WORKER")]
|
||||
Worker(#[source] tokio::task::JoinError),
|
||||
}
|
||||
|
||||
impl crate::Store {
|
||||
fn durable_source_identity(&self) -> Result<NodeIdentity, StoreError> {
|
||||
let tx = self.database.begin_read()?;
|
||||
let meta = tx.open_table(crate::META)?;
|
||||
let identity = meta.get("identity")?.ok_or(StoreError::CorruptRecord)?;
|
||||
crate::decode_identity(identity.value())
|
||||
}
|
||||
}
|
||||
impl NativeLogWriter {
|
||||
/// Read the validated durable identity, never an owner-supplied cluster label.
|
||||
pub async fn durable_identity(&self) -> Result<NodeIdentity, NativeWriteError> {
|
||||
let owner = self.store.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
owner
|
||||
.lock()
|
||||
.map_err(|_| NativeWriteError::Poisoned)?
|
||||
.durable_source_identity()
|
||||
.map_err(NativeWriteError::Store)
|
||||
})
|
||||
.await
|
||||
.map_err(NativeWriteError::Worker)?
|
||||
}
|
||||
}
|
||||
|
||||
fn uid() -> u32 {
|
||||
rustix::process::geteuid().as_raw()
|
||||
}
|
||||
fn fingerprint(meta: &fs::Metadata) -> (u64, u64) {
|
||||
(meta.dev(), meta.ino())
|
||||
}
|
||||
fn private_parent(path: &Path) -> Result<(PathBuf, (u64, u64)), ReaderError> {
|
||||
if !path.is_absolute() || path.file_name().is_none() {
|
||||
return Err(ReaderError::Permission);
|
||||
}
|
||||
let parent = path.parent().ok_or(ReaderError::Permission)?;
|
||||
let meta = fs::symlink_metadata(parent)?;
|
||||
if !meta.is_dir() || meta.uid() != uid() || meta.mode() & 0o077 != 0 {
|
||||
return Err(ReaderError::Permission);
|
||||
}
|
||||
let resolved = fs::canonicalize(parent)?;
|
||||
let resolved_meta = fs::symlink_metadata(&resolved)?;
|
||||
if fingerprint(&meta) != fingerprint(&resolved_meta) {
|
||||
return Err(ReaderError::PathChanged);
|
||||
}
|
||||
Ok((resolved.join(path.file_name().ok_or(ReaderError::Permission)?), fingerprint(&meta)))
|
||||
}
|
||||
fn owned_socket(path: &Path) -> Result<fs::Metadata, ReaderError> {
|
||||
let meta = fs::symlink_metadata(path)?;
|
||||
if !meta.file_type().is_socket() || meta.uid() != uid() {
|
||||
return Err(ReaderError::Permission);
|
||||
}
|
||||
Ok(meta)
|
||||
}
|
||||
fn socket_metadata(path: &Path) -> Result<(u64, u64), ReaderError> {
|
||||
let meta = owned_socket(path)?;
|
||||
if meta.mode() & 0o077 != 0 {
|
||||
return Err(ReaderError::Permission);
|
||||
}
|
||||
Ok(fingerprint(&meta))
|
||||
}
|
||||
fn peer(stream: &UnixStream) -> Result<(), ReaderError> {
|
||||
if stream.peer_cred()?.uid() != uid() {
|
||||
return Err(ReaderError::Permission);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
struct SocketPath {
|
||||
path: PathBuf,
|
||||
inode: Option<(u64, u64)>,
|
||||
parent: (u64, u64),
|
||||
}
|
||||
impl SocketPath {
|
||||
fn remove(&mut self) -> Result<(), ReaderError> {
|
||||
// One cleanup attempt only: never let Drop re-evaluate a later namespace.
|
||||
let Some(inode) = self.inode.take() else {
|
||||
return Ok(());
|
||||
};
|
||||
let (_, parent) = private_parent(&self.path)?;
|
||||
if parent != self.parent || fingerprint(&owned_socket(&self.path)?) != inode {
|
||||
return Err(ReaderError::PathChanged);
|
||||
}
|
||||
fs::remove_file(&self.path)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
impl Drop for SocketPath {
|
||||
fn drop(&mut self) {
|
||||
// Explicit shutdown reports cleanup failure; cancellation preserves detected replacements.
|
||||
let _ = self.remove();
|
||||
}
|
||||
}
|
||||
|
||||
/// Signal shutdown, then await the owner's `serve` task to drain workers and clean up.
|
||||
#[derive(Clone)]
|
||||
pub struct Shutdown(watch::Sender<bool>);
|
||||
impl Shutdown {
|
||||
pub fn shutdown(&self) {
|
||||
let _ = self.0.send(true);
|
||||
}
|
||||
}
|
||||
pub struct SourceReaderServer {
|
||||
listener: UnixListener,
|
||||
writer: NativeLogWriter,
|
||||
socket: SocketPath,
|
||||
shutdown: watch::Receiver<bool>,
|
||||
}
|
||||
impl SourceReaderServer {
|
||||
pub fn bind(path: impl AsRef<Path>, writer: NativeLogWriter) -> Result<(Self, Shutdown), ReaderError> {
|
||||
let (path, parent) = private_parent(path.as_ref())?;
|
||||
match fs::symlink_metadata(&path) {
|
||||
Ok(_) => return Err(ReaderError::Occupied),
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => {}
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
// The private parent prevents a foreign UID from observing the initial socket mode.
|
||||
let listener = UnixListener::bind(&path)?;
|
||||
let inode = fingerprint(&fs::symlink_metadata(&path)?);
|
||||
let socket = SocketPath {
|
||||
path,
|
||||
inode: Some(inode),
|
||||
parent,
|
||||
};
|
||||
fs::set_permissions(&socket.path, fs::Permissions::from_mode(0o600))?;
|
||||
if private_parent(&socket.path)?.1 != parent || socket_metadata(&socket.path)? != inode {
|
||||
return Err(ReaderError::PathChanged);
|
||||
}
|
||||
let (sender, shutdown) = watch::channel(false);
|
||||
Ok((
|
||||
Self {
|
||||
listener,
|
||||
writer,
|
||||
socket,
|
||||
shutdown,
|
||||
},
|
||||
Shutdown(sender),
|
||||
))
|
||||
}
|
||||
pub async fn serve(mut self) -> Result<(), ReaderError> {
|
||||
let mut tasks = JoinSet::new();
|
||||
let outcome = async {
|
||||
loop {
|
||||
if *self.shutdown.borrow() {
|
||||
break;
|
||||
}
|
||||
// Admission is shared by every clone of this writer, including re-bound services.
|
||||
let admission = self.writer.source_reader_admission.clone();
|
||||
let permit = tokio::select! {
|
||||
result = admission.acquire_owned() => result.map_err(|_| ReaderError::Protocol)?,
|
||||
_ = self.shutdown.changed() => break,
|
||||
};
|
||||
let stream = tokio::select! {
|
||||
result = self.listener.accept() => result?.0,
|
||||
_ = self.shutdown.changed() => break,
|
||||
};
|
||||
if peer(&stream).is_err() {
|
||||
continue;
|
||||
}
|
||||
let writer = self.writer.clone();
|
||||
tasks.spawn(async move {
|
||||
let _ = connection(stream, writer, permit).await;
|
||||
});
|
||||
while let Some(result) = tasks.try_join_next() {
|
||||
result.map_err(ReaderError::Worker)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
.await;
|
||||
// Ordinary shutdown drains connections and blocking workers, never aborts storage work.
|
||||
while let Some(result) = tasks.join_next().await {
|
||||
result.map_err(ReaderError::Worker)?;
|
||||
}
|
||||
drop(self.listener);
|
||||
self.socket.remove()?;
|
||||
outcome
|
||||
}
|
||||
}
|
||||
|
||||
fn reject(error: Rejection) -> ReadPageReply {
|
||||
ReadPageReply::Rejected {
|
||||
version: WIRE_VERSION,
|
||||
error,
|
||||
}
|
||||
}
|
||||
fn read_page(writer: &NativeLogWriter, request: ReadPage) -> ReadPageReply {
|
||||
if request.version != WIRE_VERSION
|
||||
|| request.expected_cluster == [0; 16]
|
||||
|| request.limit == 0
|
||||
|| request.limit > crate::MAX_EVENT_PAGE
|
||||
{
|
||||
return reject(Rejection::InvalidRequest);
|
||||
}
|
||||
// Lock order: connection admission, then Store. Native reads take no writer-ordering lock.
|
||||
let read = || -> Result<ReadPageReply, NativeWriteError> {
|
||||
let store = writer.store.lock().map_err(|_| NativeWriteError::Poisoned)?;
|
||||
let identity = store.durable_source_identity()?;
|
||||
if identity.cluster_id != request.expected_cluster {
|
||||
return Ok(reject(Rejection::WrongCluster));
|
||||
}
|
||||
let page = match store.created_events_page(request.cursor.as_ref(), request.limit) {
|
||||
Ok(page) => page,
|
||||
Err(StoreError::IdentityMismatch) => return Ok(reject(Rejection::InvalidCursor)),
|
||||
Err(error) => return Err(error.into()),
|
||||
};
|
||||
Ok(ReadPageReply::Page {
|
||||
response: ReadPageResponse {
|
||||
version: WIRE_VERSION,
|
||||
identity,
|
||||
page,
|
||||
},
|
||||
})
|
||||
};
|
||||
read().unwrap_or_else(|_| reject(Rejection::StoreUnavailable))
|
||||
}
|
||||
|
||||
struct BoundedBytes {
|
||||
bytes: Vec<u8>,
|
||||
limit: usize,
|
||||
}
|
||||
impl io::Write for BoundedBytes {
|
||||
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
|
||||
if bytes.len() > self.limit.saturating_sub(self.bytes.len()) {
|
||||
return Err(io::Error::other("source reader frame limit"));
|
||||
}
|
||||
self.bytes.extend_from_slice(bytes);
|
||||
Ok(bytes.len())
|
||||
}
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
fn encode(value: &impl Serialize, limit: usize) -> Result<Vec<u8>, ReaderError> {
|
||||
let mut buffer = BoundedBytes {
|
||||
bytes: Vec::new(),
|
||||
limit,
|
||||
};
|
||||
serde_json::to_writer(&mut buffer, value).map_err(|_| ReaderError::Protocol)?;
|
||||
Ok(buffer.bytes)
|
||||
}
|
||||
async fn frame_read(stream: &mut UnixStream, limit: usize) -> Result<Vec<u8>, ReaderError> {
|
||||
let len = usize::try_from(stream.read_u32().await?).map_err(|_| ReaderError::Protocol)?;
|
||||
if len == 0 || len > limit {
|
||||
return Err(ReaderError::Protocol);
|
||||
}
|
||||
let mut bytes = vec![0; len];
|
||||
stream.read_exact(&mut bytes).await?;
|
||||
Ok(bytes)
|
||||
}
|
||||
async fn frame_write(stream: &mut UnixStream, bytes: &[u8]) -> Result<(), ReaderError> {
|
||||
stream
|
||||
.write_u32(u32::try_from(bytes.len()).map_err(|_| ReaderError::Protocol)?)
|
||||
.await?;
|
||||
stream.write_all(bytes).await?;
|
||||
Ok(())
|
||||
}
|
||||
async fn connection(mut stream: UnixStream, writer: NativeLogWriter, permit: OwnedSemaphorePermit) -> Result<(), ReaderError> {
|
||||
let bytes = tokio::time::timeout(IO_TIMEOUT, frame_read(&mut stream, MAX_REQUEST_BYTES))
|
||||
.await
|
||||
.map_err(|_| ReaderError::Timeout)??;
|
||||
let request: ReadPage = match serde_json::from_slice(&bytes) {
|
||||
Ok(request) => request,
|
||||
Err(_) => {
|
||||
let bytes = encode(&reject(Rejection::InvalidRequest), MAX_RESPONSE_BYTES)?;
|
||||
return tokio::time::timeout(IO_TIMEOUT, frame_write(&mut stream, &bytes))
|
||||
.await
|
||||
.map_err(|_| ReaderError::Timeout)?;
|
||||
}
|
||||
};
|
||||
// The blocking worker owns admission until read + bounded serialization finish, even
|
||||
// if a deadline, shutdown or a dropped serve task cancels its awaiting connection.
|
||||
let (bytes, _permit) = tokio::task::spawn_blocking(move || {
|
||||
let reply = read_page(&writer, request);
|
||||
let bytes = encode(&reply, MAX_RESPONSE_BYTES).or_else(|_| encode(&reject(Rejection::ResponseLimit), MAX_RESPONSE_BYTES));
|
||||
(bytes, permit)
|
||||
})
|
||||
.await
|
||||
.map_err(ReaderError::Worker)?;
|
||||
tokio::time::timeout(IO_TIMEOUT, frame_write(&mut stream, &bytes?))
|
||||
.await
|
||||
.map_err(|_| ReaderError::Timeout)?
|
||||
}
|
||||
|
||||
/// Connect only to a private socket owned by the effective UID, verified at both ends.
|
||||
#[derive(Clone)]
|
||||
pub struct SourceReaderClient {
|
||||
path: PathBuf,
|
||||
}
|
||||
impl SourceReaderClient {
|
||||
pub fn new(path: impl AsRef<Path>) -> Result<Self, ReaderError> {
|
||||
let (path, _) = private_parent(path.as_ref())?;
|
||||
socket_metadata(&path)?;
|
||||
Ok(Self { path })
|
||||
}
|
||||
pub async fn page(
|
||||
&self,
|
||||
expected_cluster: [u8; 16],
|
||||
cursor: Option<EventCursor>,
|
||||
limit: usize,
|
||||
) -> Result<ReadPageResponse, ReaderError> {
|
||||
let request = ReadPage {
|
||||
version: WIRE_VERSION,
|
||||
expected_cluster,
|
||||
cursor,
|
||||
limit,
|
||||
};
|
||||
tokio::time::timeout(IO_TIMEOUT, self.page_inner(request))
|
||||
.await
|
||||
.map_err(|_| ReaderError::Timeout)?
|
||||
}
|
||||
async fn page_inner(&self, request: ReadPage) -> Result<ReadPageResponse, ReaderError> {
|
||||
let (_, parent) = private_parent(&self.path)?;
|
||||
let inode = socket_metadata(&self.path)?;
|
||||
let mut stream = UnixStream::connect(&self.path).await?;
|
||||
peer(&stream)?;
|
||||
if private_parent(&self.path)?.1 != parent || socket_metadata(&self.path)? != inode {
|
||||
return Err(ReaderError::PathChanged);
|
||||
}
|
||||
frame_write(&mut stream, &encode(&request, MAX_REQUEST_BYTES)?).await?;
|
||||
let bytes = frame_read(&mut stream, MAX_RESPONSE_BYTES).await?;
|
||||
match serde_json::from_slice::<ReadPageReply>(&bytes).map_err(|_| ReaderError::Protocol)? {
|
||||
ReadPageReply::Page { response } => {
|
||||
if response.version != WIRE_VERSION
|
||||
|| response.identity.cluster_id != request.expected_cluster
|
||||
|| response.identity.cluster_id == [0; 16]
|
||||
|| response.page.events.len() > request.limit
|
||||
|| request.limit == 0
|
||||
|| request.limit > crate::MAX_EVENT_PAGE
|
||||
{
|
||||
return Err(ReaderError::Protocol);
|
||||
}
|
||||
let next = response
|
||||
.page
|
||||
.events
|
||||
.last()
|
||||
.map(|event| EventCursor::new(response.identity.cluster_id, event.event_id, &event.captured))
|
||||
.or(request.cursor);
|
||||
if response.page.next != next {
|
||||
return Err(ReaderError::Protocol);
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
ReadPageReply::Rejected { version, error } if version == WIRE_VERSION => Err(ReaderError::Rejected(error)),
|
||||
_ => Err(ReaderError::Protocol),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod cleanup_tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn source_reader_cleanup_is_disarmed_before_rebind() {
|
||||
let directory = tempfile::tempdir().expect("private socket owner");
|
||||
fs::set_permissions(directory.path(), fs::Permissions::from_mode(0o700)).expect("private directory mode");
|
||||
let (path, parent) = private_parent(&directory.path().join("reader.sock")).expect("private path");
|
||||
let listener = std::os::unix::net::UnixListener::bind(&path).expect("first real socket");
|
||||
fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).expect("first socket mode");
|
||||
let inode = socket_metadata(&path).expect("first socket identity");
|
||||
let mut guard = SocketPath {
|
||||
path: path.clone(),
|
||||
inode: Some(inode),
|
||||
parent,
|
||||
};
|
||||
drop(listener);
|
||||
guard.remove().expect("explicit owner cleanup");
|
||||
assert!(guard.inode.is_none(), "SOURCE_READER_CLEANUP_DISARMED");
|
||||
let listener = std::os::unix::net::UnixListener::bind(&path).expect("immediate new owner bind");
|
||||
fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).expect("new socket mode");
|
||||
let replacement = socket_metadata(&path).expect("new owner socket");
|
||||
drop(guard);
|
||||
assert_eq!(
|
||||
socket_metadata(&path).expect("new socket preserved"),
|
||||
replacement,
|
||||
"SOURCE_READER_OLD_GUARD_PRESERVES_REBIND"
|
||||
);
|
||||
let _client = std::os::unix::net::UnixStream::connect(&path).expect("new service reachable");
|
||||
let _connection = listener.accept().expect("new owner accepts");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,502 @@
|
||||
//! Real NativeLogWriter and socket owner is this test process, not a production S3 owner.
|
||||
use crate::{source_reader::*, *};
|
||||
use std::{
|
||||
collections::{BTreeMap, BTreeSet},
|
||||
os::unix::fs::{MetadataExt, PermissionsExt},
|
||||
path::PathBuf,
|
||||
};
|
||||
use tokio::{
|
||||
io::{AsyncReadExt, AsyncWriteExt},
|
||||
net::UnixStream,
|
||||
};
|
||||
fn identity() -> NodeIdentity {
|
||||
NodeIdentity {
|
||||
cluster_id: [7; 16],
|
||||
node_id: 0,
|
||||
}
|
||||
}
|
||||
fn binding() -> CaptureBinding {
|
||||
CaptureBinding {
|
||||
revision: 9,
|
||||
binding_id: [8; 16],
|
||||
target_id: [9; 16],
|
||||
}
|
||||
}
|
||||
fn id(index: u64) -> LogId {
|
||||
LogId {
|
||||
term: 8,
|
||||
leader_node: 0,
|
||||
index,
|
||||
}
|
||||
}
|
||||
fn command(event: u8) -> DecideCreated {
|
||||
DecideCreated {
|
||||
operation_id: [event; 16],
|
||||
object: ObjectIdentity {
|
||||
bucket_incarnation: [2; 16],
|
||||
key: vec![event],
|
||||
},
|
||||
expected_head: None,
|
||||
prepared: PreparedIdentity {
|
||||
version_id: [event; 16],
|
||||
preparation_id: [4; 16],
|
||||
content_digest: [5; 32],
|
||||
content_length: 123,
|
||||
},
|
||||
binding_revision: 9,
|
||||
}
|
||||
}
|
||||
fn fixture() -> (tempfile::TempDir, NativeLogWriter) {
|
||||
let dir = tempfile::tempdir().expect("real source reader directory");
|
||||
std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o700)).expect("private owner directory");
|
||||
let mut store = Store::initialize(dir.path().join("db"), identity()).expect("initialize real redb");
|
||||
store.pin_capture_binding(binding()).expect("binding");
|
||||
store.enable_native_empty().expect("native schema");
|
||||
store.promote_created_provenance().expect("provenance");
|
||||
let membership = CaptureMembership::new(
|
||||
Some(id(0)),
|
||||
vec![BTreeSet::from([0])],
|
||||
BTreeMap::from([(0, CaptureNode::new("http://127.0.0.1:7000".into()).expect("member"))]),
|
||||
)
|
||||
.expect("membership");
|
||||
let membership = openraft::StoredMembership::try_from(&membership).expect("stored membership");
|
||||
store
|
||||
.append_native(&[CaptureRaftEntry {
|
||||
log_id: id(0).into(),
|
||||
payload: openraft::EntryPayload::Membership(membership.membership().clone()),
|
||||
}])
|
||||
.expect("membership log");
|
||||
store.save_committed(id(0)).expect("membership commit");
|
||||
store.apply_next_native(id(0)).expect("membership apply");
|
||||
(dir, NativeLogWriter::new(store))
|
||||
}
|
||||
async fn apply(writer: &NativeLogWriter, index: u64, event: u8) {
|
||||
let entry = CaptureRaftEntry {
|
||||
log_id: id(index).into(),
|
||||
payload: openraft::EntryPayload::Normal(CaptureCommand::try_created(command(event)).expect("canonical native command")),
|
||||
};
|
||||
writer
|
||||
.append_native(vec![entry.clone()])
|
||||
.await
|
||||
.expect("live durable log append");
|
||||
let response = writer.apply_committed(vec![entry]).await.expect("live committed apply");
|
||||
assert!(
|
||||
matches!(response[0].created_result(), Some(CreatedResult::Created { .. })),
|
||||
"SOURCE_READER_REAL_APPLY"
|
||||
);
|
||||
}
|
||||
fn start(
|
||||
dir: &tempfile::TempDir,
|
||||
writer: &NativeLogWriter,
|
||||
) -> (PathBuf, SourceReaderClient, Shutdown, tokio::task::JoinHandle<Result<(), ReaderError>>) {
|
||||
let path = dir.path().join("reader.sock");
|
||||
let (server, shutdown) = SourceReaderServer::bind(&path, writer.clone()).expect("owner socket bind");
|
||||
let client = SourceReaderClient::new(&path).expect("trusted local client");
|
||||
let task = tokio::spawn(server.serve());
|
||||
(path, client, shutdown, task)
|
||||
}
|
||||
async fn stop(path: PathBuf, shutdown: Shutdown, task: tokio::task::JoinHandle<Result<(), ReaderError>>) {
|
||||
shutdown.shutdown();
|
||||
task.await.expect("serve task").expect("clean shutdown");
|
||||
assert!(!path.exists(), "SOURCE_READER_SOCKET_CLEANUP");
|
||||
}
|
||||
async fn raw(path: &std::path::Path, value: serde_json::Value) -> ReadPageReply {
|
||||
let mut stream = UnixStream::connect(path).await.expect("real socket connect");
|
||||
let bytes = serde_json::to_vec(&value).expect("wire request");
|
||||
stream
|
||||
.write_u32(bytes.len().try_into().expect("request length"))
|
||||
.await
|
||||
.expect("frame prefix");
|
||||
stream.write_all(&bytes).await.expect("frame bytes");
|
||||
let len = stream.read_u32().await.expect("reply prefix");
|
||||
let mut bytes = vec![0; usize::try_from(len).expect("reply length")];
|
||||
stream.read_exact(&mut bytes).await.expect("reply bytes");
|
||||
serde_json::from_slice(&bytes).expect("strict native reply")
|
||||
}
|
||||
fn request() -> serde_json::Value {
|
||||
serde_json::to_value(ReadPage {
|
||||
version: WIRE_VERSION,
|
||||
expected_cluster: identity().cluster_id,
|
||||
cursor: None,
|
||||
limit: 2,
|
||||
})
|
||||
.expect("native request")
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn source_reader_live_apply() {
|
||||
let (dir, writer) = fixture();
|
||||
let (path, client, shutdown, task) = start(&dir, &writer);
|
||||
assert_eq!(writer.durable_identity().await.expect("durable META identity"), identity());
|
||||
assert!(
|
||||
client
|
||||
.page(identity().cluster_id, None, 2)
|
||||
.await
|
||||
.expect("empty live page")
|
||||
.page
|
||||
.events
|
||||
.is_empty()
|
||||
);
|
||||
apply(&writer, 1, 9).await;
|
||||
let response = client
|
||||
.page(identity().cluster_id, None, 2)
|
||||
.await
|
||||
.expect("live committed page");
|
||||
assert_eq!(response.identity, identity(), "SOURCE_READER_DURABLE_IDENTITY");
|
||||
assert_eq!(response.page.events[0].event_id, [9; 16], "SOURCE_READER_LIVE_EVENT");
|
||||
assert_eq!(response.page.events[0].captured.binding, binding());
|
||||
{
|
||||
let mut corrupt = writer.store.lock().expect("test owner");
|
||||
let tx = corrupt.write_transaction().expect("corrupt META transaction");
|
||||
tx.open_table(META)
|
||||
.expect("META")
|
||||
.insert("identity", &[0; 24][..])
|
||||
.expect("corrupt identity");
|
||||
corrupt.commit(tx).expect("persist corruption");
|
||||
}
|
||||
assert!(writer.durable_identity().await.is_err(), "SOURCE_READER_META_CORRUPTION_REJECTED");
|
||||
assert!(
|
||||
matches!(
|
||||
client.page(identity().cluster_id, None, 2).await,
|
||||
Err(ReaderError::Rejected(Rejection::StoreUnavailable))
|
||||
),
|
||||
"SOURCE_READER_NO_CALLER_IDENTITY_SUBSTITUTION"
|
||||
);
|
||||
stop(path, shutdown, task).await;
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn source_reader_restart_cursor() {
|
||||
let (dir, writer) = fixture();
|
||||
apply(&writer, 1, 9).await;
|
||||
apply(&writer, 2, 7).await;
|
||||
let (path, client, shutdown, task) = start(&dir, &writer);
|
||||
let first = client.page(identity().cluster_id, None, 1).await.expect("first");
|
||||
stop(path, shutdown, task).await;
|
||||
drop(writer);
|
||||
let writer = NativeLogWriter::new(Store::open(dir.path().join("db"), identity()).expect("restart same durable store"));
|
||||
let (path, client, shutdown, task) = start(&dir, &writer);
|
||||
let next = client
|
||||
.page(identity().cluster_id, first.page.next, 1)
|
||||
.await
|
||||
.expect("resume durable cursor");
|
||||
assert_eq!(next.page.events[0].event_id, [7; 16], "SOURCE_READER_RESTART_CONTINUES");
|
||||
stop(path, shutdown, task).await;
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn source_reader_page_boundaries() {
|
||||
let (dir, writer) = fixture();
|
||||
for (index, event) in [(1, 9), (2, 7), (3, 1)] {
|
||||
apply(&writer, index, event).await;
|
||||
}
|
||||
let (path, client, shutdown, task) = start(&dir, &writer);
|
||||
let first = client.page(identity().cluster_id, None, 2).await.expect("first two");
|
||||
assert_eq!(
|
||||
first.page.events.iter().map(|e| e.event_id).collect::<Vec<_>>(),
|
||||
vec![[9; 16], [7; 16]],
|
||||
"SOURCE_READER_DECISION_ORDER"
|
||||
);
|
||||
let next = client
|
||||
.page(identity().cluster_id, first.page.next, 2)
|
||||
.await
|
||||
.expect("remaining");
|
||||
assert_eq!(next.page.events.len(), 1);
|
||||
let end = client
|
||||
.page(identity().cluster_id, next.page.next.clone(), 2)
|
||||
.await
|
||||
.expect("end");
|
||||
assert!(end.page.events.is_empty());
|
||||
assert_eq!(end.page.next, next.page.next, "SOURCE_READER_CURSOR_RETAINED_AT_END");
|
||||
for limit in [0, MAX_EVENT_PAGE + 1] {
|
||||
assert!(
|
||||
matches!(
|
||||
client.page(identity().cluster_id, None, limit).await,
|
||||
Err(ReaderError::Rejected(Rejection::InvalidRequest))
|
||||
),
|
||||
"SOURCE_READER_LIMIT_REJECTED"
|
||||
);
|
||||
}
|
||||
stop(path, shutdown, task).await;
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn source_reader_wrong_identity() {
|
||||
let (dir, writer) = fixture();
|
||||
apply(&writer, 1, 9).await;
|
||||
let (path, client, shutdown, task) = start(&dir, &writer);
|
||||
let mut wrong = request();
|
||||
wrong["expected_cluster"] = serde_json::to_value([6; 16]).expect("wrong cluster");
|
||||
assert!(
|
||||
matches!(
|
||||
raw(&path, wrong).await,
|
||||
ReadPageReply::Rejected {
|
||||
error: Rejection::WrongCluster,
|
||||
..
|
||||
}
|
||||
),
|
||||
"SOURCE_READER_CLUSTER: reject before reading or returning foreign events"
|
||||
);
|
||||
let first = client.page(identity().cluster_id, None, 1).await.expect("real cursor");
|
||||
let mut wrong = serde_json::to_value(first.page.next.expect("cursor")).expect("cursor wire");
|
||||
wrong["event"] = serde_json::to_value([1; 16]).expect("wrong event");
|
||||
let wrong: EventCursor = serde_json::from_value(wrong).expect("syntactically valid foreign cursor");
|
||||
assert!(
|
||||
matches!(
|
||||
client.page(identity().cluster_id, Some(wrong), 1).await,
|
||||
Err(ReaderError::Rejected(Rejection::InvalidCursor))
|
||||
),
|
||||
"SOURCE_READER_CURSOR_BOUND"
|
||||
);
|
||||
for (field, value) in [
|
||||
("version", serde_json::json!(2)),
|
||||
("unknown", serde_json::json!(true)),
|
||||
("cursor", serde_json::json!({"unknown":true})),
|
||||
] {
|
||||
let mut wrong = request();
|
||||
wrong[field] = value;
|
||||
assert!(
|
||||
matches!(
|
||||
raw(&path, wrong).await,
|
||||
ReadPageReply::Rejected {
|
||||
error: Rejection::InvalidRequest,
|
||||
..
|
||||
}
|
||||
),
|
||||
"SOURCE_READER_STRICT_REQUEST"
|
||||
);
|
||||
}
|
||||
stop(path, shutdown, task).await;
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn source_reader_socket_ownership() {
|
||||
let (dir, writer) = fixture();
|
||||
let path = dir.path().join("reader.sock");
|
||||
std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o750)).expect("insecure directory");
|
||||
assert!(
|
||||
matches!(SourceReaderServer::bind(&path, writer.clone()), Err(ReaderError::Permission)),
|
||||
"SOURCE_READER_PRIVATE_DIRECTORY"
|
||||
);
|
||||
std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o700)).expect("restore private directory");
|
||||
std::fs::write(&path, b"preserve me").expect("occupied file");
|
||||
assert!(
|
||||
matches!(SourceReaderServer::bind(&path, writer.clone()), Err(ReaderError::Occupied)),
|
||||
"SOURCE_READER_OCCUPIED_PATH"
|
||||
);
|
||||
assert_eq!(std::fs::read(&path).expect("unchanged file"), b"preserve me");
|
||||
std::fs::remove_file(&path).expect("remove own fixture");
|
||||
std::os::unix::fs::symlink(dir.path().join("missing"), &path).expect("unknown symlink");
|
||||
assert!(matches!(SourceReaderServer::bind(&path, writer.clone()), Err(ReaderError::Occupied)));
|
||||
assert!(SourceReaderClient::new(&path).is_err());
|
||||
std::fs::remove_file(&path).expect("remove own symlink");
|
||||
let long = dir.path().join("x".repeat(300));
|
||||
assert!(SourceReaderServer::bind(&long, writer.clone()).is_err(), "SOURCE_READER_LONG_SOCKET_PATH");
|
||||
let parent_link = dir.path().join("parent-link");
|
||||
std::os::unix::fs::symlink(dir.path(), &parent_link).expect("parent symlink");
|
||||
assert!(
|
||||
matches!(
|
||||
SourceReaderServer::bind(parent_link.join("socket"), writer.clone()),
|
||||
Err(ReaderError::Permission)
|
||||
),
|
||||
"SOURCE_READER_PARENT_SYMLINK_DENIED"
|
||||
);
|
||||
let (path, client, shutdown, task) = start(&dir, &writer);
|
||||
let meta = std::fs::symlink_metadata(&path).expect("socket metadata");
|
||||
assert_eq!(meta.mode() & 0o777, 0o600);
|
||||
assert_eq!(meta.uid(), rustix::process::geteuid().as_raw());
|
||||
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o660)).expect("insecure socket");
|
||||
assert!(
|
||||
matches!(client.page(identity().cluster_id, None, 1).await, Err(ReaderError::Permission)),
|
||||
"SOURCE_READER_CLIENT_VALIDATES_SOCKET"
|
||||
);
|
||||
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)).expect("restore socket");
|
||||
std::fs::remove_file(&path).expect("test owner replaces socket");
|
||||
std::fs::write(&path, b"replacement").expect("replacement file");
|
||||
shutdown.shutdown();
|
||||
assert!(task.await.expect("serve exit").is_err(), "SOURCE_READER_REPLACEMENT_CLEANUP_REJECTED");
|
||||
assert_eq!(std::fs::read(&path).expect("replacement preserved"), b"replacement");
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn source_reader_transport_bounds() {
|
||||
let (dir, writer) = fixture();
|
||||
let (path, client, shutdown, task) = start(&dir, &writer);
|
||||
let mut stream = UnixStream::connect(&path).await.expect("oversized input peer");
|
||||
stream
|
||||
.write_u32(u32::try_from(MAX_REQUEST_BYTES + 1).expect("length"))
|
||||
.await
|
||||
.expect("overlong prefix");
|
||||
let mut byte = [0];
|
||||
assert_eq!(
|
||||
stream.read(&mut byte).await.expect("bounded rejection closes"),
|
||||
0,
|
||||
"SOURCE_READER_INPUT_BOUND"
|
||||
);
|
||||
let mut idle = UnixStream::connect(&path).await.expect("idle connection");
|
||||
let start = std::time::Instant::now();
|
||||
assert_eq!(
|
||||
idle.read(&mut byte).await.expect("deadline closes connection"),
|
||||
0,
|
||||
"SOURCE_READER_READ_DEADLINE"
|
||||
);
|
||||
assert!(start.elapsed() >= IO_TIMEOUT, "SOURCE_READER_REAL_READ_DEADLINE");
|
||||
assert!(
|
||||
client.page(identity().cluster_id, None, 1).await.is_ok(),
|
||||
"SOURCE_READER_TIMEOUT_RELEASES_CONNECTION"
|
||||
);
|
||||
stop(path.clone(), shutdown, task).await;
|
||||
// A same-UID peer still cannot supply unbounded responses to the client.
|
||||
let listener = tokio::net::UnixListener::bind(&path).expect("controlled response peer");
|
||||
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)).expect("private response socket");
|
||||
let client = SourceReaderClient::new(&path).expect("response client");
|
||||
let peer = tokio::spawn(async move {
|
||||
let (mut stream, _) = listener.accept().await.expect("client connects");
|
||||
stream
|
||||
.write_u32(u32::try_from(MAX_RESPONSE_BYTES + 1).expect("oversized response"))
|
||||
.await
|
||||
.expect("response prefix");
|
||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||
});
|
||||
assert!(
|
||||
matches!(client.page(identity().cluster_id, None, 1).await, Err(ReaderError::Protocol)),
|
||||
"SOURCE_READER_RESPONSE_BOUND"
|
||||
);
|
||||
peer.await.expect("peer task");
|
||||
}
|
||||
fn hold_store(writer: &NativeLogWriter) -> (std::sync::mpsc::Sender<()>, std::thread::JoinHandle<()>) {
|
||||
let owner = writer.clone();
|
||||
let (locked_tx, locked_rx) = std::sync::mpsc::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel();
|
||||
let thread = std::thread::spawn(move || {
|
||||
let _store = owner.store.lock().expect("actual Store held by test owner");
|
||||
locked_tx.send(()).expect("lock visible");
|
||||
let _ = release_rx.recv();
|
||||
});
|
||||
locked_rx.recv().expect("store locked");
|
||||
(release_tx, thread)
|
||||
}
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn source_reader_shutdown() {
|
||||
let (dir, writer) = fixture();
|
||||
apply(&writer, 1, 9).await;
|
||||
let (path, _client, shutdown, task) = start(&dir, &writer);
|
||||
let (release_tx, blocker) = hold_store(&writer);
|
||||
let mut streams = Vec::new();
|
||||
for _ in 0..MAX_CONNECTIONS {
|
||||
let mut stream = UnixStream::connect(&path).await.expect("active read socket");
|
||||
let bytes = serde_json::to_vec(&request()).expect("request");
|
||||
stream
|
||||
.write_u32(bytes.len().try_into().expect("length"))
|
||||
.await
|
||||
.expect("prefix");
|
||||
stream.write_all(&bytes).await.expect("request bytes");
|
||||
streams.push(stream);
|
||||
}
|
||||
tokio::time::timeout(std::time::Duration::from_secs(2), async {
|
||||
while writer.source_reader_admission.available_permits() != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("all admissions active");
|
||||
drop(streams);
|
||||
tokio::time::sleep(IO_TIMEOUT + std::time::Duration::from_millis(100)).await;
|
||||
assert_eq!(
|
||||
writer.source_reader_admission.available_permits(),
|
||||
0,
|
||||
"SOURCE_READER_CLIENT_TIMEOUT_KEEPS_WORKER_ADMISSION"
|
||||
);
|
||||
shutdown.shutdown();
|
||||
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||
assert!(!task.is_finished(), "SOURCE_READER_SHUTDOWN_DRAINS_READERS");
|
||||
assert_eq!(
|
||||
writer.source_reader_admission.available_permits(),
|
||||
0,
|
||||
"SOURCE_READER_CANCELLED_CLIENTS_KEEP_ADMISSION"
|
||||
);
|
||||
release_tx.send(()).expect("release real blocking reads");
|
||||
blocker.join().expect("test lock owner exits");
|
||||
task.await.expect("serve completes").expect("shutdown clean");
|
||||
assert!(!path.exists());
|
||||
assert_eq!(
|
||||
writer.source_reader_admission.available_permits(),
|
||||
MAX_CONNECTIONS,
|
||||
"SOURCE_READER_ADMISSION_RELEASED_AFTER_WORKERS"
|
||||
);
|
||||
let (release, blocker) = hold_store(&writer);
|
||||
let (path, _, _shutdown, task) = start(&dir, &writer);
|
||||
let bytes = serde_json::to_vec(&request()).expect("native request");
|
||||
let mut sockets = Vec::new();
|
||||
for _ in 0..MAX_CONNECTIONS {
|
||||
let mut stream = UnixStream::connect(&path).await.expect("cancelled service peer");
|
||||
stream
|
||||
.write_u32(bytes.len().try_into().expect("length"))
|
||||
.await
|
||||
.expect("prefix");
|
||||
stream.write_all(&bytes).await.expect("request");
|
||||
sockets.push(stream);
|
||||
}
|
||||
tokio::time::timeout(std::time::Duration::from_secs(2), async {
|
||||
while writer.source_reader_admission.available_permits() != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("cancelled service admits bounded workers");
|
||||
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||
task.abort();
|
||||
assert!(task.await.expect_err("serve cancellation").is_cancelled());
|
||||
drop(sockets);
|
||||
assert_eq!(
|
||||
writer.source_reader_admission.available_permits(),
|
||||
0,
|
||||
"SOURCE_READER_CANCELLED_SERVE_RETAINS_WORKER_ADMISSION"
|
||||
);
|
||||
let (path, client, shutdown, task) = start(&dir, &writer);
|
||||
let pending = tokio::spawn(async move { client.page(identity().cluster_id, None, 1).await });
|
||||
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||
assert!(!pending.is_finished(), "SOURCE_READER_REBIND_CANNOT_BYPASS_ADMISSION");
|
||||
assert_eq!(writer.source_reader_admission.available_permits(), 0);
|
||||
release.send(()).expect("release cancelled worker store lock");
|
||||
blocker.join().expect("lock owner completes");
|
||||
assert_eq!(
|
||||
pending
|
||||
.await
|
||||
.expect("new service client")
|
||||
.expect("new service waits for old workers")
|
||||
.page
|
||||
.events
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
stop(path, shutdown, task).await;
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn source_reader_checkpoint_replay() {
|
||||
let (dir, writer) = fixture();
|
||||
apply(&writer, 1, 9).await;
|
||||
apply(&writer, 2, 7).await;
|
||||
let (path, client, shutdown, task) = start(&dir, &writer);
|
||||
let first = client
|
||||
.page(identity().cluster_id, None, 1)
|
||||
.await
|
||||
.expect("pre-checkpoint cursor");
|
||||
let checkpoint = writer
|
||||
.checkpoint_local(CheckpointLimits::default())
|
||||
.await
|
||||
.expect("real checkpoint");
|
||||
assert_eq!(checkpoint.cut(), id(2));
|
||||
let next = client
|
||||
.page(identity().cluster_id, first.page.next.clone(), 1)
|
||||
.await
|
||||
.expect("checkpoint retained events");
|
||||
assert_eq!(next.page.events[0].event_id, [7; 16], "SOURCE_READER_CHECKPOINT_REPLAY");
|
||||
stop(path, shutdown, task).await;
|
||||
drop(writer);
|
||||
let writer = NativeLogWriter::new(Store::open(dir.path().join("db"), identity()).expect("reopen checkpointed store"));
|
||||
let (path, client, shutdown, task) = start(&dir, &writer);
|
||||
assert_eq!(
|
||||
client
|
||||
.page(identity().cluster_id, first.page.next, 1)
|
||||
.await
|
||||
.expect("replay after restart"),
|
||||
next,
|
||||
"SOURCE_READER_CHECKPOINT_RESTART"
|
||||
);
|
||||
stop(path, shutdown, task).await;
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
use crate::{LOGS, META, Store, StoreError, decode_id, decode_log, encode_id};
|
||||
use redb::ReadableTable;
|
||||
|
||||
impl Store {
|
||||
/// Atomically remove every log entry at or after `from`.
|
||||
pub fn truncate_suffix(&mut self, from: u64) -> Result<(), StoreError> {
|
||||
let tx = self.write_transaction()?;
|
||||
{
|
||||
let mut meta = tx.open_table(META)?;
|
||||
let logs = tx.open_table(LOGS)?;
|
||||
if crate::committed::validate_committed(&meta, &logs)?.is_some_and(|id| from <= id.index) {
|
||||
return Err(StoreError::CommittedLogProtected);
|
||||
}
|
||||
drop(logs);
|
||||
let tail = meta.get("tail")?.map(|v| decode_id(v.value())).transpose()?;
|
||||
if tail.is_none_or(|id| from > id.index) {
|
||||
return Ok(());
|
||||
}
|
||||
let mut logs = tx.open_table(LOGS)?;
|
||||
let new_tail = if from == 0 {
|
||||
None
|
||||
} else {
|
||||
let index = from - 1;
|
||||
let id = match logs.get(index)? {
|
||||
Some(record) => decode_log(record.value())?.id,
|
||||
None => crate::checkpoint_baseline::read_purged(&meta)?
|
||||
.filter(|id| id.index == index)
|
||||
.ok_or(StoreError::InvalidLogSequence)?,
|
||||
};
|
||||
if id.index != index {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
Some(id)
|
||||
};
|
||||
logs.retain(|index, _| index < from)?;
|
||||
match new_tail {
|
||||
Some(id) => {
|
||||
meta.insert("tail", encode_id(id).as_slice())?;
|
||||
}
|
||||
None => {
|
||||
meta.remove("tail")?;
|
||||
}
|
||||
}
|
||||
}
|
||||
self.commit(tx)
|
||||
}
|
||||
}
|
||||
@@ -18,10 +18,8 @@ use const_str::concat;
|
||||
/// Default value: RustFS
|
||||
/// Environment variable: RUSTFS_APP_NAME
|
||||
pub const APP_NAME: &str = "RustFS";
|
||||
/// Application version
|
||||
/// Default value: 1.0.0
|
||||
/// Environment variable: RUSTFS_VERSION
|
||||
pub const VERSION: &str = "1.0.0";
|
||||
/// Cargo package version for this workspace member.
|
||||
pub const VERSION: &str = env!("CARGO_PKG_VERSION");
|
||||
|
||||
/// Default configuration logger level
|
||||
/// Default value: error
|
||||
@@ -39,12 +37,6 @@ pub const SAMPLE_RATIO: f64 = 1.0;
|
||||
/// Default value: 30
|
||||
pub const METER_INTERVAL: u64 = 30;
|
||||
|
||||
/// Default configuration service version
|
||||
/// Default value: 1.0.0
|
||||
/// Environment variable: RUSTFS_OBS_SERVICE_VERSION
|
||||
/// Uses the same value as VERSION constant
|
||||
pub const SERVICE_VERSION: &str = "1.0.0";
|
||||
|
||||
/// Default configuration environment
|
||||
/// Default value: production
|
||||
pub const ENVIRONMENT: &str = "production";
|
||||
@@ -410,7 +402,6 @@ mod tests {
|
||||
APP_NAME,
|
||||
VERSION,
|
||||
DEFAULT_LOG_LEVEL,
|
||||
SERVICE_VERSION,
|
||||
ENVIRONMENT,
|
||||
RUSTFS_TLS_KEY,
|
||||
RUSTFS_TLS_CERT,
|
||||
@@ -446,8 +437,7 @@ mod tests {
|
||||
fn test_configuration_consistency() {
|
||||
// Test configuration consistency
|
||||
|
||||
// Version consistency
|
||||
assert_eq!(VERSION, SERVICE_VERSION, "Application version should match service version");
|
||||
assert_eq!(VERSION, env!("CARGO_PKG_VERSION"));
|
||||
|
||||
// Port conflict check
|
||||
let ports = [DEFAULT_PORT, DEFAULT_CONSOLE_PORT];
|
||||
|
||||
@@ -207,3 +207,9 @@ pub const ENV_TLS_RELOAD_INTERVAL: &str = "RUSTFS_TLS_RELOAD_INTERVAL";
|
||||
|
||||
/// Default interval for TLS certificate reload check
|
||||
pub const DEFAULT_TLS_RELOAD_INTERVAL: u64 = 30;
|
||||
|
||||
/// Environment variable to enable experimental HTTP/3 support.
|
||||
pub const ENV_HTTP3_ENABLE: &str = "RUSTFS_HTTP3_ENABLE";
|
||||
|
||||
/// HTTP/3 is disabled unless explicitly enabled.
|
||||
pub const DEFAULT_HTTP3_ENABLE: bool = false;
|
||||
|
||||
@@ -33,6 +33,10 @@
|
||||
//! tracked separately.
|
||||
|
||||
use crate::common::{ClusterTopology, RustFSTestClusterEnvironment};
|
||||
use crate::fault_proxy::FaultMode;
|
||||
use aws_sdk_s3::Client;
|
||||
use std::time::Duration;
|
||||
use tokio::time::{Instant, timeout};
|
||||
|
||||
type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
|
||||
|
||||
@@ -144,7 +148,99 @@ async fn cluster_volume_fault_proxy_pass_smoke() -> TestResult {
|
||||
cluster.start().await?;
|
||||
cluster.create_test_bucket(BUCKET).await?;
|
||||
let payload = vec![0x6Du8; 256 * 1024];
|
||||
put_get_roundtrip(&cluster, "volume-proxy/object", &payload).await
|
||||
put_get_roundtrip(&cluster, "volume-proxy/object", &payload).await?;
|
||||
|
||||
// Node 1 owns only two disks; a successful three-vote write must
|
||||
// authenticate at least one disk mutation through node 0's proxy.
|
||||
let client = cluster.create_s3_client(1)?;
|
||||
let client = Client::from_conf(
|
||||
client
|
||||
.config()
|
||||
.to_builder()
|
||||
.retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(1))
|
||||
.build(),
|
||||
);
|
||||
let forwarded_before = proxy.forwarded_bytes();
|
||||
client
|
||||
.put_object()
|
||||
.bucket(BUCKET)
|
||||
.key("volume-proxy/authenticated")
|
||||
.body(bytes::Bytes::copy_from_slice(&payload).into())
|
||||
.send()
|
||||
.await?;
|
||||
assert!(
|
||||
proxy.forwarded_bytes() > forwarded_before,
|
||||
"successful peer-disk publication must traverse the proxy"
|
||||
);
|
||||
|
||||
let dropped_before = proxy.dropped_bytes();
|
||||
proxy.set_mode(FaultMode::Blackhole);
|
||||
assert_eq!(proxy.mode(), FaultMode::Blackhole);
|
||||
let error = timeout(
|
||||
Duration::from_secs(30),
|
||||
client
|
||||
.put_object()
|
||||
.bucket(BUCKET)
|
||||
.key("volume-proxy/rejected")
|
||||
.body(bytes::Bytes::copy_from_slice(&payload).into())
|
||||
.send(),
|
||||
)
|
||||
.await?
|
||||
.expect_err("two local disks cannot commit a write while the peer is blackholed");
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(503));
|
||||
assert_eq!(error.as_service_error().and_then(|error| error.meta().code()), Some("ServiceUnavailable"));
|
||||
assert!(
|
||||
proxy.dropped_bytes() > dropped_before,
|
||||
"the injected blackhole must discard actual peer traffic"
|
||||
);
|
||||
|
||||
proxy.set_mode(FaultMode::Pass);
|
||||
assert_eq!(proxy.mode(), FaultMode::Pass);
|
||||
let http = reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.timeout(Duration::from_secs(3))
|
||||
.build()?;
|
||||
let deadline = Instant::now() + Duration::from_secs(30);
|
||||
loop {
|
||||
let response = http.get(format!("{}/health/ready", cluster.nodes[1].url)).send().await?;
|
||||
let status = response.status();
|
||||
let body: serde_json::Value = response.json().await?;
|
||||
// Reachable peers can still have Returning disk handles that reject
|
||||
// writes. Wait for this node's live writer inventory to be Online.
|
||||
if status.as_u16() == 200
|
||||
&& body["details"]["storage"]["source"] == "local_runtime"
|
||||
&& body["details"]["storage"]["writeQuorum"] == true
|
||||
&& body["details"]["storage"]["unavailableDrives"]
|
||||
.as_array()
|
||||
.is_some_and(Vec::is_empty)
|
||||
&& body["details"]["lock"]["ready"] == true
|
||||
{
|
||||
break;
|
||||
}
|
||||
assert!(Instant::now() < deadline, "peer write quorum did not recover: HTTP {status}, {body}");
|
||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
||||
}
|
||||
client
|
||||
.put_object()
|
||||
.bucket(BUCKET)
|
||||
.key("volume-proxy/recovered")
|
||||
.body(bytes::Bytes::copy_from_slice(&payload).into())
|
||||
.send()
|
||||
.await?;
|
||||
for key in ["volume-proxy/object", "volume-proxy/authenticated", "volume-proxy/recovered"] {
|
||||
let got = client.get_object().bucket(BUCKET).key(key).send().await?;
|
||||
let body = got.body.collect().await?.into_bytes();
|
||||
assert_eq!(body.as_ref(), payload, "proxy recovery must preserve the full object body for {key}");
|
||||
}
|
||||
let rejected = client
|
||||
.head_object()
|
||||
.bucket(BUCKET)
|
||||
.key("volume-proxy/rejected")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a rejected proxied write must never become visible after recovery");
|
||||
assert_eq!(rejected.raw_response().map(|response| response.status().as_u16()), Some(404));
|
||||
Ok(())
|
||||
}
|
||||
.await;
|
||||
|
||||
|
||||
@@ -35,7 +35,7 @@ use s3s::Body;
|
||||
use serde_json;
|
||||
use std::fs as stdfs;
|
||||
use std::io::ErrorKind;
|
||||
use std::net::SocketAddr;
|
||||
use std::net::{Ipv6Addr, SocketAddr};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Child, Command, Stdio};
|
||||
use std::sync::Once;
|
||||
@@ -1442,7 +1442,11 @@ impl RustFSTestClusterEnvironment {
|
||||
return Err(format!("a volume proxy is already configured for node {node_idx}").into());
|
||||
}
|
||||
let target = self.nodes[node_idx].address.parse::<SocketAddr>()?;
|
||||
let proxy = crate::fault_proxy::FaultProxy::start(target).await?;
|
||||
// Endpoint locality and RPC audience both use the advertised authority.
|
||||
// Keep the node's port while the second loopback IP separates the proxy
|
||||
// listener from the direct IPv4 server and S3 client address.
|
||||
let listen = SocketAddr::from((Ipv6Addr::LOCALHOST, target.port()));
|
||||
let proxy = crate::fault_proxy::FaultProxy::start_on(listen, target).await?;
|
||||
self.volume_proxy_addresses[node_idx] = Some(proxy.local_addr());
|
||||
Ok(proxy)
|
||||
}
|
||||
@@ -2467,6 +2471,20 @@ mod tests {
|
||||
|
||||
assert!(volumes.contains(&proxied), "volumes must use the proxy address: {volumes}");
|
||||
assert!(!volumes.contains(&direct), "volumes must not retain the direct address: {volumes}");
|
||||
assert_eq!(proxy.local_addr().port(), proxy.target_addr().port());
|
||||
assert_ne!(proxy.local_addr().ip(), proxy.target_addr().ip());
|
||||
assert_eq!(env.nodes[0].address, direct, "S3 clients must retain the direct address");
|
||||
let mut endpoint =
|
||||
crate::storage_api::Endpoint::try_from(volumes.as_str()).expect("proxied volume should be a valid endpoint");
|
||||
endpoint
|
||||
.update_is_local(proxy.target_addr().port())
|
||||
.expect("proxied endpoint locality should resolve");
|
||||
assert!(endpoint.is_local, "the proxied node must keep its local disk and first-disk authority");
|
||||
assert_eq!(
|
||||
endpoint.host_port(),
|
||||
proxied,
|
||||
"the local RPC audience must be the advertised proxy authority"
|
||||
);
|
||||
|
||||
proxy.shutdown().await;
|
||||
}
|
||||
|
||||
@@ -31,6 +31,7 @@ use super::harness::{
|
||||
use crate::common::{
|
||||
AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, init_logging,
|
||||
};
|
||||
use crate::upgrade_compatibility_test::wait_for_upgrade_write_readiness;
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use std::path::{Path, PathBuf};
|
||||
@@ -400,6 +401,19 @@ async fn four_node_direct_upgrade_preserves_history_and_iam_credentials() -> Tes
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn replace_node_and_wait_for_write_readiness(dist: &mut DistCluster, seed: &UpgradeSeed, node_idx: usize) -> TestResult {
|
||||
dist.replace_node_with_current_binary(node_idx).await?;
|
||||
// Node readiness proves reads can succeed; a rolling writer must also
|
||||
// recover before replacing the next peer or checking upgrade contracts.
|
||||
wait_for_upgrade_write_readiness(
|
||||
&dist.clients()?,
|
||||
&seed.history_bucket,
|
||||
&format!("replaced-node-{node_idx}"),
|
||||
CREDENTIAL_TIMEOUT,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> TestResult {
|
||||
init_logging();
|
||||
@@ -410,15 +424,15 @@ async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> Te
|
||||
|
||||
let seed = seed_history_and_iam(&dist).await?;
|
||||
|
||||
dist.replace_node_with_current_binary(0).await?;
|
||||
replace_node_and_wait_for_write_readiness(&mut dist, &seed, 0).await?;
|
||||
assert_history_and_iam(&dist, &seed, "one-current-node").await?;
|
||||
|
||||
for node_idx in [1, 2] {
|
||||
dist.replace_node_with_current_binary(node_idx).await?;
|
||||
replace_node_and_wait_for_write_readiness(&mut dist, &seed, node_idx).await?;
|
||||
}
|
||||
assert_history_and_iam(&dist, &seed, "one-previous-node").await?;
|
||||
|
||||
dist.replace_node_with_current_binary(3).await?;
|
||||
replace_node_and_wait_for_write_readiness(&mut dist, &seed, 3).await?;
|
||||
assert_history_and_iam(&dist, &seed, "homogeneous-current").await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -50,8 +50,8 @@ use std::collections::{BTreeMap, HashMap, VecDeque};
|
||||
use std::error::Error;
|
||||
use std::io;
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::{Arc, Mutex, MutexGuard};
|
||||
use std::time::{Duration, SystemTime};
|
||||
use std::sync::{Arc, Mutex, MutexGuard, OnceLock};
|
||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::sync::{OwnedSemaphorePermit, Semaphore, watch};
|
||||
use tokio::task::JoinHandle;
|
||||
@@ -363,6 +363,14 @@ pub struct RequestRecord {
|
||||
pub upload_id: Option<String>,
|
||||
pub part_number: Option<i32>,
|
||||
pub content_length: Option<u64>,
|
||||
pub source_version_id: Option<String>,
|
||||
pub source_mtime: Option<String>,
|
||||
pub source_etag: Option<String>,
|
||||
pub source_replication_request: bool,
|
||||
pub journaled_at_unix_millis: Option<u64>,
|
||||
/// Computed by the target, before response stall/disconnect faults. This
|
||||
/// does not prove that any response bytes reached the client.
|
||||
pub prepared_response: Option<PreparedResponse>,
|
||||
pub consumed_bytes: Option<usize>,
|
||||
pub replication_timestamps: ReplicationTimestampHeaders,
|
||||
pub proxy_headers: ProxyHeaderSnapshot,
|
||||
@@ -381,6 +389,20 @@ pub struct RequestRecord {
|
||||
pub fault: Option<FaultAction>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct PreparedResponse {
|
||||
pub status: u16,
|
||||
pub version_id: Option<String>,
|
||||
pub etag: Option<String>,
|
||||
pub last_modified: Option<String>,
|
||||
pub content_length: Option<u64>,
|
||||
pub prepared_at_unix_millis: Option<u64>,
|
||||
pub elapsed: Duration,
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct RequestSequence(Arc<OnceLock<u64>>);
|
||||
|
||||
/// Integrity and framing headers of an upload. A remote target's acceptance
|
||||
/// rules key off exactly these (rustfs#6853: `aws-chunked` framing stored
|
||||
/// verbatim; rustfs#7082: Object Lock PUTs need `Content-MD5` or
|
||||
@@ -430,6 +452,10 @@ impl TransportSnapshot {
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
struct JournaledHeaders {
|
||||
content_length: Option<u64>,
|
||||
source_version_id: Option<String>,
|
||||
source_mtime: Option<String>,
|
||||
source_etag: Option<String>,
|
||||
source_replication_request: bool,
|
||||
replication_timestamps: ReplicationTimestampHeaders,
|
||||
proxy_headers: ProxyHeaderSnapshot,
|
||||
transport: TransportSnapshot,
|
||||
@@ -444,6 +470,10 @@ impl JournaledHeaders {
|
||||
.get(CONTENT_LENGTH)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(|value| value.parse().ok()),
|
||||
source_version_id: header_value(headers, &SOURCE_VERSION_ID_HEADERS).map(bounded_journal_value),
|
||||
source_mtime: header_value(headers, &SOURCE_MTIME_HEADERS).map(bounded_journal_value),
|
||||
source_etag: header_value(headers, &SOURCE_ETAG_HEADERS).map(bounded_journal_value),
|
||||
source_replication_request: header_value(headers, &SOURCE_REPLICATION_REQUEST_HEADERS).as_deref() == Some("true"),
|
||||
replication_timestamps: ReplicationTimestampHeaders::from_headers(headers),
|
||||
proxy_headers: ProxyHeaderSnapshot::from_headers(headers),
|
||||
transport: TransportSnapshot::from_headers(headers),
|
||||
@@ -511,6 +541,10 @@ struct StoreState {
|
||||
/// version id the target never had answers 404 `NoSuchVersion` instead of
|
||||
/// the idempotent 204 RustFS/MinIO give.
|
||||
reject_unknown_version_deletes: bool,
|
||||
/// Models MinIO and its derivatives (Storj): `X-Minio-Internal-` is a reserved
|
||||
/// metadata prefix, so a PutObject or UploadPart carrying any such request
|
||||
/// header is refused with `InvalidArgument` (rustfs#8362).
|
||||
reject_reserved_internal_headers: bool,
|
||||
limits: StoreLimits,
|
||||
buckets: HashMap<String, BucketState>,
|
||||
uploads: HashMap<String, MultipartState>,
|
||||
@@ -738,6 +772,7 @@ impl FakeS3Target {
|
||||
let connection_limit = Arc::new(Semaphore::new(MAX_CONNECTIONS));
|
||||
let (active_connections_tx, active_connections) = watch::channel(0usize);
|
||||
let task_connection_limit = Arc::clone(&connection_limit);
|
||||
let task_control = Arc::clone(&control);
|
||||
|
||||
let task = tokio::spawn(async move {
|
||||
let mut connections = tokio::task::JoinSet::new();
|
||||
@@ -751,6 +786,7 @@ impl FakeS3Target {
|
||||
};
|
||||
active_connections_tx.send_modify(|count| *count += 1);
|
||||
let service = service.clone();
|
||||
let control = Arc::clone(&task_control);
|
||||
let active_connections = active_connections_tx.clone();
|
||||
let mut connection_gate = connection_gate.clone();
|
||||
connections.spawn(async move {
|
||||
@@ -764,7 +800,7 @@ impl FakeS3Target {
|
||||
{
|
||||
return;
|
||||
}
|
||||
let handler = service_fn(move |request| handle_request(request, service.clone()));
|
||||
let handler = service_fn(move |request| handle_request(request, service.clone(), Arc::clone(&control)));
|
||||
let mut connection = http1::Builder::new();
|
||||
connection
|
||||
.keep_alive(false)
|
||||
@@ -975,6 +1011,43 @@ impl FakeS3Target {
|
||||
lock(&self.backend.store).reject_unknown_version_deletes = enabled;
|
||||
}
|
||||
|
||||
/// Refuse every PutObject / UploadPart that carries an `x-minio-internal-*`
|
||||
/// request header, the way MinIO and Storj do.
|
||||
pub fn reject_reserved_internal_headers(&self, enabled: bool) {
|
||||
lock(&self.backend.store).reject_reserved_internal_headers = enabled;
|
||||
}
|
||||
|
||||
/// Keys of every stored, non-delete object in the bucket, sorted.
|
||||
pub fn stored_keys(&self, bucket: &str) -> Vec<String> {
|
||||
let state = lock(&self.backend.store);
|
||||
let mut keys: Vec<String> = state
|
||||
.buckets
|
||||
.get(bucket)
|
||||
.map(|bucket| {
|
||||
bucket
|
||||
.objects
|
||||
.iter()
|
||||
.filter(|(_, versions)| versions.last().is_some_and(|version| !version.delete_marker))
|
||||
.map(|(key, _)| key.clone())
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
keys.sort();
|
||||
keys
|
||||
}
|
||||
|
||||
/// Body and user metadata (names as stored, without the `x-amz-meta-` prefix)
|
||||
/// of the latest non-delete version of an object.
|
||||
pub fn stored_object(&self, bucket: &str, key: &str) -> Option<(Bytes, HashMap<String, String>)> {
|
||||
lock(&self.backend.store)
|
||||
.buckets
|
||||
.get(bucket)
|
||||
.and_then(|bucket| bucket.objects.get(key))
|
||||
.and_then(|versions| versions.last())
|
||||
.filter(|version| !version.delete_marker)
|
||||
.map(|version| (version.body.clone(), version.metadata.clone().unwrap_or_default()))
|
||||
}
|
||||
|
||||
pub fn require_checksum_for_object_lock(&self, enabled: bool) {
|
||||
lock(&self.backend.store).require_checksum_for_object_lock = enabled;
|
||||
}
|
||||
@@ -1161,7 +1234,10 @@ impl S3Access for FaultAccess {
|
||||
let operation = operation_from_s3_name(context.s3_op().name());
|
||||
let journaled = JournaledHeaders::from_headers(context.headers());
|
||||
let aws_chunked = journaled.transport.aws_chunked;
|
||||
let fault = record_request(&self.control, operation, context.method().clone(), parsed, journaled);
|
||||
let (sequence, fault) = record_request(&self.control, operation, context.method().clone(), parsed, journaled);
|
||||
if let Some(slot) = context.extensions_mut().get::<RequestSequence>() {
|
||||
let _ = slot.0.set(sequence);
|
||||
}
|
||||
if let Some(status) = fault.as_ref().and_then(|fault| scripted_status(&fault.action)) {
|
||||
return Err(scripted_status_error(status));
|
||||
}
|
||||
@@ -1174,6 +1250,15 @@ impl S3Access for FaultAccess {
|
||||
"this target does not decode aws-chunked request bodies; send a plain signed payload with an exact Content-Length"
|
||||
));
|
||||
}
|
||||
if matches!(operation, Operation::PutObject | Operation::UploadPart)
|
||||
&& lock(&self.store).reject_reserved_internal_headers
|
||||
&& context
|
||||
.headers()
|
||||
.keys()
|
||||
.any(|name| name.as_str().starts_with("x-minio-internal-"))
|
||||
{
|
||||
return Err(s3s::s3_error!(InvalidArgument, "Your metadata headers are not supported."));
|
||||
}
|
||||
let prebody_permit = if operation == Operation::CompleteMultipartUpload {
|
||||
Some(
|
||||
timeout(
|
||||
@@ -1228,7 +1313,7 @@ fn record_request(
|
||||
method: Method,
|
||||
parsed: ParsedRequest,
|
||||
headers: JournaledHeaders,
|
||||
) -> Option<RequestFault> {
|
||||
) -> (u64, Option<RequestFault>) {
|
||||
let mut state = lock(control);
|
||||
let action = parsed
|
||||
.key
|
||||
@@ -1251,6 +1336,12 @@ fn record_request(
|
||||
upload_id: parsed.upload_id.map(bounded_journal_value),
|
||||
part_number: parsed.part_number,
|
||||
content_length: headers.content_length,
|
||||
source_version_id: headers.source_version_id,
|
||||
source_mtime: headers.source_mtime,
|
||||
source_etag: headers.source_etag,
|
||||
source_replication_request: headers.source_replication_request,
|
||||
journaled_at_unix_millis: unix_millis(),
|
||||
prepared_response: None,
|
||||
consumed_bytes: None,
|
||||
replication_timestamps: headers.replication_timestamps,
|
||||
proxy_headers: headers.proxy_headers,
|
||||
@@ -1261,7 +1352,14 @@ fn record_request(
|
||||
continuation_token: parsed.continuation_token.map(bounded_journal_value),
|
||||
fault: action.clone(),
|
||||
});
|
||||
action.map(|action| RequestFault { sequence, action })
|
||||
(sequence, action.map(|action| RequestFault { sequence, action }))
|
||||
}
|
||||
|
||||
fn unix_millis() -> Option<u64> {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.ok()
|
||||
.and_then(|elapsed| u64::try_from(elapsed.as_millis()).ok())
|
||||
}
|
||||
|
||||
fn bounded_journal_value(mut value: String) -> String {
|
||||
@@ -1269,14 +1367,31 @@ fn bounded_journal_value(mut value: String) -> String {
|
||||
while !value.is_char_boundary(end) {
|
||||
end -= 1;
|
||||
}
|
||||
value.truncate(end);
|
||||
value
|
||||
if value.capacity() > MAX_RETAINED_IDENTIFIER_BYTES {
|
||||
value[..end].to_owned()
|
||||
} else {
|
||||
value.truncate(end);
|
||||
value
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_request(request: Request<Incoming>, service: S3Service) -> Result<Response<Body>, BoxError> {
|
||||
let mut response = timeout(MAX_REQUEST_DURATION, call_s3(service, request.map(Body::from)))
|
||||
.await
|
||||
.map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "fake target request exceeded 65 seconds"))??;
|
||||
async fn handle_request(
|
||||
mut request: Request<Incoming>,
|
||||
service: S3Service,
|
||||
control: Arc<Mutex<ControlState>>,
|
||||
) -> Result<Response<Body>, BoxError> {
|
||||
let started = Instant::now();
|
||||
let sequence = RequestSequence::default();
|
||||
request.extensions_mut().insert(sequence.clone());
|
||||
let mut response = timeout(MAX_REQUEST_DURATION, async {
|
||||
let result = call_s3(service, request.map(Body::from)).await;
|
||||
if let (Ok(response), Some(sequence)) = (&result, sequence.0.get()) {
|
||||
update_prepared_response(&control, *sequence, response, started.elapsed());
|
||||
}
|
||||
result
|
||||
})
|
||||
.await
|
||||
.map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "fake target request exceeded 65 seconds"))??;
|
||||
if response.headers_mut().remove(DISCONNECT_HEADER).is_some() {
|
||||
return Err(io::Error::new(io::ErrorKind::ConnectionAborted, "fake target scripted disconnect").into());
|
||||
}
|
||||
@@ -1531,6 +1646,25 @@ fn update_consumed(control: &Mutex<ControlState>, sequence: u64, consumed: usize
|
||||
}
|
||||
}
|
||||
|
||||
fn update_prepared_response(control: &Mutex<ControlState>, sequence: u64, response: &Response<Body>, elapsed: Duration) {
|
||||
let prepared = PreparedResponse {
|
||||
status: response.status().as_u16(),
|
||||
version_id: header_value(response.headers(), &["x-amz-version-id"]).map(bounded_journal_value),
|
||||
etag: header_value(response.headers(), &[ETAG.as_str()]).map(bounded_journal_value),
|
||||
last_modified: header_value(response.headers(), &[LAST_MODIFIED.as_str()]).map(bounded_journal_value),
|
||||
content_length: response
|
||||
.headers()
|
||||
.get(CONTENT_LENGTH)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(|value| value.parse().ok()),
|
||||
prepared_at_unix_millis: unix_millis(),
|
||||
elapsed,
|
||||
};
|
||||
if let Some(record) = lock(control).requests.iter_mut().find(|record| record.sequence == sequence) {
|
||||
record.prepared_response = Some(prepared);
|
||||
}
|
||||
}
|
||||
|
||||
async fn apply_non_body_fault(fault: Option<&RequestFault>, control: &Mutex<ControlState>) -> S3Result<()> {
|
||||
match fault.map(|fault| &fault.action) {
|
||||
Some(action @ (FaultAction::Status(_) | FaultAction::ResponseStatus(_))) => {
|
||||
@@ -3230,10 +3364,38 @@ mod tests {
|
||||
let retry = client.clone();
|
||||
requests.spawn(async move { get_bytes(&retry, bucket, "held", None).await });
|
||||
timeout(Duration::from_secs(2), gate.entered.wait_for(|count| *count == 2)).await??;
|
||||
let held_retry = target
|
||||
.requests()
|
||||
.into_iter()
|
||||
.rev()
|
||||
.find(|record| record.operation == Operation::GetObject && record.key.as_deref() == Some("held"))
|
||||
.expect("the held retry must be journaled");
|
||||
assert_eq!(held_retry.prepared_response, None);
|
||||
assert_eq!(
|
||||
timeout(Duration::from_secs(2), get_bytes(&client, bucket, "unrelated", None)).await??,
|
||||
Bytes::from_static(b"payload")
|
||||
);
|
||||
let unrelated = target
|
||||
.requests()
|
||||
.into_iter()
|
||||
.find(|record| record.operation == Operation::GetObject && record.key.as_deref() == Some("unrelated"))
|
||||
.expect("the unrelated GET must be journaled");
|
||||
let unrelated_response = unrelated.prepared_response.expect("the unrelated GET prepared a response");
|
||||
assert_eq!(unrelated_response.status, 200);
|
||||
assert_eq!(
|
||||
unrelated_response.version_id.as_deref(),
|
||||
Some(target.stored_versions(bucket, "unrelated")[0].0.as_str())
|
||||
);
|
||||
assert!(held_retry.sequence < unrelated.sequence);
|
||||
assert_eq!(
|
||||
target
|
||||
.requests()
|
||||
.into_iter()
|
||||
.find(|record| record.sequence == held_retry.sequence)
|
||||
.expect("the held retry remains journaled")
|
||||
.prepared_response,
|
||||
None
|
||||
);
|
||||
assert!(requests.try_join_next().is_none(), "the retry must remain behind the gate");
|
||||
drop(gate);
|
||||
assert_eq!(
|
||||
@@ -3242,6 +3404,18 @@ mod tests {
|
||||
.expect("retried GET task")??,
|
||||
Bytes::from_static(b"payload")
|
||||
);
|
||||
let held_response = target
|
||||
.requests()
|
||||
.into_iter()
|
||||
.find(|record| record.sequence == held_retry.sequence)
|
||||
.and_then(|record| record.prepared_response)
|
||||
.expect("the released retry prepared its own response");
|
||||
assert_eq!(held_response.status, 200);
|
||||
assert_eq!(
|
||||
held_response.version_id.as_deref(),
|
||||
Some(target.stored_versions(bucket, "held")[0].0.as_str())
|
||||
);
|
||||
assert_ne!(held_response.version_id, unrelated_response.version_id);
|
||||
assert_eq!(get_bytes(&client, bucket, "held", None).await?, Bytes::from_static(b"payload"));
|
||||
assert_eq!(target.count_requests(Operation::GetObject, "held"), 3);
|
||||
Ok(())
|
||||
@@ -3586,6 +3760,23 @@ mod tests {
|
||||
.await?;
|
||||
assert_eq!(first_head.content_length(), Some(5));
|
||||
assert_eq!(first_head.version_id(), first.version_id.as_deref());
|
||||
let journaled_head = target
|
||||
.requests()
|
||||
.into_iter()
|
||||
.rev()
|
||||
.find(|record| record.operation == Operation::HeadObject)
|
||||
.expect("the version HEAD must be journaled");
|
||||
assert_eq!(journaled_head.version_id, first.version_id);
|
||||
assert!(journaled_head.journaled_at_unix_millis.is_some());
|
||||
let prepared_head = journaled_head
|
||||
.prepared_response
|
||||
.expect("the version HEAD prepared a response");
|
||||
assert_eq!(prepared_head.status, 200);
|
||||
assert_eq!(prepared_head.version_id.as_deref(), first_head.version_id());
|
||||
assert_eq!(prepared_head.etag.as_deref(), first_head.e_tag());
|
||||
assert_eq!(prepared_head.content_length, Some(5));
|
||||
assert!(prepared_head.last_modified.is_some());
|
||||
assert!(prepared_head.prepared_at_unix_millis.is_some());
|
||||
assert_eq!(
|
||||
get_bytes(&client, "target-bucket", "nested/key", None).await?,
|
||||
Bytes::from_static(b"second")
|
||||
@@ -3611,6 +3802,20 @@ mod tests {
|
||||
.await
|
||||
.expect_err("deleted version must not remain readable");
|
||||
assert_sdk_error!(deleted, 404, "NoSuchVersion");
|
||||
let deleted_record = target
|
||||
.requests()
|
||||
.into_iter()
|
||||
.rev()
|
||||
.find(|record| record.operation == Operation::GetObject)
|
||||
.expect("the deleted version GET must be journaled");
|
||||
assert_eq!(deleted_record.version_id, first.version_id);
|
||||
assert_eq!(
|
||||
deleted_record
|
||||
.prepared_response
|
||||
.expect("the error response was prepared")
|
||||
.status,
|
||||
404
|
||||
);
|
||||
let marker = client
|
||||
.delete_object()
|
||||
.bucket("target-bucket")
|
||||
@@ -4042,7 +4247,21 @@ mod tests {
|
||||
.await
|
||||
.expect_err("wrong part ETag must fail without consuming the upload");
|
||||
assert_sdk_error!(invalid_complete, 400, "InvalidPart");
|
||||
client
|
||||
let rejected_complete = target
|
||||
.requests()
|
||||
.into_iter()
|
||||
.rev()
|
||||
.find(|record| record.operation == Operation::CompleteMultipartUpload)
|
||||
.expect("the rejected completion must be journaled");
|
||||
assert_eq!(rejected_complete.upload_id.as_deref(), Some(upload_id.as_str()));
|
||||
assert_eq!(
|
||||
rejected_complete
|
||||
.prepared_response
|
||||
.expect("the rejection was prepared")
|
||||
.status,
|
||||
400
|
||||
);
|
||||
let successful_complete = client
|
||||
.complete_multipart_upload()
|
||||
.bucket("target-bucket")
|
||||
.key("multipart")
|
||||
@@ -4050,6 +4269,18 @@ mod tests {
|
||||
.multipart_upload(CompletedMultipartUpload::builder().set_parts(Some(parts)).build())
|
||||
.send()
|
||||
.await?;
|
||||
let completed_record = target
|
||||
.requests()
|
||||
.into_iter()
|
||||
.rev()
|
||||
.find(|record| record.operation == Operation::CompleteMultipartUpload)
|
||||
.expect("the successful completion must be journaled");
|
||||
assert_eq!(completed_record.upload_id.as_deref(), Some(upload_id.as_str()));
|
||||
let prepared_complete = completed_record
|
||||
.prepared_response
|
||||
.expect("the completion response was prepared");
|
||||
assert_eq!(prepared_complete.status, 200);
|
||||
assert_eq!(prepared_complete.version_id.as_deref(), successful_complete.version_id());
|
||||
let completed = get_bytes(&client, "target-bucket", "multipart", None).await?;
|
||||
assert_eq!(completed.len(), MIN_MULTIPART_PART_BYTES + 5);
|
||||
assert_eq!(&completed[..MIN_MULTIPART_PART_BYTES], first_body.as_ref());
|
||||
@@ -4430,6 +4661,32 @@ mod tests {
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(completed.version_id(), Some(completed_version.as_str()));
|
||||
let receipt_records = target.requests();
|
||||
let receipt_create = receipt_records
|
||||
.iter()
|
||||
.find(|record| {
|
||||
record.operation == Operation::CreateMultipartUpload && record.key.as_deref() == Some("multipart-receipt-time")
|
||||
})
|
||||
.expect("the source version must be retained on MPU creation");
|
||||
assert_eq!(receipt_create.source_version_id.as_deref(), Some(completed_version.as_str()));
|
||||
let receipt_complete = receipt_records
|
||||
.iter()
|
||||
.find(|record| {
|
||||
record.operation == Operation::CompleteMultipartUpload
|
||||
&& record.upload_id.as_deref() == Some(receipt_upload.as_str())
|
||||
})
|
||||
.expect("the source mtime must be retained on completion");
|
||||
assert_eq!(receipt_complete.source_mtime.as_deref(), Some("1970-01-01T00:00:00Z"));
|
||||
assert!(receipt_complete.source_replication_request);
|
||||
assert_eq!(
|
||||
receipt_complete
|
||||
.prepared_response
|
||||
.as_ref()
|
||||
.expect("the receipt-time response was prepared")
|
||||
.version_id
|
||||
.as_deref(),
|
||||
completed.version_id()
|
||||
);
|
||||
assert_eq!(
|
||||
get_bytes(&client, "target-bucket", "multipart-receipt-time", None).await?,
|
||||
Bytes::from_static(b"completed"),
|
||||
@@ -4648,6 +4905,23 @@ mod tests {
|
||||
.expect_err("disconnected request must not store an object");
|
||||
assert_sdk_error!(disconnected, 404, "NotFound");
|
||||
|
||||
target.inject_for_key(Operation::PutObject, "lost-response", FaultAction::DisconnectAfterResponse, 1);
|
||||
client
|
||||
.put_object()
|
||||
.bucket("target-bucket")
|
||||
.key("lost-response")
|
||||
.body(ByteStream::from_static(b"stored"))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a prepared response does not prove client delivery");
|
||||
let lost_response_head = client
|
||||
.head_object()
|
||||
.bucket("target-bucket")
|
||||
.key("lost-response")
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(lost_response_head.content_length(), Some(6));
|
||||
|
||||
assert_eq!(
|
||||
parse_request(&Method::PUT, &"/target-bucket/key?uploadId=upload&partNumber=bad".parse::<Uri>()?).operation,
|
||||
Operation::Unknown
|
||||
@@ -4661,6 +4935,28 @@ mod tests {
|
||||
.count(),
|
||||
4
|
||||
);
|
||||
assert!(
|
||||
requests
|
||||
.iter()
|
||||
.filter(|request| request.fault == Some(FaultAction::Status(StatusCode::SERVICE_UNAVAILABLE)))
|
||||
.all(|request| {
|
||||
request
|
||||
.prepared_response
|
||||
.as_ref()
|
||||
.is_some_and(|response| response.status == 503)
|
||||
})
|
||||
);
|
||||
let lost_response = requests
|
||||
.iter()
|
||||
.find(|request| request.operation == Operation::PutObject && request.key.as_deref() == Some("lost-response"))
|
||||
.expect("the disconnected response must be journaled");
|
||||
assert_eq!(lost_response.fault, Some(FaultAction::DisconnectAfterResponse));
|
||||
let prepared_lost_response = lost_response
|
||||
.prepared_response
|
||||
.as_ref()
|
||||
.expect("the target prepared the successful response");
|
||||
assert_eq!(prepared_lost_response.status, 200);
|
||||
assert_eq!(prepared_lost_response.version_id.as_deref(), lost_response_head.version_id());
|
||||
assert!(
|
||||
requests
|
||||
.iter()
|
||||
@@ -4851,8 +5147,44 @@ mod tests {
|
||||
assert_eq!(records.len(), MAX_REQUEST_RECORDS);
|
||||
assert_eq!(records.front().map(|record| record.sequence), Some(2));
|
||||
assert_eq!(records.back().map(|record| record.sequence), Some((MAX_REQUEST_RECORDS + 1) as u64));
|
||||
let mut oversized_allocation = String::with_capacity(MAX_RETAINED_IDENTIFIER_BYTES * 16);
|
||||
oversized_allocation.push_str("short");
|
||||
let short_value = bounded_journal_value(oversized_allocation);
|
||||
assert_eq!(short_value, "short");
|
||||
assert!(short_value.capacity() <= MAX_RETAINED_IDENTIFIER_BYTES);
|
||||
let mut response = Response::new(Body::empty());
|
||||
for name in ["x-amz-version-id", "etag", "last-modified"] {
|
||||
response.headers_mut().insert(
|
||||
name,
|
||||
HeaderValue::from_str(&"r".repeat(MAX_RETAINED_IDENTIFIER_BYTES + 1)).expect("header"),
|
||||
);
|
||||
}
|
||||
response.headers_mut().insert(CONTENT_LENGTH, HeaderValue::from_static("17"));
|
||||
response
|
||||
.headers_mut()
|
||||
.insert("authorization", HeaderValue::from_static("unrecorded-response-authorization"));
|
||||
response.headers_mut().insert(
|
||||
"x-amz-server-side-encryption-customer-key",
|
||||
HeaderValue::from_static("unrecorded-response-customer-key"),
|
||||
);
|
||||
update_prepared_response(&control, 1, &response, Duration::ZERO);
|
||||
update_prepared_response(&control, u64::MAX, &response, Duration::ZERO);
|
||||
assert_eq!(lock(&control).requests, records, "misses and evicted records must not be resurrected");
|
||||
let bounded_control = Mutex::new(ControlState::default());
|
||||
let utf8_boundary = format!("{}é", "a".repeat(MAX_RETAINED_IDENTIFIER_BYTES - 1));
|
||||
let mut source_headers = HeaderMap::new();
|
||||
for name in ["x-rustfs-source-version-id", "x-minio-source-mtime", "x-rustfs-source-etag"] {
|
||||
source_headers.insert(
|
||||
name,
|
||||
HeaderValue::from_str(&"s".repeat(MAX_RETAINED_IDENTIFIER_BYTES + 1)).expect("header"),
|
||||
);
|
||||
}
|
||||
source_headers.insert("x-minio-source-replication-request", HeaderValue::from_static("true"));
|
||||
source_headers.insert("authorization", HeaderValue::from_static("unrecorded-request-authorization"));
|
||||
source_headers.insert(
|
||||
"x-amz-server-side-encryption-customer-key",
|
||||
HeaderValue::from_static("unrecorded-request-customer-key"),
|
||||
);
|
||||
record_request(
|
||||
&bounded_control,
|
||||
Operation::GetObject,
|
||||
@@ -4865,8 +5197,9 @@ mod tests {
|
||||
upload_id: Some("u".repeat(MAX_RETAINED_IDENTIFIER_BYTES + 1)),
|
||||
..Default::default()
|
||||
},
|
||||
JournaledHeaders::default(),
|
||||
JournaledHeaders::from_headers(&source_headers),
|
||||
);
|
||||
update_prepared_response(&bounded_control, 1, &response, Duration::from_millis(7));
|
||||
{
|
||||
let bounded_records = lock(&bounded_control);
|
||||
let bounded = &bounded_records.requests[0];
|
||||
@@ -4874,7 +5207,39 @@ mod tests {
|
||||
assert_eq!(bounded.key.as_ref().map(String::len), Some(MAX_RETAINED_IDENTIFIER_BYTES - 1));
|
||||
assert_eq!(bounded.version_id.as_ref().map(String::len), Some(MAX_RETAINED_IDENTIFIER_BYTES));
|
||||
assert_eq!(bounded.upload_id.as_ref().map(String::len), Some(MAX_RETAINED_IDENTIFIER_BYTES));
|
||||
assert_eq!(bounded.source_version_id.as_ref().map(String::len), Some(MAX_RETAINED_IDENTIFIER_BYTES));
|
||||
assert_eq!(bounded.source_mtime.as_ref().map(String::len), Some(MAX_RETAINED_IDENTIFIER_BYTES));
|
||||
assert_eq!(bounded.source_etag.as_ref().map(String::len), Some(MAX_RETAINED_IDENTIFIER_BYTES));
|
||||
for value in [&bounded.source_version_id, &bounded.source_mtime, &bounded.source_etag]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
{
|
||||
assert!(value.capacity() <= MAX_RETAINED_IDENTIFIER_BYTES);
|
||||
}
|
||||
assert!(bounded.source_replication_request);
|
||||
let prepared = bounded
|
||||
.prepared_response
|
||||
.as_ref()
|
||||
.expect("the retained record receives its response");
|
||||
assert_eq!(prepared.version_id.as_ref().map(String::len), Some(MAX_RETAINED_IDENTIFIER_BYTES));
|
||||
assert_eq!(prepared.etag.as_ref().map(String::len), Some(MAX_RETAINED_IDENTIFIER_BYTES));
|
||||
assert_eq!(prepared.last_modified.as_ref().map(String::len), Some(MAX_RETAINED_IDENTIFIER_BYTES));
|
||||
for value in [&prepared.version_id, &prepared.etag, &prepared.last_modified]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
{
|
||||
assert!(value.capacity() <= MAX_RETAINED_IDENTIFIER_BYTES);
|
||||
}
|
||||
assert_eq!(prepared.content_length, Some(17));
|
||||
assert_eq!(prepared.elapsed, Duration::from_millis(7));
|
||||
assert!(
|
||||
!format!("{bounded:?}").contains("unrecorded-"),
|
||||
"authorization and customer keys must not enter the journal"
|
||||
);
|
||||
}
|
||||
lock(&bounded_control).requests.clear();
|
||||
update_prepared_response(&bounded_control, 1, &response, Duration::ZERO);
|
||||
assert!(lock(&bounded_control).requests.is_empty(), "a drained record must not be resurrected");
|
||||
let target = FakeS3Target::start().await.expect("target");
|
||||
target.create_bucket("b".repeat(63));
|
||||
let empty_bucket = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
|
||||
|
||||
@@ -61,6 +61,8 @@
|
||||
|
||||
use std::io;
|
||||
use std::net::{Ipv4Addr, SocketAddr};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
@@ -108,19 +110,33 @@ pub struct FaultProxy {
|
||||
mode_tx: watch::Sender<FaultMode>,
|
||||
shutdown_tx: watch::Sender<bool>,
|
||||
accept_task: JoinHandle<()>,
|
||||
traffic: Arc<ProxyTraffic>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct ProxyTraffic {
|
||||
forwarded_bytes: AtomicUsize,
|
||||
dropped_bytes: AtomicUsize,
|
||||
}
|
||||
|
||||
impl FaultProxy {
|
||||
/// Bind a listener on `127.0.0.1:0` and start forwarding accepted
|
||||
/// connections to `target`. Starts in [`FaultMode::Pass`].
|
||||
pub async fn start(target: SocketAddr) -> io::Result<Self> {
|
||||
let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?;
|
||||
Self::start_on(SocketAddr::from((Ipv4Addr::LOCALHOST, 0)), target).await
|
||||
}
|
||||
|
||||
/// Bind a specific address so advertised volume endpoints can retain the
|
||||
/// server's port and authenticated RPC identity on another loopback IP.
|
||||
pub async fn start_on(listen: SocketAddr, target: SocketAddr) -> io::Result<Self> {
|
||||
let listener = TcpListener::bind(listen).await?;
|
||||
let listen_addr = listener.local_addr()?;
|
||||
|
||||
let (mode_tx, mode_rx) = watch::channel(FaultMode::Pass);
|
||||
let (shutdown_tx, shutdown_rx) = watch::channel(false);
|
||||
|
||||
let accept_task = tokio::spawn(accept_loop(listener, target, mode_rx, shutdown_rx));
|
||||
let traffic = Arc::new(ProxyTraffic::default());
|
||||
let accept_task = tokio::spawn(accept_loop(listener, target, mode_rx, shutdown_rx, traffic.clone()));
|
||||
|
||||
Ok(Self {
|
||||
listen_addr,
|
||||
@@ -128,9 +144,20 @@ impl FaultProxy {
|
||||
mode_tx,
|
||||
shutdown_tx,
|
||||
accept_task,
|
||||
traffic,
|
||||
})
|
||||
}
|
||||
|
||||
/// Bytes actually forwarded in either direction, excluding dropped traffic.
|
||||
pub fn forwarded_bytes(&self) -> usize {
|
||||
self.traffic.forwarded_bytes.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Bytes actually discarded by a blackhole or one-way partition.
|
||||
pub fn dropped_bytes(&self) -> usize {
|
||||
self.traffic.dropped_bytes.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Local address the proxy is listening on. Point clients here instead of
|
||||
/// the real target to route their traffic through the proxy.
|
||||
pub fn local_addr(&self) -> SocketAddr {
|
||||
@@ -171,6 +198,7 @@ async fn accept_loop(
|
||||
target: SocketAddr,
|
||||
mode_rx: watch::Receiver<FaultMode>,
|
||||
mut shutdown_rx: watch::Receiver<bool>,
|
||||
traffic: Arc<ProxyTraffic>,
|
||||
) {
|
||||
loop {
|
||||
tokio::select! {
|
||||
@@ -188,6 +216,7 @@ async fn accept_loop(
|
||||
target,
|
||||
mode_rx.clone(),
|
||||
shutdown_rx.clone(),
|
||||
traffic.clone(),
|
||||
));
|
||||
}
|
||||
// A transient accept error should not tear the proxy down.
|
||||
@@ -205,6 +234,7 @@ async fn handle_connection(
|
||||
target: SocketAddr,
|
||||
mode_rx: watch::Receiver<FaultMode>,
|
||||
shutdown_rx: watch::Receiver<bool>,
|
||||
traffic: Arc<ProxyTraffic>,
|
||||
) {
|
||||
let server = match TcpStream::connect(target).await {
|
||||
Ok(s) => s,
|
||||
@@ -221,8 +251,9 @@ async fn handle_connection(
|
||||
server_wr,
|
||||
mode_rx.clone(),
|
||||
shutdown_rx.clone(),
|
||||
traffic.clone(),
|
||||
));
|
||||
let down = tokio::spawn(pump(Direction::ServerToClient, server_rd, client_wr, mode_rx, shutdown_rx));
|
||||
let down = tokio::spawn(pump(Direction::ServerToClient, server_rd, client_wr, mode_rx, shutdown_rx, traffic));
|
||||
|
||||
let _ = up.await;
|
||||
let _ = down.await;
|
||||
@@ -237,6 +268,7 @@ async fn pump<R, W>(
|
||||
mut to: W,
|
||||
mut mode_rx: watch::Receiver<FaultMode>,
|
||||
mut shutdown_rx: watch::Receiver<bool>,
|
||||
traffic: Arc<ProxyTraffic>,
|
||||
) where
|
||||
R: AsyncReadExt + Unpin,
|
||||
W: AsyncWriteExt + Unpin,
|
||||
@@ -257,15 +289,22 @@ async fn pump<R, W>(
|
||||
let mode = *mode_rx.borrow_and_update();
|
||||
match mode {
|
||||
// Whole connection is a black hole: drain and drop.
|
||||
FaultMode::Blackhole => continue,
|
||||
FaultMode::Blackhole => {
|
||||
traffic.dropped_bytes.fetch_add(n, Ordering::Relaxed);
|
||||
continue;
|
||||
}
|
||||
// This direction is partitioned off: drop its bytes; the peer keeps
|
||||
// reading nothing while the other direction may still flow.
|
||||
FaultMode::Partition(blocked) if blocked == dir => continue,
|
||||
FaultMode::Partition(blocked) if blocked == dir => {
|
||||
traffic.dropped_bytes.fetch_add(n, Ordering::Relaxed);
|
||||
continue;
|
||||
}
|
||||
FaultMode::Latency(delay) => {
|
||||
tokio::time::sleep(delay).await;
|
||||
if to.write_all(&buf[..n]).await.is_err() {
|
||||
break;
|
||||
}
|
||||
traffic.forwarded_bytes.fetch_add(n, Ordering::Relaxed);
|
||||
if to.flush().await.is_err() {
|
||||
break;
|
||||
}
|
||||
@@ -274,6 +313,7 @@ async fn pump<R, W>(
|
||||
if to.write_all(&buf[..n]).await.is_err() {
|
||||
break;
|
||||
}
|
||||
traffic.forwarded_bytes.fetch_add(n, Ordering::Relaxed);
|
||||
if to.flush().await.is_err() {
|
||||
break;
|
||||
}
|
||||
@@ -285,7 +325,6 @@ async fn pump<R, W>(
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
@@ -375,6 +414,8 @@ mod tests {
|
||||
|
||||
assert_eq!(got, payload, "pass mode must forward bytes unchanged");
|
||||
assert_eq!(echo.received().await, payload, "server must have received the bytes");
|
||||
assert_eq!(proxy.forwarded_bytes(), payload.len() * 2);
|
||||
assert_eq!(proxy.dropped_bytes(), 0);
|
||||
|
||||
proxy.shutdown().await;
|
||||
}
|
||||
@@ -414,6 +455,8 @@ mod tests {
|
||||
|
||||
assert!(got.is_empty(), "blackhole mode must not return any bytes, got {got:?}");
|
||||
assert!(echo.received().await.is_empty(), "blackhole must not forward to the server");
|
||||
assert_eq!(proxy.forwarded_bytes(), 0);
|
||||
assert_eq!(proxy.dropped_bytes(), b"into-the-void".len());
|
||||
|
||||
// Proxy is still alive and steerable after a blackholed connection.
|
||||
proxy.set_mode(FaultMode::Pass);
|
||||
|
||||
@@ -562,17 +562,23 @@ mod tests {
|
||||
create_bucket(&client, bucket).await.expect("Failed to create bucket");
|
||||
|
||||
let object_count = 1002;
|
||||
for i in 0..object_count {
|
||||
let key = format!("object{:04}.txt", i);
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(&key)
|
||||
.body(ByteStream::from_static(b"test content"))
|
||||
.send()
|
||||
.await
|
||||
.expect("Failed to put object");
|
||||
}
|
||||
// Finish every durable PUT before listing, with the same bounded fanout as the delimiter fixtures.
|
||||
stream::iter(0..object_count)
|
||||
.for_each_concurrent(16, |i| {
|
||||
let client = &client;
|
||||
async move {
|
||||
let key = format!("object{i:04}.txt");
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(&key)
|
||||
.body(ByteStream::from_static(b"test content"))
|
||||
.send()
|
||||
.await
|
||||
.unwrap_or_else(|err| panic!("Failed to put fixture object {key}: {err}"));
|
||||
}
|
||||
})
|
||||
.await;
|
||||
|
||||
eprintln!("Seeded {object_count} objects in {bucket}; starting ListObjectsV2 pagination");
|
||||
let deadline = Instant::now() + PAGINATION_TIMEOUT;
|
||||
|
||||
@@ -300,17 +300,28 @@ async fn test_degraded_cluster_read_quorum_follows_erasure_layout() -> TestResul
|
||||
}
|
||||
|
||||
cluster.stop_node(read_quorum - 1)?;
|
||||
let lock_read_quorum = node_count - node_count / 2;
|
||||
// EC:2 on six nodes still has three shared lock votes after losing
|
||||
// its fourth data shard. Preserve the distinct data-quorum error.
|
||||
let expected_code = if read_quorum - 1 < lock_read_quorum {
|
||||
"ServiceUnavailable"
|
||||
} else {
|
||||
"SlowDownRead"
|
||||
};
|
||||
for client in clients.iter().take(read_quorum - 1) {
|
||||
match client.get_object().bucket(&bucket).key("below-quorum").send().await {
|
||||
Ok(response) => assert!(
|
||||
response.body.collect().await.is_err(),
|
||||
"fewer than {read_quorum} valid fragments must not reconstruct an uncached object"
|
||||
),
|
||||
Err(error) => assert_eq!(
|
||||
error.as_service_error().and_then(|error| error.meta().code()),
|
||||
Some("ServiceUnavailable"),
|
||||
"a quorum loss must not be mistaken for a missing object"
|
||||
),
|
||||
Err(error) => {
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(503));
|
||||
assert_eq!(
|
||||
error.as_service_error().and_then(|error| error.meta().code()),
|
||||
Some(expected_code),
|
||||
"a quorum loss must not be mistaken for a missing object"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -425,6 +436,60 @@ async fn test_concurrent_cluster_overwrites_do_not_fail_namespace_lock_quorum()
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn unavailable_readiness_drive_indices(payload: &serde_json::Value) -> Vec<usize> {
|
||||
let mut indices = payload["details"]["storage"]["unavailableDrives"]
|
||||
.as_array()
|
||||
.expect("node readiness must report unavailable drives")
|
||||
.iter()
|
||||
.map(|drive| {
|
||||
usize::try_from(drive["diskIndex"].as_u64().expect("drive index must be unsigned"))
|
||||
.expect("drive index must fit usize")
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
indices.sort_unstable();
|
||||
indices
|
||||
}
|
||||
|
||||
fn assert_node_readiness_contract(payload: &serde_json::Value, survivors: usize) {
|
||||
let read_ready = survivors >= 2;
|
||||
let write_ready = survivors >= 3;
|
||||
let storage = &payload["details"]["storage"];
|
||||
assert_eq!(payload["ready"], read_ready);
|
||||
assert_eq!(storage["ready"], read_ready);
|
||||
assert_eq!(storage["readQuorum"], read_ready);
|
||||
assert_eq!(storage["writeQuorum"], write_ready);
|
||||
assert_eq!(storage["readinessScope"], "read_quorum");
|
||||
assert_eq!(storage["source"], "local_runtime");
|
||||
assert_eq!(storage["status"], if read_ready { "connected" } else { "disconnected" });
|
||||
assert_eq!(payload["details"]["poolMetadata"]["ready"], true);
|
||||
assert_eq!(payload["details"]["iam"]["ready"], true);
|
||||
assert_eq!(payload["details"]["lock"]["ready"], read_ready);
|
||||
assert_eq!(unavailable_readiness_drive_indices(payload), (survivors..4).collect::<Vec<_>>());
|
||||
for drive in storage["unavailableDrives"].as_array().expect("unavailable drives") {
|
||||
assert_eq!(drive["poolIndex"], 0);
|
||||
assert_eq!(drive["setIndex"], 0);
|
||||
let state = drive["runtimeState"].as_str().expect("runtime state must be reported");
|
||||
let host_online = drive["hostOnline"].as_bool().expect("host reachability must be reported");
|
||||
assert!(
|
||||
matches!(state, "online" | "suspect" | "offline" | "returning"),
|
||||
"unknown runtime state: {state}"
|
||||
);
|
||||
assert!(
|
||||
state != "online" || !host_online,
|
||||
"a reachable online drive cannot be excluded from quorum"
|
||||
);
|
||||
}
|
||||
if !read_ready {
|
||||
assert!(
|
||||
payload["degradedReasons"]
|
||||
.as_array()
|
||||
.expect("degraded reasons")
|
||||
.iter()
|
||||
.any(|reason| reason == "storage_and_lock_unavailable")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async fn assert_node_readiness_tracks_quorum(cluster: &mut RustFSTestClusterEnvironment) -> TestResult {
|
||||
let clients: Vec<_> = cluster
|
||||
.create_all_clients()?
|
||||
@@ -483,27 +548,14 @@ async fn assert_node_readiness_tracks_quorum(cluster: &mut RustFSTestClusterEnvi
|
||||
&& payload["details"]["poolMetadata"]["ready"] == true
|
||||
&& payload["details"]["iam"]["ready"] == true
|
||||
&& payload["details"]["lock"]["ready"] == read_quorum
|
||||
&& unavailable_readiness_drive_indices(&payload) == (survivors..4).collect::<Vec<_>>()
|
||||
{
|
||||
break payload;
|
||||
}
|
||||
assert!(Instant::now() < deadline, "node {idx}, survivors={survivors}: HTTP {status}, {payload}");
|
||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
||||
};
|
||||
assert_eq!(payload["details"]["storage"]["readinessScope"], "read_quorum");
|
||||
assert_eq!(payload["details"]["storage"]["source"], "local_runtime");
|
||||
assert_eq!(
|
||||
payload["details"]["storage"]["status"],
|
||||
if node_ready { "connected" } else { "disconnected" }
|
||||
);
|
||||
if !node_ready {
|
||||
assert!(
|
||||
payload["degradedReasons"]
|
||||
.as_array()
|
||||
.expect("degraded reasons")
|
||||
.iter()
|
||||
.any(|reason| reason == "storage_and_lock_unavailable")
|
||||
);
|
||||
}
|
||||
assert_node_readiness_contract(&payload, survivors);
|
||||
|
||||
for path in ["/health/ready", "/minio/health/ready"] {
|
||||
let head = http.head(format!("{url}{path}")).send().await?;
|
||||
@@ -512,7 +564,9 @@ async fn assert_node_readiness_tracks_quorum(cluster: &mut RustFSTestClusterEnvi
|
||||
let response = http.get(format!("{url}{path}")).send().await?;
|
||||
assert_eq!(response.status().as_u16(), node_status);
|
||||
let body: serde_json::Value = response.json().await?;
|
||||
assert_eq!(body["details"]["storage"], payload["details"]["storage"]);
|
||||
// Each response samples live diagnostics; suspect -> offline
|
||||
// must not change the quorum contract or excluded topology.
|
||||
assert_node_readiness_contract(&body, survivors);
|
||||
assert_eq!(body["details"]["poolMetadata"], payload["details"]["poolMetadata"]);
|
||||
}
|
||||
let live = http.get(format!("{url}/health/live")).send().await?;
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::disk::{RUSTFS_META_BUCKET, VolumeInfo, WalkDirOptions};
|
||||
pub(crate) use rustfs_ecstore::api::disk::{Endpoint, RUSTFS_META_BUCKET, VolumeInfo, WalkDirOptions};
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{AuthenticatedChannel, TonicInterceptor, node_service_time_out_client_no_auth};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{
|
||||
|
||||
@@ -165,4 +165,210 @@ mod tests {
|
||||
info!("RT-13c PASS: scanner/config persists across restart");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// RT-14: a transition PUT must not carry reserved `x-minio-internal-*` headers.
|
||||
///
|
||||
/// Regression pattern: transition forwarded the object's internal metadata as raw
|
||||
/// request headers, so MinIO-derived remotes (Storj) refused every transition with
|
||||
/// `400 InvalidArgument` while `tier add` still succeeded (rustfs#8362). The fake
|
||||
/// target refuses any PutObject carrying such a header, like MinIO and Storj do.
|
||||
#[tokio::test]
|
||||
async fn test_transition_put_omits_reserved_internal_headers() -> TestResult {
|
||||
use crate::fake_s3_target::{BucketMode, FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target, Operation};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
BucketLifecycleConfiguration, ExpirationStatus, LifecycleRule, LifecycleRuleFilter, Transition,
|
||||
TransitionStorageClass,
|
||||
};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
init_logging();
|
||||
info!("RT-14: transition PUT omits reserved internal headers");
|
||||
|
||||
const TIER: &str = "RT14COLD";
|
||||
const TARGET_BUCKET: &str = "rt14-cold";
|
||||
const SOURCE_BUCKET: &str = "rt14-hot";
|
||||
|
||||
let target = FakeS3Target::start().await.expect("start fake target");
|
||||
target.create_bucket_with_mode(TARGET_BUCKET, BucketMode::Unversioned);
|
||||
target.reject_reserved_internal_headers(true);
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
|
||||
env.start_rustfs_server_with_env(
|
||||
vec![],
|
||||
&[
|
||||
("RUSTFS_CONSOLE_ENABLE", "false"),
|
||||
("RUSTFS_TIER_RUSTFS_ALLOW_LOOPBACK_ENDPOINT", "true"),
|
||||
],
|
||||
)
|
||||
.await
|
||||
.expect("start RustFS");
|
||||
|
||||
let tier_body = serde_json::json!({
|
||||
"type": "rustfs",
|
||||
"rustfs": {
|
||||
"name": TIER,
|
||||
"endpoint": target.endpoint(),
|
||||
"accessKey": FAKE_ACCESS_KEY,
|
||||
"secretKey": FAKE_SECRET_KEY,
|
||||
"bucket": TARGET_BUCKET,
|
||||
"prefix": "cold/",
|
||||
"region": "us-east-1",
|
||||
"storageClass": ""
|
||||
}
|
||||
})
|
||||
.to_string();
|
||||
let deadline = Instant::now() + Duration::from_secs(30);
|
||||
loop {
|
||||
match admin_ok(&env, http::Method::PUT, "/rustfs/admin/v3/tier", Some(tier_body.clone())).await {
|
||||
Ok(_) => break,
|
||||
Err(err) if Instant::now() < deadline => {
|
||||
info!(" add tier not ready yet: {err}");
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
Err(err) => panic!("RT-14 FAIL: add tier: {err}"),
|
||||
}
|
||||
}
|
||||
|
||||
let client = env.create_s3_client();
|
||||
client
|
||||
.create_bucket()
|
||||
.bucket(SOURCE_BUCKET)
|
||||
.send()
|
||||
.await
|
||||
.expect("create bucket");
|
||||
|
||||
let objects: Vec<(&str, Vec<u8>)> = vec![
|
||||
("small.bin", (0..1024u32).map(|i| (i % 251) as u8).collect()),
|
||||
("medium.bin", (0..200_000u32).map(|i| (i.wrapping_mul(31) % 253) as u8).collect()),
|
||||
];
|
||||
for (key, body) in &objects {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(SOURCE_BUCKET)
|
||||
.key(*key)
|
||||
.body(ByteStream::from(body.clone()))
|
||||
.send()
|
||||
.await
|
||||
.expect("put source object");
|
||||
}
|
||||
|
||||
let rule = LifecycleRule::builder()
|
||||
.id("to-cold")
|
||||
.status(ExpirationStatus::Enabled)
|
||||
.filter(LifecycleRuleFilter::builder().prefix("").build())
|
||||
.transitions(
|
||||
Transition::builder()
|
||||
.days(0)
|
||||
.storage_class(TransitionStorageClass::from(TIER))
|
||||
.build(),
|
||||
)
|
||||
.build()
|
||||
.expect("build rule");
|
||||
client
|
||||
.put_bucket_lifecycle_configuration()
|
||||
.bucket(SOURCE_BUCKET)
|
||||
.lifecycle_configuration(
|
||||
BucketLifecycleConfiguration::builder()
|
||||
.rules(rule)
|
||||
.build()
|
||||
.expect("build config"),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.expect("put lifecycle");
|
||||
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::POST,
|
||||
&format!("/rustfs/admin/v3/ilm/transition/run?bucket={SOURCE_BUCKET}&prefix=&tier={TIER}&dryRun=false"),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("run transition");
|
||||
|
||||
let deadline = Instant::now() + Duration::from_secs(60);
|
||||
while target.stored_keys(TARGET_BUCKET).len() < objects.len() {
|
||||
assert!(
|
||||
Instant::now() < deadline,
|
||||
"RT-14 FAIL: transition did not reach the target; stored keys: {:?}",
|
||||
target.stored_keys(TARGET_BUCKET)
|
||||
);
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
|
||||
let keys = target.stored_keys(TARGET_BUCKET);
|
||||
assert_eq!(keys.len(), objects.len(), "RT-14 FAIL: unexpected remote objects: {keys:?}");
|
||||
|
||||
let mut stored_bodies = Vec::new();
|
||||
for key in &keys {
|
||||
let (body, metadata) = target.stored_object(TARGET_BUCKET, key).expect("remote object");
|
||||
for name in [
|
||||
"x-rustfs-internal-transition-transaction-id",
|
||||
"x-minio-internal-transition-transaction-id",
|
||||
"x-rustfs-internal-transition-tier-destination-id",
|
||||
"x-minio-internal-transition-tier-destination-id",
|
||||
] {
|
||||
assert!(
|
||||
metadata.contains_key(name),
|
||||
"RT-14 FAIL: remote object {key} lost transition identity {name}: {:?}",
|
||||
metadata.keys().collect::<Vec<_>>()
|
||||
);
|
||||
}
|
||||
stored_bodies.push(body.to_vec());
|
||||
}
|
||||
let mut expected_bodies: Vec<Vec<u8>> = objects.iter().map(|(_, body)| body.clone()).collect();
|
||||
expected_bodies.sort();
|
||||
stored_bodies.sort();
|
||||
assert_eq!(stored_bodies, expected_bodies, "RT-14 FAIL: remote bytes differ from the source objects");
|
||||
|
||||
let transition_puts = target
|
||||
.requests()
|
||||
.iter()
|
||||
.filter(|record| {
|
||||
record.operation == Operation::PutObject
|
||||
&& record
|
||||
.key
|
||||
.as_deref()
|
||||
.is_some_and(|key| key.contains("transition-transactions"))
|
||||
})
|
||||
.count();
|
||||
assert_eq!(
|
||||
transition_puts,
|
||||
objects.len(),
|
||||
"RT-14 FAIL: transition PUTs were retried or refused by the target"
|
||||
);
|
||||
|
||||
for (key, body) in &objects {
|
||||
let head = client
|
||||
.head_object()
|
||||
.bucket(SOURCE_BUCKET)
|
||||
.key(*key)
|
||||
.send()
|
||||
.await
|
||||
.expect("head transitioned");
|
||||
assert_eq!(
|
||||
head.storage_class().map(|class| class.as_str()),
|
||||
Some(TIER),
|
||||
"RT-14 FAIL: {key} is not reported as transitioned"
|
||||
);
|
||||
let got = client
|
||||
.get_object()
|
||||
.bucket(SOURCE_BUCKET)
|
||||
.key(*key)
|
||||
.send()
|
||||
.await
|
||||
.expect("get transitioned")
|
||||
.body
|
||||
.collect()
|
||||
.await
|
||||
.expect("read body")
|
||||
.into_bytes();
|
||||
assert_eq!(got.as_ref(), body.as_slice(), "RT-14 FAIL: read-back of {key} differs");
|
||||
}
|
||||
|
||||
target.shutdown().await;
|
||||
info!("RT-14 PASS: transition PUT omits reserved internal headers");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -33,6 +33,7 @@ use aws_sdk_s3::types::{
|
||||
};
|
||||
use http::{Method, StatusCode};
|
||||
use serde_json::Value;
|
||||
use std::future::Future;
|
||||
use std::io::Write;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
@@ -57,6 +58,10 @@ const UPGRADE_READINESS_BODY: &[u8] = b"upgrade write readiness";
|
||||
const PREVIOUS_RELEASE_SEED_KEY: &str = ".upgrade-readiness/previous-seed/node-0";
|
||||
const MULTIPART_WORKERS: usize = 16;
|
||||
const MULTIPART_UPLOADS_PER_WORKER: usize = 16;
|
||||
// Bound each 64 KiB upload, including completion acknowledgement. A peer
|
||||
// that accepts the request but never responds must fail the compatibility
|
||||
// check with the writer and object, rather than consume the whole CI job.
|
||||
const MIXED_MULTIPART_UPLOAD_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
// Peers keep a restarted node's drive in Suspect/Returning for roughly
|
||||
// probe_interval (2s) x success_threshold (3) after it comes back; 30s
|
||||
// comfortably covers that window plus CI scheduling jitter.
|
||||
@@ -335,14 +340,21 @@ fn configure_cluster_logs(cluster: &mut RustFSTestClusterEnvironment) -> TestRes
|
||||
async fn write_multipart_load(clients: &[Client], phase: &str) -> Result<Vec<String>, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let mut tasks = JoinSet::new();
|
||||
for worker in 0..MULTIPART_WORKERS {
|
||||
let client = clients[worker % clients.len()].clone();
|
||||
let node = worker % clients.len();
|
||||
let client = clients[node].clone();
|
||||
let phase = phase.to_string();
|
||||
tasks.spawn(async move {
|
||||
let mut keys = Vec::with_capacity(MULTIPART_UPLOADS_PER_WORKER);
|
||||
for upload in 0..MULTIPART_UPLOADS_PER_WORKER {
|
||||
let key = format!("{phase}/multipart/{worker:02}/{upload:02}");
|
||||
let part = vec![u8::try_from(worker)?; 64 * 1024];
|
||||
write_multipart(&client, MIXED_BUCKET, &key, &[part]).await?;
|
||||
mixed_multipart_before(
|
||||
Instant::now() + MIXED_MULTIPART_UPLOAD_TIMEOUT,
|
||||
node,
|
||||
&key,
|
||||
write_multipart(&client, MIXED_BUCKET, &key, &[part]),
|
||||
)
|
||||
.await?;
|
||||
keys.push(key);
|
||||
}
|
||||
Ok::<_, Box<dyn std::error::Error + Send + Sync>>(keys)
|
||||
@@ -356,6 +368,172 @@ async fn write_multipart_load(clients: &[Client], phase: &str) -> Result<Vec<Str
|
||||
Ok(keys)
|
||||
}
|
||||
|
||||
async fn mixed_multipart_before(
|
||||
deadline: Instant,
|
||||
node: usize,
|
||||
key: &str,
|
||||
upload: impl Future<Output = TestResult>,
|
||||
) -> TestResult {
|
||||
tokio::time::timeout_at(deadline, upload)
|
||||
.await
|
||||
.map_err(|_| MultipartAcknowledgementDeadline {
|
||||
node,
|
||||
key: key.to_owned(),
|
||||
})?
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct MultipartAcknowledgementDeadline {
|
||||
node: usize,
|
||||
key: String,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for MultipartAcknowledgementDeadline {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(
|
||||
formatter,
|
||||
"node {}: multipart upload {MIXED_BUCKET}/{} acknowledgement deadline exceeded",
|
||||
self.node, self.key
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for MultipartAcknowledgementDeadline {}
|
||||
|
||||
// Decode only the public lock snapshot fields. Unknown fields, including
|
||||
// credential/configuration payloads, never enter the failure artifact.
|
||||
#[derive(serde::Deserialize, serde::Serialize)]
|
||||
struct MixedLockSnapshot {
|
||||
total: Option<u64>,
|
||||
truncated: Option<bool>,
|
||||
capability_note: Option<String>,
|
||||
locks: Vec<MixedLockEntry>,
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize, serde::Serialize)]
|
||||
struct MixedLockEntry {
|
||||
resource: Option<String>,
|
||||
bucket: Option<String>,
|
||||
object: Option<String>,
|
||||
version: Option<String>,
|
||||
owner: Option<String>,
|
||||
#[serde(rename = "type")]
|
||||
lock_type: Option<String>,
|
||||
priority: Option<String>,
|
||||
since: Option<String>,
|
||||
elapsed_secs: Option<u64>,
|
||||
ttl_secs: Option<u64>,
|
||||
}
|
||||
|
||||
async fn read_mixed_lock_snapshot(response: &mut reqwest::Response) -> Result<MixedLockSnapshot, &'static str> {
|
||||
const MAX_BODY_BYTES: usize = 1024 * 1024;
|
||||
let mut body = Vec::new();
|
||||
while let Some(chunk) = response.chunk().await.map_err(|_| "body_read_error")? {
|
||||
if chunk.len() > MAX_BODY_BYTES - body.len() {
|
||||
return Err("body_limit_exceeded");
|
||||
}
|
||||
body.extend_from_slice(&chunk);
|
||||
}
|
||||
serde_json::from_slice(&body).map_err(|_| "invalid_lock_snapshot")
|
||||
}
|
||||
|
||||
async fn probe_mixed_multipart_failure(
|
||||
cluster: &RustFSTestClusterEnvironment,
|
||||
node: usize,
|
||||
path: &str,
|
||||
head: bool,
|
||||
deadline: Instant,
|
||||
) -> Value {
|
||||
let started = Instant::now();
|
||||
let method = if head { Method::HEAD } else { Method::GET };
|
||||
let mut observation = serde_json::json!({
|
||||
"node": node, "method": method.as_str(), "path": path,
|
||||
"status": null, "observation": "unknown", "timeout": false,
|
||||
});
|
||||
// The shared absolute deadline bounds both headers and a slow/infinite
|
||||
// response body; a send-only timeout would leave the cluster alive forever.
|
||||
let result = tokio::time::timeout_at(deadline, async {
|
||||
let mut response = signed_request(
|
||||
method,
|
||||
&format!("{}{path}", cluster.nodes[node].url.trim_end_matches('/')),
|
||||
&cluster.access_key,
|
||||
&cluster.secret_key,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.map_err(|_| "request_error")?;
|
||||
let status = response.status();
|
||||
observation["status"] = serde_json::json!(status.as_u16());
|
||||
if head {
|
||||
observation["observation"] = serde_json::json!(match status {
|
||||
StatusCode::OK => "visible",
|
||||
StatusCode::NOT_FOUND => "not_visible",
|
||||
_ => "unknown",
|
||||
});
|
||||
for (field, header) in [
|
||||
("content_length", "content-length"),
|
||||
("etag", "etag"),
|
||||
("version", "x-amz-version-id"),
|
||||
] {
|
||||
observation[field] = serde_json::json!(response.headers().get(header).and_then(|value| value.to_str().ok()));
|
||||
}
|
||||
} else if status == StatusCode::OK {
|
||||
observation["snapshot"] =
|
||||
serde_json::to_value(read_mixed_lock_snapshot(&mut response).await?).map_err(|_| "snapshot_encode_error")?;
|
||||
observation["observation"] = serde_json::json!("captured");
|
||||
}
|
||||
Ok::<_, &'static str>(())
|
||||
})
|
||||
.await;
|
||||
match result {
|
||||
Ok(Ok(())) => {}
|
||||
Ok(Err(reason)) => observation["error"] = serde_json::json!(reason),
|
||||
Err(_) => observation["timeout"] = serde_json::json!(true),
|
||||
}
|
||||
observation["elapsed_secs"] = serde_json::json!(started.elapsed().as_secs_f64());
|
||||
observation
|
||||
}
|
||||
|
||||
async fn observe_mixed_multipart_failure(
|
||||
cluster: &RustFSTestClusterEnvironment,
|
||||
phase: &str,
|
||||
error: BoxError,
|
||||
log_dir: Option<&Path>,
|
||||
) -> TestResult {
|
||||
if let (Some(log_dir), Some(failure)) = (log_dir, error.downcast_ref::<MultipartAcknowledgementDeadline>()) {
|
||||
let started = Instant::now();
|
||||
let deadline = started + Duration::from_secs(5);
|
||||
// Reserve the last second for serialization and bounded artifact I/O.
|
||||
// All probes run directly in this future; no diagnostic task is spawned.
|
||||
let network_deadline = started + Duration::from_secs(4);
|
||||
let path = format!("/{MIXED_BUCKET}/{}", failure.key);
|
||||
let _ = tokio::time::timeout_at(deadline, async {
|
||||
let (heads, locks) = tokio::join!(
|
||||
futures::future::join_all((0..cluster.nodes.len()).map(|node| probe_mixed_multipart_failure(
|
||||
cluster,
|
||||
node,
|
||||
&path,
|
||||
true,
|
||||
network_deadline
|
||||
))),
|
||||
probe_mixed_multipart_failure(cluster, 0, "/rustfs/admin/v3/top/locks?count=1000", false, network_deadline),
|
||||
);
|
||||
let snapshot = serde_json::json!({
|
||||
"phase": phase, "bucket": MIXED_BUCKET, "key": failure.key,
|
||||
"writer_node": failure.node, "heads": heads, "top_locks": locks,
|
||||
});
|
||||
let body = serde_json::to_vec_pretty(&snapshot).map_err(|_| ())?;
|
||||
tokio::fs::write(log_dir.join("mixed-multipart-failure.json"), body)
|
||||
.await
|
||||
.map_err(|_| ())
|
||||
})
|
||||
.await;
|
||||
}
|
||||
// Visibility and diagnostic failures cannot acknowledge the original MPU.
|
||||
Err(error)
|
||||
}
|
||||
|
||||
/// Assert that `client` eventually lists exactly `expected` objects under
|
||||
/// `{phase}/`, polling until [`LISTING_CONVERGENCE_TIMEOUT`].
|
||||
///
|
||||
@@ -400,7 +578,7 @@ async fn exercise_mixed_cluster(
|
||||
previous_node: usize,
|
||||
) -> TestResult {
|
||||
let clients = cluster.create_all_clients()?;
|
||||
wait_for_upgrade_write_readiness(&clients, phase, LISTING_CONVERGENCE_TIMEOUT).await?;
|
||||
wait_for_upgrade_write_readiness(&clients, MIXED_BUCKET, phase, LISTING_CONVERGENCE_TIMEOUT).await?;
|
||||
let current_client = &clients[current_node];
|
||||
let previous_client = &clients[previous_node];
|
||||
|
||||
@@ -426,7 +604,13 @@ async fn exercise_mixed_cluster(
|
||||
.await?;
|
||||
assert_eq!(read_object(current_client, MIXED_BUCKET, &previous_key, None).await?.1, previous_body);
|
||||
|
||||
let multipart_keys = write_multipart_load(&clients, phase).await?;
|
||||
let multipart_keys = match write_multipart_load(&clients, phase).await {
|
||||
Ok(keys) => keys,
|
||||
Err(error) => {
|
||||
let log_dir = std::env::var_os("RUSTFS_E2E_LOG_DIR");
|
||||
return observe_mixed_multipart_failure(cluster, phase, error, log_dir.as_deref().map(Path::new)).await;
|
||||
}
|
||||
};
|
||||
let expected_count = multipart_keys.len() + 2;
|
||||
for (label, client) in [("current", current_client), ("previous", previous_client)] {
|
||||
assert_eq!(
|
||||
@@ -462,7 +646,12 @@ fn upgrade_probe_client(client: &Client) -> Client {
|
||||
)
|
||||
}
|
||||
|
||||
async fn wait_for_upgrade_write_readiness(clients: &[Client], phase: &str, budget: Duration) -> TestResult {
|
||||
pub(crate) async fn wait_for_upgrade_write_readiness(
|
||||
clients: &[Client],
|
||||
bucket: &str,
|
||||
phase: &str,
|
||||
budget: Duration,
|
||||
) -> TestResult {
|
||||
// ListBuckets can succeed before peers recover a restarted disk. Cluster
|
||||
// health also accepts Returning disks whose write health is still FAULTY.
|
||||
// Probe every writer outside the asserted phase prefix; compatibility
|
||||
@@ -482,7 +671,7 @@ async fn wait_for_upgrade_write_readiness(clients: &[Client], phase: &str, budge
|
||||
deadline,
|
||||
client
|
||||
.put_object()
|
||||
.bucket(MIXED_BUCKET)
|
||||
.bucket(bucket)
|
||||
.key(&key)
|
||||
.body(ByteStream::from_static(UPGRADE_READINESS_BODY))
|
||||
.send(),
|
||||
@@ -517,19 +706,20 @@ async fn prepare_previous_release_baseline(cluster: &mut RustFSTestClusterEnviro
|
||||
// waits for pool metadata (#7473). First prove the elected writer can
|
||||
// persist data, then restart each old process once with three peers still
|
||||
// readable. This preparation ends before any current binary is started.
|
||||
wait_for_upgrade_write_readiness(&clients[..1], "previous-seed", LISTING_CONVERGENCE_TIMEOUT).await?;
|
||||
wait_for_upgrade_write_readiness(&clients[..1], MIXED_BUCKET, "previous-seed", LISTING_CONVERGENCE_TIMEOUT).await?;
|
||||
for node in [1, 2, 3, 0] {
|
||||
cluster.stop_node(node)?;
|
||||
cluster.start_node_from_binary(node, previous_binary).await?;
|
||||
wait_for_upgrade_write_readiness(
|
||||
std::slice::from_ref(&clients[node]),
|
||||
MIXED_BUCKET,
|
||||
&format!("previous-restart-{node}"),
|
||||
LISTING_CONVERGENCE_TIMEOUT,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
wait_for_upgrade_write_readiness(&clients, "previous-baseline", LISTING_CONVERGENCE_TIMEOUT).await?;
|
||||
wait_for_upgrade_write_readiness(&clients, MIXED_BUCKET, "previous-baseline", LISTING_CONVERGENCE_TIMEOUT).await?;
|
||||
for (reader, client) in clients.iter().enumerate() {
|
||||
assert_eq!(
|
||||
read_object(client, MIXED_BUCKET, PREVIOUS_RELEASE_SEED_KEY, None).await?.1,
|
||||
@@ -556,9 +746,10 @@ mod upgrade_write_readiness_tests {
|
||||
#[tokio::test]
|
||||
async fn waits_for_each_writer_after_metadata_is_ready() -> TestResult {
|
||||
let target = FakeS3Target::start().await?;
|
||||
target.create_bucket(MIXED_BUCKET);
|
||||
let bucket = "distributed-upgrade-history";
|
||||
target.create_bucket(bucket);
|
||||
let client = fake_source_client(&target);
|
||||
client.head_bucket().bucket(MIXED_BUCKET).send().await?;
|
||||
client.head_bucket().bucket(bucket).send().await?;
|
||||
|
||||
let phase = "one-previous-node";
|
||||
let first_key = format!(".upgrade-readiness/{phase}/node-0");
|
||||
@@ -578,7 +769,7 @@ mod upgrade_write_readiness_tests {
|
||||
// Metadata readiness does not prove that a data write can succeed.
|
||||
let premature = client
|
||||
.put_object()
|
||||
.bucket(MIXED_BUCKET)
|
||||
.bucket(bucket)
|
||||
.key(&first_key)
|
||||
.body(ByteStream::from_static(b"upgrade write readiness"))
|
||||
.send()
|
||||
@@ -586,15 +777,15 @@ mod upgrade_write_readiness_tests {
|
||||
.expect_err("metadata readiness does not prove write readiness");
|
||||
assert_eq!(premature.raw_response().map(|response| response.status().as_u16()), Some(503));
|
||||
|
||||
wait_for_upgrade_write_readiness(&[client.clone(), client.clone()], phase, Duration::from_secs(5)).await?;
|
||||
wait_for_upgrade_write_readiness(&[client.clone(), client.clone()], bucket, phase, Duration::from_secs(5)).await?;
|
||||
assert_eq!(target.count_requests(FakeTargetOperation::PutObject, &first_key), 3);
|
||||
assert_eq!(target.count_requests(FakeTargetOperation::PutObject, &second_key), 2);
|
||||
assert!(target.has_object(MIXED_BUCKET, &first_key));
|
||||
assert!(target.has_object(MIXED_BUCKET, &second_key));
|
||||
assert!(target.has_object(bucket, &first_key));
|
||||
assert!(target.has_object(bucket, &second_key));
|
||||
assert!(
|
||||
client
|
||||
.list_objects_v2()
|
||||
.bucket(MIXED_BUCKET)
|
||||
.bucket(bucket)
|
||||
.prefix(format!("{phase}/"))
|
||||
.send()
|
||||
.await?
|
||||
@@ -624,9 +815,10 @@ mod upgrade_write_readiness_tests {
|
||||
let phase = format!("permanent-{}", status.as_u16());
|
||||
let key = format!(".upgrade-readiness/{phase}/node-0");
|
||||
target.inject_for_key(FakeTargetOperation::PutObject, &key, FaultAction::Status(status), 1);
|
||||
let error = wait_for_upgrade_write_readiness(std::slice::from_ref(&client), &phase, Duration::from_secs(5))
|
||||
.await
|
||||
.expect_err("a permanent error must not be retried into success");
|
||||
let error =
|
||||
wait_for_upgrade_write_readiness(std::slice::from_ref(&client), MIXED_BUCKET, &phase, Duration::from_secs(5))
|
||||
.await
|
||||
.expect_err("a permanent error must not be retried into success");
|
||||
assert!(error.to_string().contains("node 0 write readiness failed"), "{error}");
|
||||
assert_eq!(target.count_requests(FakeTargetOperation::PutObject, &key), 1);
|
||||
assert!(!target.has_object(MIXED_BUCKET, &key));
|
||||
@@ -649,7 +841,7 @@ mod upgrade_write_readiness_tests {
|
||||
FaultAction::Status(StatusCode::SERVICE_UNAVAILABLE),
|
||||
10,
|
||||
);
|
||||
let error = wait_for_upgrade_write_readiness(&[client], phase, Duration::from_secs(1))
|
||||
let error = wait_for_upgrade_write_readiness(&[client], MIXED_BUCKET, phase, Duration::from_secs(1))
|
||||
.await
|
||||
.expect_err("persistent unavailability must exhaust the shared deadline");
|
||||
// Transport scheduling consumes the same budget; do not require a
|
||||
@@ -668,10 +860,134 @@ mod upgrade_write_readiness_tests {
|
||||
let phase = "stalled";
|
||||
let key = format!(".upgrade-readiness/{phase}/node-0");
|
||||
target.inject_for_key(FakeTargetOperation::PutObject, &key, FaultAction::Stall(Duration::from_secs(30)), 1);
|
||||
let error = wait_for_upgrade_write_readiness(&[client], phase, Duration::from_secs(1))
|
||||
let error = wait_for_upgrade_write_readiness(std::slice::from_ref(&client), MIXED_BUCKET, phase, Duration::from_secs(1))
|
||||
.await
|
||||
.expect_err("a stalled request must not outlive the readiness deadline");
|
||||
assert!(error.to_string().contains("deadline exceeded during PutObject"), "{error}");
|
||||
|
||||
let part = vec![2_u8; 64 * 1024];
|
||||
let successful_key = "stalled/multipart/02/00";
|
||||
mixed_multipart_before(
|
||||
Instant::now() + MIXED_MULTIPART_UPLOAD_TIMEOUT,
|
||||
2,
|
||||
successful_key,
|
||||
write_multipart(&client, MIXED_BUCKET, successful_key, std::slice::from_ref(&part)),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(read_object(&client, MIXED_BUCKET, successful_key, None).await?.1, part);
|
||||
|
||||
let stalled_key = "stalled/multipart/02/01";
|
||||
target.inject_for_key(
|
||||
FakeTargetOperation::CompleteMultipartUpload,
|
||||
stalled_key,
|
||||
FaultAction::Stall(Duration::from_secs(30)),
|
||||
1,
|
||||
);
|
||||
let error = {
|
||||
let upload = write_multipart(&client, MIXED_BUCKET, stalled_key, std::slice::from_ref(&part));
|
||||
tokio::pin!(upload);
|
||||
tokio::select! {
|
||||
result = &mut upload => panic!("stalled completion unexpectedly returned: {result:?}"),
|
||||
observed = tokio::time::timeout(Duration::from_secs(10), async {
|
||||
while target.count_requests(FakeTargetOperation::CompleteMultipartUpload, stalled_key) == 0
|
||||
|| !target.has_object(MIXED_BUCKET, stalled_key)
|
||||
{
|
||||
sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
}) => observed.expect("multipart completion must reach and commit on the fake peer"),
|
||||
}
|
||||
// An already committed object is not an acknowledged client success.
|
||||
// Expire only after observing the real completion request, so host
|
||||
// scheduling cannot turn this into a timeout before admission.
|
||||
tokio::time::timeout(Duration::from_secs(1), mixed_multipart_before(Instant::now(), 2, stalled_key, upload))
|
||||
.await
|
||||
.expect("expired deadline must stop the in-flight completion")
|
||||
.expect_err("unacknowledged completion must fail even if the peer committed")
|
||||
};
|
||||
assert!(error.to_string().contains("node 2"), "{error}");
|
||||
assert!(error.to_string().contains(stalled_key), "{error}");
|
||||
assert!(error.to_string().contains("acknowledgement deadline exceeded"), "{error}");
|
||||
assert_eq!(target.count_requests(FakeTargetOperation::CompleteMultipartUpload, stalled_key), 1);
|
||||
|
||||
let failure = error
|
||||
.downcast_ref::<MultipartAcknowledgementDeadline>()
|
||||
.expect("the timeout must retain typed writer/object context");
|
||||
assert_eq!(failure.node, 2);
|
||||
assert_eq!(failure.key, stalled_key);
|
||||
let original_message = error.to_string();
|
||||
// These four probe addresses share an in-process peer. This verifies
|
||||
// bounded diagnostic fan-out, not a four-node RustFS cluster.
|
||||
let mut probes = RustFSTestClusterEnvironment::new(MIXED_NODE_COUNT).await?;
|
||||
probes.access_key = FAKE_ACCESS_KEY.to_owned();
|
||||
probes.secret_key = FAKE_SECRET_KEY.to_owned();
|
||||
for node in &mut probes.nodes {
|
||||
node.url = target.endpoint().to_owned();
|
||||
}
|
||||
let log_dir = tempfile::tempdir()?;
|
||||
target.inject_for_key(
|
||||
FakeTargetOperation::HeadObject,
|
||||
stalled_key,
|
||||
FaultAction::Stall(Duration::from_secs(30)),
|
||||
1,
|
||||
);
|
||||
let error = tokio::time::timeout(
|
||||
Duration::from_secs(7),
|
||||
observe_mixed_multipart_failure(&probes, phase, error, Some(log_dir.path())),
|
||||
)
|
||||
.await
|
||||
.expect("the shared diagnostic budget must include artifact writing")
|
||||
.expect_err("observing committed data cannot acknowledge the stalled upload");
|
||||
assert_eq!(error.to_string(), original_message);
|
||||
assert!(error.downcast_ref::<MultipartAcknowledgementDeadline>().is_some());
|
||||
let snapshot: Value =
|
||||
serde_json::from_slice(&tokio::fs::read(log_dir.path().join("mixed-multipart-failure.json")).await?)?;
|
||||
assert_eq!(snapshot["key"], stalled_key);
|
||||
assert_eq!(snapshot["writer_node"], 2);
|
||||
let heads = snapshot["heads"]
|
||||
.as_array()
|
||||
.expect("failure artifact must retain every probe");
|
||||
assert_eq!(heads.len(), MIXED_NODE_COUNT);
|
||||
assert_eq!(
|
||||
heads
|
||||
.iter()
|
||||
.filter(|head| head["timeout"] == true && head["observation"] == "unknown")
|
||||
.count(),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
heads
|
||||
.iter()
|
||||
.filter(|head| head["status"] == 200 && head["observation"] == "visible")
|
||||
.count(),
|
||||
3
|
||||
);
|
||||
assert_eq!(target.count_requests(FakeTargetOperation::HeadObject, stalled_key), MIXED_NODE_COUNT);
|
||||
assert_eq!(target.count_requests(FakeTargetOperation::CompleteMultipartUpload, stalled_key), 1);
|
||||
assert_eq!(snapshot["top_locks"]["observation"], "unknown", "the fake peer has no admin lock API");
|
||||
|
||||
let locks: MixedLockSnapshot = serde_json::from_value(serde_json::json!({
|
||||
"total": 1, "truncated": true, "capability_note": "partial snapshot",
|
||||
"credentials": { "secret_key": "must not persist" },
|
||||
"locks": [{
|
||||
"resource": format!("{MIXED_BUCKET}/{stalled_key}"), "owner": "writer", "type": "WRITE",
|
||||
"elapsed_secs": 30, "ttl_secs": 1, "merged_config": { "secret_key": "must not persist" },
|
||||
}],
|
||||
}))?;
|
||||
let safe_locks = serde_json::to_value(locks)?;
|
||||
assert_eq!(safe_locks["total"], 1);
|
||||
assert_eq!(safe_locks["truncated"], true);
|
||||
assert_eq!(safe_locks["locks"][0]["owner"], "writer");
|
||||
assert_eq!(safe_locks["locks"][0]["elapsed_secs"], 30);
|
||||
assert!(safe_locks.get("credentials").is_none());
|
||||
assert!(safe_locks["locks"][0].get("merged_config").is_none());
|
||||
|
||||
let mut body = br#"{"locks":[]}"#.to_vec();
|
||||
body.resize(1024 * 1024, b' ');
|
||||
let mut at_limit = reqwest::Response::from(http::Response::new(body.clone()));
|
||||
assert!(read_mixed_lock_snapshot(&mut at_limit).await?.locks.is_empty());
|
||||
body.push(b' ');
|
||||
let mut oversized = reqwest::Response::from(http::Response::new(body));
|
||||
assert_eq!(read_mixed_lock_snapshot(&mut oversized).await.err(), Some("body_limit_exceeded"));
|
||||
target.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
@@ -2181,9 +2497,9 @@ struct LayoutTransport {
|
||||
uploaded_parts: Vec<i32>,
|
||||
single_puts: usize,
|
||||
completes: usize,
|
||||
/// Raw per-key journal in target order: (sequence, operation, part number,
|
||||
/// upload id), so duplicate drives can be told apart from retries.
|
||||
journal: Vec<(u64, String, Option<i32>, Option<String>)>,
|
||||
/// Allowlisted request identity and prepared response metadata. A prepared
|
||||
/// response does not prove that the client received it.
|
||||
journal: Vec<Value>,
|
||||
}
|
||||
|
||||
/// Configure every layout bucket to replicate its existing objects to a fresh
|
||||
@@ -2245,12 +2561,28 @@ async fn replicate_layouts(
|
||||
journal: key_requests
|
||||
.iter()
|
||||
.map(|record| {
|
||||
(
|
||||
record.sequence,
|
||||
format!("{:?}", record.operation),
|
||||
record.part_number,
|
||||
record.upload_id.as_ref().map(|id| id.chars().take(12).collect()),
|
||||
)
|
||||
serde_json::json!({
|
||||
"sequence": record.sequence,
|
||||
"operation": format!("{:?}", record.operation),
|
||||
"part_number": record.part_number,
|
||||
"upload_id": record.upload_id,
|
||||
"requested_version_id": record.version_id,
|
||||
"source_version_id": record.source_version_id,
|
||||
"source_mtime": record.source_mtime,
|
||||
"source_etag": record.source_etag,
|
||||
"source_replication_request": record.source_replication_request,
|
||||
"request_content_length": record.content_length,
|
||||
"journaled_at_unix_millis": record.journaled_at_unix_millis,
|
||||
"prepared_response": record.prepared_response.as_ref().map(|response| serde_json::json!({
|
||||
"status": response.status,
|
||||
"version_id": response.version_id,
|
||||
"etag": response.etag,
|
||||
"last_modified": response.last_modified,
|
||||
"content_length": response.content_length,
|
||||
"prepared_at_unix_millis": response.prepared_at_unix_millis,
|
||||
"elapsed_secs": response.elapsed.as_secs_f64(),
|
||||
})),
|
||||
})
|
||||
})
|
||||
.collect(),
|
||||
};
|
||||
|
||||
@@ -122,6 +122,8 @@ test-util = []
|
||||
e2e-test-hooks = []
|
||||
|
||||
[dependencies]
|
||||
rustfs-capture-authority = { workspace = true }
|
||||
openraft = { workspace = true, features = ["storage-v2"] }
|
||||
hotpath.workspace = true
|
||||
rustfs-filemeta.workspace = true
|
||||
rustfs-utils = { workspace = true, features = ["full"] }
|
||||
@@ -176,7 +178,7 @@ regex = { workspace = true }
|
||||
path-absolutize = { workspace = true }
|
||||
rmp.workspace = true
|
||||
rmp-serde.workspace = true
|
||||
tokio-util = { workspace = true, features = ["io", "compat"] }
|
||||
tokio-util = { workspace = true, features = ["io", "compat", "rt"] }
|
||||
tokio-stream = { workspace = true, features = ["sync"] }
|
||||
hmac = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
|
||||
@@ -240,7 +240,10 @@ pub mod bucket {
|
||||
}
|
||||
|
||||
pub mod quota {
|
||||
pub use crate::bucket::quota::{BucketQuota, QuotaCheckResult, QuotaError, QuotaOperation};
|
||||
pub use crate::bucket::quota::{
|
||||
BucketQuota, QUOTA_RESERVATION_PROTOCOL_V1, QUOTA_RESERVATION_PROTOCOL_V2, QuotaCheckResult, QuotaError,
|
||||
QuotaOperation,
|
||||
};
|
||||
|
||||
pub mod checker {
|
||||
pub use crate::bucket::quota::checker::QuotaChecker;
|
||||
@@ -362,11 +365,49 @@ pub mod config {
|
||||
ServerConfigCorruptError, ServerConfigSaveResult, ServerConfigSnapshot, delete_config, delete_config_no_lock,
|
||||
is_server_config_corrupt_error, lookup_configs, read_config, read_config_no_lock, read_config_with_metadata,
|
||||
read_config_without_migrate, read_config_without_migrate_no_lock, read_existing_server_config_no_lock,
|
||||
read_server_config_snapshot, save_config, save_config_no_lock, save_config_with_opts, save_server_config,
|
||||
save_server_config_no_lock, save_server_config_snapshot, save_server_config_snapshot_with_generation,
|
||||
server_config_path, try_migrate_server_config, with_config_object_read_lock, with_config_object_write_lock,
|
||||
with_server_config_read_lock, with_server_config_write_lock,
|
||||
read_server_config_snapshot, save_config, save_config_no_lock, save_config_with_opts, save_config_with_opts_quiet,
|
||||
save_server_config, save_server_config_no_lock, save_server_config_snapshot,
|
||||
save_server_config_snapshot_with_generation, server_config_path, try_migrate_server_config,
|
||||
with_config_object_read_lock, with_config_object_write_lock, with_server_config_read_lock,
|
||||
with_server_config_write_lock,
|
||||
};
|
||||
|
||||
/// Observe persisted config bytes and their reader-associated metadata within a byte limit.
|
||||
pub async fn read_config_observation(
|
||||
api: std::sync::Arc<crate::store::ECStore>,
|
||||
file: &str,
|
||||
max_bytes: usize,
|
||||
) -> crate::error::Result<(Vec<u8>, crate::object_api::ObjectInfo)> {
|
||||
crate::config::com::read_config_observation(api, file, max_bytes).await
|
||||
}
|
||||
|
||||
/// Observe bounded config bytes while the caller retains the object's write owner.
|
||||
pub async fn read_config_observation_no_lock(
|
||||
api: std::sync::Arc<crate::store::ECStore>,
|
||||
file: &str,
|
||||
max_bytes: usize,
|
||||
guard: &crate::object_api::WriteCommitGuard,
|
||||
) -> crate::error::Result<(Vec<u8>, crate::object_api::ObjectInfo)> {
|
||||
crate::config::com::read_config_observation_no_lock(api, file, max_bytes, guard).await
|
||||
}
|
||||
|
||||
/// Read config bytes and metadata without treating an empty payload as absent.
|
||||
pub async fn read_config_preserve_empty_with_metadata_opts(
|
||||
api: std::sync::Arc<crate::store::ECStore>,
|
||||
file: &str,
|
||||
opts: &crate::object_api::ObjectOptions,
|
||||
) -> crate::error::Result<(Vec<u8>, crate::object_api::ObjectInfo)> {
|
||||
crate::config::com::read_config_preserve_empty_with_metadata_opts(api, file, opts).await
|
||||
}
|
||||
|
||||
pub async fn read_config_limited_preserve_empty_with_metadata_opts(
|
||||
api: std::sync::Arc<crate::store::ECStore>,
|
||||
file: &str,
|
||||
opts: &crate::object_api::ObjectOptions,
|
||||
max_bytes: usize,
|
||||
) -> crate::error::Result<(Vec<u8>, crate::object_api::ObjectInfo)> {
|
||||
crate::config::com::read_config_limited_preserve_empty_with_metadata_opts(api, file, opts, max_bytes).await
|
||||
}
|
||||
}
|
||||
|
||||
pub mod storageclass {
|
||||
@@ -468,7 +509,7 @@ pub mod event {
|
||||
pub mod global {
|
||||
pub use crate::runtime::global::{
|
||||
set_global_endpoints, set_global_region, set_global_rustfs_port, set_object_store_resolver, shutdown_background_services,
|
||||
update_erasure_type,
|
||||
update_erasure_type, wait_for_detached_mutations,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -556,6 +597,7 @@ pub mod rio {
|
||||
}
|
||||
|
||||
pub mod rpc {
|
||||
pub use crate::cluster::rpc::capture_raft::{CaptureRaftError, CaptureRaftLimits, CaptureRaftNetworkFactory, CaptureRaftRpc};
|
||||
pub use crate::cluster::rpc::{
|
||||
AuthenticatedChannel, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, MAX_NETWORK_PROBE_BYTES, MAX_NETWORK_PROBE_DURATION,
|
||||
NetworkPeerProbeClient, NetworkPeerProbeError, NetworkPeerProbeMeasurement, NetworkPeerTarget, PEER_RESTDRY_RUN,
|
||||
@@ -602,7 +644,13 @@ pub mod store_list {
|
||||
|
||||
pub mod storage {
|
||||
pub use crate::core::pools::{HealLifecycleExpiryContext, POOL_META_NAME};
|
||||
pub use crate::disk::local::native_prepare::{
|
||||
NativePreparationDescription, NativePreparationProbe, NativePreparationReceipt,
|
||||
};
|
||||
pub use crate::store::HealWalkVersion;
|
||||
pub use crate::store::native_prepare::{
|
||||
NativePreparationFinalizeRequest, NativePreparationRecovery, NativePreparationTarget,
|
||||
};
|
||||
pub use crate::store::{
|
||||
BootstrapLocalTarget, ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk,
|
||||
all_local_disk_path, find_local_disk_by_ref, init_local_disks, init_local_disks_with_instance_ctx, init_lock_clients,
|
||||
|
||||
@@ -87,6 +87,7 @@ use rustfs_filemeta::{
|
||||
FileInfo, FileInfoOpts, NULL_VERSION_ID, RestoreStatus, RestoreStatusOps, TRANSITION_COMPLETE, get_file_info,
|
||||
is_restored_object_on_disk,
|
||||
};
|
||||
use rustfs_lock::{NamespaceLockGuard, error::LockError};
|
||||
use rustfs_scanner_metrics::metrics::{
|
||||
IlmAction, Metrics, ScannerLifecycleExpiryStateUpdate, ScannerLifecycleTransitionStateUpdate, global_metrics,
|
||||
};
|
||||
@@ -190,6 +191,8 @@ const TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL: StdDuration = StdDuration::from_
|
||||
// before another full sweep can discover work persisted by another node.
|
||||
const TIER_FREE_VERSION_RECOVERY_MAX_IDLE_INTERVAL: StdDuration = StdDuration::from_secs(10 * 60);
|
||||
const TIER_FREE_VERSION_RECOVERY_JITTER_PERCENT: u64 = 10;
|
||||
const TIER_FREE_VERSION_RECOVERY_LEADER_LOCK: &str = "tier-free-version-recovery/leader.lock";
|
||||
const TIER_FREE_VERSION_RECOVERY_LOCK_TIMEOUT: StdDuration = StdDuration::from_secs(1);
|
||||
const DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS: i64 = 5;
|
||||
const EXPIRY_WORKER_QUEUE_CAPACITY: usize = 1000;
|
||||
const DEFAULT_MANUAL_TRANSITION_JOB_RECOVERY_LIMIT: usize = 100;
|
||||
@@ -1441,13 +1444,24 @@ pub(crate) async fn enqueue_committed_free_versions(api: &ECStore, free_versions
|
||||
queued
|
||||
}
|
||||
|
||||
async fn enqueue_recovered_free_version_with_state(state: &Arc<RwLock<ExpiryState>>, oi: ObjectInfo) -> bool {
|
||||
async fn enqueue_recovered_free_version_with_state(
|
||||
state: &Arc<RwLock<ExpiryState>>,
|
||||
oi: ObjectInfo,
|
||||
cancel: &CancellationToken,
|
||||
) -> bool {
|
||||
let task = FreeVersionTask(oi);
|
||||
let hash = task.op_hash();
|
||||
let (wrkr, stats) = {
|
||||
let state = state.read().await;
|
||||
let state = select! {
|
||||
biased;
|
||||
_ = cancel.cancelled() => return false,
|
||||
state = state.read() => state,
|
||||
};
|
||||
(state.get_worker_ch(hash), Arc::clone(&state.stats))
|
||||
};
|
||||
if cancel.is_cancelled() {
|
||||
return false;
|
||||
}
|
||||
let Some(wrkr) = wrkr else {
|
||||
stats.increment_missed_freevers_tasks();
|
||||
stats.record_scanner_expiry_state();
|
||||
@@ -1497,9 +1511,13 @@ fn set_recovered_free_version_enqueue_observer(
|
||||
RecoveredFreeVersionEnqueueObserverGuard
|
||||
}
|
||||
|
||||
pub async fn enqueue_recovered_free_version(api: &ECStore, oi: ObjectInfo) -> bool {
|
||||
pub(super) async fn enqueue_recovered_free_version_with_cancel(
|
||||
api: &ECStore,
|
||||
oi: ObjectInfo,
|
||||
cancel: &CancellationToken,
|
||||
) -> bool {
|
||||
let expiry_state = api.ctx.expiry_state();
|
||||
let queued = enqueue_recovered_free_version_with_state(&expiry_state, oi).await;
|
||||
let queued = enqueue_recovered_free_version_with_state(&expiry_state, oi, cancel).await;
|
||||
|
||||
#[cfg(test)]
|
||||
if let Some(observer) = RECOVERED_FREE_VERSION_ENQUEUE_OBSERVER
|
||||
@@ -2921,6 +2939,20 @@ fn resolve_tier_free_version_recovery_enabled(value: Result<String, env::VarErro
|
||||
}
|
||||
|
||||
fn spawn_tier_free_version_recovery_once(api: Arc<ECStore>, started: &OnceLock<()>) -> Option<JoinHandle<()>> {
|
||||
spawn_tier_free_version_recovery_once_with_options(
|
||||
api,
|
||||
started,
|
||||
TIER_FREE_VERSION_RECOVERY_LEADER_LOCK.to_string(),
|
||||
tier_free_version_recovery_leader_retry_delay,
|
||||
)
|
||||
}
|
||||
|
||||
fn spawn_tier_free_version_recovery_once_with_options(
|
||||
api: Arc<ECStore>,
|
||||
started: &OnceLock<()>,
|
||||
leader_lock_name: String,
|
||||
retry_delay: fn() -> StdDuration,
|
||||
) -> Option<JoinHandle<()>> {
|
||||
if !tier_free_version_recovery_enabled() {
|
||||
warn!(
|
||||
event = EVENT_LIFECYCLE_WORKER_STATE,
|
||||
@@ -2937,20 +2969,43 @@ fn spawn_tier_free_version_recovery_once(api: Arc<ECStore>, started: &OnceLock<(
|
||||
|
||||
Some(tokio::spawn(async move {
|
||||
let cancel_token = api.ctx.background_cancel_token().unwrap_or_default();
|
||||
let expiry_state = api.ctx.expiry_state();
|
||||
run_tier_free_version_recovery_loop(
|
||||
let acquire_api = Arc::clone(&api);
|
||||
let acquire_lock_name = Arc::<str>::from(leader_lock_name);
|
||||
run_tier_free_version_recovery_coordinator(
|
||||
cancel_token,
|
||||
expiry_state,
|
||||
jitter_tier_free_version_recovery_delay,
|
||||
move |bucket_marker, object_marker, recovery_cancel| {
|
||||
retry_delay,
|
||||
move || {
|
||||
let api = Arc::clone(&acquire_api);
|
||||
let lock_name = Arc::clone(&acquire_lock_name);
|
||||
async move {
|
||||
let lock = api.new_ns_lock(RUSTFS_META_BUCKET, &lock_name).await?;
|
||||
match lock.get_write_lock_quiet(TIER_FREE_VERSION_RECOVERY_LOCK_TIMEOUT).await {
|
||||
Ok(guard) => Ok(Some(guard)),
|
||||
Err(LockError::Timeout { .. }) => Ok(None),
|
||||
Err(err) => Err(err.into()),
|
||||
}
|
||||
}
|
||||
},
|
||||
move |leader_cancel| {
|
||||
let api = Arc::clone(&api);
|
||||
async move {
|
||||
recover_tier_free_versions_with_cancel(
|
||||
api,
|
||||
DEFAULT_FREE_VERSION_RECOVERY_LIMIT,
|
||||
bucket_marker,
|
||||
object_marker,
|
||||
recovery_cancel,
|
||||
run_tier_free_version_recovery_loop(
|
||||
leader_cancel,
|
||||
api.ctx.expiry_state(),
|
||||
jitter_tier_free_version_recovery_delay,
|
||||
move |bucket_marker, object_marker, recovery_cancel| {
|
||||
let api = Arc::clone(&api);
|
||||
async move {
|
||||
recover_tier_free_versions_with_cancel(
|
||||
api,
|
||||
DEFAULT_FREE_VERSION_RECOVERY_LIMIT,
|
||||
bucket_marker,
|
||||
object_marker,
|
||||
recovery_cancel,
|
||||
)
|
||||
.await
|
||||
}
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -2960,6 +3015,86 @@ fn spawn_tier_free_version_recovery_once(api: Arc<ECStore>, started: &OnceLock<(
|
||||
}))
|
||||
}
|
||||
|
||||
fn tier_free_version_recovery_leader_retry_delay() -> StdDuration {
|
||||
StdDuration::from_secs(rand::rng().random_range(15..=30))
|
||||
}
|
||||
|
||||
async fn run_tier_free_version_recovery_coordinator<A, AF, R, RF>(
|
||||
cancel: CancellationToken,
|
||||
retry_delay: fn() -> StdDuration,
|
||||
mut acquire: A,
|
||||
mut recover: R,
|
||||
) where
|
||||
A: FnMut() -> AF,
|
||||
AF: Future<Output = crate::error::Result<Option<NamespaceLockGuard>>>,
|
||||
R: FnMut(CancellationToken) -> RF,
|
||||
RF: Future<Output = ()>,
|
||||
{
|
||||
loop {
|
||||
let acquired = select! {
|
||||
biased;
|
||||
_ = cancel.cancelled() => return,
|
||||
result = acquire() => result,
|
||||
};
|
||||
match acquired {
|
||||
Ok(Some(mut guard)) => {
|
||||
let leader_cancel = cancel.child_token();
|
||||
// Hold ownership across every page, cooldown and idle wait. A
|
||||
// follower taking over starts a fresh sweep of persisted markers.
|
||||
let recovery = recover(leader_cancel.clone());
|
||||
tokio::pin!(recovery);
|
||||
debug!(
|
||||
event = EVENT_LIFECYCLE_WORKER_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
state = "free_version_recovery_leader_acquired",
|
||||
lock_name = TIER_FREE_VERSION_RECOVERY_LEADER_LOCK,
|
||||
"Tier free-version recovery leader acquired"
|
||||
);
|
||||
select! {
|
||||
biased;
|
||||
_ = cancel.cancelled() => {
|
||||
leader_cancel.cancel();
|
||||
recovery.await;
|
||||
}
|
||||
_ = guard.lock_lost_notified() => {
|
||||
debug!(
|
||||
event = EVENT_LIFECYCLE_WORKER_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
state = "free_version_recovery_leader_lost",
|
||||
lock_name = TIER_FREE_VERSION_RECOVERY_LEADER_LOCK,
|
||||
"Tier free-version recovery leader lost"
|
||||
);
|
||||
// Dropping the recovery future would detach its active
|
||||
// walk. Cancel it and poll the same future through cleanup.
|
||||
leader_cancel.cancel();
|
||||
recovery.await;
|
||||
}
|
||||
_ = &mut recovery => {}
|
||||
}
|
||||
leader_cancel.cancel();
|
||||
guard.release();
|
||||
}
|
||||
Ok(None) => {}
|
||||
Err(err) => warn!(
|
||||
event = EVENT_LIFECYCLE_WORKER_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
state = "free_version_recovery_leader_acquire_failed",
|
||||
lock_name = TIER_FREE_VERSION_RECOVERY_LEADER_LOCK,
|
||||
error = %err,
|
||||
"Tier free-version recovery leader acquisition failed"
|
||||
),
|
||||
}
|
||||
select! {
|
||||
biased;
|
||||
_ = cancel.cancelled() => return,
|
||||
_ = tokio::time::sleep(retry_delay()) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_tier_free_version_recovery_loop<F, Fut>(
|
||||
cancel_token: CancellationToken,
|
||||
expiry_state: Arc<RwLock<ExpiryState>>,
|
||||
@@ -2969,7 +3104,11 @@ async fn run_tier_free_version_recovery_loop<F, Fut>(
|
||||
F: FnMut(Option<String>, Option<String>, CancellationToken) -> Fut,
|
||||
Fut: Future<Output = crate::error::Result<FreeVersionRecoveryStats>>,
|
||||
{
|
||||
let recovery_notify = Arc::clone(&expiry_state.read().await.recovery_notify);
|
||||
let recovery_notify = select! {
|
||||
biased;
|
||||
_ = cancel_token.cancelled() => return,
|
||||
state = expiry_state.read() => Arc::clone(&state.recovery_notify),
|
||||
};
|
||||
let mut schedule = TierFreeVersionRecoverySchedule::default();
|
||||
|
||||
loop {
|
||||
@@ -2992,7 +3131,11 @@ async fn run_tier_free_version_recovery_loop<F, Fut>(
|
||||
schedule.record_success(&stats);
|
||||
rustfs_io_metrics::record_stage_duration("lifecycle_free_version_recovery", elapsed.as_secs_f64() * 1000.0);
|
||||
let (pending_tasks, active_tasks) = {
|
||||
let state = expiry_state.read().await;
|
||||
let state = select! {
|
||||
biased;
|
||||
_ = cancel_token.cancelled() => return,
|
||||
state = expiry_state.read() => state,
|
||||
};
|
||||
(state.pending_tasks(), state.stats.active_tasks())
|
||||
};
|
||||
debug!(
|
||||
@@ -4748,7 +4891,7 @@ async fn expire_transitioned_object_with_lock_lost_signal(
|
||||
bucket_incarnation_id: Uuid,
|
||||
lock_lost_signal: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
|
||||
) -> Result<ObjectInfo, std::io::Error> {
|
||||
let publication_guard = lifecycle_expiry_publication_guard(&api, oi, bucket_incarnation_id)
|
||||
let publication_guard = lifecycle_expiry_publication_guard(&api, &oi.bucket, &oi.name, bucket_incarnation_id)
|
||||
.await
|
||||
.ok_or_else(|| std::io::Error::other("lifecycle expiry is not allowed for this bucket"))?;
|
||||
let snapshot = lifecycle_delete_config_snapshot(&api, oi)
|
||||
@@ -4838,10 +4981,14 @@ pub fn gen_transition_objname(bucket: &str) -> Result<String, Error> {
|
||||
pub async fn transition_object(api: Arc<ECStore>, oi: &ObjectInfo, lae: LcAuditEvent) -> Result<(), Error> {
|
||||
let time_ilm = Metrics::time_ilm(lae.event.action);
|
||||
|
||||
let (publication_guard, bucket_incarnation_id) = lifecycle_transition_publication_guard(&api, oi)
|
||||
.await
|
||||
.ok_or_else(|| Error::other("lifecycle transition is not allowed for this bucket"))?;
|
||||
|
||||
let etag = if let Some(etag) = &oi.etag { etag } else { "" };
|
||||
let etag = etag.to_string();
|
||||
|
||||
let opts = ObjectOptions {
|
||||
let mut opts = ObjectOptions {
|
||||
transition: TransitionOptions {
|
||||
status: lifecycle::TRANSITION_PENDING.to_string(),
|
||||
tier: lae.event.storage_class,
|
||||
@@ -4853,8 +5000,10 @@ pub async fn transition_object(api: Arc<ECStore>, oi: &ObjectInfo, lae: LcAuditE
|
||||
versioned: BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await,
|
||||
version_suspended: BucketVersioningSys::prefix_suspended(&oi.bucket, &oi.name).await,
|
||||
mod_time: oi.mod_time,
|
||||
expected_bucket_incarnation_id: Some(bucket_incarnation_id),
|
||||
..Default::default()
|
||||
};
|
||||
opts.add_namespace_lock_guard(&publication_guard);
|
||||
let result = api.transition_object(&oi.bucket, &oi.name, &opts).await;
|
||||
time_ilm(1)();
|
||||
result
|
||||
@@ -5458,20 +5607,26 @@ pub async fn apply_transition_rule(event: &lifecycle::Event, src: &LcEventSrc, o
|
||||
.await
|
||||
}
|
||||
|
||||
async fn lifecycle_expiry_publication_guard(
|
||||
async fn acquire_lifecycle_publication_guard(api: &ECStore, bucket: &str) -> Result<rustfs_lock::NamespaceLockGuard, Error> {
|
||||
let lock = api
|
||||
.new_ns_lock(bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
|
||||
.await?;
|
||||
let guard = lock.get_read_lock(get_lock_acquire_timeout()).await.map_err(Error::other)?;
|
||||
if guard.is_lock_lost() {
|
||||
return Err(Error::other("table-bucket publication lock was lost before lifecycle admission"));
|
||||
}
|
||||
Ok(guard)
|
||||
}
|
||||
|
||||
pub(crate) async fn lifecycle_expiry_publication_guard(
|
||||
api: &ECStore,
|
||||
oi: &ObjectInfo,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
bucket_incarnation_id: Uuid,
|
||||
) -> Option<rustfs_lock::NamespaceLockGuard> {
|
||||
let result = async {
|
||||
let lock = api
|
||||
.new_ns_lock(&oi.bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
|
||||
.await?;
|
||||
let guard = lock.get_read_lock(get_lock_acquire_timeout()).await.map_err(Error::other)?;
|
||||
if guard.is_lock_lost() {
|
||||
return Err(Error::other("table-bucket publication lock was lost before lifecycle delete admission"));
|
||||
}
|
||||
if !metadata_boundary::lifecycle_expiry_allowed(api, &oi.bucket, bucket_incarnation_id).await? {
|
||||
let result: std::result::Result<Option<rustfs_lock::NamespaceLockGuard>, Error> = async {
|
||||
let guard = acquire_lifecycle_publication_guard(api, bucket).await?;
|
||||
if !metadata_boundary::lifecycle_expiry_allowed(api, bucket, bucket_incarnation_id).await? {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(guard))
|
||||
@@ -5484,8 +5639,8 @@ async fn lifecycle_expiry_publication_guard(
|
||||
event = EVENT_LIFECYCLE_DELETE_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
bucket = %oi.bucket,
|
||||
object = %oi.name,
|
||||
bucket,
|
||||
object,
|
||||
operation = "authorize_lifecycle_expiry",
|
||||
error = %err,
|
||||
"Lifecycle delete admission failed"
|
||||
@@ -5495,6 +5650,34 @@ async fn lifecycle_expiry_publication_guard(
|
||||
}
|
||||
}
|
||||
|
||||
async fn lifecycle_transition_publication_guard(
|
||||
api: &ECStore,
|
||||
oi: &ObjectInfo,
|
||||
) -> Option<(rustfs_lock::NamespaceLockGuard, Uuid)> {
|
||||
let result: std::result::Result<(rustfs_lock::NamespaceLockGuard, Uuid), Error> = async {
|
||||
let guard = acquire_lifecycle_publication_guard(api, &oi.bucket).await?;
|
||||
let configs = metadata_boundary::get_expiry_configs(api, &oi.bucket).await?;
|
||||
Ok((guard, configs.bucket_incarnation_id))
|
||||
}
|
||||
.await;
|
||||
match result {
|
||||
Ok(state) => Some(state),
|
||||
Err(err) => {
|
||||
warn!(
|
||||
event = EVENT_LIFECYCLE_TIER_OPERATION_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
bucket = %oi.bucket,
|
||||
object = %oi.name,
|
||||
operation = "authorize_lifecycle_transition",
|
||||
error = %err,
|
||||
"Lifecycle transition admission failed"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn apply_expiry_on_transitioned_object(
|
||||
api: Arc<ECStore>,
|
||||
oi: &ObjectInfo,
|
||||
@@ -5551,7 +5734,8 @@ async fn apply_expiry_on_non_transitioned_objects_with_lock_lost_signal(
|
||||
bucket_incarnation_id: Uuid,
|
||||
lock_lost_signal: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
|
||||
) -> bool {
|
||||
let Some(publication_guard) = lifecycle_expiry_publication_guard(&api, oi, bucket_incarnation_id).await else {
|
||||
let Some(publication_guard) = lifecycle_expiry_publication_guard(&api, &oi.bucket, &oi.name, bucket_incarnation_id).await
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
let snapshot = match lifecycle_delete_config_snapshot(&api, oi).await {
|
||||
@@ -5611,8 +5795,10 @@ async fn apply_expiry_on_non_transitioned_objects_with_lock_lost_signal(
|
||||
} else {
|
||||
None
|
||||
};
|
||||
// The delete owns the publication guard from here on, so a cancelled
|
||||
// expiry worker cannot let table-bucket publication overtake it.
|
||||
let mut dobj = match api
|
||||
.delete_object_with_tier_delete_journal(&oi.bucket, &encode_dir_object(&oi.name), opts)
|
||||
.delete_object_with_tier_delete_journal_and_guards(&oi.bucket, &encode_dir_object(&oi.name), opts, [publication_guard])
|
||||
.await
|
||||
{
|
||||
Ok(dobj) => dobj,
|
||||
@@ -5844,6 +6030,7 @@ pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc,
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::expiry_worker_count;
|
||||
use super::lifecycle_transition_publication_guard;
|
||||
use super::{
|
||||
DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS, DEFAULT_TRANSITION_QUEUE_CAPACITY, DEFAULT_TRANSITION_WORKERS_ABSOLUTE_MAX,
|
||||
DEFAULT_TRANSITION_WORKERS_CAP, EVENT_LIFECYCLE_EVALUATION_FAILED, EVENT_LIFECYCLE_EXPIRED_DETECTED,
|
||||
@@ -5862,10 +6049,10 @@ mod tests {
|
||||
merge_stale_multipart_candidate, persist_manual_transition_job_progress_if_owned,
|
||||
persist_manual_transition_page_checkpoint, recover_manual_transition_job, recover_manual_transition_jobs,
|
||||
resolve_tier_free_version_recovery_enabled, resolve_transition_queue_capacity, resolve_transition_queue_send_timeout,
|
||||
resolve_transition_worker_count, resolve_transition_workers_absolute_max, run_tier_free_version_recovery_loop,
|
||||
select_restore_s3_location, set_lifecycle_observability_observer, set_recovered_free_version_enqueue_observer,
|
||||
should_defer_date_expiry_for_recent_config_update, transitioned_cleanup_tuple, transitioned_object_delete_opts,
|
||||
wait_for_tier_free_version_recovery,
|
||||
resolve_transition_worker_count, resolve_transition_workers_absolute_max, run_tier_free_version_recovery_coordinator,
|
||||
run_tier_free_version_recovery_loop, select_restore_s3_location, set_lifecycle_observability_observer,
|
||||
set_recovered_free_version_enqueue_observer, should_defer_date_expiry_for_recent_config_update,
|
||||
transitioned_cleanup_tuple, transitioned_object_delete_opts, wait_for_tier_free_version_recovery,
|
||||
};
|
||||
#[cfg(feature = "test-util")]
|
||||
use super::{delete_free_version_remote_object_then, encode_dir_object, get_transitioned_object_reader_with_tier_manager};
|
||||
@@ -5959,6 +6146,231 @@ mod tests {
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use uuid::Uuid;
|
||||
|
||||
fn recovery_test_lock() -> (Arc<rustfs_lock::NamespaceLock>, Vec<Arc<dyn rustfs_lock::LockClient>>) {
|
||||
let clients: Vec<Arc<dyn rustfs_lock::LockClient>> = (0..3)
|
||||
.map(|_| {
|
||||
Arc::new(rustfs_lock::LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())))
|
||||
as Arc<dyn rustfs_lock::LockClient>
|
||||
})
|
||||
.collect();
|
||||
let lock =
|
||||
rustfs_lock::NamespaceLock::with_clients_and_quorum(format!("recovery-test-{}", Uuid::new_v4()), clients.clone(), 2);
|
||||
(Arc::new(lock), clients)
|
||||
}
|
||||
|
||||
async fn acquire_recovery_test_lock(
|
||||
lock: Arc<rustfs_lock::NamespaceLock>,
|
||||
owner: String,
|
||||
) -> crate::error::Result<Option<rustfs_lock::NamespaceLockGuard>> {
|
||||
// Keep this lease longer than the virtual-time cooldown/idle advances;
|
||||
// production uses the namespace wrapper's renewable 30-second lease.
|
||||
lock.lock_guard(
|
||||
rustfs_lock::ObjectKey::new(crate::disk::RUSTFS_META_BUCKET, super::TIER_FREE_VERSION_RECOVERY_LEADER_LOCK),
|
||||
&owner,
|
||||
StdDuration::from_millis(5),
|
||||
StdDuration::from_secs(3600),
|
||||
)
|
||||
.await
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
fn recovery_test_retry_delay() -> StdDuration {
|
||||
StdDuration::from_secs(15)
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial]
|
||||
async fn tier_free_version_recovery_eight_coordinators_hold_one_leader_and_take_over() {
|
||||
let (lock, _clients) = recovery_test_lock();
|
||||
let (pages_tx, mut pages_rx) = tokio::sync::mpsc::unbounded_channel();
|
||||
let mut tasks = Vec::new();
|
||||
let mut cancels = Vec::new();
|
||||
for node in 0..8 {
|
||||
let cancel = CancellationToken::new();
|
||||
cancels.push(cancel.clone());
|
||||
let lock = Arc::clone(&lock);
|
||||
let pages_tx = pages_tx.clone();
|
||||
tasks.push(tokio::spawn(async move {
|
||||
run_tier_free_version_recovery_coordinator(
|
||||
cancel,
|
||||
recovery_test_retry_delay,
|
||||
move || acquire_recovery_test_lock(Arc::clone(&lock), format!("node-{node}")),
|
||||
move |leader_cancel| {
|
||||
let pages_tx = pages_tx.clone();
|
||||
async move {
|
||||
let mut pages = 0;
|
||||
run_tier_free_version_recovery_loop(
|
||||
leader_cancel,
|
||||
ExpiryState::new(),
|
||||
std::convert::identity,
|
||||
move |bucket, object, _| {
|
||||
pages += 1;
|
||||
pages_tx.send((node, bucket, object)).expect("page observer should be open");
|
||||
std::future::ready(Ok(free_version_recovery_stats(0, 0, pages == 1)))
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}));
|
||||
}
|
||||
let (leader, bucket, object) = pages_rx.recv().await.expect("one coordinator should start recovery");
|
||||
assert!(bucket.is_none() && object.is_none());
|
||||
// The same leader must survive both a truncated page's cooldown and
|
||||
// complete idle sweeps. All other coordinators keep probing the lock.
|
||||
let mut observed_pages = 1;
|
||||
for _ in 0..27 {
|
||||
tokio::time::advance(StdDuration::from_secs(60)).await;
|
||||
for _ in 0..100 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
while let Ok((node, bucket, object)) = pages_rx.try_recv() {
|
||||
assert_eq!(node, leader, "followers must not repeat a namespace sweep");
|
||||
if observed_pages == 1 {
|
||||
assert_eq!(bucket.as_deref(), Some("bucket"));
|
||||
assert_eq!(object.as_deref(), Some("object"));
|
||||
} else {
|
||||
assert!(bucket.is_none() && object.is_none(), "complete sweeps should reset their cursors");
|
||||
}
|
||||
observed_pages += 1;
|
||||
}
|
||||
}
|
||||
assert!(observed_pages >= 7, "the leader should finish a full ten-minute idle wait");
|
||||
cancels[leader].cancel();
|
||||
let (successor, bucket, object) = tokio::time::timeout(StdDuration::from_secs(60), pages_rx.recv())
|
||||
.await
|
||||
.expect("a follower should take over after leader shutdown")
|
||||
.expect("page observer should remain open");
|
||||
assert_ne!(successor, leader);
|
||||
assert!(bucket.is_none() && object.is_none(), "a new leader must restart discovery safely");
|
||||
for cancel in cancels {
|
||||
cancel.cancel();
|
||||
}
|
||||
for task in tasks {
|
||||
task.await.expect("all coordinators should shut down cleanly");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial]
|
||||
async fn tier_free_version_recovery_lock_loss_waits_for_active_page_cleanup() {
|
||||
let (lock, _clients) = recovery_test_lock();
|
||||
let cancel = CancellationToken::new();
|
||||
let loop_cancel = cancel.clone();
|
||||
let (page_tx, mut page_rx) = tokio::sync::mpsc::unbounded_channel();
|
||||
let cleanup_started = Arc::new(tokio::sync::Notify::new());
|
||||
let finish_cleanup = Arc::new(tokio::sync::Notify::new());
|
||||
let observed_cleanup = Arc::clone(&cleanup_started);
|
||||
let release_cleanup = Arc::clone(&finish_cleanup);
|
||||
let task = tokio::spawn(async move {
|
||||
run_tier_free_version_recovery_coordinator(
|
||||
loop_cancel,
|
||||
recovery_test_retry_delay,
|
||||
move || acquire_recovery_test_lock(Arc::clone(&lock), "old-leader".to_string()),
|
||||
move |leader_cancel| {
|
||||
let page_tx = page_tx.clone();
|
||||
let cleanup_started = Arc::clone(&cleanup_started);
|
||||
let finish_cleanup = Arc::clone(&finish_cleanup);
|
||||
async move {
|
||||
run_tier_free_version_recovery_loop(
|
||||
leader_cancel,
|
||||
ExpiryState::new(),
|
||||
std::convert::identity,
|
||||
move |_, _, page_cancel| {
|
||||
let cleanup_started = Arc::clone(&cleanup_started);
|
||||
let finish_cleanup = Arc::clone(&finish_cleanup);
|
||||
page_tx.send(()).expect("page observer should remain open");
|
||||
async move {
|
||||
page_cancel.cancelled().await;
|
||||
cleanup_started.notify_one();
|
||||
finish_cleanup.notified().await;
|
||||
Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "cancelled page").into())
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
});
|
||||
page_rx.recv().await.expect("leader should start an active page");
|
||||
// Let authoritative backend leases expire before the next heartbeat
|
||||
// can renew them. A failed refresh must cancel the active page.
|
||||
tokio::time::advance(StdDuration::from_secs(3600)).await;
|
||||
tokio::time::timeout(StdDuration::from_secs(1), observed_cleanup.notified())
|
||||
.await
|
||||
.expect("expired backend leases must trigger recovery cancellation");
|
||||
assert!(!task.is_finished(), "lock loss must wait for the active page's cleanup");
|
||||
assert!(page_rx.try_recv().is_err(), "the old leader must not start another page");
|
||||
cancel.cancel();
|
||||
release_cleanup.notify_one();
|
||||
task.await.expect("coordinator should finish cleanup before exiting");
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn tier_free_version_recovery_coordinator_retries_without_scanning_after_lock_error() {
|
||||
let cancel = CancellationToken::new();
|
||||
let loop_cancel = cancel.clone();
|
||||
let attempts = Arc::new(AtomicUsize::new(0));
|
||||
let observed_attempts = Arc::clone(&attempts);
|
||||
let task = tokio::spawn(async move {
|
||||
run_tier_free_version_recovery_coordinator(
|
||||
loop_cancel,
|
||||
recovery_test_retry_delay,
|
||||
move || {
|
||||
observed_attempts.fetch_add(1, Ordering::SeqCst);
|
||||
std::future::ready(Err(rustfs_lock::LockError::InsufficientNodes {
|
||||
required: 2,
|
||||
available: 1,
|
||||
}
|
||||
.into()))
|
||||
},
|
||||
|_| async { panic!("a node without ownership must never start discovery") },
|
||||
)
|
||||
.await;
|
||||
});
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 1);
|
||||
tokio::time::advance(StdDuration::from_secs(14)).await;
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 1, "lock errors must not cause a tight retry loop");
|
||||
tokio::time::advance(StdDuration::from_secs(1)).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 2);
|
||||
cancel.cancel();
|
||||
task.await.expect("a follower should respond to shutdown during retry wait");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tier_free_version_recovery_enqueue_cancels_while_waiting_for_state_lock() {
|
||||
let state = ExpiryState::new();
|
||||
let (tx, mut rx) = tokio::sync::mpsc::channel(1);
|
||||
let mut held = state.write().await;
|
||||
held.tasks_tx.push(tx);
|
||||
let cancel = CancellationToken::new();
|
||||
let enqueue_cancel = cancel.clone();
|
||||
let enqueue_state = Arc::clone(&state);
|
||||
let enqueue = tokio::spawn(async move {
|
||||
enqueue_recovered_free_version_with_state(&enqueue_state, ObjectInfo::default(), &enqueue_cancel).await
|
||||
});
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!enqueue.is_finished(), "enqueue should be blocked on the state lock");
|
||||
cancel.cancel();
|
||||
let queued = tokio::time::timeout(StdDuration::from_secs(1), enqueue)
|
||||
.await
|
||||
.expect("cancellation must not wait for the state write guard")
|
||||
.expect("enqueue should return cleanly");
|
||||
assert!(!queued);
|
||||
assert_eq!(held.pending_tasks(), 0);
|
||||
assert_eq!(held.stats.missed_free_vers_tasks(), 0);
|
||||
assert!(rx.try_recv().is_err(), "a cancelled recovery must not enqueue a task");
|
||||
drop(held);
|
||||
assert!(!enqueue_recovered_free_version_with_state(&state, ObjectInfo::default(), &cancel).await);
|
||||
assert!(rx.try_recv().is_err(), "cancellation must win even when the lock is available");
|
||||
}
|
||||
|
||||
fn free_version_recovery_stats(enqueued: usize, failed: usize, truncated: bool) -> FreeVersionRecoveryStats {
|
||||
FreeVersionRecoveryStats {
|
||||
scanned: enqueued.saturating_add(failed),
|
||||
@@ -7638,7 +8050,7 @@ mod tests {
|
||||
assert_eq!(state.read().await.stats.pending_tasks(), 1);
|
||||
|
||||
// The single-slot queue is full for both enqueue paths.
|
||||
assert!(!enqueue_recovered_free_version_with_state(&state, oi.clone()).await);
|
||||
assert!(!enqueue_recovered_free_version_with_state(&state, oi.clone(), &CancellationToken::new()).await);
|
||||
assert_eq!(state.read().await.stats.pending_tasks(), 1);
|
||||
assert_eq!(state.read().await.stats.missed_free_vers_tasks(), 1);
|
||||
|
||||
@@ -7670,7 +8082,7 @@ mod tests {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let queued = enqueue_recovered_free_version_with_state(&state, oi).await;
|
||||
let queued = enqueue_recovered_free_version_with_state(&state, oi, &CancellationToken::new()).await;
|
||||
let state = state.read().await;
|
||||
|
||||
assert!(!queued);
|
||||
@@ -7799,8 +8211,8 @@ mod tests {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let first = enqueue_recovered_free_version_with_state(&state, oi.clone()).await;
|
||||
let second = enqueue_recovered_free_version_with_state(&state, oi).await;
|
||||
let first = enqueue_recovered_free_version_with_state(&state, oi.clone(), &CancellationToken::new()).await;
|
||||
let second = enqueue_recovered_free_version_with_state(&state, oi, &CancellationToken::new()).await;
|
||||
let state = state.read().await;
|
||||
|
||||
assert!(first);
|
||||
@@ -7901,7 +8313,7 @@ mod tests {
|
||||
};
|
||||
|
||||
assert!(
|
||||
super::enqueue_recovered_free_version(&ecstore, oi).await,
|
||||
super::enqueue_recovered_free_version_with_cancel(&ecstore, oi, &CancellationToken::new()).await,
|
||||
"the resized production worker queue should accept the task"
|
||||
);
|
||||
stop_tx.send(None).await.expect("worker stop signal should be delivered");
|
||||
@@ -13028,6 +13440,171 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn queued_noncurrent_batch_expiry_does_not_delete_from_table_bucket() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let bucket = format!("table-noncurrent-lifecycle-{}", Uuid::new_v4().simple());
|
||||
let object = "tables/table-id/data/part-00001.parquet";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
metadata_sys::update_in(
|
||||
&ecstore.ctx,
|
||||
&bucket,
|
||||
BUCKET_VERSIONING_CONFIG,
|
||||
b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
|
||||
)
|
||||
.await
|
||||
.expect("bucket versioning should be enabled");
|
||||
|
||||
let mut old_reader = PutObjReader::from_vec(b"old table data".to_vec());
|
||||
let old = ecstore
|
||||
.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut old_reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("the historical table version should be created");
|
||||
let old_version_id = old.version_id.expect("the historical version should have an identity");
|
||||
let mut current_reader = PutObjReader::from_vec(b"current table data".to_vec());
|
||||
ecstore
|
||||
.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut current_reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("the current table version should be created");
|
||||
|
||||
let incarnation = ecstore
|
||||
.bucket_incarnation_id_from_disk(&bucket)
|
||||
.await
|
||||
.expect("bucket incarnation should be available");
|
||||
let publication_lock = ecstore
|
||||
.new_ns_lock(&bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
|
||||
.await
|
||||
.expect("table-bucket publication lock should be created");
|
||||
let enable_guard = publication_lock
|
||||
.get_write_lock(get_lock_acquire_timeout())
|
||||
.await
|
||||
.expect("table-bucket enablement should acquire the publication lock");
|
||||
|
||||
let delete_store = ecstore.clone();
|
||||
let delete_bucket = bucket.clone();
|
||||
let delete_object = object.to_string();
|
||||
let mut delete = tokio::spawn(async move {
|
||||
let target = ObjectToDelete {
|
||||
object_name: delete_object,
|
||||
version_id: Some(old_version_id),
|
||||
..Default::default()
|
||||
};
|
||||
crate::bucket::lifecycle::object_handlers_common::delete_object_versions(
|
||||
&delete_store,
|
||||
&delete_bucket,
|
||||
std::slice::from_ref(&target),
|
||||
lifecycle::Event::default(),
|
||||
incarnation,
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::task::yield_now().await;
|
||||
assert!(
|
||||
tokio::time::timeout(StdDuration::from_millis(100), &mut delete)
|
||||
.await
|
||||
.is_err(),
|
||||
"noncurrent lifecycle cleanup must wait for the table-bucket publication fence"
|
||||
);
|
||||
|
||||
let sys = metadata_sys::bucket_metadata_sys_of(&ecstore.ctx).expect("metadata system should be initialized");
|
||||
let sys = sys.read().await.clone();
|
||||
let mut metadata = (*sys.get(&bucket).await.expect("bucket metadata should exist")).clone();
|
||||
metadata.table_bucket_config_json = br#"{"enabled":true}"#.to_vec();
|
||||
sys.persist_and_set(metadata)
|
||||
.await
|
||||
.expect("table bucket marker should be persisted");
|
||||
sys.reload_from_store(&bucket)
|
||||
.await
|
||||
.expect("table bucket marker should become authoritative");
|
||||
drop(enable_guard);
|
||||
|
||||
let failed = tokio::time::timeout(StdDuration::from_secs(2), delete)
|
||||
.await
|
||||
.expect("noncurrent lifecycle cleanup should resume after the fence is released")
|
||||
.expect("noncurrent lifecycle cleanup task should join");
|
||||
assert_eq!(failed, 1, "table-bucket noncurrent cleanup must be rejected");
|
||||
assert!(
|
||||
ecstore
|
||||
.get_object_info(
|
||||
&bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
version_id: Some(old_version_id.to_string()),
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.is_ok(),
|
||||
"table data must remain after queued noncurrent cleanup is rejected"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn lifecycle_transition_admission_holds_publication_reader() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let bucket = format!("lifecycle-transition-{}", Uuid::new_v4().simple());
|
||||
let object = "tables/table-id/data/part-00001.parquet";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
let object_info = ObjectInfo {
|
||||
bucket: bucket.clone(),
|
||||
name: object.to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let publication_lock = ecstore
|
||||
.new_ns_lock(&bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
|
||||
.await
|
||||
.expect("table-bucket publication lock should be created");
|
||||
let enable_guard = publication_lock
|
||||
.get_write_lock(get_lock_acquire_timeout())
|
||||
.await
|
||||
.expect("publication writer should be acquired");
|
||||
let admission_store = ecstore.clone();
|
||||
let admission_object = object_info.clone();
|
||||
let mut admission =
|
||||
tokio::spawn(async move { lifecycle_transition_publication_guard(&admission_store, &admission_object).await });
|
||||
assert!(
|
||||
tokio::time::timeout(StdDuration::from_millis(100), &mut admission)
|
||||
.await
|
||||
.is_err(),
|
||||
"transition admission must wait for an in-progress table publication"
|
||||
);
|
||||
drop(enable_guard);
|
||||
|
||||
let (transition_guard, incarnation) = tokio::time::timeout(StdDuration::from_secs(2), admission)
|
||||
.await
|
||||
.expect("transition admission should resume after publication")
|
||||
.expect("transition admission task should join")
|
||||
.expect("transition admission should succeed");
|
||||
assert_ne!(incarnation, Uuid::nil(), "transition admission must bind a bucket incarnation");
|
||||
let blocked_writer = publication_lock.get_write_lock(StdDuration::from_millis(100)).await;
|
||||
assert!(blocked_writer.is_err(), "transition admission must retain the publication read lock");
|
||||
drop(transition_guard);
|
||||
publication_lock
|
||||
.get_write_lock(get_lock_acquire_timeout())
|
||||
.await
|
||||
.expect("publication writer should proceed after transition admission releases its guard");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn queued_delete_all_rechecks_a_same_id_rule_moved_into_the_future() {
|
||||
@@ -13630,6 +14207,10 @@ mod tests {
|
||||
.await;
|
||||
}
|
||||
|
||||
fn fast_tier_free_version_recovery_leader_retry_delay() -> StdDuration {
|
||||
StdDuration::from_millis(10)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn tier_free_version_recovery_production_entrypoint_enqueues_seeded_item() {
|
||||
@@ -13645,8 +14226,13 @@ mod tests {
|
||||
seed_recoverable_free_version(&disk_paths, &bucket, object, None, None).await;
|
||||
|
||||
let started = OnceLock::new();
|
||||
let recovery = super::spawn_tier_free_version_recovery_once(Arc::clone(&ecstore), &started)
|
||||
.expect("production recovery entrypoint should start once");
|
||||
let recovery = super::spawn_tier_free_version_recovery_once_with_options(
|
||||
Arc::clone(&ecstore),
|
||||
&started,
|
||||
format!("tier-free-version-recovery/test/{}", Uuid::new_v4()),
|
||||
fast_tier_free_version_recovery_leader_retry_delay,
|
||||
)
|
||||
.expect("production recovery entrypoint should start once");
|
||||
let task = tokio::time::timeout(StdDuration::from_secs(30), async { recovery_rx.lock().await.recv().await })
|
||||
.await
|
||||
.expect("production recovery entrypoint should enqueue the seeded item")
|
||||
@@ -13689,13 +14275,14 @@ mod tests {
|
||||
};
|
||||
let mut recovery_rx = recovery_rx.lock().await;
|
||||
assert!(
|
||||
super::enqueue_recovered_free_version(
|
||||
super::enqueue_recovered_free_version_with_cancel(
|
||||
&ecstore,
|
||||
ObjectInfo {
|
||||
bucket: "prefill".to_string(),
|
||||
name: "prefill".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
&CancellationToken::new(),
|
||||
)
|
||||
.await,
|
||||
"the production recovery queue should accept its first task"
|
||||
|
||||
@@ -36,6 +36,10 @@ pub async fn delete_object_versions(
|
||||
_lc_event: lifecycle::Event,
|
||||
bucket_incarnation_id: Uuid,
|
||||
) -> usize {
|
||||
if to_del.is_empty() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if to_del.iter().any(|target| {
|
||||
target.version_id.is_none()
|
||||
|| (target.version_id.is_some_and(|version_id| version_id.is_nil()) && target.expected_identity.is_none())
|
||||
@@ -52,6 +56,25 @@ pub async fn delete_object_versions(
|
||||
return to_del.len();
|
||||
}
|
||||
|
||||
let Some(publication_guard) = super::bucket_lifecycle_ops::lifecycle_expiry_publication_guard(
|
||||
api,
|
||||
bucket,
|
||||
to_del[0].object_name.as_str(),
|
||||
bucket_incarnation_id,
|
||||
)
|
||||
.await
|
||||
else {
|
||||
debug!(
|
||||
event = EVENT_LIFECYCLE_CLEANUP_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
bucket,
|
||||
target_count = to_del.len(),
|
||||
reason = "publication_admission_rejected",
|
||||
"Skipped lifecycle noncurrent version cleanup"
|
||||
);
|
||||
return to_del.len();
|
||||
};
|
||||
let delete_config_snapshot = match ReplicationObjectBridge::delete_request_config(api, bucket).await {
|
||||
Ok(snapshot) => Arc::new(snapshot),
|
||||
Err(err) => {
|
||||
@@ -78,17 +101,13 @@ pub async fn delete_object_versions(
|
||||
remaining = &[];
|
||||
}
|
||||
|
||||
let (mut deleted_objs, errors) = api
|
||||
.delete_objects(
|
||||
bucket,
|
||||
to_del.to_vec(),
|
||||
ObjectOptions {
|
||||
delete_replication_config_snapshot: Some(Arc::clone(&delete_config_snapshot)),
|
||||
expected_bucket_incarnation_id: Some(bucket_incarnation_id),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let mut delete_options = ObjectOptions {
|
||||
delete_replication_config_snapshot: Some(Arc::clone(&delete_config_snapshot)),
|
||||
expected_bucket_incarnation_id: Some(bucket_incarnation_id),
|
||||
..Default::default()
|
||||
};
|
||||
delete_options.add_namespace_lock_guard(&publication_guard);
|
||||
let (mut deleted_objs, errors) = api.delete_objects(bucket, to_del.to_vec(), delete_options).await;
|
||||
failed += errors.iter().filter(|err| err.is_some()).count();
|
||||
|
||||
for (i, deleted_obj) in deleted_objs.iter_mut().enumerate() {
|
||||
|
||||
@@ -20,7 +20,7 @@ use std::time::Duration;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::bucket::lifecycle::bucket_lifecycle_ops::enqueue_recovered_free_version;
|
||||
use crate::bucket::lifecycle::bucket_lifecycle_ops::enqueue_recovered_free_version_with_cancel;
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::Result;
|
||||
use crate::object_api::ObjectInfo;
|
||||
@@ -201,7 +201,11 @@ pub(super) async fn recover_tier_free_versions_with_cancel(
|
||||
return Err(tier_free_version_recovery_cancelled());
|
||||
}
|
||||
retry_cursor.visit(&oi);
|
||||
if !record_recovered_free_version_enqueue(&mut stats, enqueue_recovered_free_version(&api, oi).await) {
|
||||
let queued = enqueue_recovered_free_version_with_cancel(&api, oi, &cancel_token).await;
|
||||
if cancel_token.is_cancelled() {
|
||||
return Err(tier_free_version_recovery_cancelled());
|
||||
}
|
||||
if !record_recovered_free_version_enqueue(&mut stats, queued) {
|
||||
let (bucket_marker, object_marker) = retry_cursor.retry_markers();
|
||||
stats.truncated = true;
|
||||
stats.next_bucket_marker = bucket_marker;
|
||||
|
||||
@@ -1210,6 +1210,7 @@ impl BucketMetadata {
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
}
|
||||
self.notification_config = None;
|
||||
if !self.notification_config_xml.is_empty()
|
||||
&& let Err(e) = deserialize::<NotificationConfiguration>(&self.notification_config_xml)
|
||||
.map(|c| self.notification_config = Some(c))
|
||||
@@ -1224,6 +1225,7 @@ impl BucketMetadata {
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
}
|
||||
self.lifecycle_config = None;
|
||||
if !self.lifecycle_config_xml.is_empty()
|
||||
&& let Err(e) =
|
||||
deserialize::<BucketLifecycleConfiguration>(&self.lifecycle_config_xml).map(|c| self.lifecycle_config = Some(c))
|
||||
@@ -1238,6 +1240,7 @@ impl BucketMetadata {
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
}
|
||||
self.object_lock_config = None;
|
||||
if !self.object_lock_config_xml.is_empty()
|
||||
&& let Err(e) =
|
||||
deserialize::<ObjectLockConfiguration>(&self.object_lock_config_xml).map(|c| self.object_lock_config = Some(c))
|
||||
@@ -1267,6 +1270,7 @@ impl BucketMetadata {
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
}
|
||||
self.sse_config = None;
|
||||
if !self.encryption_config_xml.is_empty()
|
||||
&& let Err(e) =
|
||||
deserialize::<ServerSideEncryptionConfiguration>(&self.encryption_config_xml).map(|c| self.sse_config = Some(c))
|
||||
@@ -1281,6 +1285,7 @@ impl BucketMetadata {
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
}
|
||||
self.tagging_config = None;
|
||||
if !self.tagging_config_xml.is_empty()
|
||||
&& let Err(e) = deserialize::<Tagging>(&self.tagging_config_xml).map(|c| self.tagging_config = Some(c))
|
||||
{
|
||||
@@ -1346,6 +1351,7 @@ impl BucketMetadata {
|
||||
} else {
|
||||
self.bucket_target_config = Some(BucketTargets::default());
|
||||
}
|
||||
self.cors_config = None;
|
||||
if !self.cors_config_xml.is_empty()
|
||||
&& let Err(e) = deserialize::<CORSConfiguration>(&self.cors_config_xml).map(|c| self.cors_config = Some(c))
|
||||
{
|
||||
@@ -1359,6 +1365,7 @@ impl BucketMetadata {
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
}
|
||||
self.logging_config = None;
|
||||
if !self.logging_config_xml.is_empty()
|
||||
&& let Err(e) = deserialize::<BucketLoggingStatus>(&self.logging_config_xml).map(|c| self.logging_config = Some(c))
|
||||
{
|
||||
@@ -1372,6 +1379,7 @@ impl BucketMetadata {
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
}
|
||||
self.website_config = None;
|
||||
if !self.website_config_xml.is_empty()
|
||||
&& let Err(e) = deserialize::<WebsiteConfiguration>(&self.website_config_xml).map(|c| self.website_config = Some(c))
|
||||
{
|
||||
@@ -1385,6 +1393,7 @@ impl BucketMetadata {
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
}
|
||||
self.accelerate_config = None;
|
||||
if !self.accelerate_config_xml.is_empty()
|
||||
&& let Err(e) =
|
||||
deserialize::<AccelerateConfiguration>(&self.accelerate_config_xml).map(|c| self.accelerate_config = Some(c))
|
||||
@@ -1399,6 +1408,7 @@ impl BucketMetadata {
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
}
|
||||
self.request_payment_config = None;
|
||||
if !self.request_payment_config_xml.is_empty()
|
||||
&& let Err(e) = deserialize::<RequestPaymentConfiguration>(&self.request_payment_config_xml)
|
||||
.map(|c| self.request_payment_config = Some(c))
|
||||
@@ -1413,6 +1423,7 @@ impl BucketMetadata {
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
}
|
||||
self.public_access_block_config = None;
|
||||
if !self.public_access_block_config_xml.is_empty()
|
||||
&& let Err(e) = deserialize::<PublicAccessBlockConfiguration>(&self.public_access_block_config_xml)
|
||||
.map(|c| self.public_access_block_config = Some(c))
|
||||
@@ -1973,6 +1984,156 @@ mod test {
|
||||
assert_eq!(bm.lifecycle_config_xml, malformed_xml, "raw bytes are retained");
|
||||
}
|
||||
|
||||
const REPARSE_XML_FIXTURES: [(&str, &[u8]); 13] = [
|
||||
(BUCKET_NOTIFICATION_CONFIG, br#"<NotificationConfiguration/>"#),
|
||||
(
|
||||
BUCKET_LIFECYCLE_CONFIG,
|
||||
br#"<LifecycleConfiguration><Rule><ID>keep</ID><Status>Enabled</Status><Expiration><Days>30</Days></Expiration></Rule></LifecycleConfiguration>"#,
|
||||
),
|
||||
(
|
||||
OBJECT_LOCK_CONFIG,
|
||||
br#"<ObjectLockConfiguration><ObjectLockEnabled>Enabled</ObjectLockEnabled><Rule><DefaultRetention><Mode>GOVERNANCE</Mode><Days>7</Days></DefaultRetention></Rule></ObjectLockConfiguration>"#,
|
||||
),
|
||||
(BUCKET_VERSIONING_CONFIG, br#"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>"#),
|
||||
(
|
||||
BUCKET_SSECONFIG,
|
||||
br#"<ServerSideEncryptionConfiguration><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>AES256</SSEAlgorithm></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>"#,
|
||||
),
|
||||
(BUCKET_TAGGING_CONFIG, br#"<Tagging><TagSet><Tag><Key>team</Key><Value>blue</Value></Tag></TagSet></Tagging>"#),
|
||||
(
|
||||
BUCKET_REPLICATION_CONFIG,
|
||||
br#"<ReplicationConfiguration><Role>arn:aws:iam::123456789012:role/replication-role</Role><Rule><ID>copy</ID><Status>Enabled</Status><Prefix>documents/</Prefix><Destination><Bucket>arn:aws:s3:::destination-bucket</Bucket></Destination></Rule></ReplicationConfiguration>"#,
|
||||
),
|
||||
(
|
||||
BUCKET_CORS_CONFIG,
|
||||
br#"<CORSConfiguration><CORSRule><AllowedOrigin>https://example.test</AllowedOrigin><AllowedMethod>GET</AllowedMethod></CORSRule></CORSConfiguration>"#,
|
||||
),
|
||||
(BUCKET_LOGGING_CONFIG, br#"<BucketLoggingStatus/>"#),
|
||||
(BUCKET_WEBSITE_CONFIG, br#"<WebsiteConfiguration><IndexDocument><Suffix>index.html</Suffix></IndexDocument></WebsiteConfiguration>"#),
|
||||
(BUCKET_ACCELERATE_CONFIG, br#"<AccelerateConfiguration><Status>Enabled</Status></AccelerateConfiguration>"#),
|
||||
(BUCKET_REQUEST_PAYMENT_CONFIG, br#"<RequestPaymentConfiguration><Payer>BucketOwner</Payer></RequestPaymentConfiguration>"#),
|
||||
(
|
||||
BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG,
|
||||
br#"<PublicAccessBlockConfiguration><BlockPublicAcls>true</BlockPublicAcls><IgnorePublicAcls>true</IgnorePublicAcls><BlockPublicPolicy>true</BlockPublicPolicy><RestrictPublicBuckets>true</RestrictPublicBuckets></PublicAccessBlockConfiguration>"#,
|
||||
),
|
||||
];
|
||||
|
||||
fn reparse_xml_state(bm: &BucketMetadata, config: &str) -> &'static str {
|
||||
fn state<T>(raw: &[u8], parsed: &Option<T>) -> &'static str {
|
||||
match ConfigState::of(raw, parsed) {
|
||||
ConfigState::Absent => "absent",
|
||||
ConfigState::Valid(_) => "valid",
|
||||
ConfigState::Unreadable { .. } => "unreadable",
|
||||
}
|
||||
}
|
||||
match config {
|
||||
BUCKET_NOTIFICATION_CONFIG => state(&bm.notification_config_xml, &bm.notification_config),
|
||||
BUCKET_LIFECYCLE_CONFIG => state(&bm.lifecycle_config_xml, &bm.lifecycle_config),
|
||||
OBJECT_LOCK_CONFIG => state(&bm.object_lock_config_xml, &bm.object_lock_config),
|
||||
BUCKET_VERSIONING_CONFIG => state(&bm.versioning_config_xml, &bm.versioning_config),
|
||||
BUCKET_SSECONFIG => state(&bm.encryption_config_xml, &bm.sse_config),
|
||||
BUCKET_TAGGING_CONFIG => state(&bm.tagging_config_xml, &bm.tagging_config),
|
||||
BUCKET_REPLICATION_CONFIG => state(&bm.replication_config_xml, &bm.replication_config),
|
||||
BUCKET_CORS_CONFIG => state(&bm.cors_config_xml, &bm.cors_config),
|
||||
BUCKET_LOGGING_CONFIG => state(&bm.logging_config_xml, &bm.logging_config),
|
||||
BUCKET_WEBSITE_CONFIG => state(&bm.website_config_xml, &bm.website_config),
|
||||
BUCKET_ACCELERATE_CONFIG => state(&bm.accelerate_config_xml, &bm.accelerate_config),
|
||||
BUCKET_REQUEST_PAYMENT_CONFIG => state(&bm.request_payment_config_xml, &bm.request_payment_config),
|
||||
BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG => state(&bm.public_access_block_config_xml, &bm.public_access_block_config),
|
||||
_ => panic!("unknown XML config: {config}"),
|
||||
}
|
||||
}
|
||||
|
||||
// A persisted payload can change without going through the request decoder.
|
||||
fn replace_xml_raw(bm: &mut BucketMetadata, config: &str, raw: Vec<u8>) {
|
||||
match config {
|
||||
BUCKET_NOTIFICATION_CONFIG => bm.notification_config_xml = raw,
|
||||
BUCKET_LIFECYCLE_CONFIG => bm.lifecycle_config_xml = raw,
|
||||
OBJECT_LOCK_CONFIG => bm.object_lock_config_xml = raw,
|
||||
BUCKET_VERSIONING_CONFIG => bm.versioning_config_xml = raw,
|
||||
BUCKET_SSECONFIG => bm.encryption_config_xml = raw,
|
||||
BUCKET_TAGGING_CONFIG => bm.tagging_config_xml = raw,
|
||||
BUCKET_REPLICATION_CONFIG => bm.replication_config_xml = raw,
|
||||
BUCKET_CORS_CONFIG => bm.cors_config_xml = raw,
|
||||
BUCKET_LOGGING_CONFIG => bm.logging_config_xml = raw,
|
||||
BUCKET_WEBSITE_CONFIG => bm.website_config_xml = raw,
|
||||
BUCKET_ACCELERATE_CONFIG => bm.accelerate_config_xml = raw,
|
||||
BUCKET_REQUEST_PAYMENT_CONFIG => bm.request_payment_config_xml = raw,
|
||||
BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG => bm.public_access_block_config_xml = raw,
|
||||
_ => panic!("unknown XML config: {config}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reparse_xml_rejects_stale_views_after_corruption() {
|
||||
let corrupt = b"<not-a-valid-document";
|
||||
for (config, valid) in REPARSE_XML_FIXTURES {
|
||||
let mut bm = BucketMetadata::new("reparse-corrupt");
|
||||
bm.update_config(config, valid.to_vec()).expect("store valid XML");
|
||||
bm.parse_all_configs().expect("parse valid XML");
|
||||
assert_eq!(reparse_xml_state(&bm, config), "valid", "{config}: valid setup");
|
||||
|
||||
replace_xml_raw(&mut bm, config, corrupt.to_vec());
|
||||
let before = bm.marshal_msg().expect("marshal corrupt raw metadata");
|
||||
bm.parse_all_configs()
|
||||
.expect("one corrupt config must not fail the metadata load");
|
||||
|
||||
assert_eq!(reparse_xml_state(&bm, config), "unreadable", "{config}: stale view accepted");
|
||||
assert_eq!(bm.xml_config_unreadable_len(config), Some(corrupt.len()), "{config}: missing refusal");
|
||||
assert_eq!(
|
||||
bm.marshal_msg().expect("marshal reparsed metadata"),
|
||||
before,
|
||||
"{config}: persisted bytes changed"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reparse_xml_clears_stale_views_after_raw_is_removed() {
|
||||
for (config, valid) in REPARSE_XML_FIXTURES {
|
||||
let mut bm = BucketMetadata::new("reparse-removed");
|
||||
bm.update_config(config, valid.to_vec()).expect("store valid XML");
|
||||
bm.parse_all_configs().expect("parse valid XML");
|
||||
assert_eq!(reparse_xml_state(&bm, config), "valid", "{config}: valid setup");
|
||||
|
||||
replace_xml_raw(&mut bm, config, Vec::new());
|
||||
let before = bm.marshal_msg().expect("marshal absent raw metadata");
|
||||
bm.parse_all_configs().expect("parse removed XML");
|
||||
|
||||
assert_eq!(reparse_xml_state(&bm, config), "absent", "{config}: stale view survived removal");
|
||||
assert!(!bm.xml_config_unreadable(config), "{config}: absence is not corruption");
|
||||
assert_eq!(
|
||||
bm.marshal_msg().expect("marshal reparsed metadata"),
|
||||
before,
|
||||
"{config}: persisted bytes changed"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reparse_xml_recovers_after_corrupt_raw_is_repaired() {
|
||||
for (config, valid) in REPARSE_XML_FIXTURES {
|
||||
let mut bm = BucketMetadata::new("reparse-repaired");
|
||||
bm.update_config(config, valid.to_vec()).expect("store valid XML");
|
||||
bm.parse_all_configs().expect("parse valid XML");
|
||||
replace_xml_raw(&mut bm, config, b"<not-a-valid-document".to_vec());
|
||||
bm.parse_all_configs().expect("parse corrupt XML");
|
||||
assert_eq!(reparse_xml_state(&bm, config), "unreadable", "{config}: corrupt setup");
|
||||
|
||||
replace_xml_raw(&mut bm, config, valid.to_vec());
|
||||
let before = bm.marshal_msg().expect("marshal repaired raw metadata");
|
||||
bm.parse_all_configs().expect("parse repaired XML");
|
||||
|
||||
assert_eq!(reparse_xml_state(&bm, config), "valid", "{config}: repaired config must become readable");
|
||||
assert!(!bm.xml_config_unreadable(config), "{config}: repaired config is not corrupt");
|
||||
assert_eq!(
|
||||
bm.marshal_msg().expect("marshal reparsed metadata"),
|
||||
before,
|
||||
"{config}: persisted bytes changed"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lifecycle_update_config_clears_parsed_config_on_delete() {
|
||||
let mut bm = BucketMetadata::new("test-bucket");
|
||||
|
||||
@@ -576,6 +576,27 @@ pub(crate) async fn update_in(
|
||||
Box::pin(update_with_sys(require_bucket_metadata_sys_in(ctx)?, bucket, config_file, data)).await
|
||||
}
|
||||
|
||||
/// Validate a caller's fence after acquiring the lifecycle and metadata
|
||||
/// transaction locks, before changing the configuration.
|
||||
pub(crate) async fn update_validated_in<F>(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
config_file: &str,
|
||||
data: Vec<u8>,
|
||||
validate: F,
|
||||
) -> Result<OffsetDateTime>
|
||||
where
|
||||
F: FnOnce() -> Result<()> + Send,
|
||||
{
|
||||
Box::pin(async {
|
||||
let sys = require_bucket_metadata_sys_in(ctx)?;
|
||||
let guard = acquire_config_write_guard(sys.clone(), bucket).await?;
|
||||
validate()?;
|
||||
update_under_config_write_guard(sys, &guard, config_file, data, None).await
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn delete(bucket: &str, config_file: &str) -> Result<OffsetDateTime> {
|
||||
delete_with_sys(get_bucket_metadata_sys()?, bucket, config_file).await
|
||||
}
|
||||
@@ -1132,7 +1153,7 @@ async fn acquire_transaction_lock_with_sys(
|
||||
/// agree on, so renaming it would leave a mixed-version cluster with two
|
||||
/// disjoint keys — and old and new nodes would stop excluding each other on
|
||||
/// the very writes that are serialized today.
|
||||
fn bucket_metadata_transaction_lock_key(bucket: &str) -> String {
|
||||
pub(crate) fn bucket_metadata_transaction_lock_key(bucket: &str) -> String {
|
||||
format!("bucket-targets/{bucket}/transaction.lock")
|
||||
}
|
||||
|
||||
@@ -1271,14 +1292,34 @@ pub(crate) async fn get_object_lock_config_state_in(
|
||||
pub(crate) async fn get_object_lock_config_and_incarnation_from_disk_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
) -> Result<(ObjectLockConfigState, Uuid, OffsetDateTime)> {
|
||||
get_object_lock_config_and_incarnation_from_disk_with_guard_in(ctx, bucket, None).await
|
||||
}
|
||||
|
||||
/// Bucket deletion already owns the namespace/lifecycle fences. Reuse its
|
||||
/// live guard rather than recursively acquiring the bucket namespace lock.
|
||||
/// The caller must also hold the metadata transaction read lock.
|
||||
pub(crate) async fn get_object_lock_config_and_incarnation_from_disk_with_guard_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
guard: Option<&rustfs_lock::NamespaceLockGuard>,
|
||||
) -> Result<(ObjectLockConfigState, Uuid, OffsetDateTime)> {
|
||||
let bucket_meta_sys_lock = bucket_metadata_sys_of(ctx)?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await.clone();
|
||||
|
||||
match bucket_meta_sys
|
||||
.read_authoritative_metadata_from_disk_under_transaction_lock(bucket)
|
||||
.await?
|
||||
{
|
||||
let authority = match guard {
|
||||
Some(guard) => {
|
||||
bucket_meta_sys
|
||||
.read_authoritative_metadata_from_disk_under_guard(bucket, guard)
|
||||
.await?
|
||||
}
|
||||
None => {
|
||||
bucket_meta_sys
|
||||
.read_authoritative_metadata_from_disk_under_transaction_lock(bucket)
|
||||
.await?
|
||||
}
|
||||
};
|
||||
match authority {
|
||||
BucketMetadataAuthority::Authoritative(metadata)
|
||||
if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() =>
|
||||
{
|
||||
@@ -1331,10 +1372,54 @@ pub(crate) async fn get_quota_config_and_incarnation_from_disk_in(
|
||||
let bucket_meta_sys_lock = bucket_metadata_sys_of(ctx)?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await.clone();
|
||||
|
||||
match bucket_meta_sys
|
||||
.read_authoritative_metadata_from_disk_under_transaction_lock(bucket)
|
||||
.await?
|
||||
quota_config_and_incarnation_from_authority(
|
||||
bucket,
|
||||
bucket_meta_sys
|
||||
.read_authoritative_metadata_from_disk_under_transaction_lock(bucket)
|
||||
.await?,
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) async fn get_quota_config_and_incarnation_from_disk_for_options_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
opts: &ObjectOptions,
|
||||
transaction_guard: &Arc<rustfs_lock::NamespaceLockGuard>,
|
||||
) -> Result<(Option<BucketQuota>, Uuid, OffsetDateTime)> {
|
||||
let Some(snapshot) = opts.object_lock_config_snapshot.as_ref() else {
|
||||
return get_quota_config_and_incarnation_from_disk_in(ctx, bucket).await;
|
||||
};
|
||||
let sys = bucket_metadata_sys_of(ctx)?.read().await.clone();
|
||||
let store = sys.object_store();
|
||||
let snapshot_guard = snapshot
|
||||
.metadata_transaction_guard_for(store.id, bucket, opts.expected_bucket_incarnation_id)
|
||||
.filter(|guard| Arc::ptr_eq(guard, transaction_guard))
|
||||
.ok_or_else(|| Error::other("quota snapshot does not hold this bucket's metadata transaction fence"))?;
|
||||
if snapshot_guard.is_released() {
|
||||
return Err(Error::other("quota snapshot metadata transaction fence was released"));
|
||||
}
|
||||
|
||||
// The snapshot carries the caller's lifecycle fence, excluding deletion.
|
||||
// Its transaction guard excludes config writes, so a same-name object
|
||||
// owner need not recursively acquire the bucket namespace here.
|
||||
let authority = sys
|
||||
.read_authoritative_metadata_from_disk_under_guard(bucket, transaction_guard)
|
||||
.await?;
|
||||
if snapshot_guard.is_lock_lost()
|
||||
|| snapshot_guard.is_released()
|
||||
|| matches!(&authority, BucketMetadataAuthority::Authoritative(metadata)
|
||||
if !snapshot.is_valid_for_destructive_put(store.id, bucket, metadata.bucket_incarnation_id))
|
||||
{
|
||||
return Err(Error::other("quota snapshot fence or bucket incarnation changed during the disk read"));
|
||||
}
|
||||
quota_config_and_incarnation_from_authority(bucket, authority)
|
||||
}
|
||||
|
||||
fn quota_config_and_incarnation_from_authority(
|
||||
bucket: &str,
|
||||
authority: BucketMetadataAuthority,
|
||||
) -> Result<(Option<BucketQuota>, Uuid, OffsetDateTime)> {
|
||||
match authority {
|
||||
BucketMetadataAuthority::Authoritative(metadata)
|
||||
if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() =>
|
||||
{
|
||||
@@ -2703,29 +2788,30 @@ impl BucketMetadataSys {
|
||||
async fn read_authoritative_metadata_from_disk_under_transaction_lock(
|
||||
&self,
|
||||
bucket: &str,
|
||||
) -> Result<BucketMetadataAuthority> {
|
||||
let namespace_lock = self.object_store().new_ns_lock(bucket, bucket).await?;
|
||||
let namespace_guard = namespace_lock
|
||||
.get_read_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await?;
|
||||
self.read_authoritative_metadata_from_disk_under_guard(bucket, &namespace_guard)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn read_authoritative_metadata_from_disk_under_guard(
|
||||
&self,
|
||||
bucket: &str,
|
||||
guard: &rustfs_lock::NamespaceLockGuard,
|
||||
) -> Result<BucketMetadataAuthority> {
|
||||
#[cfg(test)]
|
||||
if self.object_lock_disk_read_errors.write().await.remove(bucket) {
|
||||
return Err(Error::other(format!("injected Object Lock metadata disk read failure: {bucket}")));
|
||||
}
|
||||
|
||||
let namespace_lock = self.object_store().new_ns_lock(bucket, bucket).await?;
|
||||
let namespace_guard = namespace_lock
|
||||
.get_read_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await?;
|
||||
match await_bucket_namespace_operation(
|
||||
Some(&namespace_guard),
|
||||
bucket,
|
||||
"bucket metadata snapshot existence check",
|
||||
async {
|
||||
self.object_store()
|
||||
.get_bucket_info_from_sets_at_read_quorum(
|
||||
bucket,
|
||||
&crate::storage_api_contracts::bucket::BucketOptions::default(),
|
||||
)
|
||||
.await
|
||||
},
|
||||
)
|
||||
match await_bucket_namespace_operation(Some(guard), bucket, "bucket metadata snapshot existence check", async {
|
||||
self.object_store()
|
||||
.get_bucket_info_from_sets_at_read_quorum(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
|
||||
.await
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(_) => {}
|
||||
@@ -2734,7 +2820,7 @@ impl BucketMetadataSys {
|
||||
}
|
||||
|
||||
let (metadata, persisted) = await_bucket_namespace_operation(
|
||||
Some(&namespace_guard),
|
||||
Some(guard),
|
||||
bucket,
|
||||
"bucket metadata authoritative snapshot",
|
||||
load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true),
|
||||
@@ -4121,6 +4207,114 @@ mod tests {
|
||||
sys.persist_new_and_set(bm).await.expect("initial metadata should persist");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reparse_xml_rmw_corruption_refuses_cached_and_reloaded_encryption() {
|
||||
use crate::bucket::metadata::{BUCKET_SSECONFIG, is_unreadable_config_error};
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
let (dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||
let sys = BucketMetadataSys::new(ecstore);
|
||||
let bucket = "reparse-encryption-corrupt";
|
||||
let valid = br#"<ServerSideEncryptionConfiguration><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>AES256</SSEAlgorithm></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>"#;
|
||||
let corrupt = b"<ServerSideEncryptionConfiguration";
|
||||
persist_bucket_with_raw_config(&sys, &dirs, bucket, BUCKET_SSECONFIG, valid).await;
|
||||
sys.get_sse_config(bucket).await.expect("valid encryption must be readable");
|
||||
|
||||
let updated = sys
|
||||
.update_config_with(bucket, BUCKET_SSECONFIG, |bm| {
|
||||
assert!(bm.sse_config.is_some(), "rewrite must start with parsed encryption");
|
||||
Ok(corrupt.to_vec())
|
||||
})
|
||||
.await
|
||||
.expect("persist future or corrupt raw encryption without failing the whole bucket");
|
||||
|
||||
let err = sys
|
||||
.get_sse_config(bucket)
|
||||
.await
|
||||
.expect_err("cached old encryption must not hide corrupt raw");
|
||||
assert!(is_unreadable_config_error(&err), "cached refusal must retain its type: {err}");
|
||||
let (cached, _) = sys.get_config(bucket).await.expect("bucket remains readable");
|
||||
assert_eq!(cached.encryption_config_xml, corrupt);
|
||||
assert_eq!(cached.encryption_config_updated_at, updated);
|
||||
|
||||
sys.metadata_map.write().await.clear();
|
||||
let err = sys
|
||||
.get_sse_config(bucket)
|
||||
.await
|
||||
.expect_err("reloaded encryption must still refuse corrupt raw");
|
||||
assert!(is_unreadable_config_error(&err), "reloaded refusal must retain its type: {err}");
|
||||
|
||||
let calls = AtomicUsize::new(0);
|
||||
let err = sys
|
||||
.update_config_with(bucket, BUCKET_SSECONFIG, |_| {
|
||||
calls.fetch_add(1, Ordering::SeqCst);
|
||||
Ok(Vec::new())
|
||||
})
|
||||
.await
|
||||
.expect_err("another rewrite must refuse before it consumes corrupt encryption");
|
||||
assert!(is_unreadable_config_error(&err));
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0, "refused rewrite must not erase the original raw");
|
||||
let disk = sys
|
||||
.get_config_from_disk(bucket)
|
||||
.await
|
||||
.expect("read unchanged metadata from disk");
|
||||
assert_eq!(disk.encryption_config_xml, corrupt);
|
||||
assert_eq!(disk.encryption_config_updated_at, updated);
|
||||
|
||||
sys.update(bucket, BUCKET_SSECONFIG, valid.to_vec())
|
||||
.await
|
||||
.expect("explicit repair must remain possible");
|
||||
sys.get_sse_config(bucket)
|
||||
.await
|
||||
.expect("repaired cached encryption must be readable");
|
||||
sys.metadata_map.write().await.clear();
|
||||
sys.get_sse_config(bucket).await.expect("repair must survive a disk reload");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reparse_xml_rmw_removal_clears_cached_and_reloaded_encryption() {
|
||||
use crate::bucket::metadata::BUCKET_SSECONFIG;
|
||||
|
||||
let (dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||
let sys = BucketMetadataSys::new(ecstore);
|
||||
let bucket = "reparse-encryption-removed";
|
||||
let valid = br#"<ServerSideEncryptionConfiguration><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>AES256</SSEAlgorithm></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>"#;
|
||||
persist_bucket_with_raw_config(&sys, &dirs, bucket, BUCKET_SSECONFIG, valid).await;
|
||||
sys.get_sse_config(bucket).await.expect("valid encryption must be readable");
|
||||
|
||||
let updated = sys
|
||||
.update_config_with(bucket, BUCKET_SSECONFIG, |bm| {
|
||||
assert!(bm.sse_config.is_some(), "rewrite must start with parsed encryption");
|
||||
Ok(Vec::new())
|
||||
})
|
||||
.await
|
||||
.expect("removing encryption must persist");
|
||||
|
||||
assert_eq!(
|
||||
sys.get_sse_config(bucket)
|
||||
.await
|
||||
.expect_err("old cached encryption must be gone"),
|
||||
Error::ConfigNotFound
|
||||
);
|
||||
let (cached, _) = sys.get_config(bucket).await.expect("bucket remains readable");
|
||||
assert!(cached.encryption_config_xml.is_empty());
|
||||
assert_eq!(cached.encryption_config_updated_at, updated);
|
||||
|
||||
sys.metadata_map.write().await.clear();
|
||||
assert_eq!(
|
||||
sys.get_sse_config(bucket)
|
||||
.await
|
||||
.expect_err("removed encryption must stay absent after reload"),
|
||||
Error::ConfigNotFound
|
||||
);
|
||||
let disk = sys
|
||||
.get_config_from_disk(bucket)
|
||||
.await
|
||||
.expect("read removed encryption from disk");
|
||||
assert!(disk.encryption_config_xml.is_empty());
|
||||
assert_eq!(disk.encryption_config_updated_at, updated);
|
||||
}
|
||||
|
||||
/// rustfs/backlog#1734: a read-modify-write of a stored config that cannot
|
||||
/// be parsed must be refused before `mutate` runs. Otherwise `mutate` sees
|
||||
/// the unreadable config as absent, rebuilds it from nothing, and the
|
||||
@@ -4736,6 +4930,231 @@ mod tests {
|
||||
.expect("DeleteBucket should succeed");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_snapshot_reuses_same_name_owner_and_rereads_disk_authority() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = "quota-same-name-owner";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("quota fixture bucket");
|
||||
let snapshot = store
|
||||
.object_lock_config_snapshot(bucket)
|
||||
.await
|
||||
.expect("guarded Object Lock snapshot");
|
||||
let incarnation = store.bucket_incarnation_id(bucket).await.expect("fixture incarnation");
|
||||
let transaction_guard = snapshot
|
||||
.metadata_transaction_guard_for(store.id, bucket, Some(incarnation))
|
||||
.expect("snapshot's actual transaction guard");
|
||||
let expected = get_quota_config_and_incarnation_from_disk_for_options_in(
|
||||
&store.ctx,
|
||||
bucket,
|
||||
&ObjectOptions::default(),
|
||||
&transaction_guard,
|
||||
)
|
||||
.await
|
||||
.expect("a reader without a snapshot must retain the ordinary namespace path");
|
||||
|
||||
let sys = bucket_metadata_sys_of(&store.ctx).expect("fixture metadata system");
|
||||
let sys = sys.read().await.clone();
|
||||
let mut stale = (*sys.get(bucket).await.expect("cached fixture metadata")).clone();
|
||||
let metadata_path = stale.save_file_path();
|
||||
stale.quota_config = Some(BucketQuota {
|
||||
quota: Some(123),
|
||||
..Default::default()
|
||||
});
|
||||
sys.set(bucket.to_string(), Arc::new(stale)).await;
|
||||
let namespace = store.new_ns_lock(bucket, bucket).await.expect("same-name object namespace");
|
||||
let _owner = namespace
|
||||
.get_write_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await
|
||||
.expect("same-name object write owner");
|
||||
let opts = ObjectOptions {
|
||||
object_lock_config_snapshot: Some(snapshot),
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(
|
||||
get_quota_config_and_incarnation_from_disk_for_options_in(&store.ctx, bucket, &opts, &transaction_guard)
|
||||
.await
|
||||
.expect("quota disk read must reuse the held lifecycle and transaction fences"),
|
||||
expected,
|
||||
"the cached quota must not replace disk authority"
|
||||
);
|
||||
|
||||
crate::config::com::save_config_with_opts(
|
||||
store.clone(),
|
||||
&metadata_path,
|
||||
b"corrupt bucket metadata".to_vec(),
|
||||
&ObjectOptions {
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("inject committed metadata corruption");
|
||||
assert!(
|
||||
get_quota_config_and_incarnation_from_disk_for_options_in(&store.ctx, bucket, &opts, &transaction_guard)
|
||||
.await
|
||||
.is_err(),
|
||||
"a valid snapshot must not hide corrupt disk metadata"
|
||||
);
|
||||
crate::config::com::delete_config(store.clone(), &metadata_path)
|
||||
.await
|
||||
.expect("remove the persisted metadata while retaining its sidecar");
|
||||
assert!(
|
||||
get_quota_config_and_incarnation_from_disk_for_options_in(&store.ctx, bucket, &opts, &transaction_guard)
|
||||
.await
|
||||
.is_err(),
|
||||
"a valid snapshot must not promote fabricated metadata to authority"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_snapshot_rejects_wrong_scope_incarnation_and_transaction_guard() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = "quota-snapshot-scope";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("quota fixture bucket");
|
||||
let incarnation = store.bucket_incarnation_id(bucket).await.expect("fixture incarnation");
|
||||
let snapshot = store
|
||||
.object_lock_config_snapshot(bucket)
|
||||
.await
|
||||
.expect("guarded Object Lock snapshot");
|
||||
let transaction_guard = snapshot
|
||||
.metadata_transaction_guard_for(store.id, bucket, Some(incarnation))
|
||||
.expect("snapshot's actual transaction guard");
|
||||
let mut opts = ObjectOptions {
|
||||
object_lock_config_snapshot: Some(snapshot),
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(
|
||||
get_quota_config_and_incarnation_from_disk_for_options_in(&store.ctx, "another-bucket", &opts, &transaction_guard)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
opts.expected_bucket_incarnation_id = Some(Uuid::new_v4());
|
||||
assert!(
|
||||
get_quota_config_and_incarnation_from_disk_for_options_in(&store.ctx, bucket, &opts, &transaction_guard)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
opts.expected_bucket_incarnation_id = Some(incarnation);
|
||||
let other_guard = Arc::new(
|
||||
acquire_bucket_metadata_transaction_read_lock_in(&store.ctx, bucket)
|
||||
.await
|
||||
.expect("distinct real transaction read guard"),
|
||||
);
|
||||
assert!(
|
||||
get_quota_config_and_incarnation_from_disk_for_options_in(&store.ctx, bucket, &opts, &other_guard)
|
||||
.await
|
||||
.is_err(),
|
||||
"the same namespace key does not prove that this is the snapshot's actual guard"
|
||||
);
|
||||
let (_other_dirs, other_store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(other_store.clone(), Vec::new()).await;
|
||||
assert!(
|
||||
get_quota_config_and_incarnation_from_disk_for_options_in(&other_store.ctx, bucket, &opts, &transaction_guard)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
opts.object_lock_config_snapshot = Some(Arc::new(crate::object_api::ObjectLockConfigSnapshot::new(
|
||||
ObjectLockConfigState::ConfirmedAbsent,
|
||||
)));
|
||||
assert!(
|
||||
get_quota_config_and_incarnation_from_disk_for_options_in(&store.ctx, bucket, &opts, &transaction_guard)
|
||||
.await
|
||||
.is_err(),
|
||||
"an unguarded snapshot must not authorize namespace reuse"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_snapshot_rejects_lifecycle_loss_during_read_and_changed_disk_incarnation() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = "quota-snapshot-loss";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("quota fixture bucket");
|
||||
let sys = bucket_metadata_sys_of(&store.ctx).expect("fixture metadata system");
|
||||
let sys = sys.read().await.clone();
|
||||
let mut metadata = (*sys.get(bucket).await.expect("fixture metadata")).clone();
|
||||
let incarnation = metadata.bucket_incarnation_id;
|
||||
let _lifecycle_guard = store
|
||||
.acquire_bucket_lifecycle_read_lock(bucket)
|
||||
.await
|
||||
.expect("real lifecycle owner");
|
||||
let transaction_guard = Arc::new(
|
||||
acquire_bucket_metadata_transaction_read_lock_in(&store.ctx, bucket)
|
||||
.await
|
||||
.expect("real metadata transaction owner"),
|
||||
);
|
||||
let (lifecycle_fence, lost) = crate::object_api::NamespaceLockFence::loss_handle_for_test();
|
||||
let snapshot = crate::object_api::ObjectLockConfigSnapshot::for_store_bucket_under_lifecycle_fence(
|
||||
store.id,
|
||||
bucket,
|
||||
incarnation,
|
||||
metadata.object_lock_config_updated_at,
|
||||
ObjectLockConfigState::ConfirmedAbsent,
|
||||
lifecycle_fence,
|
||||
Arc::clone(&transaction_guard),
|
||||
);
|
||||
let opts = ObjectOptions {
|
||||
object_lock_config_snapshot: Some(Arc::new(snapshot)),
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
..Default::default()
|
||||
};
|
||||
let mut read = Box::pin(get_quota_config_and_incarnation_from_disk_for_options_in(
|
||||
&store.ctx,
|
||||
bucket,
|
||||
&opts,
|
||||
&transaction_guard,
|
||||
));
|
||||
assert!(
|
||||
futures::poll!(read.as_mut()).is_pending(),
|
||||
"the authoritative disk read must be in flight"
|
||||
);
|
||||
lost.store(true, std::sync::atomic::Ordering::Release);
|
||||
assert!(read.await.is_err(), "a lifecycle fence lost during the disk read must reject its result");
|
||||
|
||||
let snapshot = store
|
||||
.object_lock_config_snapshot(bucket)
|
||||
.await
|
||||
.expect("fresh guarded snapshot");
|
||||
let transaction_guard = snapshot
|
||||
.metadata_transaction_guard_for(store.id, bucket, Some(incarnation))
|
||||
.expect("snapshot's actual transaction guard");
|
||||
let opts = ObjectOptions {
|
||||
object_lock_config_snapshot: Some(snapshot),
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
..Default::default()
|
||||
};
|
||||
metadata.bucket_incarnation_id = Uuid::new_v4();
|
||||
save_bucket_incarnation(store.clone(), bucket, metadata.bucket_incarnation_id)
|
||||
.await
|
||||
.expect("inject a new disk incarnation");
|
||||
metadata
|
||||
.save_with_store_committed(store.clone())
|
||||
.await
|
||||
.expect("persist its matching metadata blob");
|
||||
assert!(
|
||||
get_quota_config_and_incarnation_from_disk_for_options_in(&store.ctx, bucket, &opts, &transaction_guard)
|
||||
.await
|
||||
.is_err(),
|
||||
"a matching sidecar and blob from another incarnation must not reuse the old snapshot"
|
||||
);
|
||||
}
|
||||
|
||||
/// Writers on different nodes updating *different* config files of one
|
||||
/// bucket must both survive. Each rewrites the whole metadata blob, so
|
||||
/// without a lock spanning the read-modify-write the later save carries
|
||||
|
||||
@@ -32,7 +32,10 @@ pub enum QuotaType {
|
||||
Hard,
|
||||
}
|
||||
|
||||
pub(crate) const QUOTA_RESERVATION_PROTOCOL_V1: u32 = 1;
|
||||
pub const QUOTA_RESERVATION_PROTOCOL_V1: u32 = 1;
|
||||
/// Sharded durable reservations. Older nodes reject this protocol instead of
|
||||
/// silently falling back to the single-ledger protocol.
|
||||
pub const QUOTA_RESERVATION_PROTOCOL_V2: u32 = 2;
|
||||
|
||||
/// Bucket quota configuration. quota_type defaults to Hard when omitted.
|
||||
#[derive(Debug, Default, Clone, PartialEq)]
|
||||
@@ -89,7 +92,10 @@ impl<'de> Deserialize<'de> for BucketQuota {
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
let wire = BucketQuotaWire::deserialize(deserializer)?;
|
||||
let quota = if wire.reservation_protocol == Some(QUOTA_RESERVATION_PROTOCOL_V1) {
|
||||
let quota = if matches!(
|
||||
wire.reservation_protocol,
|
||||
Some(QUOTA_RESERVATION_PROTOCOL_V1) | Some(QUOTA_RESERVATION_PROTOCOL_V2)
|
||||
) {
|
||||
Some(
|
||||
wire.reservation_quota
|
||||
.ok_or_else(|| D::Error::custom("reservation_quota is required for reservation protocol v1"))?,
|
||||
@@ -129,17 +135,33 @@ impl BucketQuota {
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates a quota using the sharded durable reservation protocol.
|
||||
/// Deployment must complete the mixed-version rollout before enabling it;
|
||||
/// older nodes fail closed when they see protocol v2.
|
||||
pub fn new_sharded(quota: u64) -> Self {
|
||||
let mut config = Self::new(Some(quota));
|
||||
config.reservation_protocol = Some(QUOTA_RESERVATION_PROTOCOL_V2);
|
||||
config
|
||||
}
|
||||
|
||||
pub fn get_quota_limit(&self) -> Option<u64> {
|
||||
self.quota
|
||||
}
|
||||
|
||||
pub fn uses_durable_reservations(&self) -> bool {
|
||||
self.reservation_protocol == Some(QUOTA_RESERVATION_PROTOCOL_V1)
|
||||
matches!(
|
||||
self.reservation_protocol,
|
||||
Some(QUOTA_RESERVATION_PROTOCOL_V1) | Some(QUOTA_RESERVATION_PROTOCOL_V2)
|
||||
)
|
||||
}
|
||||
|
||||
pub fn uses_sharded_reservations(&self) -> bool {
|
||||
self.reservation_protocol == Some(QUOTA_RESERVATION_PROTOCOL_V2)
|
||||
}
|
||||
|
||||
pub fn has_unsupported_reservation_protocol(&self) -> bool {
|
||||
self.reservation_protocol
|
||||
.is_some_and(|version| version != QUOTA_RESERVATION_PROTOCOL_V1)
|
||||
.is_some_and(|version| !matches!(version, QUOTA_RESERVATION_PROTOCOL_V1 | QUOTA_RESERVATION_PROTOCOL_V2))
|
||||
}
|
||||
|
||||
pub fn check_operation_allowed(&self, current_usage: u64, operation_size: u64) -> bool {
|
||||
@@ -322,9 +344,21 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_reservation_protocol_does_not_activate_v1() {
|
||||
fn sharded_quota_roundtrips_with_authoritative_reservation_quota() {
|
||||
let quota = BucketQuota::new_sharded(4096);
|
||||
let json = serde_json::to_value("a).expect("sharded quota should serialize");
|
||||
assert_eq!(json["quota"], 0);
|
||||
assert_eq!(json["reservation_protocol"], QUOTA_RESERVATION_PROTOCOL_V2);
|
||||
assert_eq!(json["reservation_quota"], 4096);
|
||||
let restored: BucketQuota = serde_json::from_value(json).expect("sharded quota should parse");
|
||||
assert_eq!(restored.quota, Some(4096));
|
||||
assert!(restored.uses_sharded_reservations());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_reservation_protocol_does_not_activate_any_known_protocol() {
|
||||
let quota: BucketQuota =
|
||||
serde_json::from_str(r#"{"quota":0,"quota_type":"Hard","reservation_protocol":2,"reservation_quota":2048}"#)
|
||||
serde_json::from_str(r#"{"quota":0,"quota_type":"Hard","reservation_protocol":3,"reservation_quota":2048}"#)
|
||||
.expect("future protocol should remain parseable");
|
||||
|
||||
assert!(!quota.uses_durable_reservations());
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -12,9 +12,11 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#[cfg(test)]
|
||||
use crate::disk::DiskIoClass;
|
||||
use crate::disk::disk_store::{get_drive_walkdir_peek_timeout, get_drive_walkdir_stall_timeout};
|
||||
use crate::disk::error::DiskError;
|
||||
use crate::disk::{self, DiskAPI, DiskStore, WalkDirOptions};
|
||||
use crate::disk::{self, DiskAPI, DiskStore, WalkDirOptions, current_disk_io_context, with_optional_disk_io_context};
|
||||
use futures::future::join_all;
|
||||
use metrics::counter;
|
||||
use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetacacheReader, is_io_eof};
|
||||
@@ -245,6 +247,8 @@ pub struct ListPathRawOptions {
|
||||
pub(crate) test_fallback_reader_behaviors: Vec<TestReaderBehavior>,
|
||||
#[cfg(test)]
|
||||
pub(crate) peek_timeout: Option<Duration>,
|
||||
#[cfg(test)]
|
||||
pub(crate) test_disk_io_context_observed: Option<Arc<AtomicBool>>,
|
||||
// pub agreed: Option<Arc<dyn Fn(MetaCacheEntry) + Send + Sync>>,
|
||||
// pub partial: Option<Arc<dyn Fn(MetaCacheEntries, &[Option<Error>]) + Send + Sync>>,
|
||||
// pub finished: Option<Arc<dyn Fn(&[Option<Error>]) + Send + Sync>>,
|
||||
@@ -276,6 +280,8 @@ impl Clone for ListPathRawOptions {
|
||||
test_fallback_reader_behaviors: self.test_fallback_reader_behaviors.clone(),
|
||||
#[cfg(test)]
|
||||
peek_timeout: self.peek_timeout,
|
||||
#[cfg(test)]
|
||||
test_disk_io_context_observed: self.test_disk_io_context_observed.clone(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
@@ -344,6 +350,7 @@ async fn list_path_raw_inner(
|
||||
let log_path = opts.path.clone();
|
||||
|
||||
let mut jobs = JoinSet::new();
|
||||
let disk_io_context = current_disk_io_context();
|
||||
let mut readers = Vec::with_capacity(opts.disks.len());
|
||||
let fds = Arc::new(TokioMutex::new(opts.fallback_disks.iter().flatten().cloned().collect::<VecDeque<_>>()));
|
||||
#[cfg(test)]
|
||||
@@ -364,11 +371,19 @@ async fn list_path_raw_inner(
|
||||
let test_fallbacks_clone = test_fallbacks.clone();
|
||||
let cancel_rx_clone = cancel_rx.clone();
|
||||
let producer_errs_clone = producer_errs.clone();
|
||||
let disk_io_context_clone = disk_io_context.clone();
|
||||
let (rd, wr) = tokio::io::duplex(64);
|
||||
readers.push(MetacacheReader::new(rd));
|
||||
jobs.spawn(async move {
|
||||
jobs.spawn(with_optional_disk_io_context(disk_io_context_clone, async move {
|
||||
let mut wr = PublishedBytesWriter::new(wr);
|
||||
#[cfg(test)]
|
||||
if let Some(observer) = &opts_clone.test_disk_io_context_observed {
|
||||
observer.store(
|
||||
crate::disk::current_disk_io_context().is_some_and(|context| context.class == DiskIoClass::Background),
|
||||
std::sync::atomic::Ordering::Release,
|
||||
);
|
||||
}
|
||||
#[cfg(test)]
|
||||
let test_primary_error = if let Some(behavior) = opts_clone.test_reader_behaviors.get(disk_idx).cloned() {
|
||||
match behavior {
|
||||
TestReaderBehavior::Eof => return Ok(()),
|
||||
@@ -652,7 +667,7 @@ async fn list_path_raw_inner(
|
||||
|
||||
// warn!("list_path_raw: while need_fallback done");
|
||||
Ok(())
|
||||
});
|
||||
}));
|
||||
}
|
||||
|
||||
let revjob_rx = rx.clone();
|
||||
@@ -1168,6 +1183,68 @@ mod tests {
|
||||
assert_eq!(err, DiskError::ErasureReadQuorum);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_path_raw_joinset_producers_inherit_scanner_disk_io_class() {
|
||||
let observed = Arc::new(AtomicBool::new(false));
|
||||
crate::disk::with_background_disk_io(
|
||||
CancellationToken::new(),
|
||||
list_path_raw(
|
||||
CancellationToken::new(),
|
||||
ListPathRawOptions {
|
||||
disks: vec![None],
|
||||
min_disks: 1,
|
||||
test_reader_behaviors: vec![TestReaderBehavior::Eof],
|
||||
test_disk_io_context_observed: Some(observed.clone()),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("empty test producer should complete the listing");
|
||||
|
||||
assert!(
|
||||
observed.load(std::sync::atomic::Ordering::Acquire),
|
||||
"the per-disk JoinSet producer must retain the scanner background class"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_joinset_walk_uses_the_local_disk_background_gate() {
|
||||
let root = tempfile::tempdir().expect("local disk root should be created");
|
||||
let endpoint = crate::disk::endpoint::Endpoint::try_from(root.path().to_str().expect("disk path should be UTF-8"))
|
||||
.expect("endpoint should parse");
|
||||
let local = Arc::new(
|
||||
crate::disk::local::LocalDisk::new(&endpoint, false)
|
||||
.await
|
||||
.expect("local disk should open"),
|
||||
);
|
||||
let wrapper = crate::disk::disk_store::LocalDiskWrapper::new(local, false);
|
||||
wrapper.make_volume("bucket").await.expect("bucket volume should be created");
|
||||
let disk = Arc::new(crate::disk::Disk::Local(Box::new(wrapper.clone())));
|
||||
let cancellation = CancellationToken::new();
|
||||
|
||||
crate::disk::with_background_disk_io(
|
||||
cancellation.clone(),
|
||||
list_path_raw(
|
||||
cancellation,
|
||||
ListPathRawOptions {
|
||||
disks: vec![Some(disk)],
|
||||
bucket: "bucket".to_string(),
|
||||
min_disks: 1,
|
||||
recursive: true,
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("scanner metacache walk should complete");
|
||||
|
||||
assert!(
|
||||
wrapper.background_io_admissions_for_tests() > 0,
|
||||
"the LocalDiskWrapper walk should acquire background admission inside its JoinSet producer"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_path_raw_returns_typed_quorum_error_for_multiple_drive_failures() {
|
||||
let err = list_path_raw(
|
||||
|
||||
@@ -0,0 +1,806 @@
|
||||
//! Bounded authenticated Raft RPC, mountable without a second storage owner.
|
||||
use super::{client, http_auth};
|
||||
use openraft::{
|
||||
DefensiveError, ErrorSubject, ErrorVerb, RaftNetwork, RaftNetworkFactory, StorageError, StorageIOError, Violation,
|
||||
error::{Fatal, InstallSnapshotError, RPCError, RaftError, RemoteError, Unreachable},
|
||||
network::RPCOption,
|
||||
raft::{
|
||||
AppendEntriesRequest, AppendEntriesResponse, InstallSnapshotRequest, InstallSnapshotResponse, VoteRequest, VoteResponse,
|
||||
},
|
||||
};
|
||||
use rustfs_capture_authority::{CaptureMembership, CaptureNode, CaptureTypeConfig, NativeRaftOwner, NodeIdentity};
|
||||
use rustfs_protos::proto_gen::node_service::{
|
||||
CaptureRaftRequest, CaptureRaftResponse,
|
||||
capture_raft_service_server::{CaptureRaftService, CaptureRaftServiceServer},
|
||||
};
|
||||
use serde::{Deserialize, Serialize, de::DeserializeOwned};
|
||||
use serde_json::value::RawValue;
|
||||
use std::{collections::BTreeMap, io::Write, sync::Arc, time::Duration};
|
||||
use tonic::{Request, Response, Status};
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct CaptureRaftLimits {
|
||||
pub max_request_bytes: usize,
|
||||
pub max_response_bytes: usize,
|
||||
pub max_entries: usize,
|
||||
pub max_snapshot_chunk_bytes: usize,
|
||||
pub max_inflight: usize,
|
||||
pub connect_timeout: Duration,
|
||||
pub request_timeout: Duration,
|
||||
}
|
||||
impl Default for CaptureRaftLimits {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_request_bytes: 16 * 1024 * 1024,
|
||||
max_response_bytes: 1024 * 1024,
|
||||
max_entries: 300,
|
||||
max_snapshot_chunk_bytes: 3 * 1024 * 1024,
|
||||
max_inflight: 16,
|
||||
connect_timeout: Duration::from_secs(3),
|
||||
request_timeout: Duration::from_secs(5),
|
||||
}
|
||||
}
|
||||
}
|
||||
impl CaptureRaftLimits {
|
||||
fn validate(&self) -> Result<(), CaptureRaftError> {
|
||||
let chunk = self
|
||||
.max_snapshot_chunk_bytes
|
||||
.checked_mul(4)
|
||||
.and_then(|n| n.checked_add(16384))
|
||||
.ok_or(CaptureRaftError::Limits)?;
|
||||
if self.max_request_bytes < chunk
|
||||
|| self.max_request_bytes > 64 * 1024 * 1024
|
||||
|| self.max_response_bytes < 4096
|
||||
|| self.max_response_bytes > 4 * 1024 * 1024
|
||||
|| self.max_entries == 0
|
||||
|| self.max_entries > 1024
|
||||
|| self.max_snapshot_chunk_bytes == 0
|
||||
|| self.max_inflight == 0
|
||||
|| self.max_inflight > 128
|
||||
|| self.connect_timeout.is_zero()
|
||||
|| self.request_timeout.is_zero()
|
||||
|| self.request_timeout > Duration::from_secs(60)
|
||||
|| self.connect_timeout > Duration::from_secs(60)
|
||||
{
|
||||
return Err(CaptureRaftError::Limits);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum CaptureRaftError {
|
||||
#[error("capture Raft message limit exceeded")]
|
||||
Limits,
|
||||
#[error("capture Raft request requires a smaller append batch")]
|
||||
RequestTooLarge,
|
||||
#[error("invalid capture Raft message")]
|
||||
Protocol,
|
||||
#[error("capture Raft identity mismatch")]
|
||||
Identity,
|
||||
#[error("capture Raft authentication failed")]
|
||||
Authentication,
|
||||
#[error("capture Raft channel unavailable")]
|
||||
ChannelUnavailable,
|
||||
#[error("capture Raft deadline exceeded")]
|
||||
Deadline,
|
||||
#[error(transparent)]
|
||||
Io(#[from] std::io::Error),
|
||||
#[error(transparent)]
|
||||
Json(#[from] serde_json::Error),
|
||||
#[error(transparent)]
|
||||
Node(#[from] rustfs_capture_authority::CaptureNodeError),
|
||||
#[error(transparent)]
|
||||
Status(Box<Status>),
|
||||
}
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub(crate) enum Method {
|
||||
AppendEntries,
|
||||
Vote,
|
||||
InstallSnapshot,
|
||||
}
|
||||
impl Method {
|
||||
pub(crate) fn path(self) -> &'static str {
|
||||
match self {
|
||||
Self::AppendEntries => "/node_service.CaptureRaftService/AppendEntries",
|
||||
Self::Vote => "/node_service.CaptureRaftService/Vote",
|
||||
Self::InstallSnapshot => "/node_service.CaptureRaftService/InstallSnapshot",
|
||||
}
|
||||
}
|
||||
}
|
||||
#[derive(Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub(crate) struct WireRequest {
|
||||
pub version: u32,
|
||||
pub cluster: [u8; 16],
|
||||
pub source: u64,
|
||||
pub target: u64,
|
||||
pub method: Method,
|
||||
pub nonce: [u8; 16],
|
||||
pub payload: Box<RawValue>,
|
||||
}
|
||||
struct Bounded {
|
||||
bytes: Vec<u8>,
|
||||
max: usize,
|
||||
}
|
||||
impl Write for Bounded {
|
||||
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
|
||||
let end = self
|
||||
.bytes
|
||||
.len()
|
||||
.checked_add(bytes.len())
|
||||
.ok_or_else(|| std::io::Error::other("capture RPC limit"))?;
|
||||
if end > self.max {
|
||||
return Err(std::io::Error::other("capture RPC limit"));
|
||||
}
|
||||
self.bytes.try_reserve(bytes.len()).map_err(std::io::Error::other)?;
|
||||
self.bytes.extend_from_slice(bytes);
|
||||
Ok(bytes.len())
|
||||
}
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
fn encoded<T: Serialize>(value: &T, max: usize) -> Result<Vec<u8>, CaptureRaftError> {
|
||||
let mut out = Bounded { bytes: Vec::new(), max };
|
||||
serde_json::to_writer(&mut out, value).map_err(|error| {
|
||||
if error.is_io() {
|
||||
CaptureRaftError::Limits
|
||||
} else {
|
||||
CaptureRaftError::Json(error)
|
||||
}
|
||||
})?;
|
||||
Ok(out.bytes)
|
||||
}
|
||||
fn raw<T: Serialize>(value: &T, max: usize) -> Result<Box<RawValue>, CaptureRaftError> {
|
||||
Ok(RawValue::from_string(
|
||||
String::from_utf8(encoded(value, max)?).map_err(|_| CaptureRaftError::Protocol)?,
|
||||
)?)
|
||||
}
|
||||
pub(crate) fn encode_request<P: Serialize>(
|
||||
local: NodeIdentity,
|
||||
target: u64,
|
||||
method: Method,
|
||||
payload: &P,
|
||||
limits: CaptureRaftLimits,
|
||||
) -> Result<CaptureRaftRequest, CaptureRaftError> {
|
||||
let wire = WireRequest {
|
||||
version: 1,
|
||||
cluster: local.cluster_id,
|
||||
source: local.node_id,
|
||||
target,
|
||||
method,
|
||||
nonce: *uuid::Uuid::new_v4().as_bytes(),
|
||||
payload: raw(payload, limits.max_request_bytes)?,
|
||||
};
|
||||
Ok(CaptureRaftRequest {
|
||||
body: encoded(&wire, limits.max_request_bytes)?.into(),
|
||||
})
|
||||
}
|
||||
pub(crate) fn canonical_request(body: &[u8]) -> Vec<u8> {
|
||||
let mut out = b"rustfs.capture-raft.request.v1\0".to_vec();
|
||||
out.extend_from_slice(body);
|
||||
out
|
||||
}
|
||||
fn decode_wire(body: &[u8], max: usize) -> Result<WireRequest, CaptureRaftError> {
|
||||
if body.len() > max {
|
||||
return Err(CaptureRaftError::Limits);
|
||||
}
|
||||
let wire: WireRequest = serde_json::from_slice(body)?;
|
||||
if wire.version != 1 || wire.cluster == [0; 16] || wire.nonce == [0; 16] || encoded(&wire, max)? != body {
|
||||
return Err(CaptureRaftError::Protocol);
|
||||
}
|
||||
Ok(wire)
|
||||
}
|
||||
fn decode_payload<P: Serialize + DeserializeOwned>(wire: &WireRequest, max: usize) -> Result<P, CaptureRaftError> {
|
||||
let value: P = serde_json::from_str(wire.payload.get())?;
|
||||
if encoded(&value, max)?.as_slice() != wire.payload.get().as_bytes() {
|
||||
return Err(CaptureRaftError::Protocol);
|
||||
}
|
||||
Ok(value)
|
||||
}
|
||||
#[derive(Serialize, Deserialize)]
|
||||
#[serde(tag = "kind", deny_unknown_fields)]
|
||||
pub(crate) enum RemoteFailure {
|
||||
Stopped,
|
||||
Panicked,
|
||||
StorageIo {
|
||||
operation: Option<IoOperation>,
|
||||
},
|
||||
StorageDefensive {
|
||||
subject: ErrorSubject<u64>,
|
||||
violation: Violation<u64>,
|
||||
},
|
||||
SnapshotMismatch {
|
||||
error: InstallSnapshotError,
|
||||
},
|
||||
}
|
||||
#[derive(Serialize, Deserialize)]
|
||||
pub(crate) struct IoOperation {
|
||||
subject: ErrorSubject<u64>,
|
||||
verb: ErrorVerb,
|
||||
}
|
||||
fn fatal(error: Fatal<u64>) -> RemoteFailure {
|
||||
match error {
|
||||
Fatal::Stopped => RemoteFailure::Stopped,
|
||||
Fatal::Panicked => RemoteFailure::Panicked,
|
||||
Fatal::StorageError(StorageError::IO { source }) => RemoteFailure::StorageIo {
|
||||
operation: encoded(&source, 16384)
|
||||
.ok()
|
||||
.and_then(|bytes| serde_json::from_slice(&bytes).ok()),
|
||||
},
|
||||
Fatal::StorageError(StorageError::Defensive { source }) => RemoteFailure::StorageDefensive {
|
||||
subject: source.subject,
|
||||
violation: source.violation,
|
||||
},
|
||||
}
|
||||
}
|
||||
#[derive(Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
enum WireReply {
|
||||
Success { payload: Box<RawValue> },
|
||||
Failure { error: RemoteFailure },
|
||||
}
|
||||
fn response_context(request: &CaptureRaftRequest, body: &[u8]) -> Result<Vec<u8>, CaptureRaftError> {
|
||||
let mut out = b"rustfs.capture-raft.response.v1\0".to_vec();
|
||||
out.try_reserve(
|
||||
request
|
||||
.body
|
||||
.len()
|
||||
.checked_add(body.len())
|
||||
.and_then(|n| n.checked_add(16))
|
||||
.ok_or(CaptureRaftError::Limits)?,
|
||||
)
|
||||
.map_err(std::io::Error::other)?;
|
||||
out.extend(
|
||||
u64::try_from(request.body.len())
|
||||
.map_err(|_| CaptureRaftError::Limits)?
|
||||
.to_be_bytes(),
|
||||
);
|
||||
out.extend_from_slice(&request.body);
|
||||
out.extend(u64::try_from(body.len()).map_err(|_| CaptureRaftError::Limits)?.to_be_bytes());
|
||||
out.extend_from_slice(body);
|
||||
Ok(out)
|
||||
}
|
||||
fn reply<P: Serialize>(
|
||||
request: &CaptureRaftRequest,
|
||||
result: Result<P, RemoteFailure>,
|
||||
limits: CaptureRaftLimits,
|
||||
) -> Result<CaptureRaftResponse, CaptureRaftError> {
|
||||
let wire = match result {
|
||||
Ok(value) => WireReply::Success {
|
||||
payload: raw(&value, limits.max_response_bytes)?,
|
||||
},
|
||||
Err(error) => WireReply::Failure { error },
|
||||
};
|
||||
let body = encoded(&wire, limits.max_response_bytes)?;
|
||||
let response_proof = http_auth::sign_tonic_rpc_response_proof(&response_context(request, &body)?)?;
|
||||
Ok(CaptureRaftResponse {
|
||||
body: body.into(),
|
||||
response_proof: response_proof.into(),
|
||||
})
|
||||
}
|
||||
pub(crate) fn verify_response<P: Serialize + DeserializeOwned>(
|
||||
request: &CaptureRaftRequest,
|
||||
response: CaptureRaftResponse,
|
||||
limits: CaptureRaftLimits,
|
||||
) -> Result<Result<P, RemoteFailure>, CaptureRaftError> {
|
||||
if request.body.len() > limits.max_request_bytes
|
||||
|| response.body.len() > limits.max_response_bytes
|
||||
|| response.response_proof.len() != 32
|
||||
{
|
||||
return Err(CaptureRaftError::Limits);
|
||||
}
|
||||
http_auth::verify_tonic_rpc_response_proof(&response_context(request, &response.body)?, &response.response_proof)?;
|
||||
let wire: WireReply = serde_json::from_slice(&response.body)?;
|
||||
if encoded(&wire, limits.max_response_bytes)?.as_slice() != response.body {
|
||||
return Err(CaptureRaftError::Protocol);
|
||||
}
|
||||
match wire {
|
||||
WireReply::Failure { error } => Ok(Err(error)),
|
||||
WireReply::Success { payload } => {
|
||||
let value: P = serde_json::from_str(payload.get())?;
|
||||
if encoded(&value, limits.max_response_bytes)?.as_slice() != payload.get().as_bytes() {
|
||||
return Err(CaptureRaftError::Protocol);
|
||||
}
|
||||
Ok(Ok(value))
|
||||
}
|
||||
}
|
||||
}
|
||||
fn restored_fatal(error: RemoteFailure) -> Result<Fatal<u64>, CaptureRaftError> {
|
||||
Ok(match error {
|
||||
RemoteFailure::Stopped => Fatal::Stopped,
|
||||
RemoteFailure::Panicked => Fatal::Panicked,
|
||||
RemoteFailure::StorageIo { operation } => {
|
||||
let (subject, verb) = operation
|
||||
.map(|op| (op.subject, op.verb))
|
||||
.unwrap_or((ErrorSubject::None, ErrorVerb::Read));
|
||||
Fatal::StorageError(
|
||||
StorageIOError::new(
|
||||
subject,
|
||||
verb,
|
||||
openraft::AnyError::new(&std::io::Error::other("remote Raft storage I/O failed; detailed source redacted")),
|
||||
)
|
||||
.into(),
|
||||
)
|
||||
}
|
||||
RemoteFailure::StorageDefensive { subject, violation } => Fatal::StorageError(StorageError::Defensive {
|
||||
source: DefensiveError {
|
||||
subject,
|
||||
violation,
|
||||
backtrace: None,
|
||||
},
|
||||
}),
|
||||
RemoteFailure::SnapshotMismatch { .. } => return Err(CaptureRaftError::Protocol),
|
||||
})
|
||||
}
|
||||
fn peers_valid(local: NodeIdentity, peers: &BTreeMap<u64, CaptureNode>) -> Result<(), CaptureRaftError> {
|
||||
if local.cluster_id == [0; 16] || !peers.contains_key(&local.node_id) {
|
||||
return Err(CaptureRaftError::Identity);
|
||||
}
|
||||
let mut addresses = std::collections::BTreeSet::new();
|
||||
for node in peers.values() {
|
||||
node.validate()?;
|
||||
if !addresses.insert(node.rpc_uri()) {
|
||||
return Err(CaptureRaftError::Identity);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
type WorkerPause = Arc<std::sync::Mutex<Option<(Arc<tokio::sync::Notify>, Arc<tokio::sync::Notify>)>>>;
|
||||
#[derive(Clone)]
|
||||
pub struct CaptureRaftRpc {
|
||||
owner: Arc<NativeRaftOwner>,
|
||||
peers: Arc<BTreeMap<u64, CaptureNode>>,
|
||||
audience: String,
|
||||
limits: CaptureRaftLimits,
|
||||
admission: Arc<tokio::sync::Semaphore>,
|
||||
accepting: Arc<std::sync::atomic::AtomicBool>,
|
||||
#[cfg(test)]
|
||||
pub(crate) worker_pause: WorkerPause,
|
||||
}
|
||||
impl CaptureRaftRpc {
|
||||
pub fn new(
|
||||
owner: Arc<NativeRaftOwner>,
|
||||
peers: BTreeMap<u64, CaptureNode>,
|
||||
limits: CaptureRaftLimits,
|
||||
) -> Result<Self, CaptureRaftError> {
|
||||
limits.validate()?;
|
||||
let identity = owner.identity();
|
||||
peers_valid(identity, &peers)?;
|
||||
let node = peers.get(&identity.node_id).ok_or(CaptureRaftError::Identity)?;
|
||||
let uri: http::Uri = node.rpc_uri().parse().map_err(|_| CaptureRaftError::Identity)?;
|
||||
let audience = http_auth::normalize_tonic_rpc_audience(uri.authority().ok_or(CaptureRaftError::Identity)?.as_str())?;
|
||||
Ok(Self {
|
||||
owner,
|
||||
peers: Arc::new(peers),
|
||||
audience,
|
||||
limits,
|
||||
#[cfg(test)]
|
||||
worker_pause: Arc::new(std::sync::Mutex::new(None)),
|
||||
accepting: Arc::new(std::sync::atomic::AtomicBool::new(true)),
|
||||
admission: Arc::new(tokio::sync::Semaphore::new(limits.max_inflight)),
|
||||
})
|
||||
}
|
||||
pub fn into_service(self) -> CaptureRaftServiceServer<Self> {
|
||||
let request = self.limits.max_request_bytes + 64;
|
||||
let response = self.limits.max_response_bytes + 128;
|
||||
CaptureRaftServiceServer::new(self)
|
||||
.max_decoding_message_size(request)
|
||||
.max_encoding_message_size(response)
|
||||
}
|
||||
fn prepare<P: Serialize + DeserializeOwned>(
|
||||
&self,
|
||||
request: &Request<CaptureRaftRequest>,
|
||||
method: Method,
|
||||
) -> Result<(WireRequest, P, tokio::sync::OwnedSemaphorePermit), Status> {
|
||||
if !self.accepting.load(std::sync::atomic::Ordering::Acquire) {
|
||||
return Err(Status::unavailable("capture Raft service stopping"));
|
||||
}
|
||||
let permit = self
|
||||
.admission
|
||||
.clone()
|
||||
.try_acquire_owned()
|
||||
.map_err(|_| Status::resource_exhausted("capture Raft admission full"))?;
|
||||
if !self.accepting.load(std::sync::atomic::Ordering::Acquire) {
|
||||
return Err(Status::unavailable("capture Raft service stopping"));
|
||||
}
|
||||
if request.get_ref().body.len() > self.limits.max_request_bytes {
|
||||
return Err(Status::resource_exhausted("capture Raft request limit"));
|
||||
}
|
||||
if request
|
||||
.metadata()
|
||||
.get(http_auth::RPC_REPLAY_SCOPE_VERSION_HEADER)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
!= Some("3")
|
||||
{
|
||||
return Err(auth_rejected("missing_replay_scope"));
|
||||
}
|
||||
http_auth::verify_tonic_rpc_signature_with_bootstrap(&self.audience, method.path(), request.metadata().as_ref(), false)
|
||||
.map_err(|_| auth_rejected("invalid_signature"))?;
|
||||
http_auth::verify_tonic_canonical_body_digest(request, &canonical_request(&request.get_ref().body))
|
||||
.map_err(|_| auth_rejected("body_digest_mismatch"))?;
|
||||
let wire = decode_wire(&request.get_ref().body, self.limits.max_request_bytes)
|
||||
.map_err(|_| Status::invalid_argument("capture Raft wire rejected"))?;
|
||||
let local = self.owner.identity();
|
||||
if wire.cluster != local.cluster_id
|
||||
|| wire.target != local.node_id
|
||||
|| wire.method != method
|
||||
|| !self.peers.contains_key(&wire.source)
|
||||
{
|
||||
return Err(Status::permission_denied("capture Raft identity rejected"));
|
||||
}
|
||||
let payload = decode_payload(&wire, self.limits.max_request_bytes)
|
||||
.map_err(|_| Status::invalid_argument("capture Raft payload rejected"))?;
|
||||
Ok((wire, payload, permit))
|
||||
}
|
||||
#[cfg(test)]
|
||||
async fn paused(pause: WorkerPause) {
|
||||
let gate = pause.lock().expect("worker pause lock").take();
|
||||
if let Some((entered, resume)) = gate {
|
||||
entered.notify_one();
|
||||
resume.notified().await;
|
||||
}
|
||||
}
|
||||
/// Stop admission and wait for all actual RPC tasks, including cancelled HTTP waiters.
|
||||
pub async fn shutdown(&self) -> Result<(), CaptureRaftError> {
|
||||
self.accepting.store(false, std::sync::atomic::Ordering::Release);
|
||||
let count = u32::try_from(self.limits.max_inflight).map_err(|_| CaptureRaftError::Limits)?;
|
||||
let _all = self
|
||||
.admission
|
||||
.clone()
|
||||
.acquire_many_owned(count)
|
||||
.await
|
||||
.map_err(|_| CaptureRaftError::ChannelUnavailable)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
fn signed_response<P: Serialize>(
|
||||
raw: &CaptureRaftRequest,
|
||||
result: Result<P, RemoteFailure>,
|
||||
limits: CaptureRaftLimits,
|
||||
audience: &str,
|
||||
challenge: Option<uuid::Uuid>,
|
||||
) -> Result<Response<CaptureRaftResponse>, Status> {
|
||||
let mut response =
|
||||
Response::new(reply(raw, result, limits).map_err(|_| Status::internal("capture Raft response unavailable"))?);
|
||||
if let Some(challenge) = challenge {
|
||||
response.metadata_mut().as_mut().extend(
|
||||
http_auth::tonic_boot_epoch_response_headers(audience, challenge)
|
||||
.map_err(|_| Status::internal("capture Raft epoch proof unavailable"))?,
|
||||
);
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
#[tonic::async_trait]
|
||||
impl CaptureRaftService for CaptureRaftRpc {
|
||||
async fn append_entries(&self, request: Request<CaptureRaftRequest>) -> Result<Response<CaptureRaftResponse>, Status> {
|
||||
let (wire, payload, permit) = self.prepare::<AppendEntriesRequest<CaptureTypeConfig>>(&request, Method::AppendEntries)?;
|
||||
validate_append(&payload, wire.source, self.limits)
|
||||
.map_err(|_| Status::invalid_argument("capture Raft append rejected"))?;
|
||||
let challenge = http_auth::tonic_boot_epoch_challenge(request.metadata().as_ref())
|
||||
.map_err(|_| Status::unauthenticated("capture Raft challenge rejected"))?;
|
||||
let raw = request.into_inner();
|
||||
let owner = self.owner.clone();
|
||||
let audience = self.audience.clone();
|
||||
let limits = self.limits;
|
||||
#[cfg(test)]
|
||||
let pause = self.worker_pause.clone();
|
||||
tokio::spawn(async move {
|
||||
let _permit = permit;
|
||||
#[cfg(test)]
|
||||
Self::paused(pause).await;
|
||||
let result = owner.raft().append_entries(payload).await.map_err(|error| match error {
|
||||
RaftError::Fatal(error) => fatal(error),
|
||||
RaftError::APIError(error) => match error {},
|
||||
});
|
||||
drop(owner);
|
||||
signed_response(&raw, result, limits, &audience, challenge)
|
||||
})
|
||||
.await
|
||||
.map_err(|_| Status::internal("capture Raft worker failed"))?
|
||||
}
|
||||
async fn vote(&self, request: Request<CaptureRaftRequest>) -> Result<Response<CaptureRaftResponse>, Status> {
|
||||
let (wire, payload, permit) = self.prepare::<VoteRequest<u64>>(&request, Method::Vote)?;
|
||||
if payload.vote.leader_id.node_id != wire.source {
|
||||
return Err(Status::invalid_argument("capture Raft candidate rejected"));
|
||||
}
|
||||
let challenge = http_auth::tonic_boot_epoch_challenge(request.metadata().as_ref())
|
||||
.map_err(|_| Status::unauthenticated("capture Raft challenge rejected"))?;
|
||||
let raw = request.into_inner();
|
||||
let owner = self.owner.clone();
|
||||
let audience = self.audience.clone();
|
||||
let limits = self.limits;
|
||||
#[cfg(test)]
|
||||
let pause = self.worker_pause.clone();
|
||||
tokio::spawn(async move {
|
||||
let _permit = permit;
|
||||
#[cfg(test)]
|
||||
Self::paused(pause).await;
|
||||
let result = owner.raft().vote(payload).await.map_err(|error| match error {
|
||||
RaftError::Fatal(error) => fatal(error),
|
||||
RaftError::APIError(error) => match error {},
|
||||
});
|
||||
drop(owner);
|
||||
signed_response(&raw, result, limits, &audience, challenge)
|
||||
})
|
||||
.await
|
||||
.map_err(|_| Status::internal("capture Raft worker failed"))?
|
||||
}
|
||||
async fn install_snapshot(&self, request: Request<CaptureRaftRequest>) -> Result<Response<CaptureRaftResponse>, Status> {
|
||||
let (wire, payload, permit) =
|
||||
self.prepare::<InstallSnapshotRequest<CaptureTypeConfig>>(&request, Method::InstallSnapshot)?;
|
||||
validate_snapshot(&payload, wire.source, self.limits)
|
||||
.map_err(|_| Status::invalid_argument("capture Raft snapshot rejected"))?;
|
||||
let challenge = http_auth::tonic_boot_epoch_challenge(request.metadata().as_ref())
|
||||
.map_err(|_| Status::unauthenticated("capture Raft challenge rejected"))?;
|
||||
let raw = request.into_inner();
|
||||
let owner = self.owner.clone();
|
||||
let audience = self.audience.clone();
|
||||
let limits = self.limits;
|
||||
#[cfg(test)]
|
||||
let pause = self.worker_pause.clone();
|
||||
tokio::spawn(async move {
|
||||
let _permit = permit;
|
||||
#[cfg(test)]
|
||||
Self::paused(pause).await;
|
||||
let result = owner.raft().install_snapshot(payload).await.map_err(|error| match error {
|
||||
RaftError::Fatal(error) => fatal(error),
|
||||
RaftError::APIError(error) => RemoteFailure::SnapshotMismatch { error },
|
||||
});
|
||||
drop(owner);
|
||||
signed_response(&raw, result, limits, &audience, challenge)
|
||||
})
|
||||
.await
|
||||
.map_err(|_| Status::internal("capture Raft worker failed"))?
|
||||
}
|
||||
}
|
||||
fn validate_append(
|
||||
request: &AppendEntriesRequest<CaptureTypeConfig>,
|
||||
source: u64,
|
||||
limits: CaptureRaftLimits,
|
||||
) -> Result<(), CaptureRaftError> {
|
||||
if request.vote.leader_id.node_id != source {
|
||||
return Err(CaptureRaftError::Identity);
|
||||
}
|
||||
if request.entries.len() > limits.max_entries {
|
||||
return Err(CaptureRaftError::RequestTooLarge);
|
||||
}
|
||||
for entry in &request.entries {
|
||||
if let openraft::EntryPayload::Membership(value) = &entry.payload {
|
||||
CaptureMembership::try_from(&openraft::StoredMembership::new(Some(entry.log_id), value.clone()))
|
||||
.map_err(|_| CaptureRaftError::Protocol)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
fn validate_snapshot(
|
||||
request: &InstallSnapshotRequest<CaptureTypeConfig>,
|
||||
source: u64,
|
||||
limits: CaptureRaftLimits,
|
||||
) -> Result<(), CaptureRaftError> {
|
||||
if request.vote.leader_id.node_id != source
|
||||
|| request.data.len() > limits.max_snapshot_chunk_bytes
|
||||
|| request
|
||||
.offset
|
||||
.checked_add(u64::try_from(request.data.len()).map_err(|_| CaptureRaftError::Limits)?)
|
||||
.is_none()
|
||||
|| request.meta.snapshot_id.is_empty()
|
||||
|| request.meta.snapshot_id.len() > 1024
|
||||
{
|
||||
return Err(CaptureRaftError::Limits);
|
||||
}
|
||||
CaptureMembership::try_from(&request.meta.last_membership).map_err(|_| CaptureRaftError::Protocol)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct CaptureRaftNetworkFactory {
|
||||
local: NodeIdentity,
|
||||
peers: Arc<BTreeMap<u64, CaptureNode>>,
|
||||
limits: CaptureRaftLimits,
|
||||
admission: Arc<tokio::sync::Semaphore>,
|
||||
}
|
||||
impl CaptureRaftNetworkFactory {
|
||||
pub fn new(
|
||||
local: NodeIdentity,
|
||||
peers: BTreeMap<u64, CaptureNode>,
|
||||
config: Arc<openraft::Config>,
|
||||
limits: CaptureRaftLimits,
|
||||
) -> Result<Self, CaptureRaftError> {
|
||||
limits.validate()?;
|
||||
peers_valid(local, &peers)?;
|
||||
if usize::try_from(config.max_payload_entries).map_err(|_| CaptureRaftError::Limits)? > limits.max_entries {
|
||||
return Err(CaptureRaftError::Limits);
|
||||
}
|
||||
if usize::try_from(config.snapshot_max_chunk_size).map_err(|_| CaptureRaftError::Limits)?
|
||||
> limits.max_snapshot_chunk_bytes
|
||||
{
|
||||
return Err(CaptureRaftError::Limits);
|
||||
}
|
||||
Ok(Self {
|
||||
local,
|
||||
peers: Arc::new(peers),
|
||||
limits,
|
||||
admission: Arc::new(tokio::sync::Semaphore::new(limits.max_inflight)),
|
||||
})
|
||||
}
|
||||
}
|
||||
pub struct CaptureRaftPeer {
|
||||
factory: CaptureRaftNetworkFactory,
|
||||
target: u64,
|
||||
node: CaptureNode,
|
||||
valid: bool,
|
||||
}
|
||||
impl RaftNetworkFactory<CaptureTypeConfig> for CaptureRaftNetworkFactory {
|
||||
type Network = CaptureRaftPeer;
|
||||
async fn new_client(&mut self, target: u64, node: &CaptureNode) -> Self::Network {
|
||||
CaptureRaftPeer {
|
||||
factory: self.clone(),
|
||||
target,
|
||||
node: node.clone(),
|
||||
valid: self.peers.get(&target) == Some(node),
|
||||
}
|
||||
}
|
||||
}
|
||||
impl CaptureRaftPeer {
|
||||
async fn exchange<P: Serialize, R: Serialize + DeserializeOwned>(
|
||||
&self,
|
||||
method: Method,
|
||||
payload: &P,
|
||||
option: &RPCOption,
|
||||
) -> Result<Result<R, RemoteFailure>, CaptureRaftError> {
|
||||
if !self.valid {
|
||||
return Err(CaptureRaftError::Identity);
|
||||
}
|
||||
let ttl = self.factory.limits.request_timeout.min(option.soft_ttl());
|
||||
tokio::time::timeout(ttl, async {
|
||||
let _permit = self
|
||||
.factory
|
||||
.admission
|
||||
.clone()
|
||||
.acquire_owned()
|
||||
.await
|
||||
.map_err(|_| CaptureRaftError::ChannelUnavailable)?;
|
||||
let raw =
|
||||
encode_request(self.factory.local, self.target, method, payload, self.factory.limits).map_err(
|
||||
|error| match error {
|
||||
CaptureRaftError::Limits => CaptureRaftError::RequestTooLarge,
|
||||
error => error,
|
||||
},
|
||||
)?;
|
||||
let mut request = Request::new(raw.clone());
|
||||
http_auth::set_tonic_canonical_body_digest(&mut request, &canonical_request(&raw.body))?;
|
||||
request.set_timeout(ttl);
|
||||
let mut client = client::capture_raft_time_out_client(
|
||||
self.node.rpc_uri(),
|
||||
self.factory.limits.connect_timeout.min(ttl),
|
||||
self.factory.limits.max_request_bytes + 64,
|
||||
self.factory.limits.max_response_bytes + 128,
|
||||
)
|
||||
.await?;
|
||||
let response = match method {
|
||||
Method::AppendEntries => client.append_entries(request).await,
|
||||
Method::Vote => client.vote(request).await,
|
||||
Method::InstallSnapshot => client.install_snapshot(request).await,
|
||||
}
|
||||
.map_err(|status| CaptureRaftError::Status(Box::new(status)))?;
|
||||
verify_response(&raw, response.into_inner(), self.factory.limits)
|
||||
})
|
||||
.await
|
||||
.map_err(|_| CaptureRaftError::Deadline)?
|
||||
}
|
||||
fn network_error<E: std::error::Error>(
|
||||
&self,
|
||||
error: CaptureRaftError,
|
||||
method: Method,
|
||||
option: &RPCOption,
|
||||
) -> RPCError<u64, CaptureNode, E> {
|
||||
match error {
|
||||
CaptureRaftError::Deadline => RPCError::Timeout(openraft::error::Timeout {
|
||||
action: match method {
|
||||
Method::AppendEntries => openraft::RPCTypes::AppendEntries,
|
||||
Method::Vote => openraft::RPCTypes::Vote,
|
||||
Method::InstallSnapshot => openraft::RPCTypes::InstallSnapshot,
|
||||
},
|
||||
id: self.factory.local.node_id,
|
||||
target: self.target,
|
||||
timeout: self.factory.limits.request_timeout.min(option.soft_ttl()),
|
||||
}),
|
||||
error @ CaptureRaftError::ChannelUnavailable => RPCError::Unreachable(Unreachable::new(&error)),
|
||||
error => RPCError::Network(openraft::error::NetworkError::new(&error)),
|
||||
}
|
||||
}
|
||||
fn append_error<E: std::error::Error>(
|
||||
&self,
|
||||
error: CaptureRaftError,
|
||||
count: usize,
|
||||
option: &RPCOption,
|
||||
) -> RPCError<u64, CaptureNode, E> {
|
||||
if matches!(error, CaptureRaftError::RequestTooLarge) && count > 1 {
|
||||
let hint = u64::try_from((count / 2).max(1).min(self.factory.limits.max_entries)).unwrap_or(1);
|
||||
return RPCError::PayloadTooLarge(openraft::error::PayloadTooLarge::new_entries_hint(hint));
|
||||
}
|
||||
self.network_error(error, Method::AppendEntries, option)
|
||||
}
|
||||
fn remote<E: std::error::Error>(&self, error: E) -> RPCError<u64, CaptureNode, E> {
|
||||
RPCError::RemoteError(RemoteError::new_with_node(self.target, self.node.clone(), error))
|
||||
}
|
||||
}
|
||||
impl RaftNetwork<CaptureTypeConfig> for CaptureRaftPeer {
|
||||
async fn append_entries(
|
||||
&mut self,
|
||||
request: AppendEntriesRequest<CaptureTypeConfig>,
|
||||
option: RPCOption,
|
||||
) -> Result<AppendEntriesResponse<u64>, RPCError<u64, CaptureNode, RaftError<u64>>> {
|
||||
let count = request.entries.len();
|
||||
validate_append(&request, self.factory.local.node_id, self.factory.limits)
|
||||
.map_err(|error| self.append_error(error, count, &option))?;
|
||||
match self
|
||||
.exchange(Method::AppendEntries, &request, &option)
|
||||
.await
|
||||
.map_err(|error| self.append_error(error, count, &option))?
|
||||
{
|
||||
Ok(value) => Ok(value),
|
||||
Err(error) => Err(self.remote(RaftError::Fatal(
|
||||
restored_fatal(error).map_err(|error| self.network_error(error, Method::AppendEntries, &option))?,
|
||||
))),
|
||||
}
|
||||
}
|
||||
async fn vote(
|
||||
&mut self,
|
||||
request: VoteRequest<u64>,
|
||||
option: RPCOption,
|
||||
) -> Result<VoteResponse<u64>, RPCError<u64, CaptureNode, RaftError<u64>>> {
|
||||
if request.vote.leader_id.node_id != self.factory.local.node_id {
|
||||
return Err(self.network_error(CaptureRaftError::Identity, Method::Vote, &option));
|
||||
}
|
||||
match self
|
||||
.exchange(Method::Vote, &request, &option)
|
||||
.await
|
||||
.map_err(|error| self.network_error(error, Method::Vote, &option))?
|
||||
{
|
||||
Ok(value) => Ok(value),
|
||||
Err(error) => Err(self.remote(RaftError::Fatal(
|
||||
restored_fatal(error).map_err(|error| self.network_error(error, Method::Vote, &option))?,
|
||||
))),
|
||||
}
|
||||
}
|
||||
async fn install_snapshot(
|
||||
&mut self,
|
||||
request: InstallSnapshotRequest<CaptureTypeConfig>,
|
||||
option: RPCOption,
|
||||
) -> Result<InstallSnapshotResponse<u64>, RPCError<u64, CaptureNode, RaftError<u64, InstallSnapshotError>>> {
|
||||
validate_snapshot(&request, self.factory.local.node_id, self.factory.limits)
|
||||
.map_err(|error| self.network_error(error, Method::InstallSnapshot, &option))?;
|
||||
match self
|
||||
.exchange(Method::InstallSnapshot, &request, &option)
|
||||
.await
|
||||
.map_err(|error| self.network_error(error, Method::InstallSnapshot, &option))?
|
||||
{
|
||||
Ok(value) => Ok(value),
|
||||
Err(error) => {
|
||||
let source = match error {
|
||||
RemoteFailure::SnapshotMismatch { error } => RaftError::APIError(error),
|
||||
error => RaftError::Fatal(
|
||||
restored_fatal(error).map_err(|error| self.network_error(error, Method::InstallSnapshot, &option))?,
|
||||
),
|
||||
};
|
||||
Err(self.remote(source))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn auth_rejected(reason: &'static str) -> Status {
|
||||
use rustfs_io_metrics::internode_metrics::{
|
||||
INTERNODE_OPERATION_GRPC_OTHER, INTERNODE_TRANSPORT_BACKEND_GRPC, global_internode_metrics,
|
||||
};
|
||||
global_internode_metrics().record_rpc_auth_failure_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_GRPC_OTHER,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
reason,
|
||||
);
|
||||
Status::unauthenticated("capture Raft authentication rejected")
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -535,6 +535,62 @@ impl tonic::service::Interceptor for TonicInterceptor {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) type CaptureRaftClient = rustfs_protos::proto_gen::node_service::capture_raft_service_client::CaptureRaftServiceClient<
|
||||
InterceptedService<AuthenticatedChannel, TonicInterceptor>,
|
||||
>;
|
||||
|
||||
/// Capture RPCs always perform the real authenticated epoch handshake, never seed replay state.
|
||||
pub(crate) async fn capture_raft_time_out_client(
|
||||
addr: &str,
|
||||
connect_timeout: std::time::Duration,
|
||||
request_max: usize,
|
||||
response_max: usize,
|
||||
) -> Result<CaptureRaftClient, super::capture_raft::CaptureRaftError> {
|
||||
use super::capture_raft::CaptureRaftError;
|
||||
let interceptor = TonicInterceptor::Signature(gen_tonic_signature_interceptor()).with_rpc_audience(addr)?;
|
||||
let channel = tokio::time::timeout(connect_timeout, async {
|
||||
match runtime_sources::cached_node_channel(addr).await {
|
||||
Some(channel) => Ok(channel),
|
||||
None => create_new_channel(addr)
|
||||
.await
|
||||
.map_err(|_| CaptureRaftError::ChannelUnavailable),
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| CaptureRaftError::Deadline)??;
|
||||
let audience = interceptor
|
||||
.replay_scope_audience()
|
||||
.ok_or(CaptureRaftError::Authentication)?
|
||||
.to_owned();
|
||||
let authenticated = ReplayScopeChannel::new(channel, Some(audience.clone()));
|
||||
let mut ping = NodeServiceClient::with_interceptor(authenticated.clone(), interceptor)
|
||||
.max_encoding_message_size(1024)
|
||||
.max_decoding_message_size(8192);
|
||||
ping.ping(tonic::Request::new(rustfs_protos::proto_gen::node_service::PingRequest {
|
||||
version: 1,
|
||||
body: bytes::Bytes::new(),
|
||||
}))
|
||||
.await
|
||||
.map_err(|status| CaptureRaftError::Status(Box::new(status)))?;
|
||||
{
|
||||
let peers = PEER_REPLAY_STATES.lock().map_err(|_| CaptureRaftError::Authentication)?;
|
||||
let state = peers.get(&audience).ok_or(CaptureRaftError::Authentication)?;
|
||||
if !matches!(state.cache_capability, Some(PeerReplayCapability::Capable { boot_epoch }) if Some(boot_epoch) == state.boot_epoch)
|
||||
{
|
||||
return Err(CaptureRaftError::Authentication);
|
||||
}
|
||||
}
|
||||
let interceptor = TonicInterceptor::Signature(gen_tonic_signature_interceptor()).with_rpc_audience(addr)?;
|
||||
Ok(
|
||||
rustfs_protos::proto_gen::node_service::capture_raft_service_client::CaptureRaftServiceClient::with_interceptor(
|
||||
authenticated,
|
||||
interceptor,
|
||||
)
|
||||
.max_encoding_message_size(request_max)
|
||||
.max_decoding_message_size(response_max),
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -13,6 +13,9 @@
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) mod background_monitor;
|
||||
pub mod capture_raft;
|
||||
#[cfg(test)]
|
||||
mod capture_raft_tests;
|
||||
pub(crate) mod client;
|
||||
pub(crate) mod context_propagation;
|
||||
pub(crate) mod http_auth;
|
||||
|
||||
@@ -640,6 +640,7 @@ pub struct PeerLiveEventsBatch {
|
||||
pub struct ScannerPublicationLease {
|
||||
pub token: Uuid,
|
||||
pub movement_generation: u64,
|
||||
pub observational_only: bool,
|
||||
/// Stable storage owner identity. This is distinct from the activity
|
||||
/// session and is bound into both acquire and release proofs.
|
||||
pub owner_id: String,
|
||||
@@ -658,6 +659,7 @@ fn validate_scanner_publication_lease_response_fields(
|
||||
response: &ScannerPublicationLeaseResponse,
|
||||
expected_session_id: &str,
|
||||
expected_generation: u64,
|
||||
expected_observational_only: bool,
|
||||
) -> Result<(Uuid, String)> {
|
||||
if !response.success {
|
||||
return Err(Error::other(
|
||||
@@ -674,6 +676,9 @@ fn validate_scanner_publication_lease_response_fields(
|
||||
if response.session_id != expected_session_id {
|
||||
return Err(Error::other("peer returned a different scanner publication lease session"));
|
||||
}
|
||||
if response.observational_only != expected_observational_only {
|
||||
return Err(Error::other("peer returned a different scanner publication lease purpose"));
|
||||
}
|
||||
let owner_id = Uuid::parse_str(&response.owner_id)
|
||||
.ok()
|
||||
.filter(|owner_id| !owner_id.is_nil())
|
||||
@@ -2418,6 +2423,7 @@ impl PeerRestClient {
|
||||
&self,
|
||||
expected_session_id: &str,
|
||||
expected_generation: u64,
|
||||
observational_only: bool,
|
||||
) -> Result<ScannerPublicationLease> {
|
||||
let request_started = std::time::Instant::now();
|
||||
self.finalize_result(
|
||||
@@ -2434,6 +2440,7 @@ impl PeerRestClient {
|
||||
ttl_ms: crate::store::SCANNER_PUBLICATION_LEASE_TTL_MS,
|
||||
expected_session_id: expected_session_id.to_string(),
|
||||
token: Bytes::new(),
|
||||
observational_only,
|
||||
});
|
||||
let canonical = rustfs_protos::canonical_scanner_publication_lease_request_body(request.get_ref())
|
||||
.map_err(|_| Error::other("scanner publication lease request is too large to authenticate"))?;
|
||||
@@ -2444,11 +2451,16 @@ impl PeerRestClient {
|
||||
.map_err(|_| Error::other("scanner publication lease response is too large to authenticate"))?;
|
||||
verify_tonic_rpc_response_proof(&response_body, &response.response_proof)
|
||||
.map_err(|_| Error::other("peer returned an invalid scanner publication lease proof"))?;
|
||||
let (token, owner_id) =
|
||||
validate_scanner_publication_lease_response_fields(&response, expected_session_id, expected_generation)?;
|
||||
let (token, owner_id) = validate_scanner_publication_lease_response_fields(
|
||||
&response,
|
||||
expected_session_id,
|
||||
expected_generation,
|
||||
observational_only,
|
||||
)?;
|
||||
Ok(ScannerPublicationLease {
|
||||
token,
|
||||
movement_generation: response.movement_generation,
|
||||
observational_only,
|
||||
owner_id,
|
||||
session_id: response.session_id,
|
||||
expires_at: scanner_publication_lease_deadline(
|
||||
@@ -2483,6 +2495,7 @@ impl PeerRestClient {
|
||||
ttl_ms: crate::store::SCANNER_PUBLICATION_LEASE_TTL_MS,
|
||||
expected_session_id: lease.session_id.clone(),
|
||||
token: lease.token.as_bytes().to_vec().into(),
|
||||
observational_only: lease.observational_only,
|
||||
});
|
||||
let canonical = rustfs_protos::canonical_scanner_publication_lease_request_body(request.get_ref())
|
||||
.map_err(|_| Error::other("scanner publication lease validation request is too large to authenticate"))?;
|
||||
@@ -2494,8 +2507,12 @@ impl PeerRestClient {
|
||||
)?;
|
||||
verify_tonic_rpc_response_proof(&response_body, &response.response_proof)
|
||||
.map_err(|_| Error::other("peer returned an invalid scanner publication lease validation proof"))?;
|
||||
let (token, owner_id) =
|
||||
validate_scanner_publication_lease_response_fields(&response, &lease.session_id, lease.movement_generation)?;
|
||||
let (token, owner_id) = validate_scanner_publication_lease_response_fields(
|
||||
&response,
|
||||
&lease.session_id,
|
||||
lease.movement_generation,
|
||||
lease.observational_only,
|
||||
)?;
|
||||
if token != lease.token {
|
||||
return Err(Error::other("peer returned a different scanner publication lease token"));
|
||||
}
|
||||
@@ -3001,15 +3018,16 @@ mod tests {
|
||||
response_proof: Bytes::new(),
|
||||
owner_id: Uuid::new_v4().to_string(),
|
||||
session_id: "session-a".to_string(),
|
||||
observational_only: false,
|
||||
};
|
||||
|
||||
assert!(validate_scanner_publication_lease_response_fields(&response, "session-a", 7).is_ok());
|
||||
assert!(validate_scanner_publication_lease_response_fields(&response, "session-a", 7, false).is_ok());
|
||||
|
||||
let stale_generation = ScannerPublicationLeaseResponse {
|
||||
movement_generation: 6,
|
||||
..response.clone()
|
||||
};
|
||||
let error = validate_scanner_publication_lease_response_fields(&stale_generation, "session-a", 7)
|
||||
let error = validate_scanner_publication_lease_response_fields(&stale_generation, "session-a", 7, false)
|
||||
.expect_err("a response from an older movement generation must be rejected");
|
||||
assert!(error.to_string().contains("different scanner publication lease generation"));
|
||||
|
||||
@@ -3017,9 +3035,21 @@ mod tests {
|
||||
session_id: "session-b".to_string(),
|
||||
..response
|
||||
};
|
||||
let error = validate_scanner_publication_lease_response_fields(&stale_session, "session-a", 7)
|
||||
let error = validate_scanner_publication_lease_response_fields(&stale_session, "session-a", 7, false)
|
||||
.expect_err("a response from an older scanner session must be rejected");
|
||||
assert!(error.to_string().contains("different scanner publication lease session"));
|
||||
|
||||
let observational = ScannerPublicationLeaseResponse {
|
||||
observational_only: true,
|
||||
..stale_session
|
||||
};
|
||||
assert!(
|
||||
validate_scanner_publication_lease_response_fields(&observational, "session-b", 7, true).is_ok(),
|
||||
"the signed purpose echo should validate an observation lease"
|
||||
);
|
||||
let error = validate_scanner_publication_lease_response_fields(&observational, "session-b", 7, false)
|
||||
.expect_err("a peer must not upgrade an observational lease to authoritative");
|
||||
assert!(error.to_string().contains("different scanner publication lease purpose"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -58,13 +58,15 @@ use rustfs_protos::proto_gen::node_service::{
|
||||
BatchReadVersionRequest, BatchReadVersionResponse, CheckPartsRequest, CompareAndUpdateFileOutcome,
|
||||
CompareAndUpdateFileRequest, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest,
|
||||
DeleteVersionsResponse, DeleteVolumeRequest, DiskInfoRequest, ListDirRequest, ListVolumesRequest, MakeVolumeRequest,
|
||||
MakeVolumesRequest, PreparePartTransactionRequest, ReadAllRequest, ReadMetadataRequest, ReadMultipleRequest,
|
||||
ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest, RenameFileRequest,
|
||||
SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest,
|
||||
SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest,
|
||||
node_service_client::NodeServiceClient,
|
||||
MakeVolumesRequest, NativePreparationBeginRequest, NativePreparationFinalizeRequest, NativePreparationLocateRequest,
|
||||
NativePreparationProbeRequest, NativePreparationRecoverRequest, PreparePartTransactionRequest, ReadAllRequest,
|
||||
ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest,
|
||||
RenameDataRequest, RenameFileRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest,
|
||||
SnapshotLeaseRequest, SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest,
|
||||
WriteMetadataRequest, node_service_client::NodeServiceClient,
|
||||
};
|
||||
use serde::{Serialize, de::DeserializeOwned};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::{
|
||||
future::Future,
|
||||
io::Cursor,
|
||||
@@ -176,6 +178,172 @@ fn attach_mutation_body_digest<T>(
|
||||
crate::cluster::rpc::set_tonic_rolling_canonical_body_digest(request, &canonical_body).map_err(Error::other)
|
||||
}
|
||||
|
||||
fn native_response_proof(body: std::result::Result<Vec<u8>, std::num::TryFromIntError>, proof: &[u8]) -> Result<()> {
|
||||
let body = body.map_err(|_| Error::other("native preparation response is too large to authenticate"))?;
|
||||
if proof.len() != 32 {
|
||||
return Err(Error::other("native preparation response proof has invalid length"));
|
||||
}
|
||||
crate::cluster::rpc::verify_tonic_rpc_response_proof(&body, proof)
|
||||
.map_err(|_| Error::other("native preparation response proof is invalid"))
|
||||
}
|
||||
|
||||
fn native_response_error(error: Option<rustfs_protos::proto_gen::node_service::Error>) -> DiskError {
|
||||
let error = DiskError::from(error.unwrap_or_default());
|
||||
match error {
|
||||
DiskError::FileCorrupt => DiskError::Io(std::io::Error::new(std::io::ErrorKind::InvalidData, error)),
|
||||
error => error,
|
||||
}
|
||||
}
|
||||
|
||||
fn native_uuid_field(value: &[u8], field: &str) -> Result<Uuid> {
|
||||
let id = Uuid::from_slice(value).map_err(|_| Error::other(format!("native preparation returned invalid {field}")))?;
|
||||
if id.is_nil() {
|
||||
return Err(Error::other(format!("native preparation returned nil {field}")));
|
||||
}
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
fn native_version_field(value: &[u8]) -> Result<Uuid> {
|
||||
Uuid::from_slice(value).map_err(|_| Error::other("native preparation returned invalid version id"))
|
||||
}
|
||||
|
||||
fn native_object_length_from_metadata(input: &crate::disk::local::native_prepare::NativePrepareInput) -> Result<u64> {
|
||||
let raw = rustfs_filemeta::FileMeta::load(&input.metadata).map_err(Error::other)?;
|
||||
if raw.versions.len() != 1 {
|
||||
return Err(Error::other("native preparation metadata must have one version"));
|
||||
}
|
||||
let info = rustfs_filemeta::get_file_info(
|
||||
&input.metadata,
|
||||
&input.bucket,
|
||||
&input.key,
|
||||
&input.version.to_string(),
|
||||
rustfs_filemeta::FileInfoOpts {
|
||||
data: true,
|
||||
include_free_versions: false,
|
||||
include_part_checksums: true,
|
||||
},
|
||||
)
|
||||
.map_err(Error::other)?;
|
||||
if info.version_id != Some(input.version) || info.deleted || info.mark_deleted || info.size < 0 {
|
||||
return Err(Error::other("native preparation metadata identity mismatch"));
|
||||
}
|
||||
u64::try_from(info.size).map_err(Error::other)
|
||||
}
|
||||
|
||||
fn validate_native_metadata_for_version(
|
||||
metadata: &[u8],
|
||||
expected_digest: Option<&[u8; 32]>,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
version: Uuid,
|
||||
) -> Result<()> {
|
||||
validate_native_metadata(metadata, expected_digest)?;
|
||||
let raw = rustfs_filemeta::FileMeta::load(metadata).map_err(Error::other)?;
|
||||
if raw.versions.len() != 1 {
|
||||
return Err(Error::other("native preparation metadata must have one version"));
|
||||
}
|
||||
let info = rustfs_filemeta::get_file_info(
|
||||
metadata,
|
||||
bucket,
|
||||
key,
|
||||
&version.to_string(),
|
||||
rustfs_filemeta::FileInfoOpts {
|
||||
data: true,
|
||||
include_free_versions: false,
|
||||
include_part_checksums: true,
|
||||
},
|
||||
)
|
||||
.map_err(Error::other)?;
|
||||
if info.version_id != Some(version) || info.deleted || info.mark_deleted || info.size < 0 {
|
||||
return Err(Error::other("native preparation metadata identity mismatch"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn native_digest_field(value: &[u8], field: &str) -> Result<[u8; 32]> {
|
||||
value
|
||||
.try_into()
|
||||
.map_err(|_| Error::other(format!("native preparation returned invalid {field}")))
|
||||
}
|
||||
|
||||
fn native_receipt_from_wire(
|
||||
receipt: rustfs_protos::proto_gen::node_service::NativePreparationReceipt,
|
||||
expected_disk: Uuid,
|
||||
expected_store: Uuid,
|
||||
expected_preparation: Uuid,
|
||||
expected_bucket: &str,
|
||||
expected_key: &str,
|
||||
expected_incarnation: Uuid,
|
||||
expected_object: Option<(&[u8], u64)>,
|
||||
expected_version: Option<Uuid>,
|
||||
) -> Result<crate::disk::local::native_prepare::NativePreparationReceipt> {
|
||||
let disk = native_uuid_field(&receipt.disk_id, "disk id")?;
|
||||
let store = native_uuid_field(&receipt.store_id, "store id")?;
|
||||
let deployment = native_uuid_field(&receipt.deployment_id, "deployment id")?;
|
||||
let preparation = native_uuid_field(&receipt.preparation_id, "preparation id")?;
|
||||
let version = native_version_field(&receipt.version_id)?;
|
||||
let incarnation = native_uuid_field(&receipt.bucket_incarnation_id, "bucket incarnation")?;
|
||||
if disk != expected_disk
|
||||
|| store != expected_store
|
||||
|| deployment != expected_store
|
||||
|| preparation != expected_preparation
|
||||
|| incarnation != expected_incarnation
|
||||
|| receipt.bucket != expected_bucket
|
||||
|| receipt.key != expected_key
|
||||
|| expected_version.is_some_and(|expected| version != expected)
|
||||
{
|
||||
return Err(Error::other("native preparation receipt identity mismatch"));
|
||||
}
|
||||
let object_digest = native_digest_field(&receipt.object_digest, "object digest")?;
|
||||
if expected_object.is_some_and(|(digest, length)| digest != object_digest.as_slice() || length != receipt.object_length) {
|
||||
return Err(Error::other("native preparation receipt object mismatch"));
|
||||
}
|
||||
let set_digest = native_digest_field(&receipt.set_digest, "set digest")?;
|
||||
let distribution_digest = native_digest_field(&receipt.distribution_digest, "distribution digest")?;
|
||||
let metadata_digest = native_digest_field(&receipt.metadata_digest, "metadata digest")?;
|
||||
let shard_digest = native_digest_field(&receipt.shard_digest, "shard digest")?;
|
||||
let data_blocks = usize::try_from(receipt.data_blocks).map_err(Error::other)?;
|
||||
let parity_blocks = usize::try_from(receipt.parity_blocks).map_err(Error::other)?;
|
||||
let shard_index = usize::try_from(receipt.shard_index).map_err(Error::other)?;
|
||||
if data_blocks == 0
|
||||
|| parity_blocks == 0
|
||||
|| data_blocks
|
||||
.checked_add(parity_blocks)
|
||||
.is_none_or(|total| shard_index == 0 || shard_index > total)
|
||||
{
|
||||
return Err(Error::other("native preparation receipt has invalid erasure geometry"));
|
||||
}
|
||||
Ok(crate::disk::local::native_prepare::NativePreparationReceipt {
|
||||
preparation,
|
||||
version,
|
||||
disk,
|
||||
set_digest,
|
||||
distribution_digest,
|
||||
metadata_digest,
|
||||
shard_digest,
|
||||
shard_length: receipt.shard_length,
|
||||
bucket: receipt.bucket,
|
||||
key: receipt.key,
|
||||
deployment,
|
||||
incarnation,
|
||||
object_digest,
|
||||
object_length: receipt.object_length,
|
||||
data_blocks,
|
||||
parity_blocks,
|
||||
shard_index,
|
||||
})
|
||||
}
|
||||
|
||||
fn validate_native_metadata(metadata: &[u8], expected_digest: Option<&[u8; 32]>) -> Result<()> {
|
||||
if metadata.len() > rustfs_protos::NATIVE_PREPARATION_METADATA_MAX_BYTES {
|
||||
return Err(Error::other("native preparation metadata exceeds the protocol limit"));
|
||||
}
|
||||
if expected_digest.is_some_and(|expected| Sha256::digest(metadata).as_slice() != expected.as_slice()) {
|
||||
return Err(Error::other("native preparation metadata digest mismatch"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn decode_volume_infos(volume_infos: Vec<String>) -> Result<Vec<VolumeInfo>> {
|
||||
volume_infos
|
||||
.into_iter()
|
||||
@@ -896,6 +1064,339 @@ impl RemoteDisk {
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn native_begin_stream(
|
||||
&self,
|
||||
preparation: Uuid,
|
||||
store_id: Uuid,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
incarnation: Uuid,
|
||||
) -> Result<()> {
|
||||
let disk_id = self
|
||||
.get_disk_id()
|
||||
.await?
|
||||
.filter(|id| !id.is_nil())
|
||||
.ok_or(DiskError::InvalidPath)?;
|
||||
if preparation.is_nil() || store_id.is_nil() || incarnation.is_nil() || bucket.is_empty() || key.is_empty() {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
self.execute_with_timeout_for_op(
|
||||
"native_preparation_begin",
|
||||
|| async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(NativePreparationBeginRequest {
|
||||
disk: self.disk_ref().await,
|
||||
disk_id: disk_id.as_bytes().to_vec().into(),
|
||||
store_id: store_id.as_bytes().to_vec().into(),
|
||||
preparation_id: preparation.as_bytes().to_vec().into(),
|
||||
bucket: bucket.to_owned(),
|
||||
key: key.to_owned(),
|
||||
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
|
||||
});
|
||||
let canonical = rustfs_protos::canonical_native_preparation_begin_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical, "native_preparation_begin")?;
|
||||
let canonical_request = request.get_ref().clone();
|
||||
let response = client.native_preparation_begin(request).await?.into_inner();
|
||||
native_response_proof(
|
||||
rustfs_protos::canonical_native_preparation_begin_response_body(&canonical_request, &response),
|
||||
&response.response_proof,
|
||||
)?;
|
||||
if !response.success {
|
||||
return Err(native_response_error(response.error));
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn native_finalize(
|
||||
&self,
|
||||
input: crate::disk::local::native_prepare::NativePrepareInput,
|
||||
) -> Result<crate::disk::local::native_prepare::NativePreparationReceipt> {
|
||||
let disk_id = self
|
||||
.get_disk_id()
|
||||
.await?
|
||||
.filter(|id| !id.is_nil())
|
||||
.ok_or(DiskError::InvalidPath)?;
|
||||
if input.preparation.is_nil() || input.store_id.is_nil() || input.bucket_incarnation.is_nil() {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
validate_native_metadata(&input.metadata, None)?;
|
||||
let expected_object_digest = input.object_digest;
|
||||
let expected_object_length = native_object_length_from_metadata(&input)?;
|
||||
let expected_version = input.version;
|
||||
self.execute_with_timeout_for_op(
|
||||
"native_preparation_finalize",
|
||||
|| async move {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(NativePreparationFinalizeRequest {
|
||||
disk: self.disk_ref().await,
|
||||
disk_id: disk_id.as_bytes().to_vec().into(),
|
||||
store_id: input.store_id.as_bytes().to_vec().into(),
|
||||
preparation_id: input.preparation.as_bytes().to_vec().into(),
|
||||
bucket: input.bucket.clone(),
|
||||
key: input.key.clone(),
|
||||
bucket_incarnation_id: input.bucket_incarnation.as_bytes().to_vec().into(),
|
||||
object_digest: expected_object_digest.to_vec().into(),
|
||||
object_length: expected_object_length,
|
||||
version_id: expected_version.as_bytes().to_vec().into(),
|
||||
metadata: input.metadata.clone(),
|
||||
});
|
||||
let canonical = rustfs_protos::canonical_native_preparation_finalize_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical, "native_preparation_finalize")?;
|
||||
let canonical_request = request.get_ref().clone();
|
||||
let response = client.native_preparation_finalize(request).await?.into_inner();
|
||||
native_response_proof(
|
||||
rustfs_protos::canonical_native_preparation_finalize_response_body(&canonical_request, &response),
|
||||
&response.response_proof,
|
||||
)?;
|
||||
if !response.success {
|
||||
return Err(native_response_error(response.error));
|
||||
}
|
||||
let receipt = response
|
||||
.receipt
|
||||
.ok_or_else(|| Error::other("native finalize response omitted its receipt"))?;
|
||||
let receipt = native_receipt_from_wire(
|
||||
receipt,
|
||||
disk_id,
|
||||
input.store_id,
|
||||
input.preparation,
|
||||
&input.bucket,
|
||||
&input.key,
|
||||
input.bucket_incarnation,
|
||||
Some((&expected_object_digest, expected_object_length)),
|
||||
Some(expected_version),
|
||||
)?;
|
||||
validate_native_metadata_for_version(
|
||||
&input.metadata,
|
||||
Some(&receipt.metadata_digest),
|
||||
&input.bucket,
|
||||
&input.key,
|
||||
expected_version,
|
||||
)?;
|
||||
Ok(receipt)
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn native_probe(
|
||||
&self,
|
||||
preparation: Uuid,
|
||||
store_id: Uuid,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
incarnation: Uuid,
|
||||
) -> Result<crate::disk::local::native_prepare::NativePreparationProbe> {
|
||||
let disk_id = self
|
||||
.get_disk_id()
|
||||
.await?
|
||||
.filter(|id| !id.is_nil())
|
||||
.ok_or(DiskError::InvalidPath)?;
|
||||
if preparation.is_nil() || store_id.is_nil() || incarnation.is_nil() || bucket.is_empty() || key.is_empty() {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
self.execute_with_timeout_for_op(
|
||||
"native_preparation_probe",
|
||||
|| async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(NativePreparationProbeRequest {
|
||||
disk: self.disk_ref().await,
|
||||
disk_id: disk_id.as_bytes().to_vec().into(),
|
||||
store_id: store_id.as_bytes().to_vec().into(),
|
||||
preparation_id: preparation.as_bytes().to_vec().into(),
|
||||
bucket: bucket.to_owned(),
|
||||
key: key.to_owned(),
|
||||
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
|
||||
});
|
||||
let canonical = rustfs_protos::canonical_native_preparation_probe_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical, "native_preparation_probe")?;
|
||||
let canonical_request = request.get_ref().clone();
|
||||
let response = client.native_preparation_probe(request).await?.into_inner();
|
||||
native_response_proof(
|
||||
rustfs_protos::canonical_native_preparation_probe_response_body(&canonical_request, &response),
|
||||
&response.response_proof,
|
||||
)?;
|
||||
if !response.success {
|
||||
return Err(native_response_error(response.error));
|
||||
}
|
||||
validate_native_metadata(&response.metadata, None)?;
|
||||
if !response.exists {
|
||||
if response.prepared
|
||||
|| !response.metadata.is_empty()
|
||||
|| !response.version_id.is_empty()
|
||||
|| !response.object_digest.is_empty()
|
||||
|| response.object_length != 0
|
||||
|| !response.set_digest.is_empty()
|
||||
|| !response.distribution_digest.is_empty()
|
||||
{
|
||||
return Err(Error::other("native probe absent response carries object state"));
|
||||
}
|
||||
return Ok(crate::disk::local::native_prepare::NativePreparationProbe::Absent);
|
||||
}
|
||||
let version = native_version_field(&response.version_id)?;
|
||||
let object_digest = native_digest_field(&response.object_digest, "object digest")?;
|
||||
let set_digest = native_digest_field(&response.set_digest, "set digest")?;
|
||||
let distribution_digest = native_digest_field(&response.distribution_digest, "distribution digest")?;
|
||||
if !response.metadata.is_empty() {
|
||||
validate_native_metadata_for_version(&response.metadata, None, bucket, key, version)?;
|
||||
}
|
||||
Ok(crate::disk::local::native_prepare::NativePreparationProbe::Existing {
|
||||
version,
|
||||
prepared: response.prepared,
|
||||
object_digest,
|
||||
object_length: response.object_length,
|
||||
set_digest,
|
||||
distribution_digest,
|
||||
metadata: (!response.metadata.is_empty()).then_some(response.metadata),
|
||||
})
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn native_recover(
|
||||
&self,
|
||||
preparation: Uuid,
|
||||
store_id: Uuid,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
incarnation: Uuid,
|
||||
) -> Result<(crate::disk::local::native_prepare::NativePreparationReceipt, Bytes)> {
|
||||
let disk_id = self
|
||||
.get_disk_id()
|
||||
.await?
|
||||
.filter(|id| !id.is_nil())
|
||||
.ok_or(DiskError::InvalidPath)?;
|
||||
if preparation.is_nil() || store_id.is_nil() || incarnation.is_nil() || bucket.is_empty() || key.is_empty() {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
self.execute_with_timeout_for_op(
|
||||
"native_preparation_recover",
|
||||
|| async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(NativePreparationRecoverRequest {
|
||||
disk: self.disk_ref().await,
|
||||
disk_id: disk_id.as_bytes().to_vec().into(),
|
||||
store_id: store_id.as_bytes().to_vec().into(),
|
||||
preparation_id: preparation.as_bytes().to_vec().into(),
|
||||
bucket: bucket.to_owned(),
|
||||
key: key.to_owned(),
|
||||
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
|
||||
});
|
||||
let canonical = rustfs_protos::canonical_native_preparation_recover_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical, "native_preparation_recover")?;
|
||||
let canonical_request = request.get_ref().clone();
|
||||
let response = client.native_preparation_recover(request).await?.into_inner();
|
||||
native_response_proof(
|
||||
rustfs_protos::canonical_native_preparation_recover_response_body(&canonical_request, &response),
|
||||
&response.response_proof,
|
||||
)?;
|
||||
if !response.success {
|
||||
return Err(native_response_error(response.error));
|
||||
}
|
||||
let receipt = response
|
||||
.receipt
|
||||
.ok_or_else(|| Error::other("native recover response omitted its receipt"))?;
|
||||
let receipt =
|
||||
native_receipt_from_wire(receipt, disk_id, store_id, preparation, bucket, key, incarnation, None, None)?;
|
||||
validate_native_metadata_for_version(
|
||||
&response.metadata,
|
||||
Some(&receipt.metadata_digest),
|
||||
bucket,
|
||||
key,
|
||||
receipt.version,
|
||||
)?;
|
||||
Ok((receipt, response.metadata))
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn native_locate(
|
||||
&self,
|
||||
store_id: Uuid,
|
||||
preparation: Uuid,
|
||||
) -> Result<Option<crate::disk::local::native_prepare::NativePreparationDescription>> {
|
||||
let disk_id = self
|
||||
.get_disk_id()
|
||||
.await?
|
||||
.filter(|id| !id.is_nil())
|
||||
.ok_or(DiskError::InvalidPath)?;
|
||||
if preparation.is_nil() || store_id.is_nil() {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
self.execute_with_timeout_for_op(
|
||||
"native_preparation_locate",
|
||||
|| async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(NativePreparationLocateRequest {
|
||||
disk: self.disk_ref().await,
|
||||
disk_id: disk_id.as_bytes().to_vec().into(),
|
||||
store_id: store_id.as_bytes().to_vec().into(),
|
||||
preparation_id: preparation.as_bytes().to_vec().into(),
|
||||
});
|
||||
let canonical = rustfs_protos::canonical_native_preparation_locate_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical, "native_preparation_locate")?;
|
||||
let canonical_request = request.get_ref().clone();
|
||||
let response = client.native_preparation_locate(request).await?.into_inner();
|
||||
native_response_proof(
|
||||
rustfs_protos::canonical_native_preparation_locate_response_body(&canonical_request, &response),
|
||||
&response.response_proof,
|
||||
)?;
|
||||
if !response.success {
|
||||
return Err(native_response_error(response.error));
|
||||
}
|
||||
if !response.found {
|
||||
if !response.bucket.is_empty()
|
||||
|| !response.key.is_empty()
|
||||
|| !response.bucket_incarnation_id.is_empty()
|
||||
|| !response.metadata.is_empty()
|
||||
|| response.prepared
|
||||
{
|
||||
return Err(Error::other("native locate absent response carries preparation state"));
|
||||
}
|
||||
return Ok(None);
|
||||
}
|
||||
if response.bucket.is_empty()
|
||||
|| response.bucket.len() > 255
|
||||
|| response.key.is_empty()
|
||||
|| response.key.len() > 4096
|
||||
|| preparation.is_nil()
|
||||
{
|
||||
return Err(Error::other("native locate response has invalid description"));
|
||||
}
|
||||
let incarnation = native_uuid_field(&response.bucket_incarnation_id, "bucket incarnation")?;
|
||||
validate_native_metadata(&response.metadata, None)?;
|
||||
if response.metadata.is_empty() {
|
||||
return Err(Error::other("native locate response omitted metadata"));
|
||||
}
|
||||
let raw_metadata = rustfs_filemeta::FileMeta::load(&response.metadata)
|
||||
.map_err(|_| Error::other("native locate response metadata is invalid"))?;
|
||||
if raw_metadata.versions.len() != 1 {
|
||||
return Err(Error::other("native locate metadata must have one version"));
|
||||
}
|
||||
let version = raw_metadata.versions[0].header.version_id.unwrap_or_else(Uuid::nil);
|
||||
Ok(Some(crate::disk::local::native_prepare::NativePreparationDescription {
|
||||
preparation,
|
||||
bucket: response.bucket,
|
||||
key: response.key,
|
||||
incarnation,
|
||||
version,
|
||||
metadata: response.metadata,
|
||||
prepared: response.prepared,
|
||||
}))
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn ns_scanner_server_epoch(&self) -> Result<Option<Uuid>> {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
|
||||
@@ -449,6 +449,64 @@ where
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_observation<S>(api: Arc<S>, file: &str, max_bytes: usize) -> Result<(Vec<u8>, ObjectInfo)>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
let opts = ObjectOptions {
|
||||
suppress_read_repair: true,
|
||||
no_lock: false,
|
||||
skip_verify_bitrot: false,
|
||||
..Default::default()
|
||||
};
|
||||
crate::object_api::without_get_object_body_cache_hook(read_config_limited_preserve_empty_with_metadata_opts(
|
||||
api, file, &opts, max_bytes,
|
||||
))
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_observation_no_lock<S>(
|
||||
api: Arc<S>,
|
||||
file: &str,
|
||||
max_bytes: usize,
|
||||
guard: &crate::object_api::WriteCommitGuard,
|
||||
) -> Result<(Vec<u8>, ObjectInfo)>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
if !guard.matches(RUSTFS_META_BUCKET, file) {
|
||||
return Err(Error::InvalidArgument(
|
||||
RUSTFS_META_BUCKET.to_owned(),
|
||||
file.to_owned(),
|
||||
"locked observation requires a matching write owner".to_owned(),
|
||||
));
|
||||
}
|
||||
let lock_lost_error = || Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "locked_config_observation",
|
||||
bucket: RUSTFS_META_BUCKET.to_owned(),
|
||||
object: file.to_owned(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
};
|
||||
if guard.is_lock_lost() {
|
||||
return Err(lock_lost_error());
|
||||
}
|
||||
let opts = ObjectOptions {
|
||||
suppress_read_repair: true,
|
||||
no_lock: true,
|
||||
skip_verify_bitrot: false,
|
||||
..Default::default()
|
||||
};
|
||||
let result = crate::object_api::without_get_object_body_cache_hook(read_config_limited_preserve_empty_with_metadata_opts(
|
||||
api, file, &opts, max_bytes,
|
||||
))
|
||||
.await;
|
||||
if guard.is_lock_lost() {
|
||||
return Err(lock_lost_error());
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_limited_preserve_empty<S>(api: Arc<S>, file: &str, max_bytes: usize) -> Result<Vec<u8>>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
@@ -495,10 +553,22 @@ pub(crate) async fn read_config_preserve_empty<S>(api: Arc<S>, file: &str) -> Re
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
let (data, _obj) = read_config_with_metadata_inner(api, file, &ObjectOptions::default(), true, None).await?;
|
||||
let (data, _obj) = read_config_preserve_empty_with_metadata_opts(api, file, &ObjectOptions::default()).await?;
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
/// Preserve an empty config payload, its metadata, and caller-supplied read options.
|
||||
pub(crate) async fn read_config_preserve_empty_with_metadata_opts<S>(
|
||||
api: Arc<S>,
|
||||
file: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<(Vec<u8>, ObjectInfo)>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
read_config_with_metadata_inner(api, file, opts, true, None).await
|
||||
}
|
||||
|
||||
pub async fn read_config_no_lock<S>(api: Arc<S>, file: &str) -> Result<Vec<u8>>
|
||||
where
|
||||
S: ObjectIO<
|
||||
@@ -527,15 +597,13 @@ pub(crate) async fn read_config_no_lock_preserve_empty_with_metadata<S>(api: Arc
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
read_config_with_metadata_inner(
|
||||
read_config_preserve_empty_with_metadata_opts(
|
||||
api,
|
||||
file,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
true,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -2815,9 +2883,10 @@ mod tests {
|
||||
config_task_join_error, configs_semantically_equal, decode_server_config_blob, encode_server_config_blob,
|
||||
heal_config_descriptor, is_standard_object_server_config, lookup_configs, map_system_metadata_write_error,
|
||||
new_and_save_server_config, read_config, read_config_no_lock_preserve_empty_with_metadata, read_config_preserve_empty,
|
||||
read_config_with_metadata, read_config_without_migrate, read_server_config_snapshot, save_config_with_opts_inner,
|
||||
save_server_config, save_server_config_snapshot, save_server_config_snapshot_with_generation,
|
||||
server_config_transaction_lock_path, should_warn_ignored_scalar_section, storage_class_kvs_mut,
|
||||
read_config_preserve_empty_with_metadata_opts, read_config_with_metadata, read_config_without_migrate,
|
||||
read_server_config_snapshot, save_config_with_opts_inner, save_server_config, save_server_config_snapshot,
|
||||
save_server_config_snapshot_with_generation, server_config_transaction_lock_path, should_warn_ignored_scalar_section,
|
||||
storage_class_kvs_mut,
|
||||
};
|
||||
use crate::config::{audit, heal, notify, oidc, scanner};
|
||||
use crate::disk::{RUSTFS_META_BUCKET, endpoint::Endpoint};
|
||||
@@ -5065,6 +5134,7 @@ mod tests {
|
||||
heal_replacement: Option<Vec<u8>>,
|
||||
heal_calls: AtomicUsize,
|
||||
write_calls: AtomicUsize,
|
||||
last_get_no_lock: AtomicBool,
|
||||
last_put_no_lock: AtomicBool,
|
||||
last_put_preconditions: Mutex<Option<HTTPPreconditions>>,
|
||||
revision: AtomicUsize,
|
||||
@@ -5080,6 +5150,7 @@ mod tests {
|
||||
heal_replacement,
|
||||
heal_calls: AtomicUsize::new(0),
|
||||
write_calls: AtomicUsize::new(0),
|
||||
last_get_no_lock: AtomicBool::new(false),
|
||||
last_put_no_lock: AtomicBool::new(false),
|
||||
last_put_preconditions: Mutex::new(None),
|
||||
revision: AtomicUsize::new(1),
|
||||
@@ -5135,8 +5206,9 @@ mod tests {
|
||||
_object: &str,
|
||||
_range: Option<HTTPRangeSpec>,
|
||||
_h: HeaderMap,
|
||||
_opts: &ObjectOptions,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<GetObjectReader> {
|
||||
self.last_get_no_lock.store(opts.no_lock, Ordering::SeqCst);
|
||||
let data = match &*self.state.lock().expect("state lock poisoned") {
|
||||
RecoveryReadState::Missing => return Err(Error::ConfigNotFound),
|
||||
RecoveryReadState::Blob(data) => data.clone(),
|
||||
@@ -5411,6 +5483,329 @@ mod tests {
|
||||
assert_eq!(store.write_calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
struct ObservationStream {
|
||||
inner: Cursor<Vec<u8>>,
|
||||
consumed: Arc<AtomicUsize>,
|
||||
fail: bool,
|
||||
eof_gate: Option<(Option<tokio::sync::oneshot::Sender<()>>, tokio::sync::oneshot::Receiver<()>)>,
|
||||
}
|
||||
|
||||
impl AsyncRead for ObservationStream {
|
||||
fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
|
||||
if self.fail {
|
||||
return Poll::Ready(Err(std::io::Error::other("observation stream failed")));
|
||||
}
|
||||
if self.inner.position() == u64::try_from(self.inner.get_ref().len()).expect("fixture length")
|
||||
&& let Some((ready, release)) = self.eof_gate.as_mut()
|
||||
{
|
||||
if let Some(ready) = ready.take() {
|
||||
ready.send(()).expect("EOF observer remains active");
|
||||
}
|
||||
match std::future::Future::poll(Pin::new(release), cx) {
|
||||
Poll::Pending => return Poll::Pending,
|
||||
Poll::Ready(result) => result.expect("EOF gate released"),
|
||||
}
|
||||
self.eof_gate = None;
|
||||
}
|
||||
let before = buf.filled().len();
|
||||
let result = Pin::new(&mut self.inner).poll_read(cx, buf);
|
||||
self.consumed.fetch_add(buf.filled().len() - before, Ordering::SeqCst);
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
struct ObservationStore {
|
||||
body: Vec<u8>,
|
||||
declared_size: i64,
|
||||
error: Option<Error>,
|
||||
stream_error: bool,
|
||||
gets: AtomicUsize,
|
||||
eof_gate: std::sync::Mutex<Option<(Option<tokio::sync::oneshot::Sender<()>>, tokio::sync::oneshot::Receiver<()>)>>,
|
||||
consumed: Arc<AtomicUsize>,
|
||||
options: std::sync::Mutex<Option<(bool, bool, bool)>>,
|
||||
writes: AtomicUsize,
|
||||
}
|
||||
|
||||
impl Debug for ObservationStore {
|
||||
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("ObservationStore").finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl ObservationStore {
|
||||
fn new(body: Vec<u8>, declared_size: i64) -> Self {
|
||||
Self {
|
||||
body,
|
||||
declared_size,
|
||||
error: None,
|
||||
stream_error: false,
|
||||
gets: AtomicUsize::new(0),
|
||||
eof_gate: std::sync::Mutex::new(None),
|
||||
consumed: Arc::new(AtomicUsize::new(0)),
|
||||
options: std::sync::Mutex::new(None),
|
||||
writes: AtomicUsize::new(0),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::storage_api_contracts::object::ObjectIO for ObservationStore {
|
||||
type Error = Error;
|
||||
type RangeSpec = HTTPRangeSpec;
|
||||
type HeaderMap = HeaderMap;
|
||||
type ObjectOptions = ObjectOptions;
|
||||
type ObjectInfo = ObjectInfo;
|
||||
type GetObjectReader = GetObjectReader;
|
||||
type PutObjectReader = PutObjReader;
|
||||
|
||||
async fn get_object_reader(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
range: Option<HTTPRangeSpec>,
|
||||
_headers: HeaderMap,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<GetObjectReader> {
|
||||
assert_eq!(bucket, RUSTFS_META_BUCKET);
|
||||
assert_eq!(object, "config/observation.bin");
|
||||
self.gets.fetch_add(1, Ordering::SeqCst);
|
||||
assert!(range.is_none());
|
||||
*self.options.lock().expect("observation options") =
|
||||
Some((opts.suppress_read_repair, opts.no_lock, opts.skip_verify_bitrot));
|
||||
if let Some(error) = &self.error {
|
||||
return Err(error.clone());
|
||||
}
|
||||
Ok(GetObjectReader {
|
||||
stream: Box::new(ObservationStream {
|
||||
inner: Cursor::new(self.body.clone()),
|
||||
consumed: self.consumed.clone(),
|
||||
fail: self.stream_error,
|
||||
eof_gate: self.eof_gate.lock().expect("EOF gate").take(),
|
||||
}),
|
||||
object_info: ObjectInfo {
|
||||
size: self.declared_size,
|
||||
etag: Some("observation-etag".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
buffered_body: None,
|
||||
body_source: Default::default(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn put_object(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_object: &str,
|
||||
_data: &mut PutObjReader,
|
||||
_opts: &ObjectOptions,
|
||||
) -> Result<ObjectInfo> {
|
||||
self.writes.fetch_add(1, Ordering::SeqCst);
|
||||
Err(Error::other("observation must not write"))
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn locked_observation_preserves_options_and_typed_errors() {
|
||||
let object = "config/observation.bin";
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
|
||||
let wrapper = rustfs_lock::NamespaceLockWrapper::new(
|
||||
rustfs_lock::NamespaceLock::with_local_manager("locked-observation-options".to_owned(), manager.clone()),
|
||||
rustfs_lock::ObjectKey::new(RUSTFS_META_BUCKET, object),
|
||||
"locked-observation-options".to_owned(),
|
||||
);
|
||||
let guard = crate::object_api::WriteCommitGuard::acquire(&wrapper, std::time::Duration::from_secs(5))
|
||||
.await
|
||||
.expect("matching real owner");
|
||||
let store = Arc::new(ObservationStore::new(b"raw".to_vec(), 3));
|
||||
let result = super::read_config_observation_no_lock(store.clone(), object, 3, &guard).await;
|
||||
assert_eq!(result.is_ok(), true, "I06L_READ_FLAGS: {result:?}");
|
||||
assert_eq!(
|
||||
*store.options.lock().expect("options") == Some((true, true, false)),
|
||||
true,
|
||||
"I06L_READ_FLAGS"
|
||||
);
|
||||
assert_eq!(result.expect("raw observation").0, b"raw");
|
||||
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
|
||||
for (bucket, key) in [(RUSTFS_META_BUCKET, "config/other.bin"), ("wrong-bucket", object)] {
|
||||
let wrapper = rustfs_lock::NamespaceLockWrapper::new(
|
||||
rustfs_lock::NamespaceLock::with_local_manager("locked-observation-target".to_owned(), manager.clone()),
|
||||
rustfs_lock::ObjectKey::new(bucket, key),
|
||||
"locked-observation-target".to_owned(),
|
||||
);
|
||||
let wrong = crate::object_api::WriteCommitGuard::acquire(&wrapper, std::time::Duration::from_secs(5))
|
||||
.await
|
||||
.expect("wrong-target real owner");
|
||||
let store = Arc::new(ObservationStore::new(Vec::new(), 0));
|
||||
let result = super::read_config_observation_no_lock(store.clone(), object, 8, &wrong).await;
|
||||
assert_eq!(matches!(result, Err(Error::InvalidArgument(..))), true, "I06L_TARGET_REJECTED");
|
||||
assert_eq!(store.gets.load(Ordering::SeqCst) == 0, true, "I06L_TARGET_REJECTED");
|
||||
}
|
||||
for error in [
|
||||
Error::FileNotFound,
|
||||
Error::ObjectNotFound(RUSTFS_META_BUCKET.to_owned(), object.to_owned()),
|
||||
Error::ErasureReadQuorum,
|
||||
] {
|
||||
let expected = if error == Error::FileNotFound || matches!(error, Error::ObjectNotFound(..)) {
|
||||
Error::ConfigNotFound
|
||||
} else {
|
||||
error.clone()
|
||||
};
|
||||
let mut fixture = ObservationStore::new(Vec::new(), 0);
|
||||
fixture.error = Some(error);
|
||||
let store = Arc::new(fixture);
|
||||
assert_eq!(
|
||||
super::read_config_observation_no_lock(store.clone(), object, 8, &guard)
|
||||
.await
|
||||
.expect_err("typed error"),
|
||||
expected
|
||||
);
|
||||
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
let mut fixture = ObservationStore::new(Vec::new(), 0);
|
||||
fixture.stream_error = true;
|
||||
let store = Arc::new(fixture);
|
||||
let err = super::read_config_observation_no_lock(store.clone(), object, 8, &guard)
|
||||
.await
|
||||
.expect_err("stream failure");
|
||||
assert_eq!(err.to_string().contains("observation stream failed"), true, "I06L_TYPED_STREAM_ERROR");
|
||||
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
|
||||
let store = Arc::new(ObservationStore::new(vec![7; 64], 0));
|
||||
let result = super::read_config_observation_no_lock(store.clone(), object, 8, &guard).await;
|
||||
assert_eq!(matches!(result, Err(Error::CorruptedFormat)), true, "I06L_STREAM_BOUND");
|
||||
assert_eq!(store.consumed.load(Ordering::SeqCst) == 9, true, "I06L_STREAM_BOUND");
|
||||
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn locked_observation_rejects_actual_owner_loss_before_and_after_read() {
|
||||
let object = "config/observation.bin";
|
||||
for after_read in [false, true] {
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
|
||||
let client = Arc::new(RefreshControlledClient::new(manager));
|
||||
let wrapper = rustfs_lock::NamespaceLockWrapper::new(
|
||||
rustfs_lock::NamespaceLock::new("locked-observation-loss".to_owned(), client.clone()),
|
||||
rustfs_lock::ObjectKey::new(RUSTFS_META_BUCKET, object),
|
||||
"locked-observation-loss".to_owned(),
|
||||
);
|
||||
let guard = crate::object_api::WriteCommitGuard::acquire(&wrapper, std::time::Duration::from_secs(5))
|
||||
.await
|
||||
.expect("real distributed owner");
|
||||
let signal = guard.lock_lost_signal().expect("acquired distributed owner signal");
|
||||
let store = Arc::new(ObservationStore::new(b"raw".to_vec(), 3));
|
||||
if after_read {
|
||||
let (ready_tx, ready_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
|
||||
*store.eof_gate.lock().expect("EOF gate") = Some((Some(ready_tx), release_rx));
|
||||
let mut read = Box::pin(super::read_config_observation_no_lock(store.clone(), object, 3, &guard));
|
||||
tokio::time::timeout(std::time::Duration::from_secs(60), async {
|
||||
tokio::select! {
|
||||
ready = ready_rx => ready.expect("read reached EOF"),
|
||||
result = &mut read => assert_eq!(false, true, "I06L_LOSS_AFTER_READ: returned before EOF gate: {result:?}"),
|
||||
}
|
||||
}).await.expect("EOF watchdog");
|
||||
client.reject_refreshes();
|
||||
tokio::time::timeout(std::time::Duration::from_secs(60), signal.notified())
|
||||
.await
|
||||
.expect("actual owner loss watchdog");
|
||||
assert_eq!(guard.is_lock_lost(), true, "I06L_LOSS_AFTER_READ");
|
||||
release_tx.send(()).expect("release EOF");
|
||||
let result = tokio::time::timeout(std::time::Duration::from_secs(60), read)
|
||||
.await
|
||||
.expect("read watchdog");
|
||||
assert_eq!(
|
||||
matches!(
|
||||
result,
|
||||
Err(Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "locked_config_observation",
|
||||
..
|
||||
})
|
||||
),
|
||||
true,
|
||||
"I06L_LOSS_AFTER_READ"
|
||||
);
|
||||
assert_eq!(store.gets.load(Ordering::SeqCst), 1);
|
||||
} else {
|
||||
client.reject_refreshes();
|
||||
tokio::time::timeout(std::time::Duration::from_secs(60), signal.notified())
|
||||
.await
|
||||
.expect("actual owner loss watchdog");
|
||||
assert_eq!(guard.is_lock_lost(), true, "I06L_LOSS_BEFORE_GET");
|
||||
let result = super::read_config_observation_no_lock(store.clone(), object, 3, &guard).await;
|
||||
assert_eq!(
|
||||
matches!(
|
||||
result,
|
||||
Err(Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "locked_config_observation",
|
||||
..
|
||||
})
|
||||
),
|
||||
true,
|
||||
"I06L_LOSS_BEFORE_GET"
|
||||
);
|
||||
assert_eq!(store.gets.load(Ordering::SeqCst) == 0, true, "I06L_LOSS_BEFORE_GET");
|
||||
}
|
||||
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn observation_preserves_options_and_errors() {
|
||||
for body in [Vec::new(), b"raw config".to_vec()] {
|
||||
let store = Arc::new(ObservationStore::new(body.clone(), i64::try_from(body.len()).expect("fixture size")));
|
||||
let result = super::read_config_observation(store.clone(), "config/observation.bin", 32).await;
|
||||
assert!(result.is_ok(), "observation must preserve empty and nonempty objects: {result:?}");
|
||||
let (observed, info) = result.expect("successful observation");
|
||||
assert_eq!(observed, body);
|
||||
assert_eq!(info.etag.as_deref(), Some("observation-etag"));
|
||||
assert_eq!(
|
||||
*store.options.lock().expect("options"),
|
||||
Some((true, false, false)),
|
||||
"I05_REPAIR_SUPPRESSED"
|
||||
);
|
||||
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
for (source, expected) in [
|
||||
(Error::FileNotFound, Error::ConfigNotFound),
|
||||
(
|
||||
Error::ObjectNotFound(RUSTFS_META_BUCKET.to_string(), "config/observation.bin".to_string()),
|
||||
Error::ConfigNotFound,
|
||||
),
|
||||
(Error::ErasureReadQuorum, Error::ErasureReadQuorum),
|
||||
] {
|
||||
let mut fixture = ObservationStore::new(Vec::new(), 0);
|
||||
fixture.error = Some(source);
|
||||
let store = Arc::new(fixture);
|
||||
let result = super::read_config_observation(store.clone(), "config/observation.bin", 32).await;
|
||||
assert_eq!(result.expect_err("read error must propagate"), expected);
|
||||
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
let mut fixture = ObservationStore::new(Vec::new(), 0);
|
||||
fixture.stream_error = true;
|
||||
let store = Arc::new(fixture);
|
||||
let result = super::read_config_observation(store.clone(), "config/observation.bin", 32).await;
|
||||
let error = result.expect_err("stream error must propagate");
|
||||
assert!(error.to_string().contains("observation stream failed"));
|
||||
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn observation_enforces_byte_limit() {
|
||||
let store = Arc::new(ObservationStore::new(vec![7; 64], 0));
|
||||
let result = super::read_config_observation(store.clone(), "config/observation.bin", 8).await;
|
||||
assert_eq!(result.is_err(), true, "I05_BOUND_ENFORCED");
|
||||
assert_eq!(
|
||||
result.expect_err("bounded observation error"),
|
||||
Error::CorruptedFormat,
|
||||
"I05_BOUND_ENFORCED"
|
||||
);
|
||||
assert_eq!(store.consumed.load(Ordering::SeqCst), 9, "I05_BOUND_ENFORCED: drain exactly limit + 1");
|
||||
let store = Arc::new(ObservationStore::new(vec![7; 8], 8));
|
||||
let result = super::read_config_observation(store.clone(), "config/observation.bin", 8).await;
|
||||
assert!(result.is_ok(), "exact byte limit must succeed: {result:?}");
|
||||
assert_eq!(result.expect("exact limit").0, vec![7; 8]);
|
||||
assert_eq!(store.consumed.load(Ordering::SeqCst), 8);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_config_preserve_empty_distinguishes_empty_object() {
|
||||
let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Blob(Vec::new()), None));
|
||||
@@ -5431,6 +5826,60 @@ mod tests {
|
||||
assert!(data.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_config_preserve_empty_metadata_keeps_empty_and_options() {
|
||||
let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Blob(Vec::new()), None));
|
||||
for no_lock in [false, true] {
|
||||
let opts = ObjectOptions {
|
||||
no_lock,
|
||||
..Default::default()
|
||||
};
|
||||
let (data, info) = read_config_preserve_empty_with_metadata_opts(store.clone(), "config/empty.json", &opts)
|
||||
.await
|
||||
.expect("an existing empty config must retain its payload and metadata");
|
||||
assert!(data.is_empty());
|
||||
assert_eq!(info.size, 0);
|
||||
assert_eq!(info.actual_size, 0);
|
||||
assert_eq!(info.etag.as_deref(), Some("config-1"));
|
||||
assert_eq!(info.data_dir, Some(uuid::Uuid::from_u128(1)));
|
||||
assert_eq!(store.last_get_no_lock.load(Ordering::SeqCst), no_lock);
|
||||
}
|
||||
let err = read_config(store, "config/empty.json")
|
||||
.await
|
||||
.expect_err("the existing reader must still treat empty configs as missing");
|
||||
assert!(matches!(err, Error::ConfigNotFound));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_config_preserve_empty_metadata_keeps_payload_and_metadata() {
|
||||
let payload = b"original config bytes".to_vec();
|
||||
let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Blob(payload.clone()), None));
|
||||
let (data, info) =
|
||||
read_config_preserve_empty_with_metadata_opts(store, "config/nonempty.json", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("a nonempty config must retain its payload and metadata");
|
||||
assert_eq!(data, payload);
|
||||
let size = i64::try_from(payload.len()).expect("test payload size fits i64");
|
||||
assert_eq!(info.size, size);
|
||||
assert_eq!(info.actual_size, size);
|
||||
assert_eq!(info.etag.as_deref(), Some("config-1"));
|
||||
assert_eq!(info.data_dir, Some(uuid::Uuid::from_u128(1)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_config_preserve_empty_metadata_preserves_read_errors() {
|
||||
let missing = Arc::new(RecoveryMockStore::new(RecoveryReadState::Missing, None));
|
||||
let err = read_config_preserve_empty_with_metadata_opts(missing, "config/missing.json", &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("physical absence must remain ConfigNotFound");
|
||||
assert!(matches!(err, Error::ConfigNotFound));
|
||||
let quorum = Arc::new(RecoveryMockStore::new(RecoveryReadState::QuorumError, None));
|
||||
let err = read_config_preserve_empty_with_metadata_opts(quorum, "config/quorum.json", &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("a quorum failure must retain its typed read error");
|
||||
assert!(matches!(err, Error::ErasureReadQuorum));
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl StorageAdminApi for RecoveryMockStore {
|
||||
type BackendInfo = rustfs_madmin::BackendInfo;
|
||||
|
||||
@@ -738,7 +738,12 @@ where
|
||||
.filter(|etag| !etag.is_empty())
|
||||
.map(str::to_owned)
|
||||
.ok_or_else(|| Error::other("data usage snapshot has no ETag"))?;
|
||||
let mut data_usage_info = parse_usage_snapshot(&reader.read_all().await?)?;
|
||||
let mut data_usage_info = parse_usage_snapshot(
|
||||
&reader
|
||||
.read_all()
|
||||
.await
|
||||
.map_err(|err| map_data_usage_metadata_read_error(err, object))?,
|
||||
)?;
|
||||
populate_backward_compatible_usage_maps(&mut data_usage_info);
|
||||
validate_complete_usage_snapshot(&mut data_usage_info);
|
||||
Ok(Some((data_usage_info, revision)))
|
||||
@@ -2997,6 +3002,61 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct UsageSnapshotReadFailureStore;
|
||||
|
||||
struct UsageSnapshotFailureReader;
|
||||
|
||||
impl tokio::io::AsyncRead for UsageSnapshotFailureReader {
|
||||
fn poll_read(
|
||||
self: std::pin::Pin<&mut Self>,
|
||||
_cx: &mut std::task::Context<'_>,
|
||||
_buf: &mut tokio::io::ReadBuf<'_>,
|
||||
) -> std::task::Poll<std::io::Result<()>> {
|
||||
std::task::Poll::Ready(Err(std::io::Error::other(Error::VolumeNotFound)))
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::storage_api_contracts::object::ObjectIO for UsageSnapshotReadFailureStore {
|
||||
type Error = Error;
|
||||
type RangeSpec = crate::storage_api_contracts::range::HTTPRangeSpec;
|
||||
type HeaderMap = http::HeaderMap;
|
||||
type ObjectOptions = ObjectOptions;
|
||||
type ObjectInfo = ObjectInfo;
|
||||
type GetObjectReader = crate::object_api::GetObjectReader;
|
||||
type PutObjectReader = PutObjReader;
|
||||
|
||||
async fn get_object_reader(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_object: &str,
|
||||
_range: Option<Self::RangeSpec>,
|
||||
_headers: Self::HeaderMap,
|
||||
_opts: &Self::ObjectOptions,
|
||||
) -> Result<Self::GetObjectReader, Self::Error> {
|
||||
Ok(crate::object_api::GetObjectReader {
|
||||
stream: Box::new(UsageSnapshotFailureReader),
|
||||
object_info: ObjectInfo {
|
||||
etag: Some("usage-revision".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
buffered_body: None,
|
||||
body_source: Default::default(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn put_object(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_object: &str,
|
||||
_data: &mut Self::PutObjectReader,
|
||||
_opts: &Self::ObjectOptions,
|
||||
) -> Result<Self::ObjectInfo, Self::Error> {
|
||||
unimplemented!("the read-failure fixture does not write")
|
||||
}
|
||||
}
|
||||
|
||||
/// Minimal ObjectIO backing `load_data_usage_cache` tests: records the keys
|
||||
/// read and fails the first N reads with a transient (non-absence) error.
|
||||
#[derive(Debug, Default)]
|
||||
@@ -3161,6 +3221,20 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn data_usage_removal_maps_missing_system_volume_during_snapshot_stream_read() {
|
||||
let store = UsageSnapshotReadFailureStore;
|
||||
|
||||
let error = load_data_usage_for_bucket_removal(&store, "bucket-metadata/.usage.json")
|
||||
.await
|
||||
.expect_err("a system volume disappearing during body read must remain a retryable storage error");
|
||||
|
||||
assert_eq!(
|
||||
error,
|
||||
Error::InsufficientReadQuorum(RUSTFS_META_BUCKET.to_string(), "bucket-metadata/.usage.json".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn load_data_usage_cache_retries_a_transient_failure() {
|
||||
let name = "usage-cache";
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use rustfs_io_metrics::record_object_lock_acquire_failure;
|
||||
use rustfs_lock::{LockError, NamespaceLockGuard, NamespaceLockWrapper};
|
||||
use std::time::{Duration, Instant};
|
||||
use tracing::Span;
|
||||
@@ -21,6 +22,26 @@ const EVENT_OBJECT_NAMESPACE_LOCK: &str = "object_namespace_lock";
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_OBJECT_LOCK: &str = "object_lock";
|
||||
|
||||
pub(crate) fn record_lock_acquire_failure(op: &'static str, bucket: &str, object: &str, mode: &'static str, error: &LockError) {
|
||||
let resource_class = if bucket == crate::disk::RUSTFS_META_MULTIPART_BUCKET {
|
||||
"multipart_metadata"
|
||||
} else if bucket == crate::disk::RUSTFS_META_BUCKET || bucket == ".minio.sys" {
|
||||
"system_metadata"
|
||||
} else if object.is_empty() || object == bucket {
|
||||
"bucket_namespace"
|
||||
} else {
|
||||
"object_namespace"
|
||||
};
|
||||
let reason = match error {
|
||||
LockError::Timeout { .. } => "timeout",
|
||||
LockError::AlreadyLocked { .. } => "contention",
|
||||
LockError::QuorumNotReached { .. } | LockError::InsufficientNodes { .. } => "quorum",
|
||||
LockError::Network { .. } | LockError::Internal { .. } => "transport",
|
||||
_ => "other",
|
||||
};
|
||||
record_object_lock_acquire_failure(op, mode, resource_class, reason);
|
||||
}
|
||||
|
||||
/// Diagnostic lifetime only; it never owns or changes a namespace lock.
|
||||
/// Keep it alongside the real guard when ownership moves to a commit tail.
|
||||
#[derive(Default)]
|
||||
|
||||
@@ -12,6 +12,10 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::disk::io_admission::{
|
||||
AdmissionReader, BoxedChunkReader, DiskIoAdmission, DiskIoClass, DiskIoContext, current_disk_io_context,
|
||||
with_disk_io_context, with_disk_io_permit,
|
||||
};
|
||||
use crate::disk::{
|
||||
CheckPartsResp, DataDirDeleteStatus, DeleteOptions, DiskAPI, DiskError, DiskInfo, DiskInfoOptions, DiskLocation, Endpoint,
|
||||
Error, FileInfoVersions, MmapCopyStageMetrics, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, Result,
|
||||
@@ -35,13 +39,15 @@ use std::sync::atomic::AtomicBool;
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
path::PathBuf,
|
||||
pin::Pin,
|
||||
sync::{
|
||||
Arc, LazyLock, RwLock as StdRwLock,
|
||||
atomic::{AtomicI64, AtomicU32, AtomicU64, Ordering},
|
||||
},
|
||||
task::{Context, Poll},
|
||||
time::{Duration, Instant, SystemTime, UNIX_EPOCH},
|
||||
};
|
||||
use tokio::{sync::RwLock, time};
|
||||
use tokio::{io::AsyncWrite, sync::RwLock, time};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{info, warn};
|
||||
use uuid::Uuid;
|
||||
@@ -54,6 +60,121 @@ const LOG_SUBSYSTEM_DISK: &str = "disk";
|
||||
const EVENT_DISK_HEALTH_CHECK_FAILED: &str = "disk_health_check_failed";
|
||||
const EVENT_DISK_RECOVERY_PROBE_STATE: &str = "disk_recovery_probe_state";
|
||||
const EVENT_DISK_TIMEOUT_POLICY_FALLBACK: &str = "disk_timeout_policy_fallback";
|
||||
const EVENT_DISK_OPERATION_SLOW: &str = "disk_operation_slow";
|
||||
const SLOW_DISK_OPERATION_THRESHOLD: Duration = Duration::from_secs(5);
|
||||
|
||||
fn warn_slow_disk_operation(endpoint: &Endpoint, op: &'static str, elapsed: Duration) {
|
||||
warn!(
|
||||
event = EVENT_DISK_OPERATION_SLOW,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DISK,
|
||||
state = "pending",
|
||||
endpoint = %endpoint,
|
||||
op,
|
||||
elapsed_ms = elapsed.as_millis(),
|
||||
threshold_ms = SLOW_DISK_OPERATION_THRESHOLD.as_millis(),
|
||||
"Disk operation is still pending"
|
||||
);
|
||||
}
|
||||
|
||||
async fn observe_disk_operation<F: std::future::Future>(
|
||||
endpoint: impl FnOnce() -> Endpoint,
|
||||
op: &'static str,
|
||||
operation: F,
|
||||
) -> F::Output {
|
||||
if !tracing::enabled!(tracing::Level::WARN) {
|
||||
return operation.await;
|
||||
}
|
||||
let started = time::Instant::now();
|
||||
tokio::pin!(operation);
|
||||
tokio::select! {
|
||||
biased;
|
||||
result = &mut operation => result,
|
||||
_ = time::sleep(SLOW_DISK_OPERATION_THRESHOLD) => {
|
||||
warn_slow_disk_operation(&endpoint(), op, started.elapsed());
|
||||
operation.await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct SlowDiskWriter {
|
||||
inner: crate::disk::FileWriter,
|
||||
endpoint: Endpoint,
|
||||
pending: Option<(&'static str, time::Instant)>,
|
||||
timer: Option<Pin<Box<time::Sleep>>>,
|
||||
warned: bool,
|
||||
}
|
||||
|
||||
impl SlowDiskWriter {
|
||||
fn new(inner: crate::disk::FileWriter, endpoint: Endpoint) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
endpoint,
|
||||
pending: None,
|
||||
timer: None,
|
||||
warned: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn observe<T>(&mut self, cx: &mut Context<'_>, op: &'static str, result: Poll<T>) -> Poll<T> {
|
||||
if result.is_ready() {
|
||||
self.pending = None;
|
||||
self.warned = false;
|
||||
} else if tracing::enabled!(tracing::Level::WARN) {
|
||||
if self.pending.is_none_or(|(pending_op, _)| pending_op != op) {
|
||||
let started = time::Instant::now();
|
||||
self.pending = Some((op, started));
|
||||
self.warned = false;
|
||||
let deadline = started + SLOW_DISK_OPERATION_THRESHOLD;
|
||||
// Reuse one timer per stream; ready writes need no timer allocation.
|
||||
if let Some(timer) = &mut self.timer {
|
||||
timer.as_mut().reset(deadline);
|
||||
} else {
|
||||
self.timer = Some(Box::pin(time::sleep_until(deadline)));
|
||||
}
|
||||
}
|
||||
if !self.warned
|
||||
&& let Some(timer) = &mut self.timer
|
||||
&& std::future::Future::poll(timer.as_mut(), cx).is_ready()
|
||||
&& let Some((_, started)) = self.pending
|
||||
{
|
||||
warn_slow_disk_operation(&self.endpoint, op, started.elapsed());
|
||||
self.warned = true;
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncWrite for SlowDiskWriter {
|
||||
fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8]) -> Poll<std::io::Result<usize>> {
|
||||
let result = Pin::new(&mut self.inner).poll_write(cx, buf);
|
||||
self.observe(cx, "write", result)
|
||||
}
|
||||
|
||||
fn poll_write_vectored(
|
||||
mut self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
bufs: &[std::io::IoSlice<'_>],
|
||||
) -> Poll<std::io::Result<usize>> {
|
||||
let result = Pin::new(&mut self.inner).poll_write_vectored(cx, bufs);
|
||||
self.observe(cx, "write_vectored", result)
|
||||
}
|
||||
|
||||
fn is_write_vectored(&self) -> bool {
|
||||
self.inner.is_write_vectored()
|
||||
}
|
||||
|
||||
fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
|
||||
let result = Pin::new(&mut self.inner).poll_flush(cx);
|
||||
self.observe(cx, "flush", result)
|
||||
}
|
||||
|
||||
fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
|
||||
let result = Pin::new(&mut self.inner).poll_shutdown(cx);
|
||||
self.observe(cx, "shutdown", result)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
enum TimeoutHealthAction {
|
||||
@@ -813,6 +934,7 @@ impl Drop for DiskMetricWaitingGuard<'_> {
|
||||
pub(crate) struct ReconnectDiskHealthState {
|
||||
pub(crate) health: Arc<DiskHealthTracker>,
|
||||
pub(crate) metrics: Arc<DiskHealthMetricEpoch>,
|
||||
pub(crate) io_admission: Arc<DiskIoAdmission>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
@@ -1329,6 +1451,8 @@ pub struct LocalDiskWrapper {
|
||||
health: Arc<DiskHealthTracker>,
|
||||
/// Internal metrics epoch preserved across local disk reconnects.
|
||||
metrics: Arc<DiskHealthMetricEpoch>,
|
||||
/// Per-drive foreground/background admission shared across wrapper reconnects.
|
||||
io_admission: Arc<DiskIoAdmission>,
|
||||
/// Whether health checking is enabled
|
||||
health_check: bool,
|
||||
/// Cancellation token for monitoring tasks
|
||||
@@ -1342,11 +1466,13 @@ pub struct LocalDiskWrapper {
|
||||
impl LocalDiskWrapper {
|
||||
/// Create a new LocalDiskWrapper
|
||||
pub fn new(disk: Arc<LocalDisk>, health_check: bool) -> Self {
|
||||
let admission = DiskIoAdmission::new(disk.endpoint().to_string());
|
||||
Self::new_with_health_and_metrics(
|
||||
disk,
|
||||
health_check,
|
||||
Arc::new(DiskHealthTracker::new()),
|
||||
Arc::new(DiskHealthMetricEpoch::default()),
|
||||
admission,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1386,11 +1512,13 @@ impl LocalDiskWrapper {
|
||||
health_check: bool,
|
||||
reconnect: Option<ReconnectDiskHealthState>,
|
||||
) -> Self {
|
||||
let endpoint = disk.endpoint().to_string();
|
||||
let reconnect = reconnect.unwrap_or_else(|| ReconnectDiskHealthState {
|
||||
health: Arc::new(DiskHealthTracker::new()),
|
||||
metrics: Arc::new(DiskHealthMetricEpoch::default()),
|
||||
io_admission: DiskIoAdmission::new(endpoint),
|
||||
});
|
||||
Self::new_with_health_and_metrics(disk, health_check, reconnect.health, reconnect.metrics)
|
||||
Self::new_with_health_and_metrics(disk, health_check, reconnect.health, reconnect.metrics, reconnect.io_admission)
|
||||
}
|
||||
|
||||
fn new_with_health_and_metrics(
|
||||
@@ -1398,6 +1526,7 @@ impl LocalDiskWrapper {
|
||||
health_check: bool,
|
||||
health: Arc<DiskHealthTracker>,
|
||||
metrics: Arc<DiskHealthMetricEpoch>,
|
||||
io_admission: Arc<DiskIoAdmission>,
|
||||
) -> Self {
|
||||
// Check environment variable for health check override.
|
||||
// Only enable if both param and env are true.
|
||||
@@ -1408,6 +1537,7 @@ impl LocalDiskWrapper {
|
||||
disk,
|
||||
health,
|
||||
metrics,
|
||||
io_admission,
|
||||
health_check: health_check && env_health_check,
|
||||
cancel_token: CancellationToken::new(),
|
||||
disk_id: Arc::new(RwLock::new(None)),
|
||||
@@ -1421,6 +1551,7 @@ impl LocalDiskWrapper {
|
||||
ReconnectDiskHealthState {
|
||||
health: Arc::new(self.health.metric_epoch_for_reconnect()),
|
||||
metrics: self.metrics.clone(),
|
||||
io_admission: self.io_admission.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1428,6 +1559,58 @@ impl LocalDiskWrapper {
|
||||
self.disk.clone()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn background_io_admissions_for_tests(&self) -> usize {
|
||||
self.io_admission.background_admissions_for_tests()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn background_io_active_for_tests(&self) -> bool {
|
||||
self.io_admission.background_active_for_tests()
|
||||
}
|
||||
|
||||
fn disk_io_context(&self) -> DiskIoContext {
|
||||
current_disk_io_context().unwrap_or_else(|| DiskIoContext {
|
||||
class: DiskIoClass::Foreground,
|
||||
cancellation: CancellationToken::new(),
|
||||
admission: None,
|
||||
permit: None,
|
||||
})
|
||||
}
|
||||
|
||||
fn disk_walk_io_context(&self) -> DiskIoContext {
|
||||
let mut context = self.disk_io_context();
|
||||
context.admission = Some(self.io_admission.clone());
|
||||
context.permit = None;
|
||||
context
|
||||
}
|
||||
|
||||
async fn acquire_disk_io_permit(&self) -> Result<Option<crate::disk::io_admission::DiskIoPermit>> {
|
||||
if let Some(context) = current_disk_io_context() {
|
||||
if let Some(permit) = context.permit
|
||||
&& permit.is_for(&self.io_admission)
|
||||
{
|
||||
return Ok(Some(permit));
|
||||
}
|
||||
if context
|
||||
.admission
|
||||
.as_ref()
|
||||
.is_some_and(|admission| Arc::ptr_eq(admission, &self.io_admission))
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
if context.class == DiskIoClass::Background {
|
||||
return self
|
||||
.io_admission
|
||||
.background(&context.cancellation)
|
||||
.await
|
||||
.map(Some)
|
||||
.map_err(DiskError::Io);
|
||||
}
|
||||
}
|
||||
Ok(Some(self.io_admission.foreground()))
|
||||
}
|
||||
|
||||
pub fn get_object_path_if_local(&self, volume: &str, path: &str) -> crate::disk::error::Result<std::path::PathBuf> {
|
||||
self.disk.get_object_path(volume, path)
|
||||
}
|
||||
@@ -1899,6 +2082,31 @@ impl LocalDiskWrapper {
|
||||
where
|
||||
F: FnOnce() -> Fut,
|
||||
Fut: std::future::Future<Output = Result<T>>,
|
||||
{
|
||||
self.track_disk_health_with_op_timeout_action_and_mutation_with_io_permit(
|
||||
op,
|
||||
move |io_permit| async move {
|
||||
let _io_permit = io_permit;
|
||||
operation().await
|
||||
},
|
||||
timeout_duration,
|
||||
timeout_health_action,
|
||||
mutation,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn track_disk_health_with_op_timeout_action_and_mutation_with_io_permit<T, F, Fut>(
|
||||
&self,
|
||||
op: &'static str,
|
||||
operation: F,
|
||||
timeout_duration: Duration,
|
||||
timeout_health_action: TimeoutHealthAction,
|
||||
mutation: DiskMetricMutation,
|
||||
) -> Result<T>
|
||||
where
|
||||
F: FnOnce(Option<crate::disk::io_admission::DiskIoPermit>) -> Fut,
|
||||
Fut: std::future::Future<Output = Result<T>>,
|
||||
{
|
||||
self.metrics.record_operation_call(op);
|
||||
// Check if disk is faulty
|
||||
@@ -1921,14 +2129,17 @@ impl LocalDiskWrapper {
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
let io_permit = self.acquire_disk_io_permit().await?;
|
||||
|
||||
// Record operation start
|
||||
self.health.last_started.store(current_unix_nanos(), Ordering::Relaxed);
|
||||
let _waiting_guard = self.health.waiting_guard();
|
||||
let _metric_waiting_guard = self.metrics.waiting_guard();
|
||||
let started = Instant::now();
|
||||
let operation = observe_disk_operation(|| self.endpoint(), op, operation(io_permit));
|
||||
|
||||
if timeout_duration == Duration::ZERO {
|
||||
let result = operation().await;
|
||||
let result = operation.await;
|
||||
self.metrics.record_operation_latency(op, started.elapsed());
|
||||
self.record_result_error_metrics(&result);
|
||||
if result.is_ok() {
|
||||
@@ -1938,7 +2149,7 @@ impl LocalDiskWrapper {
|
||||
return result;
|
||||
}
|
||||
// Execute the operation with timeout
|
||||
let result = tokio::time::timeout(timeout_duration, operation()).await;
|
||||
let result = tokio::time::timeout(timeout_duration, operation).await;
|
||||
|
||||
match result {
|
||||
Ok(operation_result) => {
|
||||
@@ -1989,11 +2200,12 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
}
|
||||
|
||||
async fn read_metadata(&self, volume: &str, path: &str) -> Result<Bytes> {
|
||||
self.track_disk_health_with_op_and_timeout_action(
|
||||
self.track_disk_health_with_op_timeout_action_and_mutation_with_io_permit(
|
||||
"read_metadata",
|
||||
|| async { self.disk.read_metadata(volume, path).await },
|
||||
|io_permit| async { self.disk.read_metadata_with_permit(volume, path, io_permit).await },
|
||||
get_drive_metadata_timeout(),
|
||||
self.scanner_timeout_health_action(),
|
||||
DiskMetricMutation::None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -2165,12 +2377,15 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
opts.timeout_duration().unwrap_or_else(get_drive_walkdir_timeout)
|
||||
};
|
||||
|
||||
self.track_disk_health_with_op_and_timeout_action(
|
||||
"walk_dir",
|
||||
|| async { self.disk.walk_dir(opts, wr).await },
|
||||
timeout_duration,
|
||||
// Listing/scanner backpressure should fail only the current walk, not poison drive health.
|
||||
TimeoutHealthAction::IgnoreFailure,
|
||||
with_disk_io_context(
|
||||
self.disk_walk_io_context(),
|
||||
self.track_disk_health_with_op_and_timeout_action(
|
||||
"walk_dir",
|
||||
|| async { self.disk.walk_dir(opts, wr).await },
|
||||
timeout_duration,
|
||||
// Listing/scanner backpressure should fail only the current walk, not poison drive health.
|
||||
TimeoutHealthAction::IgnoreFailure,
|
||||
),
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -2193,6 +2408,7 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
}
|
||||
|
||||
async fn delete_versions(&self, volume: &str, versions: Vec<FileInfoVersions>, opts: DeleteOptions) -> Vec<Option<Error>> {
|
||||
let version_count = versions.len();
|
||||
self.metrics.record_operation_call("delete_versions");
|
||||
// Check if disk is faulty before proceeding
|
||||
if self.health.is_faulty() {
|
||||
@@ -2206,9 +2422,14 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
return vec![Some(e); versions.len()];
|
||||
}
|
||||
|
||||
let _io_permit = match self.acquire_disk_io_permit().await {
|
||||
Ok(permit) => permit,
|
||||
Err(error) => return vec![Some(error); version_count],
|
||||
};
|
||||
|
||||
// Record operation start
|
||||
self.health.last_started.store(current_unix_nanos(), Ordering::Relaxed);
|
||||
self.health.increment_waiting();
|
||||
let waiting_guard = self.health.waiting_guard();
|
||||
let metric_waiting_guard = self.metrics.waiting_guard();
|
||||
let started = Instant::now();
|
||||
|
||||
@@ -2217,7 +2438,7 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
self.metrics.record_operation_latency("delete_versions", started.elapsed());
|
||||
self.record_batch_delete_error_metrics(&result);
|
||||
|
||||
self.health.decrement_waiting();
|
||||
drop(waiting_guard);
|
||||
drop(metric_waiting_guard);
|
||||
let has_err = result.iter().any(|e| e.is_some());
|
||||
if !has_err {
|
||||
@@ -2322,19 +2543,25 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
version_id: &str,
|
||||
opts: &ReadOptions,
|
||||
) -> Result<FileInfo> {
|
||||
self.track_disk_health_with_op(
|
||||
self.track_disk_health_with_op_timeout_action_and_mutation_with_io_permit(
|
||||
"read_version",
|
||||
|| async { self.disk.read_version(org_volume, volume, path, version_id, opts).await },
|
||||
|io_permit| async move {
|
||||
with_disk_io_permit(io_permit, self.disk.read_version(org_volume, volume, path, version_id, opts)).await
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
TimeoutHealthAction::MarkFailure,
|
||||
DiskMetricMutation::None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo> {
|
||||
self.track_disk_health_with_op(
|
||||
self.track_disk_health_with_op_timeout_action_and_mutation_with_io_permit(
|
||||
"read_xl",
|
||||
|| async { self.disk.read_xl(volume, path, read_data).await },
|
||||
|io_permit| async move { with_disk_io_permit(io_permit, self.disk.read_xl(volume, path, read_data)).await },
|
||||
get_max_timeout_duration(),
|
||||
TimeoutHealthAction::MarkFailure,
|
||||
DiskMetricMutation::None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -2362,21 +2589,25 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
}
|
||||
|
||||
async fn read_file(&self, volume: &str, path: &str) -> Result<crate::disk::FileReader> {
|
||||
self.track_disk_health_with_op(
|
||||
"read_file",
|
||||
|| async { self.disk.read_file(volume, path).await },
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
let reader = self
|
||||
.track_disk_health_with_op(
|
||||
"read_file",
|
||||
|| async { self.disk.read_file(volume, path).await },
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await?;
|
||||
Ok(Box::new(AdmissionReader::new(reader, self.io_admission.clone(), self.disk_io_context())))
|
||||
}
|
||||
|
||||
async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<crate::disk::FileReader> {
|
||||
self.track_disk_health_with_op(
|
||||
"read_file_stream",
|
||||
|| async { self.disk.read_file_stream(volume, path, offset, length).await },
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
let reader = self
|
||||
.track_disk_health_with_op(
|
||||
"read_file_stream",
|
||||
|| async { self.disk.read_file_stream(volume, path, offset, length).await },
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await?;
|
||||
Ok(Box::new(AdmissionReader::new(reader, self.io_admission.clone(), self.disk_io_context())))
|
||||
}
|
||||
|
||||
async fn read_file_stream_chunks(
|
||||
@@ -2386,12 +2617,20 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
offset: usize,
|
||||
length: usize,
|
||||
) -> Result<Option<rustfs_rio::ChunkReaderBox>> {
|
||||
self.track_disk_health_with_op(
|
||||
"read_file_stream_chunks",
|
||||
|| async { self.disk.read_file_stream_chunks(volume, path, offset, length).await },
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
let reader = self
|
||||
.track_disk_health_with_op(
|
||||
"read_file_stream_chunks",
|
||||
|| async { self.disk.read_file_stream_chunks(volume, path, offset, length).await },
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await?;
|
||||
Ok(reader.map(|reader| {
|
||||
Box::new(AdmissionReader::new(
|
||||
BoxedChunkReader(reader),
|
||||
self.io_admission.clone(),
|
||||
self.disk_io_context(),
|
||||
)) as rustfs_rio::ChunkReaderBox
|
||||
}))
|
||||
}
|
||||
|
||||
async fn read_file_mmap_copy(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<bytes::Bytes> {
|
||||
@@ -2424,18 +2663,22 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
}
|
||||
|
||||
async fn append_file(&self, volume: &str, path: &str) -> Result<crate::disk::FileWriter> {
|
||||
self.track_disk_health_with_op("append_file", || async { self.disk.append_file(volume, path).await }, Duration::ZERO)
|
||||
.await
|
||||
let writer = self
|
||||
.track_disk_health_with_op("append_file", || async { self.disk.append_file(volume, path).await }, Duration::ZERO)
|
||||
.await?;
|
||||
Ok(Box::new(SlowDiskWriter::new(writer, self.endpoint())))
|
||||
}
|
||||
|
||||
async fn create_file(&self, origvolume: &str, volume: &str, path: &str, file_size: i64) -> Result<crate::disk::FileWriter> {
|
||||
self.track_disk_health_mutation(
|
||||
"create_file",
|
||||
DiskMetricMutation::Write,
|
||||
|| async { self.disk.create_file(origvolume, volume, path, file_size).await },
|
||||
Duration::ZERO,
|
||||
)
|
||||
.await
|
||||
let writer = self
|
||||
.track_disk_health_mutation(
|
||||
"create_file",
|
||||
DiskMetricMutation::Write,
|
||||
|| async { self.disk.create_file(origvolume, volume, path, file_size).await },
|
||||
Duration::ZERO,
|
||||
)
|
||||
.await?;
|
||||
Ok(Box::new(SlowDiskWriter::new(writer, self.endpoint())))
|
||||
}
|
||||
|
||||
async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> {
|
||||
@@ -2524,8 +2767,14 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
}
|
||||
|
||||
async fn read_parts(&self, bucket: &str, paths: &[String]) -> Result<Vec<ObjectPartInfo>> {
|
||||
self.track_disk_health_with_op("read_parts", || async { self.disk.read_parts(bucket, paths).await }, Duration::ZERO)
|
||||
.await
|
||||
self.track_disk_health_with_op_timeout_action_and_mutation_with_io_permit(
|
||||
"read_parts",
|
||||
|io_permit| async move { with_disk_io_permit(io_permit, self.disk.read_parts(bucket, paths)).await },
|
||||
Duration::ZERO,
|
||||
TimeoutHealthAction::MarkFailure,
|
||||
DiskMetricMutation::None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn read_multiple(&self, req: ReadMultipleReq) -> Result<Vec<ReadMultipleResp>> {
|
||||
@@ -2582,7 +2831,268 @@ mod tests {
|
||||
pin::Pin,
|
||||
task::{Context, Poll},
|
||||
};
|
||||
use tokio::io::AsyncWrite;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
use tracing_subscriber::fmt::MakeWriter;
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct SlowOperationLogs(Arc<std::sync::Mutex<Vec<u8>>>);
|
||||
|
||||
impl SlowOperationLogs {
|
||||
fn contents(&self) -> String {
|
||||
String::from_utf8(self.0.lock().expect("log buffer should not be poisoned").clone()).expect("logs should be UTF-8")
|
||||
}
|
||||
}
|
||||
|
||||
impl io::Write for SlowOperationLogs {
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
self.0
|
||||
.lock()
|
||||
.expect("log buffer should not be poisoned")
|
||||
.extend_from_slice(buf);
|
||||
Ok(buf.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> MakeWriter<'a> for SlowOperationLogs {
|
||||
type Writer = Self;
|
||||
|
||||
fn make_writer(&'a self) -> Self::Writer {
|
||||
self.clone()
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn slow_disk_operation_warns_once_while_pending_and_preserves_result() {
|
||||
let dir = tempfile::tempdir().expect("temp dir should be created");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("path should be UTF-8")).expect("endpoint should parse");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created"));
|
||||
let wrapper = LocalDiskWrapper::new(disk, false);
|
||||
let logs = SlowOperationLogs::default();
|
||||
let subscriber = tracing_subscriber::fmt()
|
||||
.with_writer(logs.clone())
|
||||
.with_ansi(false)
|
||||
.without_time()
|
||||
.finish();
|
||||
let _subscriber = tracing::subscriber::set_default(subscriber);
|
||||
let _interest = crate::test_tracing::pin_callsite_interest_for_test();
|
||||
let (release, wait) = tokio::sync::oneshot::channel();
|
||||
let operation = wrapper.track_disk_health_with_op(
|
||||
"rename_data",
|
||||
|| async { wait.await.expect("operation should be released") },
|
||||
Duration::ZERO,
|
||||
);
|
||||
tokio::pin!(operation);
|
||||
assert!(futures::poll!(&mut operation).is_pending());
|
||||
time::advance(SLOW_DISK_OPERATION_THRESHOLD - Duration::from_millis(1)).await;
|
||||
assert!(futures::poll!(&mut operation).is_pending());
|
||||
assert!(logs.contents().is_empty(), "normal disk latency should not warn");
|
||||
time::advance(Duration::from_millis(1)).await;
|
||||
assert!(futures::poll!(&mut operation).is_pending());
|
||||
let warning = logs.contents();
|
||||
assert!(warning.contains("WARN"));
|
||||
assert!(warning.contains("event=\"disk_operation_slow\""));
|
||||
assert!(warning.contains("state=\"pending\""));
|
||||
assert!(warning.contains(&format!("endpoint={endpoint}")));
|
||||
assert!(warning.contains("op=\"rename_data\""));
|
||||
assert!(warning.contains("elapsed_ms=5000"));
|
||||
time::advance(Duration::from_secs(120)).await;
|
||||
assert!(futures::poll!(&mut operation).is_pending(), "warning must not cancel the disk operation");
|
||||
assert_eq!(logs.contents(), warning, "a pending operation should warn only once");
|
||||
release
|
||||
.send(Err::<(), _>(DiskError::FileAccessDenied))
|
||||
.expect("operation should still be alive");
|
||||
assert_eq!(operation.await, Err(DiskError::FileAccessDenied));
|
||||
assert_eq!(wrapper.runtime_state(), RuntimeDriveHealthState::Online);
|
||||
assert_eq!(wrapper.metrics_snapshot().total_waiting, 0);
|
||||
|
||||
let timed = wrapper.track_disk_health_with_op(
|
||||
"write_all",
|
||||
std::future::pending::<Result<()>>,
|
||||
SLOW_DISK_OPERATION_THRESHOLD + Duration::from_secs(1),
|
||||
);
|
||||
tokio::pin!(timed);
|
||||
assert!(futures::poll!(&mut timed).is_pending());
|
||||
time::advance(SLOW_DISK_OPERATION_THRESHOLD).await;
|
||||
assert!(futures::poll!(&mut timed).is_pending());
|
||||
assert_eq!(logs.contents().matches(EVENT_DISK_OPERATION_SLOW).count(), 2);
|
||||
assert_eq!(wrapper.runtime_state(), RuntimeDriveHealthState::Online);
|
||||
time::advance(Duration::from_secs(1)).await;
|
||||
assert_eq!(
|
||||
timed.await,
|
||||
Err(DiskError::Timeout),
|
||||
"existing timeout must still apply after the warning"
|
||||
);
|
||||
assert_eq!(wrapper.metrics_snapshot().total_errors_timeout, 1);
|
||||
assert_eq!(wrapper.metrics_snapshot().total_waiting, 0);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn slow_disk_operation_fast_path_is_silent_and_does_not_resolve_endpoint() {
|
||||
let logs = SlowOperationLogs::default();
|
||||
let subscriber = tracing_subscriber::fmt()
|
||||
.with_writer(logs.clone())
|
||||
.with_ansi(false)
|
||||
.without_time()
|
||||
.finish();
|
||||
let _subscriber = tracing::subscriber::set_default(subscriber);
|
||||
let _interest = crate::test_tracing::pin_callsite_interest_for_test();
|
||||
for delay in [Duration::ZERO, SLOW_DISK_OPERATION_THRESHOLD] {
|
||||
let result =
|
||||
observe_disk_operation(|| panic!("completed operation should not clone its endpoint"), "write_all", async {
|
||||
if !delay.is_zero() {
|
||||
time::sleep(delay).await;
|
||||
}
|
||||
Ok::<_, DiskError>(17)
|
||||
})
|
||||
.await;
|
||||
assert_eq!(result, Ok(17));
|
||||
}
|
||||
assert!(logs.contents().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn slow_disk_diagnostics_are_bypassed_when_warnings_are_disabled() {
|
||||
let _subscriber = tracing::subscriber::set_default(tracing::subscriber::NoSubscriber::default());
|
||||
let endpoint = Endpoint::try_from("/slow-disk").expect("endpoint should parse");
|
||||
let mut writer = SlowDiskWriter::new(Box::new(PendingWriter), endpoint);
|
||||
{
|
||||
let write = writer.write(b"bytes");
|
||||
tokio::pin!(write);
|
||||
assert!(futures::poll!(&mut write).is_pending());
|
||||
}
|
||||
assert!(writer.timer.is_none(), "disabled warnings should not allocate a stream timer");
|
||||
let result = observe_disk_operation(|| panic!("disabled warnings should not resolve endpoints"), "write_all", async {
|
||||
time::sleep(SLOW_DISK_OPERATION_THRESHOLD * 2).await;
|
||||
17
|
||||
})
|
||||
.await;
|
||||
assert_eq!(result, 17);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn slow_disk_writer_preserves_vectored_bytes_and_warns_once_while_blocked() {
|
||||
let endpoint = Endpoint::try_from("/slow-disk").expect("endpoint should parse");
|
||||
let logs = SlowOperationLogs::default();
|
||||
let subscriber = tracing_subscriber::fmt()
|
||||
.with_writer(logs.clone())
|
||||
.with_ansi(false)
|
||||
.without_time()
|
||||
.finish();
|
||||
let _subscriber = tracing::subscriber::set_default(subscriber);
|
||||
let _interest = crate::test_tracing::pin_callsite_interest_for_test();
|
||||
let (stream, mut receiver) = tokio::io::duplex(1);
|
||||
let mut writer = SlowDiskWriter::new(Box::new(stream), endpoint.clone());
|
||||
writer.write_all(b"a").await.expect("first byte should fit in the buffer");
|
||||
assert!(writer.timer.is_none(), "ready writes should not allocate a timer");
|
||||
assert!(logs.contents().is_empty());
|
||||
let mut received = [0; 1];
|
||||
{
|
||||
let bufs = [io::IoSlice::new(b"b"), io::IoSlice::new(b"c")];
|
||||
let write = writer.write_vectored(&bufs);
|
||||
tokio::pin!(write);
|
||||
assert!(futures::poll!(&mut write).is_pending());
|
||||
time::advance(SLOW_DISK_OPERATION_THRESHOLD).await;
|
||||
assert!(futures::poll!(&mut write).is_pending());
|
||||
assert!(logs.contents().contains("op=\"write_vectored\""));
|
||||
time::advance(SLOW_DISK_OPERATION_THRESHOLD).await;
|
||||
assert!(futures::poll!(&mut write).is_pending());
|
||||
assert_eq!(logs.contents().matches(EVENT_DISK_OPERATION_SLOW).count(), 1);
|
||||
tokio::io::AsyncReadExt::read_exact(&mut receiver, &mut received)
|
||||
.await
|
||||
.expect("buffered byte should be readable");
|
||||
assert_eq!(received, *b"a");
|
||||
assert_eq!(write.await.expect("blocked write should finish when capacity returns"), 1);
|
||||
}
|
||||
tokio::io::AsyncReadExt::read_exact(&mut receiver, &mut received)
|
||||
.await
|
||||
.expect("written byte should be readable");
|
||||
assert_eq!(received, *b"b");
|
||||
|
||||
drop(receiver);
|
||||
let err = writer.write_all(b"d").await.expect_err("closed receiver should fail writes");
|
||||
assert_eq!(err.kind(), io::ErrorKind::BrokenPipe);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn slow_disk_writer_preserves_full_vectored_write_and_capability() {
|
||||
let endpoint = Endpoint::try_from("/slow-disk").expect("endpoint should parse");
|
||||
let (stream, mut receiver) = tokio::io::duplex(64);
|
||||
let vectored = stream.is_write_vectored();
|
||||
let mut writer = SlowDiskWriter::new(Box::new(stream), endpoint);
|
||||
assert_eq!(writer.is_write_vectored(), vectored);
|
||||
let bufs = [io::IoSlice::new(b"hash"), io::IoSlice::new(b"data")];
|
||||
assert_eq!(writer.write_vectored(&bufs).await.expect("vectored write should succeed"), 8);
|
||||
writer.shutdown().await.expect("shutdown should succeed");
|
||||
let mut received = Vec::new();
|
||||
tokio::io::AsyncReadExt::read_to_end(&mut receiver, &mut received)
|
||||
.await
|
||||
.expect("all bytes should be readable");
|
||||
assert_eq!(received, b"hashdata");
|
||||
assert!(writer.timer.is_none());
|
||||
}
|
||||
|
||||
struct StalledWriter(Arc<AtomicBool>);
|
||||
|
||||
impl AsyncWrite for StalledWriter {
|
||||
fn poll_write(self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &[u8]) -> Poll<io::Result<usize>> {
|
||||
if self.0.load(Ordering::Relaxed) {
|
||||
Poll::Ready(Ok(buf.len()))
|
||||
} else {
|
||||
Poll::Pending
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
if self.0.load(Ordering::Relaxed) {
|
||||
Poll::Ready(Ok(()))
|
||||
} else {
|
||||
Poll::Pending
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
self.poll_flush(cx)
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn slow_disk_writer_warns_for_write_flush_and_shutdown_and_rearms_after_progress() {
|
||||
let endpoint = Endpoint::try_from("/slow-disk").expect("endpoint should parse");
|
||||
let logs = SlowOperationLogs::default();
|
||||
let subscriber = tracing_subscriber::fmt()
|
||||
.with_writer(logs.clone())
|
||||
.with_ansi(false)
|
||||
.without_time()
|
||||
.finish();
|
||||
let _subscriber = tracing::subscriber::set_default(subscriber);
|
||||
let _interest = crate::test_tracing::pin_callsite_interest_for_test();
|
||||
let ready = Arc::new(AtomicBool::new(false));
|
||||
let mut writer = SlowDiskWriter::new(Box::new(StalledWriter(Arc::clone(&ready))), endpoint);
|
||||
for (index, op) in ["write", "flush", "shutdown"].into_iter().enumerate() {
|
||||
ready.store(false, Ordering::Relaxed);
|
||||
let operation = async {
|
||||
match op {
|
||||
"write" => writer.write_all(b"bytes").await,
|
||||
"flush" => writer.flush().await,
|
||||
"shutdown" => writer.shutdown().await,
|
||||
_ => unreachable!("only write operations are tested"),
|
||||
}
|
||||
};
|
||||
tokio::pin!(operation);
|
||||
assert!(futures::poll!(&mut operation).is_pending());
|
||||
time::advance(SLOW_DISK_OPERATION_THRESHOLD).await;
|
||||
assert!(futures::poll!(&mut operation).is_pending());
|
||||
assert!(logs.contents().contains(&format!("op=\"{op}\"")));
|
||||
assert_eq!(logs.contents().matches(EVENT_DISK_OPERATION_SLOW).count(), index + 1);
|
||||
ready.store(true, Ordering::Relaxed);
|
||||
operation.await.expect("warning should not prevent operation completion");
|
||||
}
|
||||
assert!(writer.pending.is_none());
|
||||
}
|
||||
|
||||
struct DropProbe(Arc<std::sync::atomic::AtomicUsize>);
|
||||
|
||||
@@ -2592,6 +3102,84 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
struct StalledMetacacheWriter {
|
||||
entered: Arc<tokio::sync::Notify>,
|
||||
}
|
||||
|
||||
impl AsyncWrite for StalledMetacacheWriter {
|
||||
fn poll_write(self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &[u8]) -> Poll<io::Result<usize>> {
|
||||
self.entered.notify_one();
|
||||
Poll::Pending
|
||||
}
|
||||
|
||||
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
|
||||
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_walk_releases_disk_admission_while_output_is_backpressured() {
|
||||
let root = tempfile::tempdir().expect("disk root should be created");
|
||||
let endpoint =
|
||||
Endpoint::try_from(root.path().to_str().expect("disk root should be UTF-8")).expect("endpoint should parse");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should open"));
|
||||
let wrapper = Arc::new(LocalDiskWrapper::new(disk, false));
|
||||
wrapper.make_volume("bucket").await.expect("bucket should be created");
|
||||
let object_path = root.path().join("bucket/object");
|
||||
tokio::fs::create_dir_all(&object_path)
|
||||
.await
|
||||
.expect("object directory should be created");
|
||||
tokio::fs::write(object_path.join("xl.meta"), b"scanner metadata")
|
||||
.await
|
||||
.expect("object metadata should be created");
|
||||
|
||||
let entered = Arc::new(tokio::sync::Notify::new());
|
||||
let writer = StalledMetacacheWriter {
|
||||
entered: entered.clone(),
|
||||
};
|
||||
let walk_wrapper = wrapper.clone();
|
||||
let walk = tokio::spawn(async move {
|
||||
crate::disk::with_background_disk_io(CancellationToken::new(), async move {
|
||||
let mut writer = writer;
|
||||
walk_wrapper
|
||||
.walk_dir(
|
||||
WalkDirOptions {
|
||||
bucket: "bucket".to_string(),
|
||||
recursive: true,
|
||||
skip_total_timeout: true,
|
||||
..Default::default()
|
||||
},
|
||||
&mut writer,
|
||||
)
|
||||
.await
|
||||
})
|
||||
.await
|
||||
});
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(2), entered.notified())
|
||||
.await
|
||||
.expect("walk should reach the blocked output writer");
|
||||
assert!(
|
||||
!wrapper.background_io_active_for_tests(),
|
||||
"filesystem read permits must be released before metacache output backpressure"
|
||||
);
|
||||
|
||||
let checkpoint_permit = tokio::time::timeout(
|
||||
Duration::from_secs(1),
|
||||
crate::disk::with_background_disk_io(CancellationToken::new(), wrapper.acquire_disk_io_permit()),
|
||||
)
|
||||
.await
|
||||
.expect("checkpoint admission should progress while the scanner consumer is paused")
|
||||
.expect("checkpoint should acquire the disk gate");
|
||||
drop(checkpoint_permit);
|
||||
walk.abort();
|
||||
let _ = walk.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn owned_mutation_keeps_publication_guard_after_waiter_cancellation() {
|
||||
let drops = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
@@ -2742,6 +3330,7 @@ mod tests {
|
||||
let reconnect = ReconnectDiskHealthState {
|
||||
health: Arc::new(health.metric_epoch_for_reconnect()),
|
||||
metrics: metrics.clone(),
|
||||
io_admission: DiskIoAdmission::new("test-disk"),
|
||||
};
|
||||
metrics.record_operation_call("read_all");
|
||||
reconnect.metrics.record_timeout_error();
|
||||
@@ -3023,6 +3612,40 @@ mod tests {
|
||||
assert_eq!(snapshot.total_deletes, 1);
|
||||
}
|
||||
|
||||
// A batch delete dropped while the disk call is in flight (for example by a
|
||||
// cancelled caller) must not leave the drive looking busy forever.
|
||||
#[tokio::test]
|
||||
async fn delete_versions_releases_waiting_slot_when_dropped() {
|
||||
let dir = tempfile::tempdir().expect("temp dir should be created");
|
||||
let endpoint =
|
||||
Endpoint::try_from(dir.path().to_str().expect("temp dir should be valid UTF-8")).expect("endpoint should parse");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created"));
|
||||
let wrapper = LocalDiskWrapper::new(Arc::clone(&disk), false);
|
||||
wrapper.make_volume("bucket").await.expect("volume should be created");
|
||||
let object_path = disk.get_object_path("bucket", "object").expect("object path");
|
||||
// Holding the object's metadata lease parks the disk call before it mutates anything.
|
||||
let metadata_lease = crate::disk::os::acquire_metadata_mutation_lease(&object_path, None).await;
|
||||
|
||||
let mut delete = Box::pin(wrapper.delete_versions(
|
||||
"bucket",
|
||||
vec![FileInfoVersions {
|
||||
name: "object".to_string(),
|
||||
versions: vec![FileInfo::default()],
|
||||
..Default::default()
|
||||
}],
|
||||
DeleteOptions::default(),
|
||||
));
|
||||
assert!(
|
||||
futures::poll!(delete.as_mut()).is_pending(),
|
||||
"the disk call should wait for the metadata lease"
|
||||
);
|
||||
assert_eq!(wrapper.health.waiting_count(), 1);
|
||||
|
||||
drop(delete);
|
||||
assert_eq!(wrapper.health.waiting_count(), 0, "a dropped delete must release its waiting slot");
|
||||
drop(metadata_lease);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_versions_counts_faulty_drive_availability_rejection() {
|
||||
let dir = tempfile::tempdir().expect("temp dir should be created");
|
||||
|
||||
@@ -0,0 +1,706 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Per-local-disk foreground-priority admission for scanner I/O.
|
||||
|
||||
use std::collections::VecDeque;
|
||||
use std::future::Future;
|
||||
use std::io;
|
||||
use std::pin::Pin;
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::task::{Context, Poll};
|
||||
use std::time::Duration;
|
||||
|
||||
use metrics::{counter, gauge, histogram};
|
||||
use rustfs_rio::ChunkReader;
|
||||
use tokio::io::{AsyncRead, ReadBuf};
|
||||
use tokio::sync::Notify;
|
||||
use tokio::time::Instant;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
const BACKGROUND_FOREGROUND_BURST: u32 = 8;
|
||||
const BACKGROUND_MAX_WAIT: Duration = Duration::from_millis(100);
|
||||
|
||||
tokio::task_local! {
|
||||
static DISK_IO_CONTEXT: DiskIoContext;
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub(crate) enum DiskIoClass {
|
||||
Foreground,
|
||||
Background,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct DiskIoContext {
|
||||
pub(crate) class: DiskIoClass,
|
||||
pub(crate) cancellation: CancellationToken,
|
||||
pub(crate) admission: Option<Arc<DiskIoAdmission>>,
|
||||
pub(crate) permit: Option<DiskIoPermit>,
|
||||
}
|
||||
|
||||
/// Mark disk I/O issued by `future` as background work for local-disk admission.
|
||||
#[doc(hidden)]
|
||||
pub async fn with_background_disk_io<F: Future>(cancellation: CancellationToken, future: F) -> F::Output {
|
||||
with_disk_io_context(
|
||||
DiskIoContext {
|
||||
class: DiskIoClass::Background,
|
||||
cancellation,
|
||||
admission: None,
|
||||
permit: None,
|
||||
},
|
||||
future,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn with_disk_io_context<F: Future>(context: DiskIoContext, future: F) -> F::Output {
|
||||
DISK_IO_CONTEXT.scope(context, future).await
|
||||
}
|
||||
|
||||
pub(crate) async fn with_disk_io_permit<F: Future>(permit: Option<DiskIoPermit>, future: F) -> F::Output {
|
||||
let Some(permit) = permit else {
|
||||
return future.await;
|
||||
};
|
||||
let mut context = current_disk_io_context().unwrap_or_else(|| DiskIoContext {
|
||||
class: DiskIoClass::Foreground,
|
||||
cancellation: CancellationToken::new(),
|
||||
admission: None,
|
||||
permit: None,
|
||||
});
|
||||
context.admission = Some(permit.admission());
|
||||
with_disk_io_context(
|
||||
DiskIoContext {
|
||||
permit: Some(permit),
|
||||
..context
|
||||
},
|
||||
future,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn with_optional_disk_io_context<F: Future>(context: Option<DiskIoContext>, future: F) -> F::Output {
|
||||
match context {
|
||||
Some(context) => with_disk_io_context(context, future).await,
|
||||
None => future.await,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn current_disk_io_context() -> Option<DiskIoContext> {
|
||||
DISK_IO_CONTEXT.try_with(Clone::clone).ok()
|
||||
}
|
||||
|
||||
/// Acquire one low-level permit only when the current walker explicitly owns a gate.
|
||||
/// Ordinary wrapper calls use their outer operation permit instead.
|
||||
pub(crate) async fn acquire_context_disk_io_permit() -> io::Result<Option<DiskIoPermit>> {
|
||||
let Some(context) = current_disk_io_context() else {
|
||||
return Ok(None);
|
||||
};
|
||||
let Some(admission) = context.admission else {
|
||||
return Ok(None);
|
||||
};
|
||||
if let Some(permit) = context.permit
|
||||
&& permit.is_for(&admission)
|
||||
{
|
||||
return Ok(Some(permit));
|
||||
}
|
||||
match context.class {
|
||||
DiskIoClass::Foreground => Ok(Some(admission.foreground())),
|
||||
DiskIoClass::Background => Ok(Some(admission.background(&context.cancellation).await?)),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct AdmissionState {
|
||||
foreground_completions: u32,
|
||||
next_background_ticket: u64,
|
||||
background_queue: VecDeque<u64>,
|
||||
background_front_since: Option<Instant>,
|
||||
}
|
||||
|
||||
/// One gate is shared by all handles to a local disk, including reconnects.
|
||||
/// Foreground I/O never waits on this gate. Background I/O runs exclusively
|
||||
/// while the disk is idle, or after a bounded foreground burst/wait.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct DiskIoAdmission {
|
||||
state: Mutex<AdmissionState>,
|
||||
changed: Notify,
|
||||
disk: String,
|
||||
foreground_active: AtomicUsize,
|
||||
background_active: AtomicBool,
|
||||
background_waiters: AtomicUsize,
|
||||
#[cfg(test)]
|
||||
background_admissions: AtomicUsize,
|
||||
}
|
||||
|
||||
impl DiskIoAdmission {
|
||||
pub(crate) fn new(disk: impl Into<String>) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
state: Mutex::new(AdmissionState::default()),
|
||||
changed: Notify::new(),
|
||||
disk: disk.into(),
|
||||
foreground_active: AtomicUsize::new(0),
|
||||
background_active: AtomicBool::new(false),
|
||||
background_waiters: AtomicUsize::new(0),
|
||||
#[cfg(test)]
|
||||
background_admissions: AtomicUsize::new(0),
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn background_admissions_for_tests(&self) -> usize {
|
||||
self.background_admissions.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn background_active_for_tests(&self) -> bool {
|
||||
self.background_active.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn foreground_active_for_tests(&self) -> usize {
|
||||
self.foreground_active.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub(crate) fn foreground(self: &Arc<Self>) -> DiskIoPermit {
|
||||
self.foreground_active.fetch_add(1, Ordering::AcqRel);
|
||||
if self.background_waiters.load(Ordering::Acquire) > 0 {
|
||||
self.record_state();
|
||||
}
|
||||
DiskIoPermit {
|
||||
lease: Arc::new(DiskIoPermitLease {
|
||||
admission: self.clone(),
|
||||
class: DiskIoClass::Foreground,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn background(self: &Arc<Self>, cancellation: &CancellationToken) -> std::io::Result<DiskIoPermit> {
|
||||
let started_at = Instant::now();
|
||||
let ticket = {
|
||||
let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner());
|
||||
let ticket = state.next_background_ticket;
|
||||
state.next_background_ticket = state.next_background_ticket.wrapping_add(1);
|
||||
if state.background_queue.is_empty() && !self.background_active.load(Ordering::Acquire) {
|
||||
state.background_front_since = Some(Instant::now());
|
||||
}
|
||||
state.background_queue.push_back(ticket);
|
||||
self.background_waiters.fetch_add(1, Ordering::Release);
|
||||
ticket
|
||||
};
|
||||
self.record_state();
|
||||
|
||||
let mut waiter = BackgroundWaiter {
|
||||
admission: self.clone(),
|
||||
ticket,
|
||||
waiting: true,
|
||||
};
|
||||
loop {
|
||||
let notified = self.changed.notified();
|
||||
tokio::pin!(notified);
|
||||
notified.as_mut().enable();
|
||||
|
||||
let granted = {
|
||||
let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner());
|
||||
let is_front = state.background_queue.front() == Some(&ticket);
|
||||
let bounded_wait_elapsed = is_front
|
||||
&& state
|
||||
.background_front_since
|
||||
.is_some_and(|front_since| front_since.elapsed() >= BACKGROUND_MAX_WAIT);
|
||||
if is_front
|
||||
&& !self.background_active.load(Ordering::Acquire)
|
||||
&& (self.foreground_active.load(Ordering::Acquire) == 0
|
||||
|| state.foreground_completions >= BACKGROUND_FOREGROUND_BURST
|
||||
|| bounded_wait_elapsed)
|
||||
{
|
||||
self.background_active.store(true, Ordering::Release);
|
||||
state.background_queue.pop_front();
|
||||
state.background_front_since = None;
|
||||
self.background_waiters.fetch_sub(1, Ordering::AcqRel);
|
||||
state.foreground_completions = 0;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
};
|
||||
|
||||
if granted {
|
||||
waiter.waiting = false;
|
||||
#[cfg(test)]
|
||||
self.background_admissions.fetch_add(1, Ordering::Relaxed);
|
||||
histogram!("rustfs_disk_io_admission_wait_seconds", "disk" => self.disk.clone(), "class" => "background")
|
||||
.record(started_at.elapsed().as_secs_f64());
|
||||
counter!("rustfs_disk_io_admission_total", "disk" => self.disk.clone(), "class" => "background").increment(1);
|
||||
self.record_state();
|
||||
return Ok(DiskIoPermit {
|
||||
lease: Arc::new(DiskIoPermitLease {
|
||||
admission: self.clone(),
|
||||
class: DiskIoClass::Background,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
let remaining = {
|
||||
let state = self.state.lock().unwrap_or_else(|error| error.into_inner());
|
||||
if state.background_queue.front() == Some(&ticket) {
|
||||
state
|
||||
.background_front_since
|
||||
.map(|front_since| BACKGROUND_MAX_WAIT.saturating_sub(front_since.elapsed()))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
};
|
||||
if let Some(remaining) = remaining.filter(|remaining| !remaining.is_zero()) {
|
||||
tokio::select! {
|
||||
_ = cancellation.cancelled() => {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "background disk I/O admission cancelled"));
|
||||
}
|
||||
_ = tokio::time::sleep(remaining) => {
|
||||
self.changed.notify_waiters();
|
||||
}
|
||||
_ = &mut notified => {}
|
||||
}
|
||||
} else {
|
||||
tokio::select! {
|
||||
_ = cancellation.cancelled() => {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "background disk I/O admission cancelled"));
|
||||
}
|
||||
_ = &mut notified => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn record_state(&self) {
|
||||
let (foreground_active, background_waiters) = {
|
||||
let state = self.state.lock().unwrap_or_else(|error| error.into_inner());
|
||||
(
|
||||
if state.background_queue.is_empty() {
|
||||
0
|
||||
} else {
|
||||
self.foreground_active.load(Ordering::Acquire)
|
||||
},
|
||||
state.background_queue.len(),
|
||||
)
|
||||
};
|
||||
let foreground_active = u32::try_from(foreground_active).unwrap_or(u32::MAX);
|
||||
let background_waiters = u32::try_from(background_waiters).unwrap_or(u32::MAX);
|
||||
gauge!("rustfs_disk_io_admission_active", "disk" => self.disk.clone(), "class" => "foreground")
|
||||
.set(f64::from(foreground_active));
|
||||
gauge!("rustfs_disk_io_admission_active", "disk" => self.disk.clone(), "class" => "background").set(
|
||||
if self.background_active.load(Ordering::Acquire) {
|
||||
1.0
|
||||
} else {
|
||||
0.0
|
||||
},
|
||||
);
|
||||
gauge!("rustfs_disk_io_admission_waiters", "disk" => self.disk.clone(), "class" => "background")
|
||||
.set(f64::from(background_waiters));
|
||||
}
|
||||
}
|
||||
|
||||
struct BackgroundWaiter {
|
||||
admission: Arc<DiskIoAdmission>,
|
||||
ticket: u64,
|
||||
waiting: bool,
|
||||
}
|
||||
|
||||
impl Drop for BackgroundWaiter {
|
||||
fn drop(&mut self) {
|
||||
if !self.waiting {
|
||||
return;
|
||||
}
|
||||
let mut state = self.admission.state.lock().unwrap_or_else(|error| error.into_inner());
|
||||
if let Some(index) = state.background_queue.iter().position(|ticket| *ticket == self.ticket) {
|
||||
state.background_queue.remove(index);
|
||||
if index == 0 {
|
||||
state.background_front_since =
|
||||
if state.background_queue.is_empty() || self.admission.background_active.load(Ordering::Acquire) {
|
||||
None
|
||||
} else {
|
||||
Some(Instant::now())
|
||||
};
|
||||
}
|
||||
self.admission.background_waiters.fetch_sub(1, Ordering::AcqRel);
|
||||
}
|
||||
drop(state);
|
||||
self.admission.record_state();
|
||||
self.admission.changed.notify_waiters();
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct DiskIoPermit {
|
||||
lease: Arc<DiskIoPermitLease>,
|
||||
}
|
||||
|
||||
impl DiskIoPermit {
|
||||
pub(crate) fn is_for(&self, admission: &Arc<DiskIoAdmission>) -> bool {
|
||||
Arc::ptr_eq(&self.lease.admission, admission)
|
||||
}
|
||||
|
||||
fn admission(&self) -> Arc<DiskIoAdmission> {
|
||||
self.lease.admission.clone()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct DiskIoPermitLease {
|
||||
admission: Arc<DiskIoAdmission>,
|
||||
class: DiskIoClass,
|
||||
}
|
||||
|
||||
pub(crate) fn spawn_blocking_with_disk_io_permit<T, F>(
|
||||
io_permit: Option<DiskIoPermit>,
|
||||
operation: F,
|
||||
) -> tokio::task::JoinHandle<T>
|
||||
where
|
||||
T: Send + 'static,
|
||||
F: FnOnce() -> T + Send + 'static,
|
||||
{
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let _io_permit = io_permit;
|
||||
operation()
|
||||
})
|
||||
}
|
||||
|
||||
type AdmissionWait = Pin<Box<dyn Future<Output = io::Result<DiskIoPermit>> + Send>>;
|
||||
|
||||
impl Drop for DiskIoPermitLease {
|
||||
fn drop(&mut self) {
|
||||
match self.class {
|
||||
DiskIoClass::Foreground => {
|
||||
self.admission.foreground_active.fetch_sub(1, Ordering::AcqRel);
|
||||
let record_state = self.admission.background_waiters.load(Ordering::Acquire) > 0;
|
||||
if record_state {
|
||||
let mut state = self.admission.state.lock().unwrap_or_else(|error| error.into_inner());
|
||||
if !state.background_queue.is_empty() {
|
||||
state.foreground_completions = state.foreground_completions.saturating_add(1);
|
||||
}
|
||||
}
|
||||
if record_state {
|
||||
self.admission.record_state();
|
||||
self.admission.changed.notify_waiters();
|
||||
}
|
||||
}
|
||||
DiskIoClass::Background => {
|
||||
let mut state = self.admission.state.lock().unwrap_or_else(|error| error.into_inner());
|
||||
self.admission.background_active.store(false, Ordering::Release);
|
||||
state.background_front_since = if state.background_queue.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(Instant::now())
|
||||
};
|
||||
drop(state);
|
||||
self.admission.record_state();
|
||||
self.admission.changed.notify_waiters();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct AdmissionReader<R> {
|
||||
inner: R,
|
||||
admission: Arc<DiskIoAdmission>,
|
||||
context: DiskIoContext,
|
||||
wait: Mutex<Option<AdmissionWait>>,
|
||||
permit: Option<DiskIoPermit>,
|
||||
}
|
||||
|
||||
impl<R> AdmissionReader<R> {
|
||||
pub(crate) fn new(inner: R, admission: Arc<DiskIoAdmission>, context: DiskIoContext) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
admission,
|
||||
context,
|
||||
wait: Mutex::new(None),
|
||||
permit: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_permit(&mut self, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
if self.permit.is_some() {
|
||||
return Poll::Ready(Ok(()));
|
||||
}
|
||||
|
||||
if self.context.class == DiskIoClass::Foreground {
|
||||
self.permit = Some(self.admission.foreground());
|
||||
return Poll::Ready(Ok(()));
|
||||
}
|
||||
|
||||
let mut wait = self.wait.lock().unwrap_or_else(|error| error.into_inner());
|
||||
if wait.is_none() {
|
||||
let admission = self.admission.clone();
|
||||
let cancellation = self.context.cancellation.clone();
|
||||
*wait = Some(Box::pin(async move { admission.background(&cancellation).await }));
|
||||
}
|
||||
let Some(wait_future) = wait.as_mut() else {
|
||||
return Poll::Ready(Err(io::Error::other("background disk I/O admission waiter is missing")));
|
||||
};
|
||||
let result = wait_future.as_mut().poll(cx);
|
||||
match result {
|
||||
Poll::Ready(Ok(permit)) => {
|
||||
wait.take();
|
||||
drop(wait);
|
||||
self.permit = Some(permit);
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
Poll::Ready(Err(error)) => {
|
||||
wait.take();
|
||||
Poll::Ready(Err(error))
|
||||
}
|
||||
Poll::Pending => Poll::Pending,
|
||||
}
|
||||
}
|
||||
|
||||
fn finish_poll<T>(&mut self, result: Poll<io::Result<T>>) -> Poll<io::Result<T>> {
|
||||
if result.is_ready() {
|
||||
self.permit = None;
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
impl<R: AsyncRead + Unpin> AsyncRead for AdmissionReader<R> {
|
||||
fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
|
||||
match self.poll_permit(cx) {
|
||||
Poll::Ready(Ok(())) => {}
|
||||
Poll::Ready(Err(error)) => return Poll::Ready(Err(error)),
|
||||
Poll::Pending => return Poll::Pending,
|
||||
}
|
||||
let result = Pin::new(&mut self.inner).poll_read(cx, buf);
|
||||
self.finish_poll(result)
|
||||
}
|
||||
}
|
||||
|
||||
impl<R: ChunkReader + Unpin> ChunkReader for AdmissionReader<R> {
|
||||
fn poll_read_chunk(mut self: Pin<&mut Self>, cx: &mut Context<'_>, max: usize) -> Poll<io::Result<Option<bytes::Bytes>>> {
|
||||
match self.poll_permit(cx) {
|
||||
Poll::Ready(Ok(())) => {}
|
||||
Poll::Ready(Err(error)) => return Poll::Ready(Err(error)),
|
||||
Poll::Pending => return Poll::Pending,
|
||||
}
|
||||
let result = Pin::new(&mut self.inner).poll_read_chunk(cx, max);
|
||||
self.finish_poll(result)
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct BoxedChunkReader(pub(crate) rustfs_rio::ChunkReaderBox);
|
||||
|
||||
impl AsyncRead for BoxedChunkReader {
|
||||
fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
|
||||
Pin::new(&mut self.0).poll_read(cx, buf)
|
||||
}
|
||||
}
|
||||
|
||||
impl ChunkReader for BoxedChunkReader {
|
||||
fn poll_read_chunk(self: Pin<&mut Self>, cx: &mut Context<'_>, max: usize) -> Poll<io::Result<Option<bytes::Bytes>>> {
|
||||
Pin::new(self.get_mut().0.as_mut()).poll_read_chunk(cx, max)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
|
||||
struct BlockingWorker(tokio::task::JoinHandle<()>);
|
||||
|
||||
async fn wait_for_background_waiters(admission: &DiskIoAdmission, expected: usize) {
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
loop {
|
||||
let waiters = admission
|
||||
.state
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner())
|
||||
.background_queue
|
||||
.len();
|
||||
if waiters == expected {
|
||||
return;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("background admission waiter should reach expected count");
|
||||
}
|
||||
|
||||
async fn wait_for_foreground_active(admission: &DiskIoAdmission, expected: usize) {
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
loop {
|
||||
let active = admission.foreground_active.load(Ordering::Acquire);
|
||||
if active == expected {
|
||||
return;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("foreground disk read should retain its permit while pending");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn foreground_bypasses_background_and_background_progress_is_bounded() {
|
||||
let admission = DiskIoAdmission::new("disk-a");
|
||||
let foreground = admission.foreground();
|
||||
let background_admission = admission.clone();
|
||||
let cancellation = CancellationToken::new();
|
||||
let background = tokio::spawn(async move { background_admission.background(&cancellation).await });
|
||||
|
||||
wait_for_background_waiters(&admission, 1).await;
|
||||
for _ in 0..BACKGROUND_FOREGROUND_BURST {
|
||||
drop(admission.foreground());
|
||||
}
|
||||
|
||||
let background_permit = tokio::time::timeout(Duration::from_secs(1), background)
|
||||
.await
|
||||
.expect("background admission should progress after the foreground burst")
|
||||
.expect("background waiter task should complete")
|
||||
.expect("background admission should not be cancelled");
|
||||
assert_eq!(
|
||||
admission.foreground_active.load(Ordering::Acquire),
|
||||
1,
|
||||
"foreground operations remain admitted while one background operation runs"
|
||||
);
|
||||
drop(background_permit);
|
||||
drop(foreground);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancellation_removes_a_queued_background_waiter() {
|
||||
let admission = DiskIoAdmission::new("disk-a");
|
||||
let background = admission
|
||||
.background(&CancellationToken::new())
|
||||
.await
|
||||
.expect("first background operation should be admitted");
|
||||
let cancellation = CancellationToken::new();
|
||||
let waiter_admission = admission.clone();
|
||||
let waiter_cancellation = cancellation.clone();
|
||||
let waiter = tokio::spawn(async move { waiter_admission.background(&waiter_cancellation).await });
|
||||
wait_for_background_waiters(&admission, 1).await;
|
||||
|
||||
cancellation.cancel();
|
||||
let error = tokio::time::timeout(Duration::from_secs(1), waiter)
|
||||
.await
|
||||
.expect("cancelled background waiter should stop")
|
||||
.expect("background waiter task should complete")
|
||||
.expect_err("cancelled admission must not start disk I/O");
|
||||
assert_eq!(error.kind(), io::ErrorKind::Interrupted);
|
||||
assert_eq!(
|
||||
admission
|
||||
.state
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner())
|
||||
.background_queue
|
||||
.len(),
|
||||
0
|
||||
);
|
||||
drop(background);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn background_admission_is_isolated_per_disk() {
|
||||
let disk_a = DiskIoAdmission::new("disk-a");
|
||||
let disk_b = DiskIoAdmission::new("disk-b");
|
||||
let _disk_a_permit = disk_a
|
||||
.background(&CancellationToken::new())
|
||||
.await
|
||||
.expect("disk A background operation should be admitted");
|
||||
let _disk_b_permit = tokio::time::timeout(Duration::from_secs(1), disk_b.background(&CancellationToken::new()))
|
||||
.await
|
||||
.expect("disk A must not block disk B")
|
||||
.expect("disk B background operation should be admitted");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streamed_read_holds_admission_until_disk_read_resolves() {
|
||||
let admission = DiskIoAdmission::new("disk-a");
|
||||
let (mut writer, reader) = tokio::io::duplex(8);
|
||||
let mut reader = AdmissionReader::new(
|
||||
reader,
|
||||
admission.clone(),
|
||||
DiskIoContext {
|
||||
class: DiskIoClass::Foreground,
|
||||
cancellation: CancellationToken::new(),
|
||||
admission: None,
|
||||
permit: None,
|
||||
},
|
||||
);
|
||||
let read = tokio::spawn(async move {
|
||||
let mut bytes = [0; 4];
|
||||
reader
|
||||
.read_exact(&mut bytes)
|
||||
.await
|
||||
.expect("streamed disk read should complete");
|
||||
bytes
|
||||
});
|
||||
|
||||
wait_for_foreground_active(&admission, 1).await;
|
||||
writer.write_all(b"rust").await.expect("test bytes should be written");
|
||||
assert_eq!(read.await.expect("reader task should finish"), *b"rust");
|
||||
wait_for_foreground_active(&admission, 0).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn blocking_worker_keeps_permit_after_caller_timeout() {
|
||||
let admission = DiskIoAdmission::new("disk-a");
|
||||
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
|
||||
let worker = spawn_blocking_with_disk_io_permit(Some(admission.foreground()), move || {
|
||||
started_tx.send(()).expect("worker start should be observed");
|
||||
std::thread::sleep(Duration::from_millis(100));
|
||||
});
|
||||
|
||||
started_rx.await.expect("blocking worker should start");
|
||||
assert!(tokio::time::timeout(Duration::from_millis(10), worker).await.is_err());
|
||||
assert_eq!(admission.foreground_active_for_tests(), 1);
|
||||
wait_for_foreground_active(&admission, 0).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cloned_permit_remains_active_until_all_workers_finish() {
|
||||
let admission = DiskIoAdmission::new("disk-a");
|
||||
let permit = admission.foreground();
|
||||
let worker_permit = permit.clone();
|
||||
drop(permit);
|
||||
assert_eq!(admission.foreground_active_for_tests(), 1);
|
||||
|
||||
drop(worker_permit);
|
||||
assert_eq!(admission.foreground_active_for_tests(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scoped_foreground_permit_is_inherited_by_blocking_worker() {
|
||||
let admission = DiskIoAdmission::new("disk-a");
|
||||
let permit = admission.foreground();
|
||||
let worker = with_disk_io_permit(Some(permit), async {
|
||||
let worker_permit = acquire_context_disk_io_permit()
|
||||
.await
|
||||
.expect("context permit should be available")
|
||||
.expect("scoped permit should be inherited");
|
||||
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
|
||||
let worker = spawn_blocking_with_disk_io_permit(Some(worker_permit), move || {
|
||||
started_tx.send(()).expect("worker start should be observed");
|
||||
std::thread::sleep(Duration::from_millis(100));
|
||||
});
|
||||
started_rx.await.expect("blocking worker should start");
|
||||
BlockingWorker(worker)
|
||||
})
|
||||
.await;
|
||||
|
||||
assert_eq!(admission.foreground_active_for_tests(), 1);
|
||||
assert!(tokio::time::timeout(Duration::from_millis(10), worker.0).await.is_err());
|
||||
wait_for_foreground_active(&admission, 0).await;
|
||||
}
|
||||
}
|
||||
+1346
-158
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -22,9 +22,16 @@ pub mod error_reduce;
|
||||
pub mod format;
|
||||
pub mod fs;
|
||||
pub mod health_state;
|
||||
mod io_admission;
|
||||
pub mod local;
|
||||
pub(crate) mod native_prepare_rpc;
|
||||
pub mod os;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) use io_admission::DiskIoClass;
|
||||
pub use io_admission::with_background_disk_io;
|
||||
pub(crate) use io_admission::{acquire_context_disk_io_permit, current_disk_io_context, with_optional_disk_io_context};
|
||||
|
||||
pub const RUSTFS_META_BUCKET: &str = ".rustfs.sys";
|
||||
pub const MIGRATING_META_BUCKET: &str = ".minio.sys";
|
||||
pub const RUSTFS_META_MULTIPART_BUCKET: &str = ".rustfs.sys/multipart";
|
||||
@@ -224,6 +231,152 @@ pub enum Disk {
|
||||
}
|
||||
|
||||
impl Disk {
|
||||
#[cfg(unix)]
|
||||
pub(crate) async fn native_begin_stream(
|
||||
&self,
|
||||
preparation: Uuid,
|
||||
store_id: Uuid,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
incarnation: Uuid,
|
||||
fence: Arc<crate::object_api::ObjectLockConfigSnapshot>,
|
||||
) -> Result<()> {
|
||||
match self {
|
||||
Disk::Local(local) => {
|
||||
local
|
||||
.get_disk()
|
||||
.begin_native_stream(preparation, store_id, bucket, incarnation, fence)
|
||||
.await
|
||||
}
|
||||
Disk::Remote(remote) => {
|
||||
remote
|
||||
.native_begin_stream(preparation, store_id, bucket, key, incarnation)
|
||||
.await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
pub(crate) async fn native_begin_stream(
|
||||
&self,
|
||||
_preparation: Uuid,
|
||||
_store_id: Uuid,
|
||||
_bucket: &str,
|
||||
_key: &str,
|
||||
_incarnation: Uuid,
|
||||
_fence: Arc<crate::object_api::ObjectLockConfigSnapshot>,
|
||||
) -> Result<()> {
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(crate) async fn native_finalize(
|
||||
&self,
|
||||
input: local::native_prepare::NativePrepareInput,
|
||||
) -> Result<local::native_prepare::NativePreparationReceipt> {
|
||||
match self {
|
||||
Disk::Local(local) => local.get_disk().prepare_native(input).await,
|
||||
Disk::Remote(remote) => remote.native_finalize(input).await,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
pub(crate) async fn native_finalize(
|
||||
&self,
|
||||
_input: local::native_prepare::NativePrepareInput,
|
||||
) -> Result<local::native_prepare::NativePreparationReceipt> {
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(crate) async fn native_recover(
|
||||
&self,
|
||||
preparation: Uuid,
|
||||
store_id: Uuid,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
incarnation: Uuid,
|
||||
fence: Arc<crate::object_api::ObjectLockConfigSnapshot>,
|
||||
) -> Result<(local::native_prepare::NativePreparationReceipt, Bytes)> {
|
||||
match self {
|
||||
Disk::Local(local) => {
|
||||
local
|
||||
.get_disk()
|
||||
.recover_native_with_metadata(preparation, store_id, bucket, incarnation, fence)
|
||||
.await
|
||||
}
|
||||
Disk::Remote(remote) => remote.native_recover(preparation, store_id, bucket, key, incarnation).await,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
pub(crate) async fn native_recover(
|
||||
&self,
|
||||
_preparation: Uuid,
|
||||
_store_id: Uuid,
|
||||
_bucket: &str,
|
||||
_key: &str,
|
||||
_incarnation: Uuid,
|
||||
_fence: Arc<crate::object_api::ObjectLockConfigSnapshot>,
|
||||
) -> Result<(local::native_prepare::NativePreparationReceipt, Bytes)> {
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(crate) async fn native_probe(
|
||||
&self,
|
||||
preparation: Uuid,
|
||||
store_id: Uuid,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
incarnation: Uuid,
|
||||
fence: Arc<crate::object_api::ObjectLockConfigSnapshot>,
|
||||
) -> Result<local::native_prepare::NativePreparationProbe> {
|
||||
match self {
|
||||
Disk::Local(local) => {
|
||||
local
|
||||
.get_disk()
|
||||
.probe_native(preparation, store_id, bucket, key, incarnation, fence)
|
||||
.await
|
||||
}
|
||||
Disk::Remote(remote) => remote.native_probe(preparation, store_id, bucket, key, incarnation).await,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
pub(crate) async fn native_probe(
|
||||
&self,
|
||||
_preparation: Uuid,
|
||||
_store_id: Uuid,
|
||||
_bucket: &str,
|
||||
_key: &str,
|
||||
_incarnation: Uuid,
|
||||
_fence: Arc<crate::object_api::ObjectLockConfigSnapshot>,
|
||||
) -> Result<local::native_prepare::NativePreparationProbe> {
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(crate) async fn native_locate(
|
||||
&self,
|
||||
store_id: Uuid,
|
||||
preparation: Uuid,
|
||||
) -> Result<Option<local::native_prepare::NativePreparationDescription>> {
|
||||
match self {
|
||||
Disk::Local(local) => local.get_disk().locate_native_preparation(store_id, preparation).await,
|
||||
Disk::Remote(remote) => remote.native_locate(store_id, preparation).await,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
pub(crate) async fn native_locate(
|
||||
&self,
|
||||
_store_id: Uuid,
|
||||
_preparation: Uuid,
|
||||
) -> Result<Option<local::native_prepare::NativePreparationDescription>> {
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
}
|
||||
|
||||
pub(crate) async fn set_disk_id_state(&self, id: Option<Uuid>) -> Result<()> {
|
||||
match self {
|
||||
Disk::Local(local_disk) => {
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::disk::{
|
||||
RUSTFS_META_BUCKET,
|
||||
error::{DiskError, Result},
|
||||
};
|
||||
use rustfs_utils::path::path_join_buf;
|
||||
use uuid::Uuid;
|
||||
|
||||
const NATIVE_STREAM_PREFIX: &str = "native-preparations/";
|
||||
const NATIVE_STREAM_ROOT: &str = "native-preparations";
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum NativeStreamFile {
|
||||
Part(usize),
|
||||
Staging(Uuid),
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct NativeStreamPath {
|
||||
pub(crate) preparation: Uuid,
|
||||
pub(crate) data_dir: Uuid,
|
||||
pub(crate) file: NativeStreamFile,
|
||||
}
|
||||
|
||||
fn targets_native_stream_namespace(volume: &str, path: &str) -> bool {
|
||||
if !volume.contains(RUSTFS_META_BUCKET) && !path.contains(NATIVE_STREAM_ROOT) {
|
||||
return false;
|
||||
}
|
||||
let normalized = path_join_buf(&[volume, path]);
|
||||
let prefix = format!("{RUSTFS_META_BUCKET}/{NATIVE_STREAM_PREFIX}");
|
||||
normalized == format!("{RUSTFS_META_BUCKET}/{NATIVE_STREAM_ROOT}") || normalized.starts_with(&prefix)
|
||||
}
|
||||
|
||||
/// Recognize only the reserved per-preparation shard path used by the native
|
||||
/// stream writer. Malformed paths under that namespace must not fall through to
|
||||
/// the general disk writer.
|
||||
pub(crate) fn parse_native_stream_path(volume: &str, path: &str) -> Result<Option<NativeStreamPath>> {
|
||||
if !targets_native_stream_namespace(volume, path) {
|
||||
return Ok(None);
|
||||
}
|
||||
if volume != RUSTFS_META_BUCKET {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
if !path.starts_with(NATIVE_STREAM_PREFIX) {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
|
||||
let mut components = path.split('/');
|
||||
if components.next() != Some("native-preparations") {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
let preparation = components
|
||||
.next()
|
||||
.and_then(|value| Uuid::parse_str(value).ok())
|
||||
.filter(|value| !value.is_nil())
|
||||
.ok_or(DiskError::InvalidPath)?;
|
||||
let data_dir = components
|
||||
.next()
|
||||
.and_then(|value| Uuid::parse_str(value).ok())
|
||||
.filter(|value| !value.is_nil())
|
||||
.ok_or(DiskError::InvalidPath)?;
|
||||
let file_name = components.next().ok_or(DiskError::InvalidPath)?;
|
||||
let file = file_name
|
||||
.strip_prefix("part.")
|
||||
.and_then(|value| value.parse::<usize>().ok())
|
||||
.filter(|value| *value > 0)
|
||||
.filter(|part| file_name == format!("part.{part}"))
|
||||
.map(NativeStreamFile::Part)
|
||||
.or_else(|| {
|
||||
file_name
|
||||
.strip_prefix(".rustfs-put-")
|
||||
.and_then(|value| Uuid::parse_str(value).ok())
|
||||
.filter(|value| !value.is_nil())
|
||||
.filter(|id| file_name == format!(".rustfs-put-{id}"))
|
||||
.map(NativeStreamFile::Staging)
|
||||
});
|
||||
|
||||
if components.next().is_some()
|
||||
|| file.is_none()
|
||||
|| path != format!("native-preparations/{preparation}/{data_dir}/{file_name}")
|
||||
{
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
Ok(Some(NativeStreamPath {
|
||||
preparation,
|
||||
data_dir,
|
||||
file: file.expect("validated native stream filename"),
|
||||
}))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn native_stream_path_requires_canonical_identity_and_part() {
|
||||
let preparation = Uuid::from_u128(1);
|
||||
let data_dir = Uuid::from_u128(2);
|
||||
let valid = format!("native-preparations/{preparation}/{data_dir}/part.1");
|
||||
assert!(
|
||||
parse_native_stream_path(RUSTFS_META_BUCKET, &valid)
|
||||
.expect("valid path")
|
||||
.is_some()
|
||||
);
|
||||
let staging = format!("native-preparations/{preparation}/{data_dir}/.rustfs-put-{data_dir}");
|
||||
assert!(
|
||||
parse_native_stream_path(RUSTFS_META_BUCKET, &staging)
|
||||
.expect("valid staging path")
|
||||
.is_some()
|
||||
);
|
||||
assert!(parse_native_stream_path("bucket", &valid).expect("other volume").is_none());
|
||||
assert!(
|
||||
parse_native_stream_path(RUSTFS_META_BUCKET, "ordinary/object")
|
||||
.expect("ordinary path")
|
||||
.is_none()
|
||||
);
|
||||
for invalid in [
|
||||
"native-preparations/00000000-0000-0000-0000-000000000000/00000000-0000-0000-0000-000000000002/part.1".to_owned(),
|
||||
format!("native-preparations/{preparation}/{data_dir}/part.0"),
|
||||
format!("native-preparations/{preparation}/{data_dir}/part.01"),
|
||||
format!("native-preparations/{preparation}/{data_dir}/.rustfs-put-{}", Uuid::nil()),
|
||||
format!("native-preparations/{preparation}/{data_dir}/.rustfs-put-not-a-uuid"),
|
||||
format!("native-preparations/{preparation}/{data_dir}/part.1/extra"),
|
||||
format!("native-preparations/{preparation}/../part.1"),
|
||||
] {
|
||||
assert!(matches!(
|
||||
parse_native_stream_path(RUSTFS_META_BUCKET, &invalid),
|
||||
Err(DiskError::InvalidPath)
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -497,7 +497,7 @@ where
|
||||
|
||||
/// Write a (hash+data) block. Returns the number of data bytes written.
|
||||
/// Returns an error if called after a short write or if data exceeds shard_size.
|
||||
#[hotpath::measure(label = "BitrotWriter::write", impl_type = "BitrotWriter")]
|
||||
#[hotpath::measure(impl_type = "BitrotWriter", future = true)]
|
||||
pub async fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
|
||||
if buf.is_empty() {
|
||||
return Ok(0);
|
||||
|
||||
@@ -791,7 +791,7 @@ impl Erasure {
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip_all, fields(data_len=data.len()))]
|
||||
#[hotpath::measure(label = "Erasure::encode_data", impl_type = "Erasure")]
|
||||
#[hotpath::measure(impl_type = "Erasure")]
|
||||
pub(crate) fn encode_data_block(&self, data: &[u8]) -> io::Result<EncodedBlock> {
|
||||
self.encode_data_block_inner(data)
|
||||
}
|
||||
@@ -810,7 +810,7 @@ impl Erasure {
|
||||
.map(|block| block.into_shards(self.total_shard_count()))
|
||||
}
|
||||
|
||||
#[hotpath::measure(label = "Erasure::encode_data_owned", impl_type = "Erasure")]
|
||||
#[hotpath::measure(impl_type = "Erasure")]
|
||||
pub(crate) fn encode_data_owned_block(&self, data: Vec<u8>) -> io::Result<EncodedBlock> {
|
||||
self.encode_data_owned_block_inner(data)
|
||||
}
|
||||
@@ -844,7 +844,7 @@ impl Erasure {
|
||||
.map(|block| block.into_shards(self.total_shard_count()))
|
||||
}
|
||||
|
||||
#[hotpath::measure(label = "Erasure::encode_data_bytes_mut", impl_type = "Erasure")]
|
||||
#[hotpath::measure(impl_type = "Erasure")]
|
||||
pub(crate) fn encode_data_bytes_mut_block(&self, data_buffer: BytesMut, data_len: usize) -> io::Result<EncodedBlock> {
|
||||
self.encode_buffer(data_buffer, data_len)
|
||||
}
|
||||
|
||||
@@ -442,12 +442,19 @@ impl WritePlan {
|
||||
}
|
||||
|
||||
if let Some(encryption) = self.encryption {
|
||||
let etag_nonce = match encryption.mode {
|
||||
WriteEncryptionMode::Singlepart { base_nonce } | WriteEncryptionMode::MultipartLegacy { base_nonce, .. } => {
|
||||
base_nonce
|
||||
}
|
||||
WriteEncryptionMode::SinglepartObjectKey | WriteEncryptionMode::MultipartObjectKey { .. } => [0u8; 12],
|
||||
};
|
||||
let protected_reader = rustfs_rio::EtagReader::new_encrypted(reader, encryption.key_bytes, etag_nonce)?;
|
||||
reader = match encryption.mode {
|
||||
WriteEncryptionMode::SinglepartObjectKey => HashReader::from_reader(
|
||||
#[cfg(feature = "rio-v2")]
|
||||
EncryptReader::new_with_object_key(reader, encryption.key_bytes),
|
||||
EncryptReader::new_with_object_key(protected_reader, encryption.key_bytes),
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
EncryptReader::new(reader, encryption.key_bytes, [0u8; 12]),
|
||||
EncryptReader::new(protected_reader, encryption.key_bytes, [0u8; 12]),
|
||||
HashReader::SIZE_PRESERVE_LAYER,
|
||||
actual_size,
|
||||
None,
|
||||
@@ -457,12 +464,12 @@ impl WritePlan {
|
||||
WriteEncryptionMode::Singlepart { base_nonce } => {
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
let encrypt_reader = if encryption_frame_v2_enabled() {
|
||||
EncryptReader::new_v2(reader, encryption.key_bytes, base_nonce)
|
||||
EncryptReader::new_v2(protected_reader, encryption.key_bytes, base_nonce)
|
||||
} else {
|
||||
EncryptReader::new(reader, encryption.key_bytes, base_nonce)
|
||||
EncryptReader::new(protected_reader, encryption.key_bytes, base_nonce)
|
||||
};
|
||||
#[cfg(feature = "rio-v2")]
|
||||
let encrypt_reader = EncryptReader::new(reader, encryption.key_bytes, base_nonce);
|
||||
let encrypt_reader = EncryptReader::new(protected_reader, encryption.key_bytes, base_nonce);
|
||||
HashReader::from_reader(encrypt_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false)?
|
||||
}
|
||||
WriteEncryptionMode::MultipartLegacy {
|
||||
@@ -471,20 +478,25 @@ impl WritePlan {
|
||||
} => {
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
let encrypt_reader = if encryption_frame_v2_enabled() {
|
||||
EncryptReader::new_multipart_v2(reader, encryption.key_bytes, base_nonce, multipart_part_number)
|
||||
EncryptReader::new_multipart_v2(protected_reader, encryption.key_bytes, base_nonce, multipart_part_number)
|
||||
} else {
|
||||
EncryptReader::new_multipart(reader, encryption.key_bytes, base_nonce, multipart_part_number)
|
||||
EncryptReader::new_multipart(protected_reader, encryption.key_bytes, base_nonce, multipart_part_number)
|
||||
};
|
||||
#[cfg(feature = "rio-v2")]
|
||||
let encrypt_reader =
|
||||
EncryptReader::new_multipart(reader, encryption.key_bytes, base_nonce, multipart_part_number);
|
||||
EncryptReader::new_multipart(protected_reader, encryption.key_bytes, base_nonce, multipart_part_number);
|
||||
HashReader::from_reader(encrypt_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false)?
|
||||
}
|
||||
WriteEncryptionMode::MultipartObjectKey { multipart_part_number } => HashReader::from_reader(
|
||||
#[cfg(feature = "rio-v2")]
|
||||
EncryptReader::new_multipart_with_object_key(reader, encryption.key_bytes, multipart_part_number),
|
||||
EncryptReader::new_multipart_with_object_key(protected_reader, encryption.key_bytes, multipart_part_number),
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
EncryptReader::new_multipart(reader, encryption.key_bytes, [0u8; 12], multipart_part_number as usize),
|
||||
EncryptReader::new_multipart(
|
||||
protected_reader,
|
||||
encryption.key_bytes,
|
||||
[0u8; 12],
|
||||
multipart_part_number as usize,
|
||||
),
|
||||
HashReader::SIZE_PRESERVE_LAYER,
|
||||
actual_size,
|
||||
None,
|
||||
@@ -497,6 +509,7 @@ impl WritePlan {
|
||||
// `ignore_value` deliberately avoids a second hasher over compressed or
|
||||
// encrypted bytes. The inner reader still validates the plaintext request
|
||||
// checksum while this outer reader exposes the request checksum context.
|
||||
reader.encrypted_etag = encrypted;
|
||||
reader.add_non_trailing_checksum(checksum, true)?;
|
||||
reader.set_trailer(trailer);
|
||||
|
||||
@@ -513,6 +526,36 @@ mod tests {
|
||||
use std::io::Cursor;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
#[tokio::test]
|
||||
async fn write_plan_encryption_persists_protected_etags_for_every_mode() {
|
||||
let data = b"Salary 2026: 85000 EUR";
|
||||
let size = i64::try_from(data.len()).expect("test data size");
|
||||
let mut control =
|
||||
HashReader::from_stream(Cursor::new(data), size, size, None, None, false).expect("create control reader");
|
||||
control.read_to_end(&mut Vec::new()).await.expect("read control plaintext");
|
||||
let md5 = control.try_resolve_etag().expect("control MD5");
|
||||
for encryption in [
|
||||
WriteEncryption::singlepart([1; 32], [1; 12]),
|
||||
WriteEncryption::singlepart_object_key([1; 32]),
|
||||
WriteEncryption::multipart([1; 32], [1; 12], 1),
|
||||
WriteEncryption::multipart_object_key([1; 32], 1),
|
||||
] {
|
||||
let plaintext = HashReader::from_stream(Cursor::new(data), size, size, Some(md5.clone()), None, false)
|
||||
.expect("create checked reader");
|
||||
let mut reader = WritePlan::new()
|
||||
.with_encryption(encryption)
|
||||
.apply(plaintext, size)
|
||||
.expect("apply encryption");
|
||||
let mut ciphertext = Vec::new();
|
||||
reader.read_to_end(&mut ciphertext).await.expect("stream ciphertext");
|
||||
assert_ne!(ciphertext, data);
|
||||
let tag = reader.try_resolve_etag().expect("resolve persisted encrypted ETag");
|
||||
assert_ne!(tag, md5, "stored ETag must not reveal the plaintext MD5");
|
||||
assert_eq!(tag.len(), 32, "multipart ETags require a 16-byte hexadecimal part tag");
|
||||
assert_eq!(reader.try_resolve_etag().as_ref(), Some(&tag));
|
||||
}
|
||||
}
|
||||
|
||||
async fn assert_non_trailing_checksum_survives(plan: WritePlan) {
|
||||
let plaintext = b"checksum-context-through-write-plan".repeat(256);
|
||||
let actual_size = plaintext.len() as i64;
|
||||
|
||||
@@ -55,6 +55,7 @@ mod set_disk;
|
||||
mod storage_api_contracts;
|
||||
mod store;
|
||||
|
||||
pub use disk::with_background_disk_io;
|
||||
pub use store::PoolMetaWriteGateStatus;
|
||||
|
||||
// pub mod checksum;
|
||||
|
||||
@@ -1237,7 +1237,8 @@ impl ObjectOptions {
|
||||
self.version_id.is_some() || !self.versioned || self.version_suspended
|
||||
}
|
||||
|
||||
pub(crate) fn add_namespace_lock_lost_signal(&mut self, signal: Arc<rustfs_lock::distributed_lock::LockLostSignal>) {
|
||||
#[doc(hidden)]
|
||||
pub fn add_namespace_lock_lost_signal(&mut self, signal: Arc<rustfs_lock::distributed_lock::LockLostSignal>) {
|
||||
#[cfg(test)]
|
||||
let test_fence = NAMESPACE_LOCK_SIGNAL_TEST_FENCES
|
||||
.get_or_init(|| std::sync::Mutex::new(Vec::new()))
|
||||
|
||||
@@ -58,11 +58,17 @@ impl WriteCommitGuard {
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the loss signal of the namespace owner retained by acquisition.
|
||||
#[doc(hidden)]
|
||||
pub fn lock_lost_signal(&self) -> Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>> {
|
||||
self.guards.first().and_then(|guard| guard.lock_lost_signal())
|
||||
}
|
||||
|
||||
pub fn is_lock_lost(&self) -> bool {
|
||||
self.guards.iter().any(|guard| guard.is_lock_lost() || guard.is_released())
|
||||
}
|
||||
|
||||
fn matches(&self, bucket: &str, object: &str) -> bool {
|
||||
pub(crate) fn matches(&self, bucket: &str, object: &str) -> bool {
|
||||
self.bucket.as_ref() == bucket && self.object.as_ref() == object
|
||||
}
|
||||
}
|
||||
|
||||
@@ -307,6 +307,13 @@ pub fn shutdown_background_services() {
|
||||
}
|
||||
}
|
||||
|
||||
/// Wait up to `timeout` for object mutations that outlived their callers
|
||||
/// (see `InstanceContext::wait_for_detached_mutations`). Returns `false` when
|
||||
/// some are still running at the deadline.
|
||||
pub async fn wait_for_detached_mutations(timeout: std::time::Duration) -> bool {
|
||||
current_ctx().wait_for_detached_mutations(timeout).await
|
||||
}
|
||||
|
||||
/// Set the global lock client (first LocalClient created)
|
||||
///
|
||||
/// # Arguments
|
||||
|
||||
@@ -59,6 +59,8 @@ use std::sync::{
|
||||
use tokio::sync::{Mutex, Notify, OnceCell, OwnedRwLockReadGuard, RwLock};
|
||||
use tokio::time::Instant;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tokio_util::task::TaskTracker;
|
||||
use tracing::Instrument as _;
|
||||
use uuid::Uuid;
|
||||
|
||||
const SCANNER_PUBLICATION_STATE_UNKNOWN: u8 = 0;
|
||||
@@ -72,10 +74,19 @@ pub(crate) const SCANNER_PUBLICATION_LEASE_MAX_ENTRIES: usize = 256;
|
||||
/// have started movement after the lease was abandoned.
|
||||
pub(crate) const SCANNER_PUBLICATION_LEASE_TTL: std::time::Duration = std::time::Duration::from_secs(60);
|
||||
|
||||
/// Observational leases hold data-movement fencing but may coexist with
|
||||
/// ordinary namespace commits; authoritative leases require a stable namespace.
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub(crate) enum ScannerPublicationLeasePurpose {
|
||||
Authoritative,
|
||||
Observational,
|
||||
}
|
||||
|
||||
pub(crate) struct ScannerPublicationLeaseEntry {
|
||||
pub(crate) expires_at: Instant,
|
||||
pub(crate) movement_generation: u64,
|
||||
pub(crate) namespace_generation: u64,
|
||||
pub(crate) namespace_generation: Option<u64>,
|
||||
pub(crate) purpose: ScannerPublicationLeasePurpose,
|
||||
pub(crate) _operation_guard: OwnedRwLockReadGuard<()>,
|
||||
}
|
||||
|
||||
@@ -234,6 +245,10 @@ pub struct InstanceContext {
|
||||
suppress_tier_delete_journal_recovery: bool,
|
||||
transition_transaction_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
|
||||
tier_delete_journal_recovery_wakeup: tokio::sync::Notify,
|
||||
/// Object mutations that run on their own task so that dropping the
|
||||
/// caller (for example, an S3 client disconnect) cannot stop them between
|
||||
/// their on-disk steps. Shutdown waits for them after the HTTP drain.
|
||||
detached_mutations: TaskTracker,
|
||||
}
|
||||
|
||||
impl InstanceContext {
|
||||
@@ -284,6 +299,7 @@ impl InstanceContext {
|
||||
suppress_tier_delete_journal_recovery: false,
|
||||
transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()),
|
||||
tier_delete_journal_recovery_wakeup: tokio::sync::Notify::new(),
|
||||
detached_mutations: TaskTracker::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -297,6 +313,31 @@ impl InstanceContext {
|
||||
self.lock_manager.clone()
|
||||
}
|
||||
|
||||
/// Run a multi-step mutation on its own task and wait for its output.
|
||||
///
|
||||
/// Dropping the returned future leaves the task running to completion, so
|
||||
/// the mutation, together with the locks and guards it owns, never stops
|
||||
/// between two of its on-disk steps because its caller went away.
|
||||
pub(crate) async fn run_detached_mutation<F>(&self, mutation: F) -> Result<F::Output, tokio::task::JoinError>
|
||||
where
|
||||
F: std::future::Future + Send + 'static,
|
||||
F::Output: Send + 'static,
|
||||
{
|
||||
self.detached_mutations.spawn(mutation.in_current_span()).await
|
||||
}
|
||||
|
||||
/// Wait up to `timeout` for every detached mutation to finish. Returns
|
||||
/// `false` when some are still running at the deadline.
|
||||
pub async fn wait_for_detached_mutations(&self, timeout: std::time::Duration) -> bool {
|
||||
self.detached_mutations.close();
|
||||
tokio::time::timeout(timeout, self.detached_mutations.wait()).await.is_ok()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn detached_mutation_count(&self) -> usize {
|
||||
self.detached_mutations.len()
|
||||
}
|
||||
|
||||
pub(crate) fn data_movement_operation_gate(&self) -> Arc<RwLock<()>> {
|
||||
Arc::clone(&self.data_movement_operation_gate)
|
||||
}
|
||||
@@ -306,7 +347,8 @@ impl InstanceContext {
|
||||
token: Uuid,
|
||||
expires_at: Instant,
|
||||
movement_generation: u64,
|
||||
namespace_generation: u64,
|
||||
namespace_generation: Option<u64>,
|
||||
purpose: ScannerPublicationLeasePurpose,
|
||||
operation_guard: OwnedRwLockReadGuard<()>,
|
||||
) -> bool {
|
||||
let mut leases = self.scanner_publication_leases.lock().await;
|
||||
@@ -319,6 +361,7 @@ impl InstanceContext {
|
||||
expires_at,
|
||||
movement_generation,
|
||||
namespace_generation,
|
||||
purpose,
|
||||
_operation_guard: operation_guard,
|
||||
},
|
||||
);
|
||||
@@ -329,21 +372,24 @@ impl InstanceContext {
|
||||
self.scanner_publication_leases.lock().await.remove(&token).is_some()
|
||||
}
|
||||
|
||||
/// Return both generations from the same live lease while the caller holds
|
||||
/// the movement read guard. Namespace commits do not take that guard, so
|
||||
/// the caller must compare the saved namespace generation after this await.
|
||||
/// Return the generations and purpose from the same live lease while the
|
||||
/// caller holds the movement read guard. Namespace commits do not take that
|
||||
/// guard, so strict leases compare their saved generation after this await.
|
||||
/// The process-owned table rejects tokens from a prior instance or expiry.
|
||||
pub(crate) async fn scanner_publication_lease_generations(&self, token: Uuid) -> Option<(u64, u64)> {
|
||||
pub(crate) async fn scanner_publication_lease_generations(
|
||||
&self,
|
||||
token: Uuid,
|
||||
) -> Option<(u64, Option<u64>, ScannerPublicationLeasePurpose)> {
|
||||
let mut leases = self.scanner_publication_leases.lock().await;
|
||||
let now = Instant::now();
|
||||
let (expires_at, movement_generation, namespace_generation) = leases
|
||||
let (expires_at, movement_generation, namespace_generation, purpose) = leases
|
||||
.get(&token)
|
||||
.map(|entry| (entry.expires_at, entry.movement_generation, entry.namespace_generation))?;
|
||||
.map(|entry| (entry.expires_at, entry.movement_generation, entry.namespace_generation, entry.purpose))?;
|
||||
if expires_at <= now {
|
||||
leases.remove(&token);
|
||||
return None;
|
||||
}
|
||||
Some((movement_generation, namespace_generation))
|
||||
Some((movement_generation, namespace_generation, purpose))
|
||||
}
|
||||
|
||||
pub(crate) async fn expire_scanner_publication_lease(&self, token: Uuid, expires_at: Instant) {
|
||||
@@ -859,12 +905,22 @@ mod tests {
|
||||
let gate = ctx.data_movement_operation_gate();
|
||||
let permit = gate.clone().read_owned().await;
|
||||
assert!(
|
||||
ctx.install_scanner_publication_lease(token, Instant::now() + SCANNER_PUBLICATION_LEASE_TTL, 7, 11, permit)
|
||||
.await
|
||||
ctx.install_scanner_publication_lease(
|
||||
token,
|
||||
Instant::now() + SCANNER_PUBLICATION_LEASE_TTL,
|
||||
7,
|
||||
Some(11),
|
||||
ScannerPublicationLeasePurpose::Authoritative,
|
||||
permit,
|
||||
)
|
||||
.await
|
||||
);
|
||||
drop(ctx.begin_namespace_commit());
|
||||
assert_eq!(ctx.namespace_commit_generation(), 2);
|
||||
assert_eq!(ctx.scanner_publication_lease_generations(token).await, Some((7, 11)));
|
||||
assert_eq!(
|
||||
ctx.scanner_publication_lease_generations(token).await,
|
||||
Some((7, Some(11), ScannerPublicationLeasePurpose::Authoritative))
|
||||
);
|
||||
assert!(gate.clone().try_write_owned().is_err(), "lookup must retain the stored permit");
|
||||
tokio::time::advance(SCANNER_PUBLICATION_LEASE_TTL).await;
|
||||
assert_eq!(ctx.scanner_publication_lease_generations(token).await, None);
|
||||
@@ -896,6 +952,33 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
// A detached mutation keeps running after its caller is dropped, and
|
||||
// shutdown waits for it instead of tearing it down mid-way.
|
||||
#[tokio::test]
|
||||
async fn detached_mutation_outlives_dropped_caller_and_is_awaited() {
|
||||
let ctx = InstanceContext::new();
|
||||
let finished = Arc::new(AtomicBool::new(false));
|
||||
let (release_tx, release_rx) = tokio::sync::oneshot::channel::<()>();
|
||||
let mut caller = Box::pin(ctx.run_detached_mutation({
|
||||
let finished = Arc::clone(&finished);
|
||||
async move {
|
||||
let _ = release_rx.await;
|
||||
finished.store(true, Ordering::Release);
|
||||
}
|
||||
}));
|
||||
assert!(futures::poll!(caller.as_mut()).is_pending());
|
||||
drop(caller);
|
||||
assert_eq!(ctx.detached_mutation_count(), 1, "dropping the caller must not cancel the mutation");
|
||||
|
||||
assert!(
|
||||
!ctx.wait_for_detached_mutations(std::time::Duration::from_millis(50)).await,
|
||||
"the wait must report a mutation that is still running"
|
||||
);
|
||||
release_tx.send(()).expect("the detached mutation should still be waiting");
|
||||
assert!(ctx.wait_for_detached_mutations(std::time::Duration::from_secs(5)).await);
|
||||
assert!(finished.load(Ordering::Acquire));
|
||||
}
|
||||
|
||||
// A fresh context (before any update) reflects the initial all-false state.
|
||||
#[tokio::test]
|
||||
async fn fresh_context_is_all_false() {
|
||||
|
||||
@@ -2888,8 +2888,23 @@ impl NotificationSys {
|
||||
/// leases are released before returning so a partial acquisition cannot
|
||||
/// pin movement on one peer.
|
||||
pub async fn acquire_scanner_publication_leases(
|
||||
&self,
|
||||
targets: Vec<(String, String, u64)>,
|
||||
) -> Result<Vec<ScannerPublicationLeaseGrant>> {
|
||||
self.acquire_scanner_publication_leases_with_purpose(targets, false).await
|
||||
}
|
||||
|
||||
pub async fn acquire_observational_scanner_publication_leases(
|
||||
&self,
|
||||
targets: Vec<(String, String, u64)>,
|
||||
) -> Result<Vec<ScannerPublicationLeaseGrant>> {
|
||||
self.acquire_scanner_publication_leases_with_purpose(targets, true).await
|
||||
}
|
||||
|
||||
async fn acquire_scanner_publication_leases_with_purpose(
|
||||
&self,
|
||||
mut targets: Vec<(String, String, u64)>,
|
||||
observational_only: bool,
|
||||
) -> Result<Vec<ScannerPublicationLeaseGrant>> {
|
||||
targets.sort_by(|left, right| left.0.cmp(&right.0));
|
||||
for pair in targets.windows(2) {
|
||||
@@ -2910,7 +2925,10 @@ impl NotificationSys {
|
||||
let _ = self.release_scanner_publication_leases(grants).await;
|
||||
return Err(Error::other(format!("scanner publication lease peer {host} is unavailable")));
|
||||
};
|
||||
match client.acquire_scanner_publication_lease(&session_id, generation).await {
|
||||
match client
|
||||
.acquire_scanner_publication_lease(&session_id, generation, observational_only)
|
||||
.await
|
||||
{
|
||||
Ok(lease) => grants.push(ScannerPublicationLeaseGrant { host, lease }),
|
||||
Err(err) => {
|
||||
let _ = self.release_scanner_publication_leases(grants).await;
|
||||
@@ -4949,6 +4967,7 @@ mod tests {
|
||||
lease: ScannerPublicationLease {
|
||||
token: Uuid::new_v4(),
|
||||
movement_generation: 3,
|
||||
observational_only: false,
|
||||
owner_id: Uuid::new_v4().to_string(),
|
||||
session_id: "session-a".to_string(),
|
||||
expires_at: Instant::now() + Duration::from_secs(30),
|
||||
|
||||
@@ -298,6 +298,11 @@ pub fn build_transition_put_options(storage_class: String, mut metadata: HashMap
|
||||
}
|
||||
}
|
||||
|
||||
// x-minio-internal-* is reserved by MinIO and its derivatives; they reject
|
||||
// any request carrying it with 400 InvalidArgument. The transition ids above
|
||||
// are the only internal keys the remote needs, and they travel as x-amz-meta-*.
|
||||
metadata.retain(|key, _| !rustfs_utils::http::is_internal_key(key));
|
||||
|
||||
opts.user_metadata = metadata;
|
||||
opts
|
||||
}
|
||||
@@ -2244,6 +2249,38 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_transition_put_options_drops_reserved_internal_headers() {
|
||||
let transaction_suffix = rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID;
|
||||
let mut metadata = HashMap::new();
|
||||
for suffix in [
|
||||
rustfs_utils::http::SUFFIX_INLINE_DATA,
|
||||
rustfs_utils::http::SUFFIX_ACTUAL_SIZE,
|
||||
rustfs_utils::http::SUFFIX_PART_CHECKSUMS,
|
||||
transaction_suffix,
|
||||
] {
|
||||
rustfs_utils::http::metadata_compat::insert_str(&mut metadata, suffix, "x".to_string());
|
||||
}
|
||||
metadata.insert("X-Minio-Internal-Compression".to_string(), "klauspost/compress/s2".to_string());
|
||||
metadata.insert("x-amz-meta-foo".to_string(), "bar".to_string());
|
||||
metadata.insert("name".to_string(), "object".to_string());
|
||||
|
||||
let opts = build_transition_put_options("COLD".to_string(), metadata);
|
||||
|
||||
let leaked: Vec<_> = opts
|
||||
.user_metadata
|
||||
.keys()
|
||||
.filter(|key| rustfs_utils::http::is_internal_key(key))
|
||||
.collect();
|
||||
assert!(leaked.is_empty(), "reserved internal keys must not reach the remote: {leaked:?}");
|
||||
assert_eq!(opts.user_metadata.get("x-amz-meta-foo").map(String::as_str), Some("bar"));
|
||||
assert_eq!(opts.user_metadata.get("name").map(String::as_str), Some("object"));
|
||||
assert!(opts.user_metadata.contains_key(&format!(
|
||||
"x-amz-meta-{}",
|
||||
rustfs_utils::http::metadata_compat::internal_key_rustfs(transaction_suffix)
|
||||
)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_transition_put_options_requests_no_checksum_and_content_md5() {
|
||||
// Regression for rustfs/rustfs#4811: transition uploads must leave the
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user