mirror of
https://github.com/rustfs/rustfs.git
synced 2026-10-27 22:37:13 +00:00
Compare commits
102 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| ca5be1ef95 | |||
| 9373e8fa3e | |||
| cb32965522 | |||
| fb6cf49a67 | |||
| 8c97049e68 | |||
| 17d4fe27cc | |||
| fb0c7972b2 | |||
| eba7967755 | |||
| 451c1c31ad | |||
| 8780dd670e | |||
| f29bf2c902 | |||
| fd03436090 | |||
| 1f1e7b1c97 | |||
| 45a952609e | |||
| 323190a95a | |||
| 38cdda9292 | |||
| 85c84fa73d | |||
| 51402087c5 | |||
| a994984fe2 | |||
| 48eb0e402c | |||
| 446479791e | |||
| 99af59a914 | |||
| b77bd26874 | |||
| 956c1cf027 | |||
| 8b9d44110e | |||
| 44ce8d114f | |||
| 769b0d7578 | |||
| 03e775947d | |||
| 25e73d3bd6 | |||
| a464b45367 | |||
| 5972a09cc3 | |||
| 8a5e730bc9 | |||
| 1df874b724 | |||
| 68dcad8387 | |||
| 8474feaa26 | |||
| d386f7741d | |||
| 5b4432620e | |||
| 1a450bf228 | |||
| 371241afcb | |||
| 61b31e7b45 | |||
| 0d1d1487f3 | |||
| 3824779520 | |||
| a9670ebb4d | |||
| 99ae115731 | |||
| 545e794945 | |||
| 855586255f | |||
| bf47a919ac | |||
| bdd23da034 | |||
| 89a277c3d9 | |||
| 3e87004032 | |||
| 76386572f6 | |||
| 34917c7947 | |||
| f0b2080e0c | |||
| 9c6b0e182b | |||
| ab9af5523c | |||
| b18c5afc8d | |||
| df988cfd86 | |||
| 04bc5dffd6 | |||
| 681c95f359 | |||
| 32d98ff8b2 | |||
| 6b1554003e | |||
| 19978b2cb6 | |||
| db2abcf0a6 | |||
| 14eb28dcd6 | |||
| 2f0143ef15 | |||
| 3b92708011 | |||
| 8fea8ef600 | |||
| 477f768f7e | |||
| 47ed6717de | |||
| 480312e5ad | |||
| 1c5bbc0c47 | |||
| 14adc5ed3f | |||
| 714706328b | |||
| cd0b895ad7 | |||
| 073b4d838e | |||
| 95268a3b9e | |||
| f7c4ac11b2 | |||
| 9d9e92bab8 | |||
| 0ad0a6d8de | |||
| 051a54f129 | |||
| c71e4f72ea | |||
| 003570b61e | |||
| 0c3efda7bd | |||
| 2c7e7d195c | |||
| 0764785754 | |||
| 182b2bfe98 | |||
| 00a549504c | |||
| 83fa2611f4 | |||
| e22414fe68 | |||
| 45f97bccf9 | |||
| 046fb21d86 | |||
| 4783adff36 | |||
| 68ad1a1d7b | |||
| 7c289001ed | |||
| 7b1ffbfdc1 | |||
| 562e03670e | |||
| 5e1bd498ce | |||
| 5aca750a6e | |||
| a760d567b6 | |||
| 47d208325e | |||
| d087416ff9 | |||
| 309128f034 |
@@ -50,6 +50,7 @@ Update this file only when an advisory adds or changes a reusable lesson, affect
|
||||
|
||||
- `GHSA-g8w9-qw9q-fghr` and `GHSA-xm99-m3gq-83g8`: a valid presigned `PutObject` accepted extra `x-amz-*` headers omitted from `SignedHeaders`; `x-amz-copy-source` could turn that upload capability into a cross-bucket read performed as the signer. Lesson: a presigned URL is a bounded capability; inspect all received security-sensitive headers and reject unsigned ones before selecting the S3 operation or reaching storage.
|
||||
- `GHSA-3ppv-fx5m-m749`: explicit `versionId` reads and copy sources authorized `s3:GetObject` instead of `s3:GetObjectVersion`. Lesson: version-specific object access must select version-specific actions for direct reads, `CopyObject`, and `UploadPartCopy`, with tests proving the backend is not reached on denial.
|
||||
- `GHSA-gjph-xcgw-4p7w`: `PutObject` and browser POST accepted a caller-supplied `versionId` and replaced an unlocked historical version with only `s3:PutObject`. Lesson: only replication-authorized writes may choose an existing version ID; ordinary uploads must not rewrite version history.
|
||||
- `GHSA-x298-9x87-fvjq`: anonymous `ListObjectVersions` fell back to `ListBucket` and returned before public-access-block gates. Lesson: compatibility fallbacks must converge on the same post-authorization checks as direct grants, especially `RestrictPublicBuckets` and anonymous data-plane denies.
|
||||
- `GHSA-mx42-j6wv-px98`: `UploadPartCopy` missed source authorization and allowed cross-bucket object exfiltration. Lesson: multipart copy must enforce the same source and destination contract as `CopyObject`.
|
||||
- `GHSA-wfxj-ph3v-7mjf`: `UploadPartCopy` checked source and destination independently but missed destination copy-source policy constraints. Lesson: source read and destination write checks are not sufficient when policy constrains allowed copy sources.
|
||||
@@ -60,12 +61,14 @@ Update this file only when an advisory adds or changes a reusable lesson, affect
|
||||
- `GHSA-3g29-xff2-92vp`: FTP `RETR` and `SIZE`/`MDTM` read paths authenticated the user but skipped IAM before calling storage. Lesson: non-HTTP protocol frontends must enforce the same per-operation authorization as the S3 API before backend access.
|
||||
- `GHSA-g3vq-vv42-f647`: FTPS `MKD` called `create_bucket` without checking `s3:CreateBucket`. Lesson: protocol command handlers need action-specific checks even when sibling handlers already authorize correctly.
|
||||
- `GHSA-3p3x-734c-h5vx`: FTPS and WebDAV compared secret keys with early-return string equality, while FTPS also returned distinguishable invalid-user and invalid-password failures. Lesson: password-style protocol auth needs constant-time secret comparison, indistinguishable failures where practical, and rate limiting.
|
||||
- `GHSA-rhg6-jrf2-29c4`: WebDAV, FTP/FTPS, and SFTP used identity-policy checks with empty conditions and bypassed S3 bucket-policy explicit denies and condition-key denies. Lesson: alternate listeners must evaluate the same effective policy and trustworthy request context before storage access.
|
||||
|
||||
### Filesystem paths and object key traversal
|
||||
|
||||
- `GHSA-pq29-69jg-9mxc`: RPC `read_file_stream` joined untrusted paths under a volume directory without canonical boundary checks. Lesson: `PathBuf::join` plus length checks are not path security.
|
||||
- `GHSA-8r6f-hmq2-28rg`: object keys containing traversal sequences bypassed bucket/object authorization when mapped to filesystem paths. Lesson: reject traversal at object-key parsing and verify final storage paths remain under the expected bucket/key root.
|
||||
- `GHSA-f4vq-9ffr-m8m3`: Snowball auto-extract accepted archive entries such as `../victim-bucket/object`, authorized the raw attacker-bucket path, then storage path cleaning crossed bucket boundaries. Lesson: archive entries become object keys and need traversal rejection plus consistent authz/storage normalization before writes.
|
||||
- `GHSA-g7fq-6xjm-mxf4`: versioned `DeleteObjects` accepted dot-segment keys, authorized the literal prefix, then wrote delete markers under a resolved sibling prefix. Lesson: validate each batch key before authorization and storage normalization, using the same object-name rules as single delete.
|
||||
|
||||
### Secrets, defaults, and cryptographic misuse
|
||||
|
||||
@@ -106,6 +109,7 @@ Update this file only when an advisory adds or changes a reusable lesson, affect
|
||||
### Object Lock and retention invariants
|
||||
|
||||
- `GHSA-j548-9grx-fh4f`: Object Lock enforcement treated unreadable, fabricated, or unparsable bucket metadata as absent configuration and allowed retained objects to be deleted or expired. Lesson: retention must fail closed unless Object Lock absence is authoritative, and every delete, lifecycle, scanner, force-delete, and default-retention path needs the same state distinction.
|
||||
- `GHSA-28w4-rp2p-636w` and `GHSA-c7hx-vr97-45rg`: an authorized `DeleteBucket` with `x-rustfs-force-delete: true` removed active COMPLIANCE-retained versions even though ordinary version deletion was denied. Lesson: bucket force-delete must enforce retention across every version, including against root/owner callers.
|
||||
|
||||
### Serde deserialization and input validation
|
||||
|
||||
@@ -135,16 +139,17 @@ rg -n "deny_unknown_fields|serde.default|as u32|as usize|as i32" rustfs crates
|
||||
## Minimum Regression Test Expectations
|
||||
|
||||
- Authz fixes: include unauthenticated, valid low-privilege, wrong-action, correct-action, owner, non-owner, and root/admin cases as applicable.
|
||||
- Protocol frontend authz fixes: include denied `RETR`, `SIZE`/`MDTM`, `MKD`, bucket probe, and sibling allowed-operation cases, and assert denied paths do not reach the storage backend.
|
||||
- Protocol frontend authz fixes: include denied `RETR`, `SIZE`/`MDTM`, `MKD`, bucket probe, and sibling allowed-operation cases; test bucket-policy explicit denies and condition-key denies on each enabled listener against an S3 control, and assert denied paths do not reach storage.
|
||||
- IAM fixes: include import/update/list service-account cases with attacker-controlled parent, claims, access key, secret key, and policy.
|
||||
- Copy/upload fixes: include cross-bucket, cross-user, source-denied, destination-denied, copy-source-condition, and multipart completion cases.
|
||||
- Presigned upload fixes: include a valid presign with extra unsigned tagging, redirect, storage-class, and cross-bucket copy-source headers; require rejection before source or destination storage access, and verify explicitly signed equivalents still work.
|
||||
- Version-action fixes: include historical UUID, explicit current version, `null`, range, partNumber, presigned, STS/session, service-account, anonymous bucket-policy, copy source, and multipart-copy source cases.
|
||||
- Version-write fixes: a `PutObject`-only principal must not replace an existing version through `PutObject` or browser POST `?versionId=`; verify replication-authorized writes and versioned, suspended, and unversioned buckets separately.
|
||||
- Policy-condition fixes: include reserved-key header collisions, missing keys, partially overlapping multi-value sets, plugin mode, and built-in policy mode.
|
||||
- Path fixes: include encoded traversal, absolute path, nested traversal, archive entries with `..`, valid object keys that resemble traversal text but should be rejected, and canonical bucket/prefix boundary checks.
|
||||
- Path fixes: include encoded traversal, absolute path, nested traversal, archive entries with `..`, valid object keys that resemble traversal text but should be rejected, and canonical bucket/prefix boundary checks; test per-key `DeleteObjects` errors and prefix-scoped denials on versioned buckets.
|
||||
- Logging fixes: assert redacted output for structs and response bodies that may contain credentials.
|
||||
- IAM export fixes: assert exported archives omit plaintext user and service-account secrets unless the format deliberately encrypts or seals them.
|
||||
- RPC auth fixes: include captured metadata replay across two concrete methods, stale timestamps, wrong path, wrong method surrogate, wrong secret, and valid same-method calls.
|
||||
- Browser/CORS fixes: assert no credentials on reflected/default origins, correct behavior for explicit allowlists, and no same-origin script execution for previewed object content.
|
||||
- SSE fixes: inspect stored bytes and verify API metadata, read-back behavior, and on-disk ciphertext together; for ETags, upload equal plaintext under different keys and assert list-only callers cannot derive or correlate the plaintext digest, including multipart objects.
|
||||
- Object Lock fixes: include unreadable metadata, fabricated metadata defaults, unparsable config, confirmed absent config, COMPLIANCE/GOVERNANCE retention, lifecycle expiry, scanner sweeps, and force-delete paths.
|
||||
- Object Lock fixes: include unreadable metadata, fabricated metadata defaults, unparsable config, confirmed absent config, COMPLIANCE/GOVERNANCE retention, lifecycle expiry, scanner sweeps, and force-delete paths; an authorized force-delete must preserve active COMPLIANCE versions.
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=98dff0961c6eb21d6128bfd2d4a6100a624ddf1fb7db05dba360a5d3d0a310c2
|
||||
sha256-linux=9ff0da7406de9b89c0e848ff29f2238b30ba9da89c7e9df97c246ce86631350f
|
||||
sha256-linux=890f5925b87e1bf3710c356253f3b2276d901e156df9ccc04ec9315374de6d22
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=55534a97fbd376f64c8f6c341d319017d11ff77cad6da8629a1a7f6a874e0315
|
||||
sha256-linux=c06fb8c19aed6f388b9dc61cb8251b7a44f8561a9bf764ad2b9e635598f8dc17
|
||||
sha256-darwin=a2cb67d91eb0d093928371d6e84a84b2dd41b2ba975a28094e81879bd65838ad
|
||||
sha256-linux=82db0561cf9b111f11782d647d0859c9d2a44c5c68481b5e39c48505828bd392
|
||||
|
||||
@@ -55,6 +55,281 @@
|
||||
"invariant": "odm-source-contract-gcs",
|
||||
"suite": "rustfs",
|
||||
"name": "on_demand_migration::gcs::tests::gcs_native_backend_satisfies_the_shared_backend_contract"
|
||||
},
|
||||
{
|
||||
"invariant": "rollback-inspection",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::core::io_primitives::tests::rename_rollback_incomplete_preserves_overwrite_data_dirs_and_staging"
|
||||
},
|
||||
{
|
||||
"invariant": "rollback-closed-inspection-channel",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::core::io_primitives::tests::rename_rollback_closed_heal_receiver_preserves_recovery_material"
|
||||
},
|
||||
{
|
||||
"invariant": "rollback-cancellation",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::core::io_primitives::tests::rename_rollback_incomplete_cancelled_task_is_not_success"
|
||||
},
|
||||
{
|
||||
"invariant": "rollback-completion-equivalence",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::core::io_primitives::tests::rename_rollback_incomplete_matches_early_ack_and_full_wait_after_reopen"
|
||||
},
|
||||
{
|
||||
"invariant": "rollback-physical-owner",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::core::io_primitives::tests::rename_rollback_children_keep_namespace_ownership_after_coordinator_panic"
|
||||
},
|
||||
{
|
||||
"invariant": "late-ack-retains-commit",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::core::io_primitives::tests::rename_data_early_ack_post_mutation_tail_error_never_rolls_back_commit"
|
||||
},
|
||||
{
|
||||
"invariant": "tail-drained-cas",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::ops::object::put_object_tmp_cleanup_tests::tail_drained_put_waits_for_tail_and_allows_immediate_cas"
|
||||
},
|
||||
{
|
||||
"invariant": "seeded-put-tail-pressure",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::ops::object::put_object_tmp_cleanup_tests::seeded_put_completion_pressure_drains_every_real_tail"
|
||||
},
|
||||
{
|
||||
"invariant": "borrowed-lock-tail",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::ops::object::put_object_tmp_cleanup_tests::no_lock_put_waits_for_rename_tail_under_outer_guard"
|
||||
},
|
||||
{
|
||||
"invariant": "cancelled-writer-owner",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::ops::object::put_object_tmp_cleanup_tests::tail_drained_put_owned_commit_survives_waiter_cancellation"
|
||||
},
|
||||
{
|
||||
"invariant": "get-body-reconstruction",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "ecstore_validation_blackbox::blackbox_get_restores_body_after_one_shard_file_is_removed"
|
||||
},
|
||||
{
|
||||
"invariant": "range-reconstruction",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "ecstore_validation_blackbox::blackbox_range_read_restores_exact_slice_with_one_offline_disk"
|
||||
},
|
||||
{
|
||||
"invariant": "multipart-abort-linearization",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::ops::multipart::tests::abort_and_complete_linearize_for_plain_sse_and_legacy_layouts"
|
||||
},
|
||||
{
|
||||
"invariant": "multipart-write-quorum",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::ops::multipart::tests::abort_enforces_delete_write_quorum_boundary"
|
||||
},
|
||||
{
|
||||
"invariant": "multipart-physical-owner",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::ops::multipart::tests::complete_multipart_timeout_keeps_namespace_owner_until_physical_publication"
|
||||
},
|
||||
{
|
||||
"invariant": "restore-abort-cleanup",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::ops::object::transition_commit_failure_tests::multipart_restore_aborts_every_post_create_failure"
|
||||
},
|
||||
{
|
||||
"invariant": "legacy-metadata",
|
||||
"suite": "rustfs-filemeta",
|
||||
"name": "filemeta::test::test_issue_2288_legacy_xlmeta_compatibility"
|
||||
},
|
||||
{
|
||||
"invariant": "legacy-nil-version",
|
||||
"suite": "rustfs-filemeta",
|
||||
"name": "filemeta::test::test_into_fileinfo_reads_legacy_nil_uuid_inline_key"
|
||||
},
|
||||
{
|
||||
"invariant": "commit-phase-schedule",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::core::io_primitives::tests::commit_fault_schedule_keeps_multiple_disk_phases_independent"
|
||||
},
|
||||
{
|
||||
"invariant": "commit-phase-ownership",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::core::io_primitives::tests::commit_fault_schedule_rejects_duplicate_checkpoint_without_disarming_owner"
|
||||
},
|
||||
{
|
||||
"invariant": "metadata-observation-all-four-slot-states-and-arrival-orders-preserve-reduction",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::core::metadata_quorum::tests::metadata_observation_all_four_slot_states_and_arrival_orders_preserve_reduction"
|
||||
},
|
||||
{
|
||||
"invariant": "metadata-observation-mixed-current-identity-arrivals-preserve-selection",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::core::metadata_quorum::tests::metadata_observation_mixed_current_identity_arrivals_preserve_selection"
|
||||
},
|
||||
{
|
||||
"invariant": "metadata-observation-pending-newer-version-and-same-time-different-directory-force-full-wait",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::core::metadata_quorum::tests::metadata_observation_pending_newer_version_and_same_time_different_directory_force_full_wait"
|
||||
},
|
||||
{
|
||||
"invariant": "metadata-observation-duplicate-shard-cannot-supply-a-read-reserve",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::core::metadata_quorum::tests::metadata_observation_duplicate_shard_cannot_supply_a_read_reserve"
|
||||
},
|
||||
{
|
||||
"invariant": "metadata-observation-null-versions-markers-and-invalid-success-keep-their-meaning",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::core::metadata_quorum::tests::metadata_observation_null_versions_markers_and_invalid_success_keep_their_meaning"
|
||||
},
|
||||
{
|
||||
"invariant": "metadata-cache-real-get-and-range-preserve-body-across-read-modes",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::read::metadata_cache_tests::metadata_cache_real_get_and_range_preserve_body_across_read_modes"
|
||||
},
|
||||
{
|
||||
"invariant": "late-metadata-refresh-rejects-changed-generation-and-mapped-shards",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::read::metadata_cache_tests::late_metadata_refresh_rejects_changed_generation_and_mapped_shards"
|
||||
},
|
||||
{
|
||||
"invariant": "late-full-get-rejects-changed-object-generation",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::prepared_get_object_metadata_tests::non_inline_two_phase_get_rejects_generation_change_before_late_refresh"
|
||||
},
|
||||
{
|
||||
"invariant": "list-objects-shared-corpus-static-namespace",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "store::list_objects::differential_tests::list_objects_shared_corpus_static_namespace"
|
||||
},
|
||||
{
|
||||
"invariant": "list-objects-shared-corpus-cross-subdirectory-limit",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "store::list_objects::differential_tests::list_objects_shared_corpus_cross_subdirectory_limit"
|
||||
},
|
||||
{
|
||||
"invariant": "list-objects-shared-corpus-max-and-max-plus-one",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "store::list_objects::differential_tests::list_objects_shared_corpus_max_and_max_plus_one"
|
||||
},
|
||||
{
|
||||
"invariant": "list-objects-shared-corpus-fixed-seed-marker-and-prefix",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "store::list_objects::differential_tests::list_objects_shared_corpus_fixed_seed_marker_and_prefix"
|
||||
},
|
||||
{
|
||||
"invariant": "list-objects-shared-corpus-offline-corrupt-and-minority-stale",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "store::list_objects::differential_tests::list_objects_shared_corpus_offline_corrupt_and_minority_stale"
|
||||
},
|
||||
{
|
||||
"invariant": "list-objects-shared-corpus-versions-delete-markers-and-free-versions",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "store::list_objects::differential_tests::list_objects_shared_corpus_versions_delete_markers_and_free_versions"
|
||||
},
|
||||
{
|
||||
"invariant": "list-objects-shared-corpus-orphan-cleanup-requires-authoritative-evidence",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "store::list_objects::differential_tests::list_objects_shared_corpus_orphan_cleanup_requires_authoritative_evidence"
|
||||
},
|
||||
{
|
||||
"invariant": "list-objects-shared-corpus-consumer-drop-joins-real-disk-producer",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "store::list_objects::differential_tests::list_objects_shared_corpus_consumer_drop_joins_real_disk_producer"
|
||||
},
|
||||
{
|
||||
"invariant": "list-objects-shared-corpus-cancelled-quorum-walk-releases-senders-and-disk-lock",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "store::list_objects::differential_tests::list_objects_shared_corpus_cancelled_quorum_walk_releases_senders_and_disk_lock"
|
||||
},
|
||||
{
|
||||
"invariant": "list-objects-shared-corpus-enabled-index-contracts",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "store::list_objects::differential_tests::list_objects_shared_corpus_enabled_index_contracts"
|
||||
},
|
||||
{
|
||||
"invariant": "borrowed-context-boundary",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::ops::object::put_object_tmp_cleanup_tests::borrowed_write_context_rejects_wrong_namespace_and_cache_flag"
|
||||
},
|
||||
{
|
||||
"invariant": "borrowed-put-cancellation",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::ops::object::put_object_tmp_cleanup_tests::borrowed_tail_drained_put_keeps_owner_after_caller_cancel"
|
||||
},
|
||||
{
|
||||
"invariant": "borrowed-mpu-cancellation",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::ops::multipart::tests::borrowed_complete_keeps_namespace_owner_after_caller_cancel"
|
||||
},
|
||||
{
|
||||
"invariant": "pool-cas-owner",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "core::pools::pools_tests::pool_meta_cas_context_retains_actual_owner_for_exact_target"
|
||||
},
|
||||
{
|
||||
"invariant": "pool-cas-released-owner",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "core::pools::pools_tests::pool_meta_cas_rejects_released_owner"
|
||||
},
|
||||
{
|
||||
"invariant": "copy-tail-drained",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "store::object::tests::borrowed_copy_tail_drained_waits_for_rename_tail"
|
||||
},
|
||||
{
|
||||
"invariant": "catalog-borrowed-publication",
|
||||
"suite": "rustfs",
|
||||
"name": "admin::handlers::table_catalog::tests::commit_backend_borrowed_put_preserves_owner_authorization_and_precondition"
|
||||
},
|
||||
{
|
||||
"invariant": "directory-marker-quorum",
|
||||
"suite": "rustfs-filemeta",
|
||||
"name": "metacache::tests::resolve_directory_marker_and_prefix_preserve_quorum_in_both_orders"
|
||||
},
|
||||
{
|
||||
"invariant": "logical-prefix-limit",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "store::list_objects::test::list_path_gather_results_counts_common_prefixes_before_page_limit"
|
||||
},
|
||||
{
|
||||
"invariant": "quota-same-name-owner-and-disk-authority",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "bucket::metadata_sys::tests::quota_snapshot_reuses_same_name_owner_and_rereads_disk_authority"
|
||||
},
|
||||
{
|
||||
"invariant": "quota-snapshot-scope-and-owner",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "bucket::metadata_sys::tests::quota_snapshot_rejects_wrong_scope_incarnation_and_transaction_guard"
|
||||
},
|
||||
{
|
||||
"invariant": "quota-snapshot-lifecycle-and-incarnation",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "bucket::metadata_sys::tests::quota_snapshot_rejects_lifecycle_loss_during_read_and_changed_disk_incarnation"
|
||||
},
|
||||
{
|
||||
"invariant": "marker-purge-receipt-reopen",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::ops::object::marker_purge_receipt_tests::quorum_marker_purge_receipt_survives_reopen_with_stale_member"
|
||||
},
|
||||
{
|
||||
"invariant": "external-minio-shards",
|
||||
"suite": "rustfs-ecstore::pinned_erasure_fixtures_test",
|
||||
"name": "pinned_minio_shards_decode_and_reconstruct_plaintext"
|
||||
},
|
||||
{
|
||||
"invariant": "external-legacy-shards",
|
||||
"suite": "rustfs-ecstore::pinned_erasure_fixtures_test",
|
||||
"name": "pinned_legacy_shards_decode_and_reconstruct_plaintext"
|
||||
},
|
||||
{
|
||||
"invariant": "external-shard-bitrot",
|
||||
"suite": "rustfs-ecstore::pinned_erasure_fixtures_test",
|
||||
"name": "pinned_erasure_shards_reject_bitrot_corruption"
|
||||
},
|
||||
{
|
||||
"invariant": "external-shard-quorum-codec",
|
||||
"suite": "rustfs-ecstore::pinned_erasure_fixtures_test",
|
||||
"name": "pinned_erasure_shards_reject_insufficient_quorum_and_wrong_codec"
|
||||
}
|
||||
],
|
||||
"fixtures": [
|
||||
@@ -82,6 +357,36 @@
|
||||
"path": "crates/ecstore/tests/fixtures/minio/bucket_metadata_full.xlmeta.hex",
|
||||
"sha256": "3b6de589519c08a1614c8bd409bb8199c17d42043861b07bce513075e6fbfc12",
|
||||
"source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md"
|
||||
},
|
||||
{
|
||||
"path": "crates/filemeta/tests/fixtures/issue_2288_legacy_xlmeta.hex",
|
||||
"sha256": "b3e8d32f9950a38b5dc71f679519cc830a2b2a807e8b0b39f78840b238d8eacf",
|
||||
"source": "Legacy xl.meta captured for rustfs/backlog#2288; filemeta::test::test_issue_2288_legacy_xlmeta_compatibility"
|
||||
},
|
||||
{
|
||||
"path": "crates/filemeta/tests/fixtures/issue_2265_legacy_meta_v2_object.hex",
|
||||
"sha256": "89dd7ec92d16e54b5d31e0c49c9edffff8a7aa58890aaeff59bc2ff84da0a62f",
|
||||
"source": "Legacy metadata v2 object captured for rustfs/backlog#2265; crates/filemeta/src/test_data.rs"
|
||||
},
|
||||
{
|
||||
"path": "crates/ecstore/tests/fixtures/list_namespace_keys.json",
|
||||
"sha256": "5afddb60565deba0938e617470255563f0e15a15fab699645c59bb7ccc84898e",
|
||||
"source": "Shared static LIST namespace corpus; ECStore differential tests and ODM list_through_static_namespace_boundary_matrix"
|
||||
},
|
||||
{
|
||||
"path": "crates/ecstore/tests/fixtures/erasure-shards/minio/manifest.json",
|
||||
"sha256": "c398f8dcb76903f6cf8eee07931c76592895ced201ff9d59ed4ff9b0ead2b13c",
|
||||
"source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/erasure-shards/README.md"
|
||||
},
|
||||
{
|
||||
"path": "crates/ecstore/tests/fixtures/erasure-shards/legacy/manifest.json",
|
||||
"sha256": "dce397d5b2f42748eb5091b4355bb9d133a50cbba003626fafac28095dbe540b",
|
||||
"source": "RustFS 1.0.0-alpha.37; crates/ecstore/tests/fixtures/erasure-shards/README.md"
|
||||
},
|
||||
{
|
||||
"path": "crates/ecstore/tests/fixtures/erasure-shards/plaintext.bin",
|
||||
"sha256": "b5a83332327961a31b65a8731c64559fd04ff8ac9f6124022201dbbdba5ffa01",
|
||||
"source": "Independent fixed SHA256 counter-stream oracle; crates/ecstore/tests/fixtures/erasure-shards/README.md"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
+14
-3
@@ -92,6 +92,12 @@ setup = 'ecstore-large-stack'
|
||||
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
# These handler/store fixtures and the relocated-pool GET compose deep storage
|
||||
# futures in libtest threads; on Linux they overflow the default 2 MiB stack.
|
||||
[[profile.default.scripts]]
|
||||
filter = 'package(rustfs) & (test(/^admin::handlers::table_catalog::tests::/) | test(/^table_catalog::store::tests::/) | test(=app::object::get::tests::execute_get_object_resumes_from_relocated_pool_without_splicing_body))'
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
[[profile.default.scripts]]
|
||||
filter = 'binary(lifecycle_integration_test) | (package(rustfs) & test(/^app::lifecycle_transition_api_test::/))'
|
||||
setup = 'lifecycle-large-stack'
|
||||
@@ -170,10 +176,11 @@ filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnost
|
||||
test-group = 'embedded-test-ports'
|
||||
|
||||
# Registration, inventory delivery, and real drive/object probes include
|
||||
# durable filesystem IO in short deadlines. Reserve capacity so sibling test
|
||||
# filesystem or HTTP IO in short deadlines. Reserve capacity so sibling test
|
||||
# processes cannot exhaust those budgets. Keep deadlines and assertions intact.
|
||||
# The short HTTP fixtures are tracked in rustfs/backlog#2714.
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_object) & test(=real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put)))'
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# This cancellation probe hashes the serving executable before acquiring its
|
||||
@@ -336,6 +343,10 @@ setup = 'ecstore-large-stack'
|
||||
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
[[profile.ci.scripts]]
|
||||
filter = 'package(rustfs) & (test(/^admin::handlers::table_catalog::tests::/) | test(/^table_catalog::store::tests::/) | test(=app::object::get::tests::execute_get_object_resumes_from_relocated_pool_without_splicing_body))'
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
[[profile.ci.scripts]]
|
||||
filter = 'binary(lifecycle_integration_test) | (package(rustfs) & test(/^app::lifecycle_transition_api_test::/))'
|
||||
setup = 'lifecycle-large-stack'
|
||||
@@ -428,7 +439,7 @@ filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnost
|
||||
test-group = 'embedded-test-ports'
|
||||
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_object) & test(=real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put)))'
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# Preserve the cancellation probe's startup budget and assertions in CI too.
|
||||
|
||||
@@ -98,6 +98,10 @@ runs:
|
||||
shell: bash
|
||||
run: make script-tests
|
||||
|
||||
- name: Check object generation protocol model
|
||||
shell: bash
|
||||
run: python3 ./scripts/test_object_generation_protocol_model.py
|
||||
|
||||
- name: Check test wiring
|
||||
shell: bash
|
||||
run: python3 ./scripts/check_test_wiring.py
|
||||
|
||||
@@ -88,7 +88,7 @@ jobs:
|
||||
# database lookup with a second ignore list to maintain.
|
||||
cargo-deny:
|
||||
name: Cargo Deny
|
||||
if: github.event_name != 'pull_request' || github.event.action != 'closed'
|
||||
if: github.repository == 'rustfs/rustfs' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
runs-on: sm-standard-2
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
@@ -131,7 +131,7 @@ jobs:
|
||||
|
||||
workflow-pin-report:
|
||||
name: Workflow Pin Report
|
||||
if: github.event_name != 'pull_request' || github.event.action != 'closed'
|
||||
if: github.repository == 'rustfs/rustfs' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
runs-on: sm-standard-2
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
@@ -203,7 +203,7 @@ jobs:
|
||||
# workflow produces, and until now it was only visible to whoever happened to
|
||||
# open the Actions tab. Same ci-8 mechanism coverage.yml and
|
||||
# e2e-replication-nightly.yml already use.
|
||||
if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
|
||||
if: github.repository == 'rustfs/rustfs' && (always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure'))
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
|
||||
@@ -440,6 +440,17 @@ jobs:
|
||||
fi
|
||||
done
|
||||
|
||||
- name: Verify macOS binary has no dynamic liblzma dependency
|
||||
if: matrix.platform == 'macos'
|
||||
shell: bash
|
||||
run: |
|
||||
binary="target/${{ matrix.target }}/release/rustfs"
|
||||
if otool -L "$binary" | grep -E 'liblzma.*\.dylib'; then
|
||||
echo "macOS release binary must not depend on a dynamic liblzma library" >&2
|
||||
exit 1
|
||||
fi
|
||||
"$binary" --version
|
||||
|
||||
- name: Generate the CPU symbol catalog
|
||||
if: endsWith(matrix.target, '-linux-gnu') && matrix.features == 'pyroscope'
|
||||
shell: bash
|
||||
|
||||
@@ -36,7 +36,9 @@ on:
|
||||
- "flake.lock"
|
||||
pull_request:
|
||||
types: [ opened, synchronize, reopened, closed ]
|
||||
branches: [ main ]
|
||||
# gateway/integration carries main's ruleset (rustfs/backlog#2734), so its
|
||||
# pull requests must produce the same required checks.
|
||||
branches: [ main, gateway/integration ]
|
||||
merge_group:
|
||||
types: [ checks_requested ]
|
||||
schedule:
|
||||
@@ -509,7 +511,7 @@ jobs:
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
strategy:
|
||||
# Preserve both independent results when one protocol fails, so diagnosis
|
||||
# Preserve independent results when one protocol fails, so diagnosis
|
||||
# and a focused fix do not require rebuilding an interrupted sibling.
|
||||
fail-fast: false
|
||||
matrix:
|
||||
@@ -518,6 +520,8 @@ jobs:
|
||||
flags: "--features swift"
|
||||
- name: sftp
|
||||
flags: "--features sftp"
|
||||
- name: http3
|
||||
flags: "--features http3,swift"
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
|
||||
@@ -78,6 +78,7 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
distributed:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Distributed 4-node 4-disk e2e
|
||||
# GitHub-hosted VM: loop and tmpfs mounts work here. sm-standard-4 is an
|
||||
# ARC pod and rejects both (`mount -o loop` ENOENT, tmpfs "read-only").
|
||||
@@ -167,11 +168,13 @@ jobs:
|
||||
FILTER: ${{ inputs.filter }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
rm -f target/nextest/e2e-distributed/junit.xml
|
||||
if [ -n "${FILTER}" ]; then
|
||||
python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-distributed -p e2e_test -E "${FILTER}"
|
||||
else
|
||||
python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-distributed -p e2e_test --no-tests=fail
|
||||
fi
|
||||
test -s target/nextest/e2e-distributed/junit.xml
|
||||
|
||||
- name: Upload distributed e2e diagnostics
|
||||
if: always()
|
||||
@@ -200,7 +203,7 @@ jobs:
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [distributed]
|
||||
if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
|
||||
if: github.repository == 'rustfs/rustfs' && (always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure'))
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
|
||||
@@ -53,6 +53,7 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
repl-nightly:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Replication e2e (nightly)
|
||||
# dual-node tests spawn two full rustfs servers each; the extra headroom
|
||||
# over the per-PR sm-standard-2 e2e job keeps the parallel process fan-out
|
||||
@@ -119,6 +120,7 @@ jobs:
|
||||
if-no-files-found: ignore
|
||||
|
||||
cluster-nightly:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Cluster fault e2e (nightly)
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
@@ -168,6 +170,7 @@ jobs:
|
||||
if-no-files-found: warn
|
||||
|
||||
protocols-nightly:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Protocol e2e (nightly)
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
@@ -229,7 +232,7 @@ jobs:
|
||||
# Only scheduled runs open/append the tracking issue (backlog#1149 ci-8);
|
||||
# manual workflow_dispatch runs stay quiet so a debugging run never files a
|
||||
# spurious alert.
|
||||
if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
|
||||
if: github.repository == 'rustfs/rustfs' && (always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure'))
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
|
||||
@@ -11,6 +11,7 @@ concurrency:
|
||||
cancel-in-progress: false
|
||||
jobs:
|
||||
collect:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
|
||||
@@ -72,9 +72,11 @@ jobs:
|
||||
fuzz-build:
|
||||
name: Build Fuzz Harness
|
||||
if: >
|
||||
(github.event_name == 'pull_request' && github.event.action != 'closed') ||
|
||||
github.event_name == 'schedule' ||
|
||||
github.event_name == 'workflow_dispatch'
|
||||
github.repository == 'rustfs/rustfs' && (
|
||||
(github.event_name == 'pull_request' && github.event.action != 'closed') ||
|
||||
github.event_name == 'schedule' ||
|
||||
github.event_name == 'workflow_dispatch'
|
||||
)
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
@@ -239,7 +241,7 @@ jobs:
|
||||
# `always()` is required: without it this job is skipped when a needed
|
||||
# job fails. Alerts only for scheduled (nightly) runs (backlog#1149
|
||||
# ci-8); PR and manual dispatch failures are already watched by a human.
|
||||
if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
|
||||
if: github.repository == 'rustfs/rustfs' && (always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure'))
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
|
||||
@@ -0,0 +1,197 @@
|
||||
# Copyright 2026 RustFS Team
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
name: Gateway Integration Sync
|
||||
|
||||
# Merges `main` into the long-lived `gateway/integration` branch every weekday
|
||||
# so the s3s -> gateway migration (rustfs/backlog#2734, task T0.1 in #2736)
|
||||
# never accumulates merge debt. A clean merge lands as a `--no-ff` merge commit
|
||||
# pushed straight to the branch. A conflicting merge is aborted and surfaced as
|
||||
# a pull request from `sync/<source>-<date>`, whose head is the un-merged
|
||||
# source commit, so the conflict is resolved the same day, in review, with no
|
||||
# other change riding along. The run itself stays green in that case: the pull
|
||||
# request is the signal. Any other failure (push rejected, pull request not
|
||||
# created) fails the run and, on the schedule, files the shared failure issue.
|
||||
#
|
||||
# Resolving a conflict pull request: merge `origin/gateway/integration` into
|
||||
# the sync branch, resolve, push, get the review. Land it with a fast-forward
|
||||
# push of the resolved merge commit to `gateway/integration`, not with the
|
||||
# squash button: a squash drops the merge ancestry and the same hunks conflict
|
||||
# again on the next run.
|
||||
#
|
||||
# Write scope: only `gateway/integration` and `sync/*` are ever pushed, and
|
||||
# both push targets below are literal refs. The pushes use a one-hour
|
||||
# installation token of the organisation's own GitHub App (contents: write)
|
||||
# because that app is the only non-admin bypass actor of the
|
||||
# `gateway/integration` ruleset; GITHUB_TOKEN cannot be a ruleset bypass actor,
|
||||
# so it cannot push through the required reviews and checks. The conflict pull
|
||||
# request is opened with FLAKE_UPDATE_TOKEN, the PAT that already opens the
|
||||
# weekly flake.lock pull requests, because this repository keeps "Allow GitHub
|
||||
# Actions to create and approve pull requests" switched off. GITHUB_TOKEN is
|
||||
# therefore read-only here.
|
||||
#
|
||||
# `schedule` fires only from the default branch, so the cron is live once this
|
||||
# file exists on `main`. `workflow_dispatch` runs from any ref that carries the
|
||||
# file and always acts on `gateway/integration`.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 2 * * 1-5" # Weekdays 02:00 UTC
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
source_ref:
|
||||
description: >-
|
||||
Branch to merge into gateway/integration. The schedule always merges
|
||||
main; only the conflict acceptance test (rustfs/backlog#2736) sets
|
||||
another branch.
|
||||
required: false
|
||||
default: main
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
TARGET_BRANCH: gateway/integration
|
||||
|
||||
jobs:
|
||||
sync:
|
||||
name: Merge main into gateway/integration
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Create the branch push token
|
||||
id: push-token
|
||||
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3
|
||||
with:
|
||||
app-id: ${{ vars.CLA_BOT_APP_ID }}
|
||||
private-key: ${{ secrets.CLA_BOT_APP_PRIVATE_KEY }}
|
||||
owner: ${{ github.repository_owner }}
|
||||
repositories: rustfs
|
||||
permission-contents: write
|
||||
|
||||
- name: Checkout gateway/integration
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
# persist-credentials-exempt: this checkout's token IS the push
|
||||
# credential — the merge commit and the sync branch are pushed with it
|
||||
# below. It is a one-hour installation token scoped to contents.
|
||||
ref: gateway/integration
|
||||
fetch-depth: 0
|
||||
token: ${{ steps.push-token.outputs.token }}
|
||||
|
||||
- name: Configure the bot identity
|
||||
run: |
|
||||
git config user.name "rustfs-cla-assistant[bot]"
|
||||
git config user.email "273121464+rustfs-cla-assistant[bot]@users.noreply.github.com"
|
||||
|
||||
- name: Merge the source branch
|
||||
id: merge
|
||||
env:
|
||||
SOURCE_REF: ${{ inputs.source_ref || 'main' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if ! [[ "$SOURCE_REF" =~ ^[A-Za-z0-9][A-Za-z0-9._/-]*$ ]] \
|
||||
|| [[ "$SOURCE_REF" == *..* ]] \
|
||||
|| [[ "$SOURCE_REF" == "$TARGET_BRANCH" ]]; then
|
||||
echo "::error::refusing source_ref '${SOURCE_REF}'"
|
||||
exit 1
|
||||
fi
|
||||
git fetch origin "refs/heads/${SOURCE_REF}:refs/remotes/origin/${SOURCE_REF}"
|
||||
source_sha=$(git rev-parse "origin/${SOURCE_REF}")
|
||||
target_sha=$(git rev-parse HEAD)
|
||||
{
|
||||
echo "source_ref=${SOURCE_REF}"
|
||||
echo "source_sha=${source_sha}"
|
||||
echo "target_sha=${target_sha}"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
if git merge --no-ff --no-edit "origin/${SOURCE_REF}"; then
|
||||
if [[ "$(git rev-parse HEAD)" == "$target_sha" ]]; then
|
||||
echo "result=up-to-date" >> "$GITHUB_OUTPUT"
|
||||
echo "\`${TARGET_BRANCH}\` already contains \`${SOURCE_REF}\` (${source_sha}); nothing to push." >> "$GITHUB_STEP_SUMMARY"
|
||||
else
|
||||
echo "result=merged" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
git diff --name-only --diff-filter=U > "$RUNNER_TEMP/conflicts.txt"
|
||||
git merge --abort
|
||||
if [[ ! -s "$RUNNER_TEMP/conflicts.txt" ]]; then
|
||||
echo "::error::git merge failed without reporting conflicting paths; this is not a conflict"
|
||||
exit 1
|
||||
fi
|
||||
echo "result=conflict" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Push the merge commit
|
||||
if: steps.merge.outputs.result == 'merged'
|
||||
env:
|
||||
SOURCE_REF: ${{ steps.merge.outputs.source_ref }}
|
||||
SOURCE_SHA: ${{ steps.merge.outputs.source_sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git push origin HEAD:refs/heads/gateway/integration
|
||||
echo "Merged \`${SOURCE_REF}\` (${SOURCE_SHA}) into \`${TARGET_BRANCH}\` as $(git rev-parse HEAD)." >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Open the conflict pull request
|
||||
if: steps.merge.outputs.result == 'conflict'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.FLAKE_UPDATE_TOKEN }}
|
||||
SOURCE_REF: ${{ steps.merge.outputs.source_ref }}
|
||||
SOURCE_SHA: ${{ steps.merge.outputs.source_sha }}
|
||||
TARGET_SHA: ${{ steps.merge.outputs.target_sha }}
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
date_utc=$(date -u +%Y-%m-%d)
|
||||
sync_branch="sync/${SOURCE_REF//\//-}-${date_utc}"
|
||||
git push origin "refs/remotes/origin/${SOURCE_REF}:refs/heads/${sync_branch}"
|
||||
{
|
||||
echo "The scheduled merge of \`${SOURCE_REF}\` into \`${TARGET_BRANCH}\` stopped on conflicts and was aborted; \`${TARGET_BRANCH}\` is unchanged."
|
||||
echo
|
||||
echo "This branch is the un-merged \`${SOURCE_REF}\` head. Merge \`origin/${TARGET_BRANCH}\` into it, resolve the files below, push, and get the review. Land the resolved merge commit with a fast-forward push to \`${TARGET_BRANCH}\`; a squash merge drops the merge ancestry and the same hunks conflict again on the next run. Do not include any change that is not conflict resolution."
|
||||
echo
|
||||
echo "- Run: ${RUN_URL}"
|
||||
echo "- Source: \`${SOURCE_REF}\` at ${SOURCE_SHA}"
|
||||
echo "- Target: \`${TARGET_BRANCH}\` at ${TARGET_SHA}"
|
||||
echo
|
||||
echo "## Conflicting files"
|
||||
echo
|
||||
echo "Output of \`git diff --name-only --diff-filter=U\` captured before \`git merge --abort\`:"
|
||||
echo
|
||||
sed 's/^/- /' "$RUNNER_TEMP/conflicts.txt"
|
||||
} > "$RUNNER_TEMP/pr-body.md"
|
||||
title="sync: merge ${SOURCE_REF} ${date_utc} (conflicts)"
|
||||
existing=$(gh pr list --repo "$GITHUB_REPOSITORY" --base "$TARGET_BRANCH" --head "$sync_branch" --state open --json number --jq '.[0].number // empty')
|
||||
if [[ -n "$existing" ]]; then
|
||||
gh pr edit "$existing" --repo "$GITHUB_REPOSITORY" --title "$title" --body-file "$RUNNER_TEMP/pr-body.md"
|
||||
pr_url=$(gh pr view "$existing" --repo "$GITHUB_REPOSITORY" --json url --jq .url)
|
||||
else
|
||||
pr_url=$(gh pr create --repo "$GITHUB_REPOSITORY" --base "$TARGET_BRANCH" --head "$sync_branch" --title "$title" --body-file "$RUNNER_TEMP/pr-body.md")
|
||||
fi
|
||||
echo "Conflicts in \`${SOURCE_REF}\` -> \`${TARGET_BRANCH}\`; pull request: ${pr_url}" >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [sync]
|
||||
# Same ci-8 mechanism audit.yml and coverage.yml use: a scheduled run that
|
||||
# could not push or could not open the conflict pull request must not stay
|
||||
# visible only in the Actions tab.
|
||||
if: github.repository == 'rustfs/rustfs' && (always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure'))
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Open or update failure-tracking issue
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}
|
||||
@@ -43,6 +43,7 @@ env:
|
||||
|
||||
jobs:
|
||||
resolve-source:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Resolve nightly source
|
||||
runs-on: sm-standard-2
|
||||
timeout-minutes: 10
|
||||
@@ -496,8 +497,10 @@ jobs:
|
||||
name: Alert on scheduled failure
|
||||
needs: [build, kms-vault-lane, kms-vault-ha-failover]
|
||||
if: >-
|
||||
always() && github.event_name == 'schedule' &&
|
||||
(contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
|
||||
github.repository == 'rustfs/rustfs' && (
|
||||
always() && github.event_name == 'schedule' &&
|
||||
(contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
|
||||
)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
|
||||
@@ -32,6 +32,7 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
update-flake:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Update flake.lock
|
||||
runs-on: sm-standard-2
|
||||
timeout-minutes: 90
|
||||
|
||||
@@ -49,6 +49,7 @@ env:
|
||||
|
||||
jobs:
|
||||
warp-ab:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Warp A/B budget gate
|
||||
runs-on: sm-standard-2
|
||||
# A normal nightly restores the last successful binary and builds only the
|
||||
@@ -364,8 +365,10 @@ jobs:
|
||||
# composite action already reports cancelled/timed-out jobs in the issue
|
||||
# body.
|
||||
if: >-
|
||||
always() && github.event_name == 'schedule' &&
|
||||
(contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
|
||||
github.repository == 'rustfs/rustfs' && (
|
||||
always() && github.event_name == 'schedule' &&
|
||||
(contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
|
||||
)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
|
||||
@@ -42,6 +42,7 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
check-ephemerality:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Check runner ephemerality
|
||||
runs-on: sm-standard-2
|
||||
timeout-minutes: 15
|
||||
@@ -65,7 +66,7 @@ jobs:
|
||||
# Same ci-8 mechanism as coverage.yml, audit.yml and the nightly lanes:
|
||||
# scheduled runs file a tracking issue, manual dispatch stays quiet so
|
||||
# debugging never produces a spurious alert.
|
||||
if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
|
||||
if: github.repository == 'rustfs/rustfs' && (always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure'))
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
|
||||
@@ -266,7 +266,10 @@ jobs:
|
||||
# the previous 45m step cap killed four consecutive runs mid-rebalance
|
||||
# with steps 1-7 green. 240m covers the observed pace (rebalance 1/4
|
||||
# in ~21m) plus one decommission pass; override per dispatch.
|
||||
timeout-minutes: ${{ inputs.pool_timeout_minutes || 240 }}
|
||||
# fromJSON coerces the string dispatch input to the number that
|
||||
# timeout-minutes requires; the bare || form yielded the string '240'
|
||||
# and the runner rejected the template (Unexpected value '240').
|
||||
timeout-minutes: ${{ fromJSON(inputs.pool_timeout_minutes || '240') }}
|
||||
# Case failures keep the run green: the report and the backlog
|
||||
# issue manager carry the product signal.
|
||||
continue-on-error: true
|
||||
@@ -583,25 +586,6 @@ jobs:
|
||||
echo "pool test script reported a failure (step FAIL verdict or die() abort)" >&2
|
||||
failed=1
|
||||
fi
|
||||
# A runner-level kill (step timeout) truncates the suite without a
|
||||
# FAIL verdict or an [ERROR] line, so require every expected
|
||||
# [POOL-STEP] marker to be present with PASS - missing markers mean
|
||||
# the run was truncated, not green. This also closes the gap for
|
||||
# standalone dispatches, where the chain driver's evidence
|
||||
# validation does not run.
|
||||
last_step=6
|
||||
if [ "${{ inputs.pools || '3' }}" = "3" ]; then
|
||||
last_step=8
|
||||
if [ "${{ inputs.run_decommission != 'false' }}" = "true" ]; then
|
||||
last_step=9
|
||||
fi
|
||||
fi
|
||||
for n in $(seq 1 "${last_step}"); do
|
||||
grep -q "^\[POOL-STEP\] ${n} .* PASS" "${POOL_ARTIFACT_DIR}/pool-test.log" 2>/dev/null || {
|
||||
echo "pool test log is missing [POOL-STEP] ${n} PASS (expected through step ${last_step}); the run was truncated" >&2
|
||||
failed=1
|
||||
}
|
||||
done
|
||||
[ "${failed}" -eq 0 ] || exit 1
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
|
||||
@@ -28,6 +28,7 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
check-freshness:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Check scheduled validation freshness
|
||||
# Stay on GitHub-hosted runners: this monitor must keep running when the
|
||||
# self-hosted sm-standard-* pool is down, which is exactly when the
|
||||
|
||||
@@ -39,6 +39,7 @@ jobs:
|
||||
alert-on-incomplete-run:
|
||||
name: Alert on incomplete scheduled run
|
||||
if: >-
|
||||
github.repository == 'rustfs/rustfs' &&
|
||||
github.event.workflow_run.event == 'schedule' &&
|
||||
github.event.workflow_run.conclusion != 'success' &&
|
||||
github.event.workflow_run.conclusion != 'failure'
|
||||
|
||||
@@ -26,6 +26,7 @@ on:
|
||||
|
||||
jobs:
|
||||
stale:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
runs-on: sm-standard-2
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
|
||||
@@ -14,6 +14,7 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
update:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
runs-on: sm-standard-2
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
|
||||
@@ -40,6 +40,7 @@ env:
|
||||
|
||||
jobs:
|
||||
targets-live:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: PostgreSQL, MySQL, AMQP, and NATS
|
||||
runs-on: sm-standard-2
|
||||
timeout-minutes: 90
|
||||
@@ -175,8 +176,10 @@ jobs:
|
||||
name: Alert on scheduled failure
|
||||
needs: [targets-live]
|
||||
if: >-
|
||||
always() && github.event_name == 'schedule' &&
|
||||
(contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
|
||||
github.repository == 'rustfs/rustfs' && (
|
||||
always() && github.event_name == 'schedule' &&
|
||||
(contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
|
||||
)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
|
||||
Generated
+369
-178
File diff suppressed because it is too large
Load Diff
+29
-19
@@ -18,6 +18,7 @@ members = [
|
||||
"crates/audit", # Audit target management system with multi-target fan-out
|
||||
"crates/checksums", # client checksums
|
||||
"crates/common", # Shared utilities and data structures
|
||||
"crates/capture-authority", # Durable capture authority storage
|
||||
"crates/data-usage", # Shared data usage models and algorithms
|
||||
"crates/config", # Configuration management
|
||||
"crates/connect-symbol-catalog", # Release-only CPU profile symbol catalog generator
|
||||
@@ -73,7 +74,7 @@ resolver = "3"
|
||||
edition = "2024"
|
||||
license = "Apache-2.0"
|
||||
repository = "https://github.com/rustfs/rustfs"
|
||||
rust-version = "1.98.1"
|
||||
rust-version = "1.99.0"
|
||||
version = "1.0.1"
|
||||
homepage = "https://rustfs.com"
|
||||
description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
|
||||
@@ -157,7 +158,7 @@ futures-util = "0.3.34"
|
||||
pollster = "1.0.1"
|
||||
pulsar = { default-features = false, version = "6.9.0" }
|
||||
lapin = { default-features = false, version = "4.12.0" }
|
||||
hyper = { version = "1.11.1" }
|
||||
hyper = { version = "1.12.0" }
|
||||
hyper-rustls = { default-features = false, version = "0.27.10" }
|
||||
hyper-util = { version = "0.1.21" }
|
||||
http = "1.5.0"
|
||||
@@ -166,9 +167,9 @@ http-body-util = "0.1.5"
|
||||
memchr = "2.8.3"
|
||||
minlz = "1.2.3"
|
||||
reqwest = "0.13.5"
|
||||
rustfs-kafka-async = { version = "1.3.1" }
|
||||
rustfs-kafka-async = { version = "1.4.0" }
|
||||
socket2 = { version = "0.6.5" }
|
||||
tokio = { version = "1.53.1" }
|
||||
tokio = { version = "1.53.2" }
|
||||
tokio-rustls = { default-features = false, version = "0.26.6" }
|
||||
tokio-stream = { version = "0.1.19" }
|
||||
tokio-test = "0.4.6"
|
||||
@@ -178,6 +179,13 @@ tonic-prost = { version = "0.14.6" }
|
||||
tonic-prost-build = { version = "0.14.6" }
|
||||
tower = { version = "0.5.3" }
|
||||
tower-http = { version = "0.7.1" }
|
||||
h3 = "0.0.8"
|
||||
h3-quinn = "0.0.10"
|
||||
quinn = { version = "0.11.11", default-features = false, features = [
|
||||
"futures-io",
|
||||
"runtime-tokio",
|
||||
"rustls-aws-lc-rs",
|
||||
] }
|
||||
|
||||
# Serialization and Data Formats
|
||||
apache-avro = { version = "0.22.0", features = ["snappy", "zstandard"] }
|
||||
@@ -219,13 +227,13 @@ x509-parser = "0.18.1"
|
||||
sha1 = "0.11.0"
|
||||
sha2 = "0.11.0"
|
||||
subtle = "2.6"
|
||||
zeroize = { version = "1.9.0" }
|
||||
zeroize = { version = "1.9.1" }
|
||||
proptest = "1"
|
||||
|
||||
# Time and Date
|
||||
chrono = { version = "0.4.45" }
|
||||
humantime = "2.4.0"
|
||||
jiff = { version = "0.2.37" }
|
||||
jiff = { version = "0.2.38" }
|
||||
time = { version = "0.3.55" }
|
||||
|
||||
# Database
|
||||
@@ -244,9 +252,9 @@ atoi = "3.1.0"
|
||||
atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.12.0" }
|
||||
aws-credential-types = { version = "1.3.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.122.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.151.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.118.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.123.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.152.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.119.0" }
|
||||
aws-smithy-async = { version = "1.3.0" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.5.0" }
|
||||
aws-smithy-runtime-api = { version = "1.19.0" }
|
||||
@@ -279,7 +287,7 @@ highway = { version = "1.3.0" }
|
||||
hostname = "0.4.2"
|
||||
ipnetwork = { version = "0.21.1" }
|
||||
lazy_static = "1.5.1"
|
||||
libc = "0.2.189"
|
||||
libc = "0.2.190"
|
||||
libsystemd = "0.7.2"
|
||||
local-ip-address = "0.6.13"
|
||||
log = "0.4"
|
||||
@@ -299,7 +307,7 @@ percent-encoding = "2.3.2"
|
||||
# Server-side QR rendering for TOTP enrollment, so neither the console nor the
|
||||
# CLI needs its own QR encoder. No default features: the image/render backends
|
||||
# pull in an image stack this only needs SVG and text output from.
|
||||
qrcode-rs = { version = "2.1.1", default-features = false, features = ["std", "svg"] }
|
||||
qrcode-rs = { version = "2.2.0", default-features = false, features = ["std", "svg"] }
|
||||
pin-project-lite = "0.2.17"
|
||||
pretty_assertions = "1.4.1"
|
||||
rand = { version = "0.10.3" }
|
||||
@@ -307,6 +315,8 @@ ratelimit = "2.0.0"
|
||||
rayon = "1.12.0"
|
||||
rustfs-erasure-codec = { version = "9.0.0" }
|
||||
reed-solomon-simd = "3.1.0"
|
||||
openraft = { version = "=0.9.25", default-features = false, features = ["serde"] }
|
||||
redb = "=4.3.0"
|
||||
regex = { version = "1.13.1" }
|
||||
rumqttc = { package = "rumqttc-next", version = "0.34.0" }
|
||||
redis = { version = "1.7.1" }
|
||||
@@ -314,15 +324,15 @@ rustify = { version = "0.7", default-features = false }
|
||||
rustix = { version = "1.1.5" }
|
||||
rust-embed = { version = "8.12.0" }
|
||||
rustc-hash = { version = "2.1.3" }
|
||||
s3s = { version = "0.17.0", features = ["minio"] }
|
||||
s3s-sigv4 = "0.17.0"
|
||||
s3s = { git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fcac87e3b99fd5bd9e176b442c0", features = ["minio"] }
|
||||
s3s-sigv4 = { git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fcac87e3b99fd5bd9e176b442c0" }
|
||||
serial_test = "4.0.1"
|
||||
shadow-rs = { default-features = false, version = "2.0.0" }
|
||||
siphasher = "1.0.4"
|
||||
smallvec = { version = "1.16.2" }
|
||||
compact_str = "0.10.0"
|
||||
snap = "1.1.2"
|
||||
starshard = { version = "2.3.0" }
|
||||
starshard = { version = "2.4.0" }
|
||||
strum = { version = "0.28.0" }
|
||||
sysinfo = "0.39.6"
|
||||
temp-env = "0.3.6"
|
||||
@@ -338,7 +348,7 @@ tracing-subscriber = { version = "0.3.23" }
|
||||
transform-stream = "0.3.1"
|
||||
url = "2.5.8"
|
||||
urlencoding = "2.1.3"
|
||||
uuid = { version = "1.26.1" }
|
||||
uuid = { version = "1.27.0" }
|
||||
vaultrs = { version = "0.8.0" }
|
||||
tar = "0.4.46"
|
||||
walkdir = "2.5.0"
|
||||
@@ -352,7 +362,7 @@ zstd = "0.14.0"
|
||||
# Observability and Metrics
|
||||
metrics = "0.24.6"
|
||||
metrics-util = "0.20"
|
||||
dial9 = { version = "0.5.2", default-features = false }
|
||||
dial9 = { version = "0.5.3", default-features = false }
|
||||
opentelemetry = { version = "0.33.0" }
|
||||
opentelemetry-appender-tracing = { version = "0.33.0" }
|
||||
opentelemetry-otlp = { version = "0.33.0" }
|
||||
@@ -367,7 +377,7 @@ libunftp = { version = "0.23.1" }
|
||||
unftp-core = "0.1.0"
|
||||
suppaftp = { version = "12.1.1" }
|
||||
rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
russh = { version = "0.63.3" }
|
||||
russh = { version = "0.64.1" }
|
||||
russh-sftp = "3.0.1"
|
||||
|
||||
# WebDAV
|
||||
@@ -376,9 +386,9 @@ dav-server = "0.11.0"
|
||||
# Performance Analysis and Memory Profiling
|
||||
rustfs-mimalloc = { version = "0.5.6" }
|
||||
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
|
||||
hotpath = { version = "0.28.0", default-features = false }
|
||||
hotpath = { version = "0.28.5", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.48" }
|
||||
insta = { version = "1.49" }
|
||||
|
||||
[workspace.metadata.cargo-shear]
|
||||
ignored = ["hotpath", "rustfs"]
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
[package]
|
||||
name = "rustfs-capture-authority"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
repository.workspace = true
|
||||
description = "Durable local storage for capture authority"
|
||||
|
||||
[dependencies]
|
||||
openraft = { workspace = true }
|
||||
redb = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
thiserror = { workspace = true }
|
||||
url = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
serde_json = { workspace = true }
|
||||
tempfile = { workspace = true }
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
@@ -0,0 +1,623 @@
|
||||
use crate::{
|
||||
LOGS, LogId, META, Store, StoreError, committed::validate_committed, created::*, decode_log, encode_id, encode_log,
|
||||
next_index, number,
|
||||
};
|
||||
use redb::{ReadTransaction, ReadableDatabase, ReadableTable, ReadableTableMetadata, TableDefinition};
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
pub(crate) const HEADS: TableDefinition<&[u8], &[u8]> = TableDefinition::new("heads");
|
||||
pub(crate) const VERSIONS: TableDefinition<&[u8], &[u8]> = TableDefinition::new("versions");
|
||||
pub(crate) const EVENTS: TableDefinition<&[u8], &[u8]> = TableDefinition::new("events");
|
||||
pub(crate) const OPERATIONS: TableDefinition<&[u8], &[u8]> = TableDefinition::new("operations");
|
||||
pub(crate) const APPLY_RESULTS: TableDefinition<u64, &[u8]> = TableDefinition::new("apply_results");
|
||||
|
||||
fn version_key(object: &ObjectIdentity, version: [u8; 16]) -> Result<Vec<u8>, StoreError> {
|
||||
let mut b = object_bytes(object)?;
|
||||
b.extend(version);
|
||||
Ok(b)
|
||||
}
|
||||
fn decode_version(b: &[u8]) -> Result<CreatedVersion, StoreError> {
|
||||
let mut d = Decoder::new(b);
|
||||
let v = CreatedVersion {
|
||||
prepared: d.prepared()?,
|
||||
decision_log_id: d.id()?,
|
||||
binding_revision: d.u64()?,
|
||||
};
|
||||
d.finish()?;
|
||||
Ok(v)
|
||||
}
|
||||
fn decode_event(b: &[u8]) -> Result<CreatedEvent, StoreError> {
|
||||
let mut d = Decoder::new(b);
|
||||
let e = CreatedEvent {
|
||||
object: d.object()?,
|
||||
prepared: d.prepared()?,
|
||||
decision_log_id: d.id()?,
|
||||
binding: d.binding()?,
|
||||
};
|
||||
d.finish()?;
|
||||
Ok(e)
|
||||
}
|
||||
fn decode_operation(b: &[u8]) -> Result<CreatedOperation, StoreError> {
|
||||
let mut d = Decoder::new(b);
|
||||
let len = usize::try_from(d.u64()?).map_err(|_| StoreError::CorruptRecord)?;
|
||||
let command = b
|
||||
.get(8..8usize.checked_add(len).ok_or(StoreError::CorruptRecord)?)
|
||||
.ok_or(StoreError::CorruptRecord)?;
|
||||
DecideCreated::decode(command)?;
|
||||
let mut d = Decoder::new(&b[8 + len..]);
|
||||
let result = d.result()?;
|
||||
d.finish()?;
|
||||
Ok(CreatedOperation {
|
||||
canonical_command: command.to_vec(),
|
||||
result,
|
||||
})
|
||||
}
|
||||
fn decode_apply(b: &[u8]) -> Result<CreatedApplyResult, StoreError> {
|
||||
let mut d = Decoder::new(b);
|
||||
let r = CreatedApplyResult {
|
||||
log_id: d.id()?,
|
||||
result: d.result()?,
|
||||
};
|
||||
d.finish()?;
|
||||
Ok(r)
|
||||
}
|
||||
fn read_binding(meta: &impl ReadableTable<&'static str, &'static [u8]>) -> Result<CaptureBinding, StoreError> {
|
||||
let b = meta.get("capture_binding")?.ok_or(StoreError::BindingNotPinned)?;
|
||||
let mut d = Decoder::new(b.value());
|
||||
let binding = d.binding()?;
|
||||
d.finish()?;
|
||||
Ok(binding)
|
||||
}
|
||||
fn read_applied(meta: &impl ReadableTable<&'static str, &'static [u8]>) -> Result<Option<LogId>, StoreError> {
|
||||
let b = meta.get("last_applied")?.ok_or(StoreError::CorruptRecord)?;
|
||||
let mut d = Decoder::new(b.value());
|
||||
let id = match d.take::<1>()?[0] {
|
||||
0 => None,
|
||||
1 => Some(d.id()?),
|
||||
_ => return Err(StoreError::CorruptRecord),
|
||||
};
|
||||
d.finish()?;
|
||||
Ok(id)
|
||||
}
|
||||
fn required<T>(value: Option<T>) -> Result<T, StoreError> {
|
||||
value.ok_or(StoreError::CorruptRecord)
|
||||
}
|
||||
fn encode_native_entry(entry: &crate::CaptureRaftEntry) -> Result<Vec<u8>, StoreError> {
|
||||
let id: LogId = entry.log_id.into();
|
||||
let (kind, body) = match &entry.payload {
|
||||
openraft::EntryPayload::Blank => (0, vec![]),
|
||||
openraft::EntryPayload::Normal(command) => (1, command.created().encode()?),
|
||||
openraft::EntryPayload::Membership(membership) => {
|
||||
let stored = openraft::StoredMembership::new(Some(entry.log_id), membership.clone());
|
||||
let checked = crate::CaptureMembership::try_from(&stored)?;
|
||||
(2, checked.encode()?)
|
||||
}
|
||||
};
|
||||
let mut payload = b"CENR\x01".to_vec();
|
||||
payload.push(kind);
|
||||
payload.extend(
|
||||
u64::try_from(body.len())
|
||||
.map_err(|_| StoreError::CorruptRecord)?
|
||||
.to_be_bytes(),
|
||||
);
|
||||
payload.extend(body);
|
||||
encode_log(&crate::LogEntry { id, payload })
|
||||
}
|
||||
|
||||
pub(crate) fn decode_native_entry(entry: crate::LogEntry) -> Result<crate::CaptureRaftEntry, StoreError> {
|
||||
let bytes = &entry.payload;
|
||||
if bytes.len() < 14 || &bytes[..5] != b"CENR\x01" {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let length = usize::try_from(number(&bytes[6..14])?).map_err(|_| StoreError::CorruptRecord)?;
|
||||
let body = &bytes[14..];
|
||||
if length != body.len() {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let payload = match bytes[5] {
|
||||
0 if body.is_empty() => openraft::EntryPayload::Blank,
|
||||
1 => openraft::EntryPayload::Normal(crate::CaptureCommand::try_created(DecideCreated::decode(body)?)?),
|
||||
2 => {
|
||||
let membership = crate::CaptureMembership::decode(body).map_err(|_| StoreError::CorruptRecord)?;
|
||||
if *membership.log_id() != Some(entry.id) {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let stored = openraft::StoredMembership::try_from(&membership).map_err(|_| StoreError::CorruptRecord)?;
|
||||
openraft::EntryPayload::Membership(stored.membership().clone())
|
||||
}
|
||||
_ => return Err(StoreError::CorruptRecord),
|
||||
};
|
||||
Ok(crate::CaptureRaftEntry {
|
||||
log_id: entry.id.into(),
|
||||
payload,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn native_application_state(tx: &ReadTransaction) -> Result<(Option<LogId>, crate::CaptureMembership), StoreError> {
|
||||
let meta = tx.open_table(META)?;
|
||||
if number(required(meta.get("schema")?)?.value())? != 4 {
|
||||
return Err(StoreError::UnsupportedSchema);
|
||||
}
|
||||
let last = read_applied(&meta)?;
|
||||
let membership = crate::CaptureMembership::decode(required(meta.get("native_membership")?)?.value())?;
|
||||
if last.is_some() || membership.log_id().is_some() || !membership.configs().is_empty() || !membership.nodes().is_empty() {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
Ok((last, membership))
|
||||
}
|
||||
|
||||
impl Store {
|
||||
/// Explicitly opt an empty, bound store into native entry persistence.
|
||||
pub fn enable_native_empty(&mut self) -> Result<(), StoreError> {
|
||||
let tx = self.write_transaction()?;
|
||||
{
|
||||
let mut meta = tx.open_table(META)?;
|
||||
let schema = number(required(meta.get("schema")?)?.value())?;
|
||||
if !matches!(schema, 3 | 4) {
|
||||
return Err(StoreError::UnsupportedSchema);
|
||||
}
|
||||
read_binding(&meta)?;
|
||||
if schema == 4 {
|
||||
let membership = crate::CaptureMembership::decode(required(meta.get("native_membership")?)?.value())?;
|
||||
if read_applied(&meta)?.is_some()
|
||||
|| membership.log_id().is_some()
|
||||
|| !membership.configs().is_empty()
|
||||
|| !membership.nodes().is_empty()
|
||||
{
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
if read_applied(&meta)?.is_some() || !tx.open_table(LOGS)?.is_empty()? {
|
||||
return Err(StoreError::UnsupportedSchema);
|
||||
}
|
||||
for table in [HEADS, VERSIONS, EVENTS, OPERATIONS] {
|
||||
if !tx.open_table(table)?.is_empty()? {
|
||||
return Err(StoreError::UnsupportedSchema);
|
||||
}
|
||||
}
|
||||
if !tx.open_table(APPLY_RESULTS)?.is_empty()? || meta.get("committed")?.is_some() || meta.get("tail")?.is_some() {
|
||||
return Err(StoreError::UnsupportedSchema);
|
||||
}
|
||||
if meta.get("native_membership")?.is_some() {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let membership = crate::CaptureMembership::new(None, vec![], BTreeMap::new())?;
|
||||
meta.insert("native_membership", membership.encode()?.as_slice())?;
|
||||
meta.insert("schema", 4u64.to_be_bytes().as_slice())?;
|
||||
}
|
||||
self.commit(tx)
|
||||
}
|
||||
pub fn append_native(&mut self, entries: &[crate::CaptureRaftEntry]) -> Result<(), StoreError> {
|
||||
let tx = self.write_transaction()?;
|
||||
{
|
||||
let mut meta = tx.open_table(META)?;
|
||||
if number(required(meta.get("schema")?)?.value())? != 4 {
|
||||
return Err(StoreError::UnsupportedSchema);
|
||||
}
|
||||
if entries.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let tail = meta.get("tail")?.map(|value| crate::decode_id(value.value())).transpose()?;
|
||||
let mut next = next_index(tail.map(|id| id.index))?;
|
||||
let mut encoded = Vec::with_capacity(entries.len());
|
||||
for (position, entry) in entries.iter().enumerate() {
|
||||
if entry.log_id.index != next {
|
||||
return Err(StoreError::InvalidLogSequence);
|
||||
}
|
||||
encoded.push(encode_native_entry(entry)?);
|
||||
if position + 1 < entries.len() {
|
||||
next = next_index(Some(next))?;
|
||||
}
|
||||
}
|
||||
let mut logs = tx.open_table(LOGS)?;
|
||||
for (entry, bytes) in entries.iter().zip(&encoded) {
|
||||
logs.insert(entry.log_id.index, bytes.as_slice())?;
|
||||
}
|
||||
let last = entries.last().ok_or(StoreError::InvalidLogSequence)?;
|
||||
meta.insert("tail", encode_id(last.log_id.into()).as_slice())?;
|
||||
}
|
||||
self.commit(tx)
|
||||
}
|
||||
pub fn read_native_entry(&self, index: u64) -> Result<Option<crate::CaptureRaftEntry>, StoreError> {
|
||||
let tx = self.database.begin_read()?;
|
||||
if number(required(tx.open_table(META)?.get("schema")?)?.value())? != 4 {
|
||||
return Err(StoreError::UnsupportedSchema);
|
||||
}
|
||||
tx.open_table(LOGS)?
|
||||
.get(index)?
|
||||
.map(|value| {
|
||||
let entry = decode_log(value.value())?;
|
||||
if entry.id.index != index {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
decode_native_entry(entry)
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
pub fn read_native_entries(&self, start: u64, end: u64) -> Result<Vec<crate::CaptureRaftEntry>, StoreError> {
|
||||
let tx = self.database.begin_read()?;
|
||||
if number(required(tx.open_table(META)?.get("schema")?)?.value())? != 4 {
|
||||
return Err(StoreError::UnsupportedSchema);
|
||||
}
|
||||
if start > end {
|
||||
return Err(StoreError::InvalidLogSequence);
|
||||
}
|
||||
let logs = tx.open_table(LOGS)?;
|
||||
let mut entries = Vec::new();
|
||||
for row in logs.range(start..end)? {
|
||||
let (key, value) = row?;
|
||||
let entry = decode_log(value.value())?;
|
||||
if entry.id.index != key.value() {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
entries.push(decode_native_entry(entry)?);
|
||||
}
|
||||
Ok(entries)
|
||||
}
|
||||
pub fn apply_next_native(&mut self, _id: LogId) -> Result<crate::CaptureResponse, StoreError> {
|
||||
let _tx = self.write_transaction()?;
|
||||
Err(StoreError::UnsupportedSchema)
|
||||
}
|
||||
pub fn native_apply_response(&self, _index: u64) -> Result<Option<crate::CaptureResponse>, StoreError> {
|
||||
Err(StoreError::UnsupportedSchema)
|
||||
}
|
||||
pub fn native_application_state(&self) -> Result<(Option<LogId>, crate::CaptureMembership), StoreError> {
|
||||
native_application_state(&self.database.begin_read()?)
|
||||
}
|
||||
/// Bootstrap the immutable capture policy. The caller must be the trusted
|
||||
/// deployment provisioning owner. This crate does not authenticate callers;
|
||||
/// do not expose this public method through an ordinary object request API.
|
||||
/// Prepared identities assert identity only, not physical data durability.
|
||||
pub fn pin_capture_binding(&mut self, binding: CaptureBinding) -> Result<(), StoreError> {
|
||||
let encoded = binding_bytes(&binding)?;
|
||||
let tx = self.write_transaction()?;
|
||||
{
|
||||
let mut meta = tx.open_table(META)?;
|
||||
if matches!(number(required(meta.get("schema")?)?.value())?, 3 | 4) {
|
||||
if read_binding(&meta)? == binding {
|
||||
return Ok(());
|
||||
}
|
||||
return Err(StoreError::BindingConflict);
|
||||
}
|
||||
if meta.get("capture_binding")?.is_some() || meta.get("last_applied")?.is_some() {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
for table in [HEADS, VERSIONS, EVENTS, OPERATIONS] {
|
||||
if !tx.open_table(table)?.is_empty()? {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
}
|
||||
if !tx.open_table(APPLY_RESULTS)?.is_empty()? {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
meta.insert("capture_binding", encoded.as_slice())?;
|
||||
meta.insert("last_applied", &[0][..])?;
|
||||
meta.insert("schema", 3u64.to_be_bytes().as_slice())?;
|
||||
}
|
||||
self.commit(tx)
|
||||
}
|
||||
/// Apply the next exact persisted committed Created command atomically.
|
||||
pub fn apply_next_created(&mut self, id: LogId) -> Result<CreatedResult, StoreError> {
|
||||
let tx = self.write_transaction()?;
|
||||
let result;
|
||||
{
|
||||
let mut meta = tx.open_table(META)?;
|
||||
if number(required(meta.get("schema")?)?.value())? != 3 {
|
||||
return Err(StoreError::UnsupportedSchema);
|
||||
}
|
||||
let binding = read_binding(&meta)?;
|
||||
let logs = tx.open_table(LOGS)?;
|
||||
let committed = validate_committed(&meta, &logs)?.ok_or(StoreError::InvalidCommittedLog)?;
|
||||
let entry = decode_log(required(logs.get(id.index)?)?.value())?;
|
||||
if entry.id != id || id.index > committed.index {
|
||||
return Err(StoreError::InvalidCommittedLog);
|
||||
}
|
||||
let last = read_applied(&meta)?;
|
||||
let mut results = tx.open_table(APPLY_RESULTS)?;
|
||||
if last.is_some_and(|last| id.index <= last.index) {
|
||||
let saved = decode_apply(required(results.get(id.index)?)?.value())?;
|
||||
if saved.log_id != id {
|
||||
return Err(StoreError::InvalidApplySequence);
|
||||
}
|
||||
return Ok(saved.result);
|
||||
}
|
||||
if id.index != next_index(last.map(|last| last.index))? {
|
||||
return Err(StoreError::InvalidApplySequence);
|
||||
}
|
||||
let command = DecideCreated::decode(&entry.payload)?;
|
||||
let mut operations = tx.open_table(OPERATIONS)?;
|
||||
let existing = operations
|
||||
.get(command.operation_id.as_slice())?
|
||||
.map(|v| decode_operation(v.value()))
|
||||
.transpose()?;
|
||||
result = if let Some(existing) = existing {
|
||||
if existing.canonical_command == entry.payload {
|
||||
existing.result
|
||||
} else {
|
||||
CreatedResult::OperationConflict
|
||||
}
|
||||
} else {
|
||||
let mut heads = tx.open_table(HEADS)?;
|
||||
let mut versions = tx.open_table(VERSIONS)?;
|
||||
let object = object_bytes(&command.object)?;
|
||||
let key = version_key(&command.object, command.prepared.version_id)?;
|
||||
let actual = heads
|
||||
.get(object.as_slice())?
|
||||
.map(|v| {
|
||||
let mut d = Decoder::new(v.value());
|
||||
let id = d.uuid()?;
|
||||
d.finish()?;
|
||||
Ok::<_, StoreError>(id)
|
||||
})
|
||||
.transpose()?;
|
||||
let outcome = if command.binding_revision != binding.revision {
|
||||
CreatedResult::BindingMismatch
|
||||
} else if actual != command.expected_head {
|
||||
CreatedResult::HeadMismatch { actual_head: actual }
|
||||
} else if versions.get(key.as_slice())?.is_some() {
|
||||
CreatedResult::VersionConflict
|
||||
} else {
|
||||
let mut version = Vec::new();
|
||||
put_prepared(&mut version, &command.prepared);
|
||||
version.extend(encode_id(id));
|
||||
version.extend(binding.revision.to_be_bytes());
|
||||
versions.insert(key.as_slice(), version.as_slice())?;
|
||||
let mut event = object;
|
||||
put_prepared(&mut event, &command.prepared);
|
||||
event.extend(encode_id(id));
|
||||
event.extend(binding_bytes(&binding)?);
|
||||
tx.open_table(EVENTS)?
|
||||
.insert(command.operation_id.as_slice(), event.as_slice())?;
|
||||
heads.insert(object_bytes(&command.object)?.as_slice(), command.prepared.version_id.as_slice())?;
|
||||
CreatedResult::Created {
|
||||
version_id: command.prepared.version_id,
|
||||
event_id: command.operation_id,
|
||||
decision_log_id: id,
|
||||
}
|
||||
};
|
||||
let mut op = u64::try_from(entry.payload.len())
|
||||
.map_err(|_| StoreError::CorruptRecord)?
|
||||
.to_be_bytes()
|
||||
.to_vec();
|
||||
op.extend(&entry.payload);
|
||||
op.extend(result_bytes(&outcome));
|
||||
operations.insert(command.operation_id.as_slice(), op.as_slice())?;
|
||||
outcome
|
||||
};
|
||||
let mut applied = encode_id(id);
|
||||
applied.extend(result_bytes(&result));
|
||||
results.insert(id.index, applied.as_slice())?;
|
||||
let mut last = vec![1];
|
||||
last.extend(encode_id(id));
|
||||
meta.insert("last_applied", last.as_slice())?;
|
||||
}
|
||||
self.commit(tx)?;
|
||||
Ok(result)
|
||||
}
|
||||
pub fn capture_binding(&self) -> Result<Option<CaptureBinding>, StoreError> {
|
||||
let tx = self.database.begin_read()?;
|
||||
let meta = tx.open_table(META)?;
|
||||
if !matches!(number(required(meta.get("schema")?)?.value())?, 3 | 4) {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(read_binding(&meta)?))
|
||||
}
|
||||
pub fn last_applied(&self) -> Result<Option<LogId>, StoreError> {
|
||||
let tx = self.database.begin_read()?;
|
||||
let meta = tx.open_table(META)?;
|
||||
if !matches!(number(required(meta.get("schema")?)?.value())?, 3 | 4) {
|
||||
return Ok(None);
|
||||
}
|
||||
read_applied(&meta)
|
||||
}
|
||||
pub fn created_head(&self, object: &ObjectIdentity) -> Result<Option<[u8; 16]>, StoreError> {
|
||||
let tx = self.database.begin_read()?;
|
||||
let t = tx.open_table(HEADS)?;
|
||||
t.get(object_bytes(object)?.as_slice())?
|
||||
.map(|v| {
|
||||
let mut d = Decoder::new(v.value());
|
||||
let id = d.uuid()?;
|
||||
d.finish()?;
|
||||
Ok(id)
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
pub fn created_version(&self, object: &ObjectIdentity, version: [u8; 16]) -> Result<Option<CreatedVersion>, StoreError> {
|
||||
let tx = self.database.begin_read()?;
|
||||
let t = tx.open_table(VERSIONS)?;
|
||||
t.get(version_key(object, version)?.as_slice())?
|
||||
.map(|v| decode_version(v.value()))
|
||||
.transpose()
|
||||
}
|
||||
pub fn created_event(&self, operation: [u8; 16]) -> Result<Option<CreatedEvent>, StoreError> {
|
||||
let tx = self.database.begin_read()?;
|
||||
let t = tx.open_table(EVENTS)?;
|
||||
t.get(operation.as_slice())?.map(|v| decode_event(v.value())).transpose()
|
||||
}
|
||||
pub fn created_operation(&self, operation: [u8; 16]) -> Result<Option<CreatedOperation>, StoreError> {
|
||||
let tx = self.database.begin_read()?;
|
||||
let t = tx.open_table(OPERATIONS)?;
|
||||
t.get(operation.as_slice())?.map(|v| decode_operation(v.value())).transpose()
|
||||
}
|
||||
pub fn created_apply_result(&self, index: u64) -> Result<Option<CreatedApplyResult>, StoreError> {
|
||||
let tx = self.database.begin_read()?;
|
||||
let t = tx.open_table(APPLY_RESULTS)?;
|
||||
t.get(index)?.map(|v| decode_apply(v.value())).transpose()
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn validate_application(tx: &ReadTransaction) -> Result<(), StoreError> {
|
||||
let meta = tx.open_table(META)?;
|
||||
let schema = number(required(meta.get("schema")?)?.value())?;
|
||||
if schema == 4 {
|
||||
read_binding(&meta)?;
|
||||
native_application_state(tx)?;
|
||||
for table in [HEADS, VERSIONS, EVENTS, OPERATIONS] {
|
||||
if !tx.open_table(table)?.is_empty()? {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
}
|
||||
if !tx.open_table(APPLY_RESULTS)?.is_empty()? {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
if schema != 3 {
|
||||
return Ok(());
|
||||
}
|
||||
let binding = read_binding(&meta).map_err(|error| match error {
|
||||
StoreError::BindingNotPinned => StoreError::CorruptRecord,
|
||||
other => other,
|
||||
})?;
|
||||
let last = read_applied(&meta)?;
|
||||
let logs = tx.open_table(LOGS)?;
|
||||
let committed = validate_committed(&meta, &logs)?;
|
||||
let heads = tx.open_table(HEADS)?;
|
||||
let versions = tx.open_table(VERSIONS)?;
|
||||
let events = tx.open_table(EVENTS)?;
|
||||
let operations = tx.open_table(OPERATIONS)?;
|
||||
let results = tx.open_table(APPLY_RESULTS)?;
|
||||
if last.is_some_and(|last| committed.is_none_or(|c| last.index > c.index)) {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let mut next = Some(0);
|
||||
let mut observed = None;
|
||||
let mut first_operations = BTreeMap::new();
|
||||
for row in results.iter()? {
|
||||
let (k, v) = row?;
|
||||
let r = decode_apply(v.value())?;
|
||||
if next != Some(k.value()) || r.log_id.index != k.value() {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let entry = decode_log(required(logs.get(k.value())?)?.value())?;
|
||||
if entry.id != r.log_id {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let c = DecideCreated::decode(&entry.payload)?;
|
||||
let op = decode_operation(required(operations.get(c.operation_id.as_slice())?)?.value())?;
|
||||
let canonical_matches = op.canonical_command == entry.payload;
|
||||
let expected = if canonical_matches {
|
||||
op.result
|
||||
} else {
|
||||
CreatedResult::OperationConflict
|
||||
};
|
||||
if expected != r.result {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
if canonical_matches {
|
||||
first_operations.entry(c.operation_id).or_insert(r.log_id);
|
||||
}
|
||||
observed = Some(r.log_id);
|
||||
next = k.value().checked_add(1)
|
||||
}
|
||||
if observed != last {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
for row in operations.iter()? {
|
||||
let (k, v) = row?;
|
||||
let op = decode_operation(v.value())?;
|
||||
let c = DecideCreated::decode(&op.canonical_command)?;
|
||||
if k.value() != c.operation_id {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let first = required(first_operations.get(&c.operation_id))?;
|
||||
if let CreatedResult::Created {
|
||||
version_id,
|
||||
event_id,
|
||||
decision_log_id,
|
||||
} = op.result
|
||||
{
|
||||
if version_id != c.prepared.version_id
|
||||
|| event_id != c.operation_id
|
||||
|| c.binding_revision != binding.revision
|
||||
|| decision_log_id != *first
|
||||
{
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let event = decode_event(required(events.get(event_id.as_slice())?)?.value())?;
|
||||
if event
|
||||
!= (CreatedEvent {
|
||||
object: c.object.clone(),
|
||||
prepared: c.prepared.clone(),
|
||||
decision_log_id,
|
||||
binding: binding.clone(),
|
||||
})
|
||||
{
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
} else if events.get(c.operation_id.as_slice())?.is_some() {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
}
|
||||
for row in events.iter()? {
|
||||
let (k, v) = row?;
|
||||
let e = decode_event(v.value())?;
|
||||
if e.binding != binding {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let op = decode_operation(required(operations.get(k.value())?)?.value())?;
|
||||
let c = DecideCreated::decode(&op.canonical_command)?;
|
||||
if c.object != e.object
|
||||
|| c.prepared != e.prepared
|
||||
|| op.result
|
||||
!= (CreatedResult::Created {
|
||||
version_id: e.prepared.version_id,
|
||||
event_id: c.operation_id,
|
||||
decision_log_id: e.decision_log_id,
|
||||
})
|
||||
{
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let version =
|
||||
decode_version(required(versions.get(version_key(&e.object, e.prepared.version_id)?.as_slice())?)?.value())?;
|
||||
if version
|
||||
!= (CreatedVersion {
|
||||
prepared: e.prepared.clone(),
|
||||
decision_log_id: e.decision_log_id,
|
||||
binding_revision: binding.revision,
|
||||
})
|
||||
{
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
}
|
||||
for row in versions.iter()? {
|
||||
let (k, v) = row?;
|
||||
let version = decode_version(v.value())?;
|
||||
let mut d = Decoder::new(k.value());
|
||||
let object = d.object()?;
|
||||
let id = d.uuid()?;
|
||||
d.finish()?;
|
||||
if id != version.prepared.version_id || version.binding_revision != binding.revision {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let entry = decode_log(required(logs.get(version.decision_log_id.index)?)?.value())?;
|
||||
let c = DecideCreated::decode(&entry.payload)?;
|
||||
if entry.id != version.decision_log_id
|
||||
|| c.object != object
|
||||
|| c.prepared != version.prepared
|
||||
|| events.get(c.operation_id.as_slice())?.is_none()
|
||||
{
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let head = required(heads.get(object_bytes(&object)?.as_slice())?)?;
|
||||
let mut d = Decoder::new(head.value());
|
||||
let head_id = d.uuid()?;
|
||||
d.finish()?;
|
||||
let current = decode_version(required(versions.get(version_key(&object, head_id)?.as_slice())?)?.value())?;
|
||||
if current.decision_log_id.index < version.decision_log_id.index {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
}
|
||||
for row in heads.iter()? {
|
||||
let (k, v) = row?;
|
||||
let mut d = Decoder::new(k.value());
|
||||
let object = d.object()?;
|
||||
d.finish()?;
|
||||
let mut d = Decoder::new(v.value());
|
||||
let id = d.uuid()?;
|
||||
d.finish()?;
|
||||
decode_version(required(versions.get(version_key(&object, id)?.as_slice())?)?.value())?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,531 @@
|
||||
use super::*;
|
||||
use crate::application::{APPLY_RESULTS, EVENTS, OPERATIONS, VERSIONS};
|
||||
use crate::tests::CrashBackend;
|
||||
use redb::ReadableTableMetadata;
|
||||
use std::{
|
||||
io::{BufRead, BufReader, Write},
|
||||
process::{Command, Stdio},
|
||||
sync::mpsc,
|
||||
time::Duration,
|
||||
};
|
||||
fn identity() -> NodeIdentity {
|
||||
NodeIdentity {
|
||||
cluster_id: [7; 16],
|
||||
node_id: 0,
|
||||
}
|
||||
}
|
||||
fn binding() -> CaptureBinding {
|
||||
CaptureBinding {
|
||||
revision: 9,
|
||||
binding_id: [8; 16],
|
||||
target_id: [9; 16],
|
||||
}
|
||||
}
|
||||
fn command() -> DecideCreated {
|
||||
DecideCreated {
|
||||
operation_id: [1; 16],
|
||||
object: ObjectIdentity {
|
||||
bucket_incarnation: [2; 16],
|
||||
key: vec![0, 255, 3],
|
||||
},
|
||||
expected_head: None,
|
||||
prepared: PreparedIdentity {
|
||||
version_id: [3; 16],
|
||||
preparation_id: [4; 16],
|
||||
content_digest: [5; 32],
|
||||
content_length: 123,
|
||||
},
|
||||
binding_revision: 9,
|
||||
}
|
||||
}
|
||||
fn id(index: u64) -> LogId {
|
||||
LogId {
|
||||
term: 8,
|
||||
leader_node: 0,
|
||||
index,
|
||||
}
|
||||
}
|
||||
fn append(store: &mut Store, index: u64, c: &DecideCreated) {
|
||||
store
|
||||
.append(&[LogEntry {
|
||||
id: id(index),
|
||||
payload: c.encode().expect("encode command"),
|
||||
}])
|
||||
.expect("append command");
|
||||
}
|
||||
fn apply(store: &mut Store, index: u64, c: &DecideCreated) -> CreatedResult {
|
||||
append(store, index, c);
|
||||
store.save_committed(id(index)).expect("commit log");
|
||||
store.apply_next_created(id(index)).expect("apply command")
|
||||
}
|
||||
fn success(c: &DecideCreated, index: u64) -> CreatedResult {
|
||||
CreatedResult::Created {
|
||||
version_id: c.prepared.version_id,
|
||||
event_id: c.operation_id,
|
||||
decision_log_id: id(index),
|
||||
}
|
||||
}
|
||||
fn complete(store: &Store) {
|
||||
let c = command();
|
||||
assert_eq!(store.capture_binding().expect("binding"), Some(binding()));
|
||||
assert_eq!(
|
||||
store.created_head(&c.object).expect("head"),
|
||||
Some(c.prepared.version_id),
|
||||
"U11 acknowledged head lost"
|
||||
);
|
||||
assert_eq!(
|
||||
store.created_version(&c.object, c.prepared.version_id).expect("version"),
|
||||
Some(CreatedVersion {
|
||||
prepared: c.prepared.clone(),
|
||||
decision_log_id: id(0),
|
||||
binding_revision: 9
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
store.created_event(c.operation_id).expect("event"),
|
||||
Some(CreatedEvent {
|
||||
object: c.object.clone(),
|
||||
prepared: c.prepared.clone(),
|
||||
decision_log_id: id(0),
|
||||
binding: binding()
|
||||
}),
|
||||
"U11 complete event missing"
|
||||
);
|
||||
assert_eq!(
|
||||
store.created_operation(c.operation_id).expect("operation"),
|
||||
Some(CreatedOperation {
|
||||
canonical_command: c.encode().expect("encode"),
|
||||
result: success(&c, 0)
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
store.created_apply_result(0).expect("result"),
|
||||
Some(CreatedApplyResult {
|
||||
log_id: id(0),
|
||||
result: success(&c, 0)
|
||||
})
|
||||
);
|
||||
assert_eq!(store.last_applied().expect("last applied"), Some(id(0)));
|
||||
}
|
||||
fn fixture() -> (tempfile::TempDir, Store) {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let mut store = Store::initialize(dir.path().join("db"), identity()).expect("initialize");
|
||||
store.pin_capture_binding(binding()).expect("pin");
|
||||
(dir, store)
|
||||
}
|
||||
fn reopen(path: &std::path::Path) -> Store {
|
||||
let result = Store::open(path, identity());
|
||||
assert!(result.is_ok(), "U11 complete reopen: {:?}", result.as_ref().err());
|
||||
result.expect("asserted successful reopen")
|
||||
}
|
||||
#[test]
|
||||
fn created_codec_is_canonical_and_strict() {
|
||||
let c = command();
|
||||
let b = c.encode().expect("encode");
|
||||
let mut golden = b"CRTD\x01".to_vec();
|
||||
golden.extend([1; 16]);
|
||||
golden.extend([2; 16]);
|
||||
golden.extend(3u32.to_be_bytes());
|
||||
golden.extend([0, 255, 3]);
|
||||
golden.push(0);
|
||||
golden.extend([3; 16]);
|
||||
golden.extend([4; 16]);
|
||||
golden.extend([5; 32]);
|
||||
golden.extend(123u64.to_be_bytes());
|
||||
golden.extend(9u64.to_be_bytes());
|
||||
assert_eq!(b, golden);
|
||||
assert_eq!(DecideCreated::decode(&b).expect("decode"), c);
|
||||
let mut variants = Vec::new();
|
||||
let mut v = c.clone();
|
||||
v.operation_id = [6; 16];
|
||||
variants.push(v);
|
||||
let mut v = c.clone();
|
||||
v.object.bucket_incarnation = [6; 16];
|
||||
variants.push(v);
|
||||
let mut v = c.clone();
|
||||
v.object.key.push(4);
|
||||
variants.push(v);
|
||||
let mut v = c.clone();
|
||||
v.expected_head = Some([6; 16]);
|
||||
variants.push(v);
|
||||
let mut v = c.clone();
|
||||
v.prepared.version_id = [6; 16];
|
||||
variants.push(v);
|
||||
let mut v = c.clone();
|
||||
v.prepared.preparation_id = [6; 16];
|
||||
variants.push(v);
|
||||
let mut v = c.clone();
|
||||
v.prepared.content_digest = [6; 32];
|
||||
variants.push(v);
|
||||
let mut v = c.clone();
|
||||
v.prepared.content_length += 1;
|
||||
variants.push(v);
|
||||
let mut v = c.clone();
|
||||
v.binding_revision += 1;
|
||||
variants.push(v);
|
||||
for v in variants {
|
||||
assert_ne!(v.encode().expect("variant"), b)
|
||||
}
|
||||
for n in 0..b.len() {
|
||||
assert!(DecideCreated::decode(&b[..n]).is_err())
|
||||
}
|
||||
let mut bad = b.clone();
|
||||
bad[4] = 2;
|
||||
assert!(DecideCreated::decode(&bad).is_err());
|
||||
let mut bad = b.clone();
|
||||
bad[44] = 2;
|
||||
assert!(DecideCreated::decode(&bad).is_err());
|
||||
let mut bad = b.clone();
|
||||
bad.push(0);
|
||||
assert!(DecideCreated::decode(&bad).is_err());
|
||||
let mut bad = b;
|
||||
bad[37..41].copy_from_slice(&4097u32.to_be_bytes());
|
||||
assert!(DecideCreated::decode(&bad).is_err());
|
||||
let mut bad = c.clone();
|
||||
bad.operation_id = [0; 16];
|
||||
assert!(bad.encode().is_err());
|
||||
bad = c;
|
||||
bad.object.key = vec![0; MAX_OBJECT_KEY_LENGTH + 1];
|
||||
assert!(bad.encode().is_err());
|
||||
}
|
||||
#[test]
|
||||
fn created_committed_order_and_identity_are_enforced() {
|
||||
let (_dir, mut s) = fixture();
|
||||
let c = command();
|
||||
append(&mut s, 0, &c);
|
||||
append(&mut s, 1, &c);
|
||||
assert!(s.apply_next_created(id(0)).is_err());
|
||||
assert_eq!(s.last_applied().expect("no apply"), None);
|
||||
s.save_committed(id(1)).expect("committed");
|
||||
assert!(matches!(s.apply_next_created(id(1)), Err(StoreError::InvalidApplySequence)));
|
||||
let mut wrong = id(0);
|
||||
wrong.term += 1;
|
||||
assert!(s.apply_next_created(wrong).is_err());
|
||||
wrong = id(0);
|
||||
wrong.leader_node += 1;
|
||||
assert!(s.apply_next_created(wrong).is_err());
|
||||
assert_eq!(s.apply_next_created(id(0)).expect("first"), success(&c, 0));
|
||||
complete(&s);
|
||||
assert_eq!(s.apply_next_created(id(0)).expect("retry"), success(&c, 0));
|
||||
complete(&s);
|
||||
assert!(s.apply_next_created(wrong).is_err());
|
||||
assert_eq!(s.apply_next_created(id(1)).expect("duplicate log"), success(&c, 0));
|
||||
assert_eq!(s.last_applied().expect("last"), Some(id(1)));
|
||||
assert_eq!(
|
||||
s.created_apply_result(1).expect("result"),
|
||||
Some(CreatedApplyResult {
|
||||
log_id: id(1),
|
||||
result: success(&c, 0)
|
||||
})
|
||||
);
|
||||
let tx = s.database.begin_read().expect("read");
|
||||
assert_eq!(tx.open_table(EVENTS).expect("events").len().expect("len"), 1);
|
||||
}
|
||||
#[test]
|
||||
fn created_head_version_event_and_dedup_reopen_together() {
|
||||
let (dir, mut s) = fixture();
|
||||
let c = command();
|
||||
assert_eq!(apply(&mut s, 0, &c), success(&c, 0));
|
||||
drop(s);
|
||||
let mut s = reopen(&dir.path().join("db"));
|
||||
complete(&s);
|
||||
assert_eq!(apply(&mut s, 1, &c), success(&c, 0));
|
||||
let mut changed = c.clone();
|
||||
changed.prepared.content_digest = [6; 32];
|
||||
assert_eq!(
|
||||
apply(&mut s, 2, &changed),
|
||||
CreatedResult::OperationConflict,
|
||||
"U11 full canonical operation conflict"
|
||||
);
|
||||
assert_eq!(
|
||||
s.created_operation(c.operation_id)
|
||||
.expect("original")
|
||||
.expect("operation")
|
||||
.result,
|
||||
success(&c, 0)
|
||||
);
|
||||
let mut stale = c.clone();
|
||||
stale.operation_id = [6; 16];
|
||||
stale.prepared.version_id = [6; 16];
|
||||
assert_eq!(
|
||||
apply(&mut s, 3, &stale),
|
||||
CreatedResult::HeadMismatch {
|
||||
actual_head: Some(c.prepared.version_id)
|
||||
}
|
||||
);
|
||||
let mut reuse = c.clone();
|
||||
reuse.operation_id = [7; 16];
|
||||
reuse.expected_head = Some(c.prepared.version_id);
|
||||
assert_eq!(apply(&mut s, 4, &reuse), CreatedResult::VersionConflict);
|
||||
let mut replacement = stale.clone();
|
||||
replacement.operation_id = [10; 16];
|
||||
replacement.expected_head = Some(c.prepared.version_id);
|
||||
assert_eq!(apply(&mut s, 5, &replacement), success(&replacement, 5));
|
||||
assert_eq!(s.created_head(&c.object).expect("head"), Some(replacement.prepared.version_id));
|
||||
assert_eq!(
|
||||
s.created_version(&c.object, c.prepared.version_id)
|
||||
.expect("old version")
|
||||
.expect("version")
|
||||
.decision_log_id,
|
||||
id(0)
|
||||
);
|
||||
assert!(s.created_event(c.operation_id).expect("old event").is_some());
|
||||
assert!(s.created_event(replacement.operation_id).expect("new event").is_some());
|
||||
assert!(s.created_event(stale.operation_id).expect("rejection event").is_none());
|
||||
assert_eq!(
|
||||
apply(&mut s, 6, &stale),
|
||||
CreatedResult::HeadMismatch {
|
||||
actual_head: Some(c.prepared.version_id)
|
||||
}
|
||||
);
|
||||
drop(s);
|
||||
let s = reopen(&dir.path().join("db"));
|
||||
assert_eq!(s.last_applied().expect("last"), Some(id(6)));
|
||||
let tx = s.database.begin_read().expect("read");
|
||||
assert_eq!(tx.open_table(EVENTS).expect("events").len().expect("len"), 2);
|
||||
}
|
||||
#[test]
|
||||
fn created_binding_is_pinned_and_not_caller_selected() {
|
||||
let (dir, mut s) = fixture();
|
||||
let mut bad = command();
|
||||
bad.binding_revision = 10;
|
||||
assert_eq!(apply(&mut s, 0, &bad), CreatedResult::BindingMismatch);
|
||||
assert!(s.created_head(&bad.object).expect("head").is_none());
|
||||
assert!(s.created_event(bad.operation_id).expect("event").is_none());
|
||||
assert_eq!(
|
||||
s.created_operation(bad.operation_id)
|
||||
.expect("operation")
|
||||
.expect("saved")
|
||||
.result,
|
||||
CreatedResult::BindingMismatch
|
||||
);
|
||||
s.pin_capture_binding(binding()).expect("idempotent pin");
|
||||
let mut changed = binding();
|
||||
changed.target_id = [10; 16];
|
||||
assert!(matches!(s.pin_capture_binding(changed), Err(StoreError::BindingConflict)));
|
||||
drop(s);
|
||||
let s = reopen(&dir.path().join("db"));
|
||||
assert_eq!(s.capture_binding().expect("binding"), Some(binding()));
|
||||
}
|
||||
#[test]
|
||||
fn created_schema_compatibility_and_corruption_are_checked() {
|
||||
let dir = tempfile::tempdir().expect("dir");
|
||||
let path = dir.path().join("db");
|
||||
let s = Store::initialize(&path, identity()).expect("schema1");
|
||||
assert_eq!(s.capture_binding().expect("unmigrated"), None);
|
||||
drop(s);
|
||||
let mut s = reopen(&path);
|
||||
assert_eq!(s.last_applied().expect("no apply"), None);
|
||||
append(&mut s, 0, &command());
|
||||
s.save_committed(id(0)).expect("schema2");
|
||||
drop(s);
|
||||
let mut s = reopen(&path);
|
||||
assert_eq!(s.capture_binding().expect("schema2 unchanged"), None);
|
||||
s.pin_capture_binding(binding()).expect("explicit upgrade");
|
||||
drop(s);
|
||||
let mut s = reopen(&path);
|
||||
assert_eq!(s.capture_binding().expect("schema3"), Some(binding()));
|
||||
s.apply_next_created(id(0)).expect("apply");
|
||||
drop(s);
|
||||
let db = Database::open(&path).expect("raw db");
|
||||
let tx = db.begin_write().expect("write corruption");
|
||||
tx.open_table(EVENTS)
|
||||
.expect("events")
|
||||
.remove(command().operation_id.as_slice())
|
||||
.expect("remove reference");
|
||||
tx.commit().expect("commit corruption");
|
||||
drop(db);
|
||||
let result = Store::open(&path, identity());
|
||||
assert!(
|
||||
matches!(result, Err(StoreError::CorruptRecord)),
|
||||
"U11 missing event rejected before expect"
|
||||
);
|
||||
let (dir, mut s) = fixture();
|
||||
assert_eq!(s.read_committed().expect("empty committed"), None);
|
||||
append(&mut s, 0, &command());
|
||||
s.save_committed(id(0)).expect("committed schema3");
|
||||
assert_eq!(s.capture_binding().expect("no downgrade"), Some(binding()));
|
||||
drop(s);
|
||||
let path = dir.path().join("db");
|
||||
let db = Database::open(&path).expect("raw db");
|
||||
let tx = db.begin_write().expect("write");
|
||||
let mut marker = vec![1];
|
||||
marker.extend(encode_id(id(0)));
|
||||
tx.open_table(META)
|
||||
.expect("meta")
|
||||
.insert("last_applied", marker.as_slice())
|
||||
.expect("corrupt applied");
|
||||
tx.commit().expect("commit");
|
||||
drop(db);
|
||||
assert!(
|
||||
matches!(Store::open(&path, identity()), Err(StoreError::CorruptRecord)),
|
||||
"U11 last applied missing result rejected"
|
||||
);
|
||||
let (dir, s) = fixture();
|
||||
drop(s);
|
||||
let path = dir.path().join("db");
|
||||
let db = Database::open(&path).expect("raw pinned db");
|
||||
let tx = db.begin_write().expect("missing binding write");
|
||||
tx.open_table(META)
|
||||
.expect("meta")
|
||||
.remove("capture_binding")
|
||||
.expect("remove binding");
|
||||
tx.commit().expect("binding corruption commit");
|
||||
drop(db);
|
||||
assert!(
|
||||
matches!(Store::open(&path, identity()), Err(StoreError::CorruptRecord)),
|
||||
"U11 missing pinned binding rejected"
|
||||
);
|
||||
|
||||
let (dir, mut s) = fixture();
|
||||
let c = command();
|
||||
assert_eq!(apply(&mut s, 0, &c), success(&c, 0));
|
||||
assert_eq!(apply(&mut s, 1, &c), success(&c, 0));
|
||||
drop(s);
|
||||
let path = dir.path().join("db");
|
||||
let db = Database::open(&path).expect("raw duplicate db");
|
||||
let tx = db.begin_write().expect("retarget decision write");
|
||||
let mut key = created::object_bytes(&c.object).expect("object key");
|
||||
key.extend(c.prepared.version_id);
|
||||
let mut version = Vec::new();
|
||||
created::put_prepared(&mut version, &c.prepared);
|
||||
version.extend(encode_id(id(1)));
|
||||
version.extend(binding().revision.to_be_bytes());
|
||||
tx.open_table(VERSIONS)
|
||||
.expect("versions")
|
||||
.insert(key.as_slice(), version.as_slice())
|
||||
.expect("retarget version");
|
||||
let mut event = created::object_bytes(&c.object).expect("event object");
|
||||
created::put_prepared(&mut event, &c.prepared);
|
||||
event.extend(encode_id(id(1)));
|
||||
event.extend(created::binding_bytes(&binding()).expect("binding bytes"));
|
||||
tx.open_table(EVENTS)
|
||||
.expect("events")
|
||||
.insert(c.operation_id.as_slice(), event.as_slice())
|
||||
.expect("retarget event");
|
||||
let canonical = c.encode().expect("canonical");
|
||||
let mut operation = u64::try_from(canonical.len()).expect("length").to_be_bytes().to_vec();
|
||||
operation.extend(canonical);
|
||||
operation.extend(created::result_bytes(&success(&c, 1)));
|
||||
tx.open_table(OPERATIONS)
|
||||
.expect("operations")
|
||||
.insert(c.operation_id.as_slice(), operation.as_slice())
|
||||
.expect("retarget original response");
|
||||
for index in 0..=1 {
|
||||
let mut result = encode_id(id(index));
|
||||
result.extend(created::result_bytes(&success(&c, 1)));
|
||||
tx.open_table(APPLY_RESULTS)
|
||||
.expect("results")
|
||||
.insert(index, result.as_slice())
|
||||
.expect("retarget duplicate response");
|
||||
}
|
||||
tx.commit().expect("coherent retarget commit");
|
||||
drop(db);
|
||||
assert!(
|
||||
matches!(Store::open(&path, identity()), Err(StoreError::CorruptRecord)),
|
||||
"U11 original decision cannot retarget duplicate log"
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn created_process_child() {
|
||||
if let Some(path) = std::env::var_os("U11_CHILD_DB") {
|
||||
let mut s = Store::initialize(path, identity()).expect("child init");
|
||||
s.pin_capture_binding(binding()).expect("child pin");
|
||||
assert_eq!(apply(&mut s, 0, &command()), success(&command(), 0));
|
||||
complete(&s);
|
||||
println!("U11_ACK");
|
||||
std::io::stdout().flush().expect("flush ACK");
|
||||
loop {
|
||||
std::thread::park()
|
||||
}
|
||||
}
|
||||
let (_dir, mut s) = fixture();
|
||||
assert_eq!(apply(&mut s, 0, &command()), success(&command(), 0));
|
||||
complete(&s);
|
||||
}
|
||||
#[test]
|
||||
fn created_acknowledged_apply_survives_child_kill() {
|
||||
let dir = tempfile::tempdir().expect("dir");
|
||||
let path = dir.path().join("db");
|
||||
let mut child = Command::new(std::env::current_exe().expect("test executable"))
|
||||
.args(["--exact", "application_tests::created_process_child", "--nocapture"])
|
||||
.env("U11_CHILD_DB", &path)
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::inherit())
|
||||
.spawn()
|
||||
.expect("spawn");
|
||||
let stdout = child.stdout.take().expect("stdout");
|
||||
let (tx, rx) = mpsc::channel();
|
||||
let reader = std::thread::spawn(move || {
|
||||
for line in BufReader::new(stdout).lines() {
|
||||
let line = line.expect("child line");
|
||||
if line.ends_with("U11_ACK") {
|
||||
let _ = tx.send(());
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
let acknowledged = rx.recv_timeout(Duration::from_secs(20));
|
||||
child.kill().expect("kill child");
|
||||
child.wait().expect("wait child");
|
||||
reader.join().expect("reader cleanup");
|
||||
assert!(acknowledged.is_ok(), "U11 bounded child ACK missing");
|
||||
complete(&reopen(&path));
|
||||
}
|
||||
#[test]
|
||||
fn created_power_cut_before_and_after_apply_flush() {
|
||||
let (dir, mut s) = fixture();
|
||||
append(&mut s, 0, &command());
|
||||
s.save_committed(id(0)).expect("committed");
|
||||
drop(s);
|
||||
let path = dir.path().join("db");
|
||||
let backend = CrashBackend::new(&path);
|
||||
let mut s = Store::from_database(backend.database(), identity()).expect("backend");
|
||||
assert_eq!(s.apply_next_created(id(0)).expect("apply ACK"), success(&command(), 0));
|
||||
backend.cut();
|
||||
drop(s);
|
||||
complete(&reopen(&path));
|
||||
for sync in 1..=2 {
|
||||
for after in [false, true] {
|
||||
let (dir, mut s) = fixture();
|
||||
append(&mut s, 0, &command());
|
||||
s.save_committed(id(0)).expect("committed");
|
||||
drop(s);
|
||||
let path = dir.path().join("db");
|
||||
let backend = CrashBackend::new(&path);
|
||||
let mut s = Store::from_database(backend.database(), identity()).expect("backend");
|
||||
let (entered, resume) = backend.pause_sync(sync, after);
|
||||
let writer = std::thread::spawn(move || {
|
||||
let result = s.apply_next_created(id(0));
|
||||
(s, result)
|
||||
});
|
||||
entered.wait();
|
||||
backend.cut();
|
||||
resume.wait();
|
||||
let (s, result) = writer.join().expect("writer");
|
||||
drop(s);
|
||||
let reopened = Store::open(&path, identity());
|
||||
assert!(
|
||||
reopened.is_ok(),
|
||||
"U11 apply cut must reopen whole old/new state sync={sync} after={after}: {:?}",
|
||||
reopened.as_ref().err()
|
||||
);
|
||||
let s = reopened.expect("asserted reopen");
|
||||
if s.last_applied().expect("last").is_some() {
|
||||
complete(&s)
|
||||
} else {
|
||||
assert!(result.is_err());
|
||||
assert_eq!(s.created_head(&command().object).expect("old head"), None);
|
||||
assert_eq!(
|
||||
s.created_version(&command().object, command().prepared.version_id)
|
||||
.expect("old version"),
|
||||
None
|
||||
);
|
||||
assert_eq!(s.created_event(command().operation_id).expect("old event"), None);
|
||||
assert_eq!(s.created_operation(command().operation_id).expect("old op"), None);
|
||||
assert_eq!(s.created_apply_result(0).expect("old result"), None)
|
||||
}
|
||||
eprintln!("U11 cut boundary sync={sync} after={after}");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
use crate::{LOGS, LogId, META, Store, StoreError, decode_id, decode_log, encode_id, number};
|
||||
use redb::{ReadableDatabase, ReadableTable};
|
||||
|
||||
pub(crate) fn validate_committed(
|
||||
meta: &impl ReadableTable<&'static str, &'static [u8]>,
|
||||
logs: &impl ReadableTable<u64, &'static [u8]>,
|
||||
) -> Result<Option<LogId>, StoreError> {
|
||||
let schema = number(meta.get("schema")?.ok_or(StoreError::CorruptRecord)?.value())?;
|
||||
let marker = meta.get("committed")?;
|
||||
match (schema, marker) {
|
||||
(1 | 3 | 4, None) => {
|
||||
if matches!(schema, 3 | 4) && meta.get("last_applied")?.is_none_or(|v| v.value() != [0]) {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
(2 | 3 | 4, Some(marker)) => {
|
||||
let id = decode_id(marker.value())?;
|
||||
let record = logs.get(id.index)?.ok_or(StoreError::CorruptRecord)?;
|
||||
if decode_log(record.value())?.id != id {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
Ok(Some(id))
|
||||
}
|
||||
(1 | 2, _) => Err(StoreError::CorruptRecord),
|
||||
_ => Err(StoreError::UnsupportedSchema),
|
||||
}
|
||||
}
|
||||
|
||||
impl Store {
|
||||
/// Read the exact committed boundary, validating it against the stored log.
|
||||
pub fn read_committed(&self) -> Result<Option<LogId>, StoreError> {
|
||||
let tx = self.database.begin_read()?;
|
||||
validate_committed(&tx.open_table(META)?, &tx.open_table(LOGS)?)
|
||||
}
|
||||
|
||||
/// Persist a monotonic committed boundary supplied by the authority owner.
|
||||
pub fn save_committed(&mut self, id: LogId) -> Result<(), StoreError> {
|
||||
let tx = self.write_transaction()?;
|
||||
{
|
||||
let mut meta = tx.open_table(META)?;
|
||||
let logs = tx.open_table(LOGS)?;
|
||||
let previous = validate_committed(&meta, &logs)?;
|
||||
let record = logs.get(id.index)?.ok_or(StoreError::InvalidCommittedLog)?;
|
||||
if decode_log(record.value())?.id != id {
|
||||
return Err(StoreError::InvalidCommittedLog);
|
||||
}
|
||||
if let Some(previous) = previous {
|
||||
if id.index < previous.index {
|
||||
return Err(StoreError::CommittedRegression);
|
||||
}
|
||||
if id == previous {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
meta.insert("committed", encode_id(id).as_slice())?;
|
||||
if previous.is_none() && !matches!(number(meta.get("schema")?.ok_or(StoreError::CorruptRecord)?.value())?, 3 | 4) {
|
||||
meta.insert("schema", 2u64.to_be_bytes().as_slice())?;
|
||||
}
|
||||
}
|
||||
self.commit(tx)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,242 @@
|
||||
use crate::{LogId, StoreError, decode_id, encode_id};
|
||||
|
||||
pub const MAX_OBJECT_KEY_LENGTH: usize = 4096;
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ObjectIdentity {
|
||||
pub bucket_incarnation: [u8; 16],
|
||||
pub key: Vec<u8>,
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct PreparedIdentity {
|
||||
pub version_id: [u8; 16],
|
||||
pub preparation_id: [u8; 16],
|
||||
pub content_digest: [u8; 32],
|
||||
pub content_length: u64,
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct DecideCreated {
|
||||
pub operation_id: [u8; 16],
|
||||
pub object: ObjectIdentity,
|
||||
pub expected_head: Option<[u8; 16]>,
|
||||
pub prepared: PreparedIdentity,
|
||||
pub binding_revision: u64,
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CaptureBinding {
|
||||
pub revision: u64,
|
||||
pub binding_id: [u8; 16],
|
||||
pub target_id: [u8; 16],
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum CreatedResult {
|
||||
Created {
|
||||
version_id: [u8; 16],
|
||||
event_id: [u8; 16],
|
||||
decision_log_id: LogId,
|
||||
},
|
||||
HeadMismatch {
|
||||
actual_head: Option<[u8; 16]>,
|
||||
},
|
||||
OperationConflict,
|
||||
BindingMismatch,
|
||||
VersionConflict,
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CreatedVersion {
|
||||
pub prepared: PreparedIdentity,
|
||||
pub decision_log_id: LogId,
|
||||
pub binding_revision: u64,
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CreatedEvent {
|
||||
pub object: ObjectIdentity,
|
||||
pub prepared: PreparedIdentity,
|
||||
pub decision_log_id: LogId,
|
||||
pub binding: CaptureBinding,
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CreatedOperation {
|
||||
pub canonical_command: Vec<u8>,
|
||||
pub result: CreatedResult,
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CreatedApplyResult {
|
||||
pub log_id: LogId,
|
||||
pub result: CreatedResult,
|
||||
}
|
||||
|
||||
pub(crate) struct Decoder<'a> {
|
||||
bytes: &'a [u8],
|
||||
}
|
||||
impl<'a> Decoder<'a> {
|
||||
pub(crate) fn new(bytes: &'a [u8]) -> Self {
|
||||
Self { bytes }
|
||||
}
|
||||
pub(crate) fn take<const N: usize>(&mut self) -> Result<[u8; N], StoreError> {
|
||||
let a = self
|
||||
.bytes
|
||||
.get(..N)
|
||||
.ok_or(StoreError::CorruptRecord)?
|
||||
.try_into()
|
||||
.map_err(|_| StoreError::CorruptRecord)?;
|
||||
self.bytes = &self.bytes[N..];
|
||||
Ok(a)
|
||||
}
|
||||
pub(crate) fn u64(&mut self) -> Result<u64, StoreError> {
|
||||
Ok(u64::from_be_bytes(self.take()?))
|
||||
}
|
||||
pub(crate) fn uuid(&mut self) -> Result<[u8; 16], StoreError> {
|
||||
let id = self.take()?;
|
||||
if id == [0; 16] {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
Ok(id)
|
||||
}
|
||||
pub(crate) fn option(&mut self) -> Result<Option<[u8; 16]>, StoreError> {
|
||||
match self.take::<1>()?[0] {
|
||||
0 => Ok(None),
|
||||
1 => Ok(Some(self.uuid()?)),
|
||||
_ => Err(StoreError::CorruptRecord),
|
||||
}
|
||||
}
|
||||
pub(crate) fn object(&mut self) -> Result<ObjectIdentity, StoreError> {
|
||||
let bucket_incarnation = self.uuid()?;
|
||||
let len = usize::try_from(u32::from_be_bytes(self.take()?)).map_err(|_| StoreError::CorruptRecord)?;
|
||||
if len > MAX_OBJECT_KEY_LENGTH {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let key = self.bytes.get(..len).ok_or(StoreError::CorruptRecord)?.to_vec();
|
||||
self.bytes = &self.bytes[len..];
|
||||
Ok(ObjectIdentity { bucket_incarnation, key })
|
||||
}
|
||||
pub(crate) fn prepared(&mut self) -> Result<PreparedIdentity, StoreError> {
|
||||
Ok(PreparedIdentity {
|
||||
version_id: self.uuid()?,
|
||||
preparation_id: self.uuid()?,
|
||||
content_digest: self.take()?,
|
||||
content_length: self.u64()?,
|
||||
})
|
||||
}
|
||||
pub(crate) fn id(&mut self) -> Result<LogId, StoreError> {
|
||||
decode_id(&self.take::<24>()?)
|
||||
}
|
||||
pub(crate) fn binding(&mut self) -> Result<CaptureBinding, StoreError> {
|
||||
Ok(CaptureBinding {
|
||||
revision: self.u64()?,
|
||||
binding_id: self.uuid()?,
|
||||
target_id: self.uuid()?,
|
||||
})
|
||||
}
|
||||
pub(crate) fn finish(self) -> Result<(), StoreError> {
|
||||
if self.bytes.is_empty() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(StoreError::CorruptRecord)
|
||||
}
|
||||
}
|
||||
pub(crate) fn result(&mut self) -> Result<CreatedResult, StoreError> {
|
||||
match self.take::<1>()?[0] {
|
||||
0 => Ok(CreatedResult::Created {
|
||||
version_id: self.uuid()?,
|
||||
event_id: self.uuid()?,
|
||||
decision_log_id: self.id()?,
|
||||
}),
|
||||
1 => Ok(CreatedResult::HeadMismatch {
|
||||
actual_head: self.option()?,
|
||||
}),
|
||||
2 => Ok(CreatedResult::OperationConflict),
|
||||
3 => Ok(CreatedResult::BindingMismatch),
|
||||
4 => Ok(CreatedResult::VersionConflict),
|
||||
_ => Err(StoreError::CorruptRecord),
|
||||
}
|
||||
}
|
||||
}
|
||||
pub(crate) fn put_option(b: &mut Vec<u8>, id: Option<[u8; 16]>) {
|
||||
b.push(u8::from(id.is_some()));
|
||||
if let Some(id) = id {
|
||||
b.extend(id)
|
||||
}
|
||||
}
|
||||
pub(crate) fn object_bytes(o: &ObjectIdentity) -> Result<Vec<u8>, StoreError> {
|
||||
if o.bucket_incarnation == [0; 16] || o.key.len() > MAX_OBJECT_KEY_LENGTH {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let capacity = 20usize.checked_add(o.key.len()).ok_or(StoreError::CorruptRecord)?;
|
||||
let mut b = Vec::with_capacity(capacity);
|
||||
b.extend(o.bucket_incarnation);
|
||||
b.extend(
|
||||
u32::try_from(o.key.len())
|
||||
.map_err(|_| StoreError::CorruptRecord)?
|
||||
.to_be_bytes(),
|
||||
);
|
||||
b.extend(&o.key);
|
||||
Ok(b)
|
||||
}
|
||||
pub(crate) fn put_prepared(b: &mut Vec<u8>, p: &PreparedIdentity) {
|
||||
b.extend(p.version_id);
|
||||
b.extend(p.preparation_id);
|
||||
b.extend(p.content_digest);
|
||||
b.extend(p.content_length.to_be_bytes())
|
||||
}
|
||||
pub(crate) fn binding_bytes(v: &CaptureBinding) -> Result<Vec<u8>, StoreError> {
|
||||
let mut b = v.revision.to_be_bytes().to_vec();
|
||||
b.extend(v.binding_id);
|
||||
b.extend(v.target_id);
|
||||
let mut d = Decoder::new(&b);
|
||||
d.binding()?;
|
||||
d.finish()?;
|
||||
Ok(b)
|
||||
}
|
||||
pub(crate) fn result_bytes(r: &CreatedResult) -> Vec<u8> {
|
||||
let mut b = Vec::new();
|
||||
match r {
|
||||
CreatedResult::Created {
|
||||
version_id,
|
||||
event_id,
|
||||
decision_log_id,
|
||||
} => {
|
||||
b.push(0);
|
||||
b.extend(version_id);
|
||||
b.extend(event_id);
|
||||
b.extend(encode_id(*decision_log_id))
|
||||
}
|
||||
CreatedResult::HeadMismatch { actual_head } => {
|
||||
b.push(1);
|
||||
put_option(&mut b, *actual_head)
|
||||
}
|
||||
CreatedResult::OperationConflict => b.push(2),
|
||||
CreatedResult::BindingMismatch => b.push(3),
|
||||
CreatedResult::VersionConflict => b.push(4),
|
||||
}
|
||||
b
|
||||
}
|
||||
impl DecideCreated {
|
||||
pub fn encode(&self) -> Result<Vec<u8>, StoreError> {
|
||||
let object = object_bytes(&self.object)?;
|
||||
let capacity = object.len().checked_add(134).ok_or(StoreError::CorruptRecord)?;
|
||||
let mut b = Vec::with_capacity(capacity);
|
||||
b.extend(b"CRTD\x01");
|
||||
b.extend(self.operation_id);
|
||||
b.extend(object);
|
||||
put_option(&mut b, self.expected_head);
|
||||
put_prepared(&mut b, &self.prepared);
|
||||
b.extend(self.binding_revision.to_be_bytes());
|
||||
Self::decode(&b)?;
|
||||
Ok(b)
|
||||
}
|
||||
pub fn decode(bytes: &[u8]) -> Result<Self, StoreError> {
|
||||
let mut d = Decoder::new(bytes);
|
||||
if d.take::<5>()? != *b"CRTD\x01" {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
let c = Self {
|
||||
operation_id: d.uuid()?,
|
||||
object: d.object()?,
|
||||
expected_head: d.option()?,
|
||||
prepared: d.prepared()?,
|
||||
binding_revision: d.u64()?,
|
||||
};
|
||||
d.finish()?;
|
||||
Ok(c)
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,371 @@
|
||||
//! Validated capture peer metadata and its independent CNOD v1 codec.
|
||||
use serde::{Deserialize, Deserializer, Serialize, Serializer, ser::SerializeStruct};
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct CaptureNode {
|
||||
rpc_uri: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum CaptureNodeError {
|
||||
#[error("invalid RPC URI")]
|
||||
InvalidRpcUri,
|
||||
#[error("invalid RPC URI")]
|
||||
InvalidRpcUriParse(#[source] url::ParseError),
|
||||
#[error("RPC URI too long")]
|
||||
RpcUriTooLong,
|
||||
#[error("invalid node magic")]
|
||||
InvalidNodeMagic,
|
||||
#[error("unsupported node version")]
|
||||
UnsupportedNodeVersion,
|
||||
#[error("invalid node length")]
|
||||
InvalidNodeLength,
|
||||
#[error("truncated node record")]
|
||||
TruncatedNodeRecord,
|
||||
#[error("invalid node UTF-8")]
|
||||
InvalidNodeUtf8(#[source] std::str::Utf8Error),
|
||||
#[error("trailing node bytes")]
|
||||
TrailingNodeBytes,
|
||||
}
|
||||
impl CaptureNodeError {
|
||||
pub fn code(&self) -> &'static str {
|
||||
match self {
|
||||
Self::InvalidRpcUri | Self::InvalidRpcUriParse(_) => "invalid_rpc_uri",
|
||||
Self::RpcUriTooLong => "rpc_uri_too_long",
|
||||
Self::InvalidNodeMagic => "invalid_node_magic",
|
||||
Self::UnsupportedNodeVersion => "unsupported_node_version",
|
||||
Self::InvalidNodeLength => "invalid_node_length",
|
||||
Self::TruncatedNodeRecord => "truncated_node_record",
|
||||
Self::InvalidNodeUtf8(_) => "invalid_node_utf8",
|
||||
Self::TrailingNodeBytes => "trailing_node_bytes",
|
||||
}
|
||||
}
|
||||
}
|
||||
impl CaptureNode {
|
||||
pub fn new(rpc_uri: String) -> Result<Self, CaptureNodeError> {
|
||||
let node = Self { rpc_uri };
|
||||
node.validate()?;
|
||||
Ok(node)
|
||||
}
|
||||
|
||||
pub fn rpc_uri(&self) -> &str {
|
||||
&self.rpc_uri
|
||||
}
|
||||
|
||||
pub fn validate(&self) -> Result<(), CaptureNodeError> {
|
||||
let uri = &self.rpc_uri;
|
||||
if uri.len() > 4096 {
|
||||
return Err(CaptureNodeError::RpcUriTooLong);
|
||||
}
|
||||
if uri.is_empty() || uri.chars().any(|c| c.is_whitespace() || c.is_control() || c == '\\') {
|
||||
return Err(CaptureNodeError::InvalidRpcUri);
|
||||
}
|
||||
let rest = uri
|
||||
.strip_prefix("http://")
|
||||
.or_else(|| uri.strip_prefix("https://"))
|
||||
.ok_or(CaptureNodeError::InvalidRpcUri)?;
|
||||
if uri.contains('?') || uri.contains('#') {
|
||||
return Err(CaptureNodeError::InvalidRpcUri);
|
||||
}
|
||||
let (authority, path) = rest.split_once('/').map_or((rest, ""), |(a, p)| (a, p));
|
||||
// An empty suffix after the first slash is the only allowed path.
|
||||
if !path.is_empty() || authority.contains('@') {
|
||||
return Err(CaptureNodeError::InvalidRpcUri);
|
||||
}
|
||||
let (host, port) = if authority.starts_with('[') {
|
||||
let end = authority.find(']').ok_or(CaptureNodeError::InvalidRpcUri)?;
|
||||
let port = authority
|
||||
.get(end + 1..)
|
||||
.and_then(|suffix| suffix.strip_prefix(':'))
|
||||
.ok_or(CaptureNodeError::InvalidRpcUri)?;
|
||||
(&authority[..=end], port)
|
||||
} else {
|
||||
authority.split_once(':').ok_or(CaptureNodeError::InvalidRpcUri)?
|
||||
};
|
||||
if host.is_empty() || port.is_empty() || !port.bytes().all(|b| b.is_ascii_digit()) {
|
||||
return Err(CaptureNodeError::InvalidRpcUri);
|
||||
}
|
||||
let port = port.parse::<u16>().map_err(|_| CaptureNodeError::InvalidRpcUri)?;
|
||||
if port == 0 {
|
||||
return Err(CaptureNodeError::InvalidRpcUri);
|
||||
}
|
||||
let parsed_url = url::Url::parse(uri).map_err(CaptureNodeError::InvalidRpcUriParse)?;
|
||||
if parsed_url.host().is_none() {
|
||||
return Err(CaptureNodeError::InvalidRpcUri);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn encode(&self) -> Result<Vec<u8>, CaptureNodeError> {
|
||||
self.validate()?;
|
||||
let length = u32::try_from(self.rpc_uri.len()).map_err(|_| CaptureNodeError::InvalidNodeLength)?;
|
||||
let mut bytes = b"CNOD\x01".to_vec();
|
||||
bytes.extend_from_slice(&length.to_be_bytes());
|
||||
bytes.extend_from_slice(self.rpc_uri.as_bytes());
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
pub fn decode(bytes: &[u8]) -> Result<Self, CaptureNodeError> {
|
||||
if bytes.len() < 9 {
|
||||
return Err(CaptureNodeError::TruncatedNodeRecord);
|
||||
}
|
||||
if &bytes[..4] != b"CNOD" {
|
||||
return Err(CaptureNodeError::InvalidNodeMagic);
|
||||
}
|
||||
if bytes[4] != 1 {
|
||||
return Err(CaptureNodeError::UnsupportedNodeVersion);
|
||||
}
|
||||
let length = u32::from_be_bytes(bytes[5..9].try_into().map_err(|_| CaptureNodeError::InvalidNodeLength)?);
|
||||
if !(1..=4096).contains(&length) {
|
||||
return Err(CaptureNodeError::InvalidNodeLength);
|
||||
}
|
||||
let length = usize::try_from(length).map_err(|_| CaptureNodeError::InvalidNodeLength)?;
|
||||
let end = 9_usize.checked_add(length).ok_or(CaptureNodeError::InvalidNodeLength)?;
|
||||
if bytes.len() < end {
|
||||
return Err(CaptureNodeError::TruncatedNodeRecord);
|
||||
}
|
||||
if bytes.len() != end {
|
||||
return Err(CaptureNodeError::TrailingNodeBytes);
|
||||
}
|
||||
let uri = std::str::from_utf8(&bytes[9..end]).map_err(CaptureNodeError::InvalidNodeUtf8)?;
|
||||
Self::new(uri.to_owned())
|
||||
}
|
||||
}
|
||||
impl Serialize for CaptureNode {
|
||||
fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
|
||||
self.validate().map_err(serde::ser::Error::custom)?;
|
||||
let mut state = serializer.serialize_struct("CaptureNode", 1)?;
|
||||
state.serialize_field("rpc_uri", &self.rpc_uri)?;
|
||||
state.end()
|
||||
}
|
||||
}
|
||||
impl<'de> Deserialize<'de> for CaptureNode {
|
||||
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct Fields {
|
||||
rpc_uri: String,
|
||||
}
|
||||
let fields = Fields::deserialize(deserializer).map_err(|_| serde::de::Error::custom("invalid capture node fields"))?;
|
||||
Self::new(fields.rpc_uri).map_err(serde::de::Error::custom)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::error::Error;
|
||||
|
||||
fn record(payload: &[u8]) -> Vec<u8> {
|
||||
let mut bytes = b"CNOD\x01".to_vec();
|
||||
bytes.extend_from_slice(&u32::try_from(payload.len()).expect("test payload length").to_be_bytes());
|
||||
bytes.extend_from_slice(payload);
|
||||
bytes
|
||||
}
|
||||
fn reject_uri(uri: &str) {
|
||||
let result = CaptureNode::new(uri.to_owned());
|
||||
assert!(result.is_err(), "URI should reject: {uri:?}");
|
||||
assert_eq!(result.expect_err("rejected URI").code(), "invalid_rpc_uri");
|
||||
}
|
||||
fn reject_record(bytes: &[u8], code: &str) {
|
||||
let result = CaptureNode::decode(bytes);
|
||||
assert!(result.is_err(), "record should reject with {code}");
|
||||
assert_eq!(result.expect_err("rejected record").code(), code);
|
||||
}
|
||||
#[test]
|
||||
fn actual_openraft_node_bound() {
|
||||
fn require_node<T: openraft::Node>() {}
|
||||
require_node::<CaptureNode>();
|
||||
assert_eq!(CaptureNode::default().rpc_uri(), "");
|
||||
}
|
||||
#[test]
|
||||
fn default_and_empty_rejected_at_every_boundary() {
|
||||
let node = CaptureNode::default();
|
||||
assert_eq!(node.rpc_uri(), "");
|
||||
reject_uri("");
|
||||
assert_eq!(node.validate().expect_err("empty validation").code(), "invalid_rpc_uri");
|
||||
assert!(serde_json::to_string(&node).is_err());
|
||||
assert!(serde_json::from_str::<CaptureNode>(r#"{"rpc_uri":""}"#).is_err());
|
||||
assert_eq!(node.encode().expect_err("empty encoding").code(), "invalid_rpc_uri");
|
||||
reject_record(&record(b""), "invalid_node_length");
|
||||
}
|
||||
#[test]
|
||||
fn accepted_uris_preserve_exact_bytes() {
|
||||
for uri in [
|
||||
"http://node:80",
|
||||
"https://node:443",
|
||||
"http://node:1",
|
||||
"https://node:65535/",
|
||||
"http://127.0.0.1:9000",
|
||||
"https://[::1]:443/",
|
||||
"http://[2001:db8::1]:1",
|
||||
"http://NoDe:00080/",
|
||||
] {
|
||||
let result = CaptureNode::new(uri.to_owned());
|
||||
assert!(result.is_ok(), "valid URI construction: {uri}");
|
||||
let node = result.expect("valid URI");
|
||||
assert_eq!(node.rpc_uri(), uri, "preserve explicit default port and raw URI");
|
||||
assert_eq!(node.encode().expect("encode URI"), record(uri.as_bytes()));
|
||||
assert_eq!(CaptureNode::decode(&record(uri.as_bytes())).expect("decode URI"), node);
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn golden_cnod_bytes() {
|
||||
let node = CaptureNode {
|
||||
rpc_uri: "http://node:80".to_owned(),
|
||||
};
|
||||
let golden = b"CNOD\x01\x00\x00\x00\x0ehttp://node:80";
|
||||
assert_eq!(node.encode().ok().as_deref(), Some(golden.as_slice()), "golden CNOD bytes");
|
||||
}
|
||||
#[test]
|
||||
fn valid_cnod_decode() {
|
||||
let golden = b"CNOD\x01\x00\x00\x00\x0ehttp://node:80";
|
||||
let decoded = CaptureNode::decode(golden);
|
||||
assert!(decoded.is_ok(), "golden decode succeeds");
|
||||
assert_eq!(decoded.expect("golden decoded node").rpc_uri(), "http://node:80");
|
||||
}
|
||||
#[test]
|
||||
fn serde_shape_roundtrip_and_strict_fields() {
|
||||
let node = CaptureNode::new("https://node:443/".to_owned()).expect("serde node");
|
||||
let json = r#"{"rpc_uri":"https://node:443/"}"#;
|
||||
assert_eq!(serde_json::to_string(&node).expect("serialize node"), json);
|
||||
assert_eq!(serde_json::from_str::<CaptureNode>(json).expect("deserialize node"), node);
|
||||
for json in [
|
||||
r#"{}"#,
|
||||
r#"{"rpc_uri":1}"#,
|
||||
r#"{"rpc_uri":null}"#,
|
||||
r#"{"rpc_uri":"http://node:1","extra":0}"#,
|
||||
r#"{"rpc_uri":"http://node:1","rpc_uri":"http://node:2"}"#,
|
||||
] {
|
||||
assert!(serde_json::from_str::<CaptureNode>(json).is_err(), "strict fields: {json}");
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn credentials_and_raw_uri_shapes_reject() {
|
||||
for uri in [
|
||||
"http://user@node:1",
|
||||
"http://user:password@node:1",
|
||||
"http://@node:1",
|
||||
"HTTP://node:1",
|
||||
"Https://node:1",
|
||||
"http://node",
|
||||
"http://node:",
|
||||
"http://node:0",
|
||||
"http://node:-1",
|
||||
"http://node:+1",
|
||||
"http://node:65536",
|
||||
"http://node:999999999999999999999",
|
||||
"http://[::1:1",
|
||||
"http://[::1]extra:1",
|
||||
"http://[invalid]:1",
|
||||
"http://::1:1",
|
||||
"http://:1",
|
||||
"/disk/path",
|
||||
"file:///disk/path",
|
||||
"http://node:1/./",
|
||||
"http://node:1/../",
|
||||
"http://node:1/%2e/",
|
||||
"http://node:1/%2E%2e/",
|
||||
"http://node:1/path",
|
||||
"http://node:1\\",
|
||||
" http://node:1",
|
||||
"http://node:1\n",
|
||||
"http://no\tde:1",
|
||||
"http://node:1\u{00a0}",
|
||||
"http://node:1\u{0085}",
|
||||
"http://node:1\0",
|
||||
] {
|
||||
reject_uri(uri);
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn query_and_fragment_markers_reject() {
|
||||
for uri in [
|
||||
"http://node:1?",
|
||||
"http://node:1?x=y",
|
||||
"http://node:1/?",
|
||||
"http://node:1#",
|
||||
"http://node:1#fragment",
|
||||
"http://node:1/#",
|
||||
] {
|
||||
reject_uri(uri);
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn utf8_byte_length_boundaries() {
|
||||
for length in [4095, 4096] {
|
||||
let prefix = "http://node:";
|
||||
let uri = format!("{prefix}{}1", "0".repeat(length - prefix.len() - 1));
|
||||
assert_eq!(uri.len(), length);
|
||||
assert!(url::Url::parse(&uri).is_ok(), "boundary fixture accepted by actual URL parser");
|
||||
let node = CaptureNode::new(uri.clone()).expect("boundary node");
|
||||
assert_eq!(node.rpc_uri(), uri);
|
||||
assert_eq!(
|
||||
CaptureNode::decode(&node.encode().expect("boundary encoding")).expect("boundary decoding"),
|
||||
node
|
||||
);
|
||||
}
|
||||
let uri = format!("http://node:{}1", "0".repeat(4097 - "http://node:".len() - 1));
|
||||
assert_eq!(CaptureNode::new(uri).expect_err("oversize URI").code(), "rpc_uri_too_long");
|
||||
let unicode = format!("http://{}:1", "é".repeat(2044));
|
||||
assert!(unicode.chars().count() < 4096);
|
||||
assert!(unicode.len() > 4096);
|
||||
assert_eq!(CaptureNode::new(unicode).expect_err("UTF-8 byte limit").code(), "rpc_uri_too_long");
|
||||
}
|
||||
#[test]
|
||||
fn cnod_headers_and_lengths_reject_exact_codes() {
|
||||
let valid = record(b"http://node:80");
|
||||
for end in 0..9 {
|
||||
reject_record(&valid[..end], "truncated_node_record");
|
||||
}
|
||||
let mut bad = valid.clone();
|
||||
bad[0] = b'X';
|
||||
reject_record(&bad, "invalid_node_magic");
|
||||
bad = valid.clone();
|
||||
bad[4] = 2;
|
||||
reject_record(&bad, "unsupported_node_version");
|
||||
for length in [0_u32, 4097, u32::MAX] {
|
||||
bad = b"CNOD\x01".to_vec();
|
||||
bad.extend_from_slice(&length.to_be_bytes());
|
||||
reject_record(&bad, "invalid_node_length");
|
||||
}
|
||||
for end in 9..valid.len() {
|
||||
reject_record(&valid[..end], "truncated_node_record");
|
||||
}
|
||||
reject_record(&record(&[0xff]), "invalid_node_utf8");
|
||||
reject_record(&record(&[0xc3, 0x28]), "invalid_node_utf8");
|
||||
reject_record(&record(b"invalid URI"), "invalid_rpc_uri");
|
||||
}
|
||||
#[test]
|
||||
fn trailing_bytes_reject() {
|
||||
for tail in [&b"x"[..], &b"xyz"[..]] {
|
||||
let mut bytes = record(b"http://node:80");
|
||||
bytes.extend_from_slice(tail);
|
||||
reject_record(&bytes, "trailing_node_bytes");
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn errors_and_sources_redact_input() {
|
||||
let sentinel = "http://SECRET_USER:SECRET_PASSWORD@node:1";
|
||||
let error = CaptureNode::new(sentinel.to_owned()).expect_err("credentials reject");
|
||||
let mut current: &dyn Error = &error;
|
||||
loop {
|
||||
let rendered = format!("{current} {current:?}");
|
||||
assert!(!rendered.contains("SECRET"));
|
||||
match current.source() {
|
||||
Some(source) => current = source,
|
||||
None => break,
|
||||
}
|
||||
}
|
||||
let json = format!(r#"{{"rpc_uri":"{sentinel}"}}"#);
|
||||
let error = serde_json::from_str::<CaptureNode>(&json).expect_err("serde credentials reject");
|
||||
assert!(!format!("{error} {error:?}").contains("SECRET"));
|
||||
let json = format!(r#"{{"{sentinel}":0}}"#);
|
||||
let error = serde_json::from_str::<CaptureNode>(&json).expect_err("unknown fields reject");
|
||||
assert!(!format!("{error} {error:?}").contains("SECRET"));
|
||||
let error = CaptureNode::new("http://[bad]:1".to_owned()).expect_err("parser rejection");
|
||||
assert!(error.source().is_some(), "retain URL parser source");
|
||||
let error = CaptureNode::decode(&record(&[0xff])).expect_err("UTF-8 rejection");
|
||||
assert!(error.source().is_some(), "retain UTF-8 source");
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,43 @@
|
||||
use crate::{LOGS, META, Store, StoreError, decode_id, decode_log, encode_id};
|
||||
use redb::ReadableTable;
|
||||
|
||||
impl Store {
|
||||
/// Atomically remove every log entry at or after `from`.
|
||||
pub fn truncate_suffix(&mut self, from: u64) -> Result<(), StoreError> {
|
||||
let tx = self.write_transaction()?;
|
||||
{
|
||||
let mut meta = tx.open_table(META)?;
|
||||
let logs = tx.open_table(LOGS)?;
|
||||
if crate::committed::validate_committed(&meta, &logs)?.is_some_and(|id| from <= id.index) {
|
||||
return Err(StoreError::CommittedLogProtected);
|
||||
}
|
||||
drop(logs);
|
||||
let tail = meta.get("tail")?.map(|v| decode_id(v.value())).transpose()?;
|
||||
if tail.is_none_or(|id| from > id.index) {
|
||||
return Ok(());
|
||||
}
|
||||
let mut logs = tx.open_table(LOGS)?;
|
||||
let new_tail = if from == 0 {
|
||||
None
|
||||
} else {
|
||||
let index = from - 1;
|
||||
let record = logs.get(index)?.ok_or(StoreError::InvalidLogSequence)?;
|
||||
let id = decode_log(record.value())?.id;
|
||||
if id.index != index {
|
||||
return Err(StoreError::CorruptRecord);
|
||||
}
|
||||
Some(id)
|
||||
};
|
||||
logs.retain(|index, _| index < from)?;
|
||||
match new_tail {
|
||||
Some(id) => {
|
||||
meta.insert("tail", encode_id(id).as_slice())?;
|
||||
}
|
||||
None => {
|
||||
meta.remove("tail")?;
|
||||
}
|
||||
}
|
||||
}
|
||||
self.commit(tx)
|
||||
}
|
||||
}
|
||||
@@ -18,10 +18,8 @@ use const_str::concat;
|
||||
/// Default value: RustFS
|
||||
/// Environment variable: RUSTFS_APP_NAME
|
||||
pub const APP_NAME: &str = "RustFS";
|
||||
/// Application version
|
||||
/// Default value: 1.0.0
|
||||
/// Environment variable: RUSTFS_VERSION
|
||||
pub const VERSION: &str = "1.0.0";
|
||||
/// Cargo package version for this workspace member.
|
||||
pub const VERSION: &str = env!("CARGO_PKG_VERSION");
|
||||
|
||||
/// Default configuration logger level
|
||||
/// Default value: error
|
||||
@@ -39,12 +37,6 @@ pub const SAMPLE_RATIO: f64 = 1.0;
|
||||
/// Default value: 30
|
||||
pub const METER_INTERVAL: u64 = 30;
|
||||
|
||||
/// Default configuration service version
|
||||
/// Default value: 1.0.0
|
||||
/// Environment variable: RUSTFS_OBS_SERVICE_VERSION
|
||||
/// Uses the same value as VERSION constant
|
||||
pub const SERVICE_VERSION: &str = "1.0.0";
|
||||
|
||||
/// Default configuration environment
|
||||
/// Default value: production
|
||||
pub const ENVIRONMENT: &str = "production";
|
||||
@@ -410,7 +402,6 @@ mod tests {
|
||||
APP_NAME,
|
||||
VERSION,
|
||||
DEFAULT_LOG_LEVEL,
|
||||
SERVICE_VERSION,
|
||||
ENVIRONMENT,
|
||||
RUSTFS_TLS_KEY,
|
||||
RUSTFS_TLS_CERT,
|
||||
@@ -446,8 +437,7 @@ mod tests {
|
||||
fn test_configuration_consistency() {
|
||||
// Test configuration consistency
|
||||
|
||||
// Version consistency
|
||||
assert_eq!(VERSION, SERVICE_VERSION, "Application version should match service version");
|
||||
assert_eq!(VERSION, env!("CARGO_PKG_VERSION"));
|
||||
|
||||
// Port conflict check
|
||||
let ports = [DEFAULT_PORT, DEFAULT_CONSOLE_PORT];
|
||||
|
||||
@@ -207,3 +207,9 @@ pub const ENV_TLS_RELOAD_INTERVAL: &str = "RUSTFS_TLS_RELOAD_INTERVAL";
|
||||
|
||||
/// Default interval for TLS certificate reload check
|
||||
pub const DEFAULT_TLS_RELOAD_INTERVAL: u64 = 30;
|
||||
|
||||
/// Environment variable to enable experimental HTTP/3 support.
|
||||
pub const ENV_HTTP3_ENABLE: &str = "RUSTFS_HTTP3_ENABLE";
|
||||
|
||||
/// HTTP/3 is disabled unless explicitly enabled.
|
||||
pub const DEFAULT_HTTP3_ENABLE: bool = false;
|
||||
|
||||
@@ -33,6 +33,10 @@
|
||||
//! tracked separately.
|
||||
|
||||
use crate::common::{ClusterTopology, RustFSTestClusterEnvironment};
|
||||
use crate::fault_proxy::FaultMode;
|
||||
use aws_sdk_s3::Client;
|
||||
use std::time::Duration;
|
||||
use tokio::time::{Instant, timeout};
|
||||
|
||||
type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
|
||||
|
||||
@@ -144,7 +148,99 @@ async fn cluster_volume_fault_proxy_pass_smoke() -> TestResult {
|
||||
cluster.start().await?;
|
||||
cluster.create_test_bucket(BUCKET).await?;
|
||||
let payload = vec![0x6Du8; 256 * 1024];
|
||||
put_get_roundtrip(&cluster, "volume-proxy/object", &payload).await
|
||||
put_get_roundtrip(&cluster, "volume-proxy/object", &payload).await?;
|
||||
|
||||
// Node 1 owns only two disks; a successful three-vote write must
|
||||
// authenticate at least one disk mutation through node 0's proxy.
|
||||
let client = cluster.create_s3_client(1)?;
|
||||
let client = Client::from_conf(
|
||||
client
|
||||
.config()
|
||||
.to_builder()
|
||||
.retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(1))
|
||||
.build(),
|
||||
);
|
||||
let forwarded_before = proxy.forwarded_bytes();
|
||||
client
|
||||
.put_object()
|
||||
.bucket(BUCKET)
|
||||
.key("volume-proxy/authenticated")
|
||||
.body(bytes::Bytes::copy_from_slice(&payload).into())
|
||||
.send()
|
||||
.await?;
|
||||
assert!(
|
||||
proxy.forwarded_bytes() > forwarded_before,
|
||||
"successful peer-disk publication must traverse the proxy"
|
||||
);
|
||||
|
||||
let dropped_before = proxy.dropped_bytes();
|
||||
proxy.set_mode(FaultMode::Blackhole);
|
||||
assert_eq!(proxy.mode(), FaultMode::Blackhole);
|
||||
let error = timeout(
|
||||
Duration::from_secs(30),
|
||||
client
|
||||
.put_object()
|
||||
.bucket(BUCKET)
|
||||
.key("volume-proxy/rejected")
|
||||
.body(bytes::Bytes::copy_from_slice(&payload).into())
|
||||
.send(),
|
||||
)
|
||||
.await?
|
||||
.expect_err("two local disks cannot commit a write while the peer is blackholed");
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(503));
|
||||
assert_eq!(error.as_service_error().and_then(|error| error.meta().code()), Some("ServiceUnavailable"));
|
||||
assert!(
|
||||
proxy.dropped_bytes() > dropped_before,
|
||||
"the injected blackhole must discard actual peer traffic"
|
||||
);
|
||||
|
||||
proxy.set_mode(FaultMode::Pass);
|
||||
assert_eq!(proxy.mode(), FaultMode::Pass);
|
||||
let http = reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.timeout(Duration::from_secs(3))
|
||||
.build()?;
|
||||
let deadline = Instant::now() + Duration::from_secs(30);
|
||||
loop {
|
||||
let response = http.get(format!("{}/health/ready", cluster.nodes[1].url)).send().await?;
|
||||
let status = response.status();
|
||||
let body: serde_json::Value = response.json().await?;
|
||||
// Reachable peers can still have Returning disk handles that reject
|
||||
// writes. Wait for this node's live writer inventory to be Online.
|
||||
if status.as_u16() == 200
|
||||
&& body["details"]["storage"]["source"] == "local_runtime"
|
||||
&& body["details"]["storage"]["writeQuorum"] == true
|
||||
&& body["details"]["storage"]["unavailableDrives"]
|
||||
.as_array()
|
||||
.is_some_and(Vec::is_empty)
|
||||
&& body["details"]["lock"]["ready"] == true
|
||||
{
|
||||
break;
|
||||
}
|
||||
assert!(Instant::now() < deadline, "peer write quorum did not recover: HTTP {status}, {body}");
|
||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
||||
}
|
||||
client
|
||||
.put_object()
|
||||
.bucket(BUCKET)
|
||||
.key("volume-proxy/recovered")
|
||||
.body(bytes::Bytes::copy_from_slice(&payload).into())
|
||||
.send()
|
||||
.await?;
|
||||
for key in ["volume-proxy/object", "volume-proxy/authenticated", "volume-proxy/recovered"] {
|
||||
let got = client.get_object().bucket(BUCKET).key(key).send().await?;
|
||||
let body = got.body.collect().await?.into_bytes();
|
||||
assert_eq!(body.as_ref(), payload, "proxy recovery must preserve the full object body for {key}");
|
||||
}
|
||||
let rejected = client
|
||||
.head_object()
|
||||
.bucket(BUCKET)
|
||||
.key("volume-proxy/rejected")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a rejected proxied write must never become visible after recovery");
|
||||
assert_eq!(rejected.raw_response().map(|response| response.status().as_u16()), Some(404));
|
||||
Ok(())
|
||||
}
|
||||
.await;
|
||||
|
||||
|
||||
@@ -35,7 +35,7 @@ use s3s::Body;
|
||||
use serde_json;
|
||||
use std::fs as stdfs;
|
||||
use std::io::ErrorKind;
|
||||
use std::net::SocketAddr;
|
||||
use std::net::{Ipv6Addr, SocketAddr};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Child, Command, Stdio};
|
||||
use std::sync::Once;
|
||||
@@ -1442,7 +1442,11 @@ impl RustFSTestClusterEnvironment {
|
||||
return Err(format!("a volume proxy is already configured for node {node_idx}").into());
|
||||
}
|
||||
let target = self.nodes[node_idx].address.parse::<SocketAddr>()?;
|
||||
let proxy = crate::fault_proxy::FaultProxy::start(target).await?;
|
||||
// Endpoint locality and RPC audience both use the advertised authority.
|
||||
// Keep the node's port while the second loopback IP separates the proxy
|
||||
// listener from the direct IPv4 server and S3 client address.
|
||||
let listen = SocketAddr::from((Ipv6Addr::LOCALHOST, target.port()));
|
||||
let proxy = crate::fault_proxy::FaultProxy::start_on(listen, target).await?;
|
||||
self.volume_proxy_addresses[node_idx] = Some(proxy.local_addr());
|
||||
Ok(proxy)
|
||||
}
|
||||
@@ -2467,6 +2471,20 @@ mod tests {
|
||||
|
||||
assert!(volumes.contains(&proxied), "volumes must use the proxy address: {volumes}");
|
||||
assert!(!volumes.contains(&direct), "volumes must not retain the direct address: {volumes}");
|
||||
assert_eq!(proxy.local_addr().port(), proxy.target_addr().port());
|
||||
assert_ne!(proxy.local_addr().ip(), proxy.target_addr().ip());
|
||||
assert_eq!(env.nodes[0].address, direct, "S3 clients must retain the direct address");
|
||||
let mut endpoint =
|
||||
crate::storage_api::Endpoint::try_from(volumes.as_str()).expect("proxied volume should be a valid endpoint");
|
||||
endpoint
|
||||
.update_is_local(proxy.target_addr().port())
|
||||
.expect("proxied endpoint locality should resolve");
|
||||
assert!(endpoint.is_local, "the proxied node must keep its local disk and first-disk authority");
|
||||
assert_eq!(
|
||||
endpoint.host_port(),
|
||||
proxied,
|
||||
"the local RPC audience must be the advertised proxy authority"
|
||||
);
|
||||
|
||||
proxy.shutdown().await;
|
||||
}
|
||||
|
||||
@@ -31,6 +31,7 @@ use super::harness::{
|
||||
use crate::common::{
|
||||
AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, init_logging,
|
||||
};
|
||||
use crate::upgrade_compatibility_test::wait_for_upgrade_write_readiness;
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use std::path::{Path, PathBuf};
|
||||
@@ -400,6 +401,19 @@ async fn four_node_direct_upgrade_preserves_history_and_iam_credentials() -> Tes
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn replace_node_and_wait_for_write_readiness(dist: &mut DistCluster, seed: &UpgradeSeed, node_idx: usize) -> TestResult {
|
||||
dist.replace_node_with_current_binary(node_idx).await?;
|
||||
// Node readiness proves reads can succeed; a rolling writer must also
|
||||
// recover before replacing the next peer or checking upgrade contracts.
|
||||
wait_for_upgrade_write_readiness(
|
||||
&dist.clients()?,
|
||||
&seed.history_bucket,
|
||||
&format!("replaced-node-{node_idx}"),
|
||||
CREDENTIAL_TIMEOUT,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> TestResult {
|
||||
init_logging();
|
||||
@@ -410,15 +424,15 @@ async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> Te
|
||||
|
||||
let seed = seed_history_and_iam(&dist).await?;
|
||||
|
||||
dist.replace_node_with_current_binary(0).await?;
|
||||
replace_node_and_wait_for_write_readiness(&mut dist, &seed, 0).await?;
|
||||
assert_history_and_iam(&dist, &seed, "one-current-node").await?;
|
||||
|
||||
for node_idx in [1, 2] {
|
||||
dist.replace_node_with_current_binary(node_idx).await?;
|
||||
replace_node_and_wait_for_write_readiness(&mut dist, &seed, node_idx).await?;
|
||||
}
|
||||
assert_history_and_iam(&dist, &seed, "one-previous-node").await?;
|
||||
|
||||
dist.replace_node_with_current_binary(3).await?;
|
||||
replace_node_and_wait_for_write_readiness(&mut dist, &seed, 3).await?;
|
||||
assert_history_and_iam(&dist, &seed, "homogeneous-current").await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -50,8 +50,8 @@ use std::collections::{BTreeMap, HashMap, VecDeque};
|
||||
use std::error::Error;
|
||||
use std::io;
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::{Arc, Mutex, MutexGuard};
|
||||
use std::time::{Duration, SystemTime};
|
||||
use std::sync::{Arc, Mutex, MutexGuard, OnceLock};
|
||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::sync::{OwnedSemaphorePermit, Semaphore, watch};
|
||||
use tokio::task::JoinHandle;
|
||||
@@ -363,6 +363,14 @@ pub struct RequestRecord {
|
||||
pub upload_id: Option<String>,
|
||||
pub part_number: Option<i32>,
|
||||
pub content_length: Option<u64>,
|
||||
pub source_version_id: Option<String>,
|
||||
pub source_mtime: Option<String>,
|
||||
pub source_etag: Option<String>,
|
||||
pub source_replication_request: bool,
|
||||
pub journaled_at_unix_millis: Option<u64>,
|
||||
/// Computed by the target, before response stall/disconnect faults. This
|
||||
/// does not prove that any response bytes reached the client.
|
||||
pub prepared_response: Option<PreparedResponse>,
|
||||
pub consumed_bytes: Option<usize>,
|
||||
pub replication_timestamps: ReplicationTimestampHeaders,
|
||||
pub proxy_headers: ProxyHeaderSnapshot,
|
||||
@@ -381,6 +389,20 @@ pub struct RequestRecord {
|
||||
pub fault: Option<FaultAction>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct PreparedResponse {
|
||||
pub status: u16,
|
||||
pub version_id: Option<String>,
|
||||
pub etag: Option<String>,
|
||||
pub last_modified: Option<String>,
|
||||
pub content_length: Option<u64>,
|
||||
pub prepared_at_unix_millis: Option<u64>,
|
||||
pub elapsed: Duration,
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct RequestSequence(Arc<OnceLock<u64>>);
|
||||
|
||||
/// Integrity and framing headers of an upload. A remote target's acceptance
|
||||
/// rules key off exactly these (rustfs#6853: `aws-chunked` framing stored
|
||||
/// verbatim; rustfs#7082: Object Lock PUTs need `Content-MD5` or
|
||||
@@ -430,6 +452,10 @@ impl TransportSnapshot {
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
struct JournaledHeaders {
|
||||
content_length: Option<u64>,
|
||||
source_version_id: Option<String>,
|
||||
source_mtime: Option<String>,
|
||||
source_etag: Option<String>,
|
||||
source_replication_request: bool,
|
||||
replication_timestamps: ReplicationTimestampHeaders,
|
||||
proxy_headers: ProxyHeaderSnapshot,
|
||||
transport: TransportSnapshot,
|
||||
@@ -444,6 +470,10 @@ impl JournaledHeaders {
|
||||
.get(CONTENT_LENGTH)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(|value| value.parse().ok()),
|
||||
source_version_id: header_value(headers, &SOURCE_VERSION_ID_HEADERS).map(bounded_journal_value),
|
||||
source_mtime: header_value(headers, &SOURCE_MTIME_HEADERS).map(bounded_journal_value),
|
||||
source_etag: header_value(headers, &SOURCE_ETAG_HEADERS).map(bounded_journal_value),
|
||||
source_replication_request: header_value(headers, &SOURCE_REPLICATION_REQUEST_HEADERS).as_deref() == Some("true"),
|
||||
replication_timestamps: ReplicationTimestampHeaders::from_headers(headers),
|
||||
proxy_headers: ProxyHeaderSnapshot::from_headers(headers),
|
||||
transport: TransportSnapshot::from_headers(headers),
|
||||
@@ -511,6 +541,10 @@ struct StoreState {
|
||||
/// version id the target never had answers 404 `NoSuchVersion` instead of
|
||||
/// the idempotent 204 RustFS/MinIO give.
|
||||
reject_unknown_version_deletes: bool,
|
||||
/// Models MinIO and its derivatives (Storj): `X-Minio-Internal-` is a reserved
|
||||
/// metadata prefix, so a PutObject or UploadPart carrying any such request
|
||||
/// header is refused with `InvalidArgument` (rustfs#8362).
|
||||
reject_reserved_internal_headers: bool,
|
||||
limits: StoreLimits,
|
||||
buckets: HashMap<String, BucketState>,
|
||||
uploads: HashMap<String, MultipartState>,
|
||||
@@ -738,6 +772,7 @@ impl FakeS3Target {
|
||||
let connection_limit = Arc::new(Semaphore::new(MAX_CONNECTIONS));
|
||||
let (active_connections_tx, active_connections) = watch::channel(0usize);
|
||||
let task_connection_limit = Arc::clone(&connection_limit);
|
||||
let task_control = Arc::clone(&control);
|
||||
|
||||
let task = tokio::spawn(async move {
|
||||
let mut connections = tokio::task::JoinSet::new();
|
||||
@@ -751,6 +786,7 @@ impl FakeS3Target {
|
||||
};
|
||||
active_connections_tx.send_modify(|count| *count += 1);
|
||||
let service = service.clone();
|
||||
let control = Arc::clone(&task_control);
|
||||
let active_connections = active_connections_tx.clone();
|
||||
let mut connection_gate = connection_gate.clone();
|
||||
connections.spawn(async move {
|
||||
@@ -764,7 +800,7 @@ impl FakeS3Target {
|
||||
{
|
||||
return;
|
||||
}
|
||||
let handler = service_fn(move |request| handle_request(request, service.clone()));
|
||||
let handler = service_fn(move |request| handle_request(request, service.clone(), Arc::clone(&control)));
|
||||
let mut connection = http1::Builder::new();
|
||||
connection
|
||||
.keep_alive(false)
|
||||
@@ -975,6 +1011,43 @@ impl FakeS3Target {
|
||||
lock(&self.backend.store).reject_unknown_version_deletes = enabled;
|
||||
}
|
||||
|
||||
/// Refuse every PutObject / UploadPart that carries an `x-minio-internal-*`
|
||||
/// request header, the way MinIO and Storj do.
|
||||
pub fn reject_reserved_internal_headers(&self, enabled: bool) {
|
||||
lock(&self.backend.store).reject_reserved_internal_headers = enabled;
|
||||
}
|
||||
|
||||
/// Keys of every stored, non-delete object in the bucket, sorted.
|
||||
pub fn stored_keys(&self, bucket: &str) -> Vec<String> {
|
||||
let state = lock(&self.backend.store);
|
||||
let mut keys: Vec<String> = state
|
||||
.buckets
|
||||
.get(bucket)
|
||||
.map(|bucket| {
|
||||
bucket
|
||||
.objects
|
||||
.iter()
|
||||
.filter(|(_, versions)| versions.last().is_some_and(|version| !version.delete_marker))
|
||||
.map(|(key, _)| key.clone())
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
keys.sort();
|
||||
keys
|
||||
}
|
||||
|
||||
/// Body and user metadata (names as stored, without the `x-amz-meta-` prefix)
|
||||
/// of the latest non-delete version of an object.
|
||||
pub fn stored_object(&self, bucket: &str, key: &str) -> Option<(Bytes, HashMap<String, String>)> {
|
||||
lock(&self.backend.store)
|
||||
.buckets
|
||||
.get(bucket)
|
||||
.and_then(|bucket| bucket.objects.get(key))
|
||||
.and_then(|versions| versions.last())
|
||||
.filter(|version| !version.delete_marker)
|
||||
.map(|version| (version.body.clone(), version.metadata.clone().unwrap_or_default()))
|
||||
}
|
||||
|
||||
pub fn require_checksum_for_object_lock(&self, enabled: bool) {
|
||||
lock(&self.backend.store).require_checksum_for_object_lock = enabled;
|
||||
}
|
||||
@@ -1161,7 +1234,10 @@ impl S3Access for FaultAccess {
|
||||
let operation = operation_from_s3_name(context.s3_op().name());
|
||||
let journaled = JournaledHeaders::from_headers(context.headers());
|
||||
let aws_chunked = journaled.transport.aws_chunked;
|
||||
let fault = record_request(&self.control, operation, context.method().clone(), parsed, journaled);
|
||||
let (sequence, fault) = record_request(&self.control, operation, context.method().clone(), parsed, journaled);
|
||||
if let Some(slot) = context.extensions_mut().get::<RequestSequence>() {
|
||||
let _ = slot.0.set(sequence);
|
||||
}
|
||||
if let Some(status) = fault.as_ref().and_then(|fault| scripted_status(&fault.action)) {
|
||||
return Err(scripted_status_error(status));
|
||||
}
|
||||
@@ -1174,6 +1250,15 @@ impl S3Access for FaultAccess {
|
||||
"this target does not decode aws-chunked request bodies; send a plain signed payload with an exact Content-Length"
|
||||
));
|
||||
}
|
||||
if matches!(operation, Operation::PutObject | Operation::UploadPart)
|
||||
&& lock(&self.store).reject_reserved_internal_headers
|
||||
&& context
|
||||
.headers()
|
||||
.keys()
|
||||
.any(|name| name.as_str().starts_with("x-minio-internal-"))
|
||||
{
|
||||
return Err(s3s::s3_error!(InvalidArgument, "Your metadata headers are not supported."));
|
||||
}
|
||||
let prebody_permit = if operation == Operation::CompleteMultipartUpload {
|
||||
Some(
|
||||
timeout(
|
||||
@@ -1228,7 +1313,7 @@ fn record_request(
|
||||
method: Method,
|
||||
parsed: ParsedRequest,
|
||||
headers: JournaledHeaders,
|
||||
) -> Option<RequestFault> {
|
||||
) -> (u64, Option<RequestFault>) {
|
||||
let mut state = lock(control);
|
||||
let action = parsed
|
||||
.key
|
||||
@@ -1251,6 +1336,12 @@ fn record_request(
|
||||
upload_id: parsed.upload_id.map(bounded_journal_value),
|
||||
part_number: parsed.part_number,
|
||||
content_length: headers.content_length,
|
||||
source_version_id: headers.source_version_id,
|
||||
source_mtime: headers.source_mtime,
|
||||
source_etag: headers.source_etag,
|
||||
source_replication_request: headers.source_replication_request,
|
||||
journaled_at_unix_millis: unix_millis(),
|
||||
prepared_response: None,
|
||||
consumed_bytes: None,
|
||||
replication_timestamps: headers.replication_timestamps,
|
||||
proxy_headers: headers.proxy_headers,
|
||||
@@ -1261,7 +1352,14 @@ fn record_request(
|
||||
continuation_token: parsed.continuation_token.map(bounded_journal_value),
|
||||
fault: action.clone(),
|
||||
});
|
||||
action.map(|action| RequestFault { sequence, action })
|
||||
(sequence, action.map(|action| RequestFault { sequence, action }))
|
||||
}
|
||||
|
||||
fn unix_millis() -> Option<u64> {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.ok()
|
||||
.and_then(|elapsed| u64::try_from(elapsed.as_millis()).ok())
|
||||
}
|
||||
|
||||
fn bounded_journal_value(mut value: String) -> String {
|
||||
@@ -1269,14 +1367,31 @@ fn bounded_journal_value(mut value: String) -> String {
|
||||
while !value.is_char_boundary(end) {
|
||||
end -= 1;
|
||||
}
|
||||
value.truncate(end);
|
||||
value
|
||||
if value.capacity() > MAX_RETAINED_IDENTIFIER_BYTES {
|
||||
value[..end].to_owned()
|
||||
} else {
|
||||
value.truncate(end);
|
||||
value
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_request(request: Request<Incoming>, service: S3Service) -> Result<Response<Body>, BoxError> {
|
||||
let mut response = timeout(MAX_REQUEST_DURATION, call_s3(service, request.map(Body::from)))
|
||||
.await
|
||||
.map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "fake target request exceeded 65 seconds"))??;
|
||||
async fn handle_request(
|
||||
mut request: Request<Incoming>,
|
||||
service: S3Service,
|
||||
control: Arc<Mutex<ControlState>>,
|
||||
) -> Result<Response<Body>, BoxError> {
|
||||
let started = Instant::now();
|
||||
let sequence = RequestSequence::default();
|
||||
request.extensions_mut().insert(sequence.clone());
|
||||
let mut response = timeout(MAX_REQUEST_DURATION, async {
|
||||
let result = call_s3(service, request.map(Body::from)).await;
|
||||
if let (Ok(response), Some(sequence)) = (&result, sequence.0.get()) {
|
||||
update_prepared_response(&control, *sequence, response, started.elapsed());
|
||||
}
|
||||
result
|
||||
})
|
||||
.await
|
||||
.map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "fake target request exceeded 65 seconds"))??;
|
||||
if response.headers_mut().remove(DISCONNECT_HEADER).is_some() {
|
||||
return Err(io::Error::new(io::ErrorKind::ConnectionAborted, "fake target scripted disconnect").into());
|
||||
}
|
||||
@@ -1531,6 +1646,25 @@ fn update_consumed(control: &Mutex<ControlState>, sequence: u64, consumed: usize
|
||||
}
|
||||
}
|
||||
|
||||
fn update_prepared_response(control: &Mutex<ControlState>, sequence: u64, response: &Response<Body>, elapsed: Duration) {
|
||||
let prepared = PreparedResponse {
|
||||
status: response.status().as_u16(),
|
||||
version_id: header_value(response.headers(), &["x-amz-version-id"]).map(bounded_journal_value),
|
||||
etag: header_value(response.headers(), &[ETAG.as_str()]).map(bounded_journal_value),
|
||||
last_modified: header_value(response.headers(), &[LAST_MODIFIED.as_str()]).map(bounded_journal_value),
|
||||
content_length: response
|
||||
.headers()
|
||||
.get(CONTENT_LENGTH)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(|value| value.parse().ok()),
|
||||
prepared_at_unix_millis: unix_millis(),
|
||||
elapsed,
|
||||
};
|
||||
if let Some(record) = lock(control).requests.iter_mut().find(|record| record.sequence == sequence) {
|
||||
record.prepared_response = Some(prepared);
|
||||
}
|
||||
}
|
||||
|
||||
async fn apply_non_body_fault(fault: Option<&RequestFault>, control: &Mutex<ControlState>) -> S3Result<()> {
|
||||
match fault.map(|fault| &fault.action) {
|
||||
Some(action @ (FaultAction::Status(_) | FaultAction::ResponseStatus(_))) => {
|
||||
@@ -3230,10 +3364,38 @@ mod tests {
|
||||
let retry = client.clone();
|
||||
requests.spawn(async move { get_bytes(&retry, bucket, "held", None).await });
|
||||
timeout(Duration::from_secs(2), gate.entered.wait_for(|count| *count == 2)).await??;
|
||||
let held_retry = target
|
||||
.requests()
|
||||
.into_iter()
|
||||
.rev()
|
||||
.find(|record| record.operation == Operation::GetObject && record.key.as_deref() == Some("held"))
|
||||
.expect("the held retry must be journaled");
|
||||
assert_eq!(held_retry.prepared_response, None);
|
||||
assert_eq!(
|
||||
timeout(Duration::from_secs(2), get_bytes(&client, bucket, "unrelated", None)).await??,
|
||||
Bytes::from_static(b"payload")
|
||||
);
|
||||
let unrelated = target
|
||||
.requests()
|
||||
.into_iter()
|
||||
.find(|record| record.operation == Operation::GetObject && record.key.as_deref() == Some("unrelated"))
|
||||
.expect("the unrelated GET must be journaled");
|
||||
let unrelated_response = unrelated.prepared_response.expect("the unrelated GET prepared a response");
|
||||
assert_eq!(unrelated_response.status, 200);
|
||||
assert_eq!(
|
||||
unrelated_response.version_id.as_deref(),
|
||||
Some(target.stored_versions(bucket, "unrelated")[0].0.as_str())
|
||||
);
|
||||
assert!(held_retry.sequence < unrelated.sequence);
|
||||
assert_eq!(
|
||||
target
|
||||
.requests()
|
||||
.into_iter()
|
||||
.find(|record| record.sequence == held_retry.sequence)
|
||||
.expect("the held retry remains journaled")
|
||||
.prepared_response,
|
||||
None
|
||||
);
|
||||
assert!(requests.try_join_next().is_none(), "the retry must remain behind the gate");
|
||||
drop(gate);
|
||||
assert_eq!(
|
||||
@@ -3242,6 +3404,18 @@ mod tests {
|
||||
.expect("retried GET task")??,
|
||||
Bytes::from_static(b"payload")
|
||||
);
|
||||
let held_response = target
|
||||
.requests()
|
||||
.into_iter()
|
||||
.find(|record| record.sequence == held_retry.sequence)
|
||||
.and_then(|record| record.prepared_response)
|
||||
.expect("the released retry prepared its own response");
|
||||
assert_eq!(held_response.status, 200);
|
||||
assert_eq!(
|
||||
held_response.version_id.as_deref(),
|
||||
Some(target.stored_versions(bucket, "held")[0].0.as_str())
|
||||
);
|
||||
assert_ne!(held_response.version_id, unrelated_response.version_id);
|
||||
assert_eq!(get_bytes(&client, bucket, "held", None).await?, Bytes::from_static(b"payload"));
|
||||
assert_eq!(target.count_requests(Operation::GetObject, "held"), 3);
|
||||
Ok(())
|
||||
@@ -3586,6 +3760,23 @@ mod tests {
|
||||
.await?;
|
||||
assert_eq!(first_head.content_length(), Some(5));
|
||||
assert_eq!(first_head.version_id(), first.version_id.as_deref());
|
||||
let journaled_head = target
|
||||
.requests()
|
||||
.into_iter()
|
||||
.rev()
|
||||
.find(|record| record.operation == Operation::HeadObject)
|
||||
.expect("the version HEAD must be journaled");
|
||||
assert_eq!(journaled_head.version_id, first.version_id);
|
||||
assert!(journaled_head.journaled_at_unix_millis.is_some());
|
||||
let prepared_head = journaled_head
|
||||
.prepared_response
|
||||
.expect("the version HEAD prepared a response");
|
||||
assert_eq!(prepared_head.status, 200);
|
||||
assert_eq!(prepared_head.version_id.as_deref(), first_head.version_id());
|
||||
assert_eq!(prepared_head.etag.as_deref(), first_head.e_tag());
|
||||
assert_eq!(prepared_head.content_length, Some(5));
|
||||
assert!(prepared_head.last_modified.is_some());
|
||||
assert!(prepared_head.prepared_at_unix_millis.is_some());
|
||||
assert_eq!(
|
||||
get_bytes(&client, "target-bucket", "nested/key", None).await?,
|
||||
Bytes::from_static(b"second")
|
||||
@@ -3611,6 +3802,20 @@ mod tests {
|
||||
.await
|
||||
.expect_err("deleted version must not remain readable");
|
||||
assert_sdk_error!(deleted, 404, "NoSuchVersion");
|
||||
let deleted_record = target
|
||||
.requests()
|
||||
.into_iter()
|
||||
.rev()
|
||||
.find(|record| record.operation == Operation::GetObject)
|
||||
.expect("the deleted version GET must be journaled");
|
||||
assert_eq!(deleted_record.version_id, first.version_id);
|
||||
assert_eq!(
|
||||
deleted_record
|
||||
.prepared_response
|
||||
.expect("the error response was prepared")
|
||||
.status,
|
||||
404
|
||||
);
|
||||
let marker = client
|
||||
.delete_object()
|
||||
.bucket("target-bucket")
|
||||
@@ -4042,7 +4247,21 @@ mod tests {
|
||||
.await
|
||||
.expect_err("wrong part ETag must fail without consuming the upload");
|
||||
assert_sdk_error!(invalid_complete, 400, "InvalidPart");
|
||||
client
|
||||
let rejected_complete = target
|
||||
.requests()
|
||||
.into_iter()
|
||||
.rev()
|
||||
.find(|record| record.operation == Operation::CompleteMultipartUpload)
|
||||
.expect("the rejected completion must be journaled");
|
||||
assert_eq!(rejected_complete.upload_id.as_deref(), Some(upload_id.as_str()));
|
||||
assert_eq!(
|
||||
rejected_complete
|
||||
.prepared_response
|
||||
.expect("the rejection was prepared")
|
||||
.status,
|
||||
400
|
||||
);
|
||||
let successful_complete = client
|
||||
.complete_multipart_upload()
|
||||
.bucket("target-bucket")
|
||||
.key("multipart")
|
||||
@@ -4050,6 +4269,18 @@ mod tests {
|
||||
.multipart_upload(CompletedMultipartUpload::builder().set_parts(Some(parts)).build())
|
||||
.send()
|
||||
.await?;
|
||||
let completed_record = target
|
||||
.requests()
|
||||
.into_iter()
|
||||
.rev()
|
||||
.find(|record| record.operation == Operation::CompleteMultipartUpload)
|
||||
.expect("the successful completion must be journaled");
|
||||
assert_eq!(completed_record.upload_id.as_deref(), Some(upload_id.as_str()));
|
||||
let prepared_complete = completed_record
|
||||
.prepared_response
|
||||
.expect("the completion response was prepared");
|
||||
assert_eq!(prepared_complete.status, 200);
|
||||
assert_eq!(prepared_complete.version_id.as_deref(), successful_complete.version_id());
|
||||
let completed = get_bytes(&client, "target-bucket", "multipart", None).await?;
|
||||
assert_eq!(completed.len(), MIN_MULTIPART_PART_BYTES + 5);
|
||||
assert_eq!(&completed[..MIN_MULTIPART_PART_BYTES], first_body.as_ref());
|
||||
@@ -4430,6 +4661,32 @@ mod tests {
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(completed.version_id(), Some(completed_version.as_str()));
|
||||
let receipt_records = target.requests();
|
||||
let receipt_create = receipt_records
|
||||
.iter()
|
||||
.find(|record| {
|
||||
record.operation == Operation::CreateMultipartUpload && record.key.as_deref() == Some("multipart-receipt-time")
|
||||
})
|
||||
.expect("the source version must be retained on MPU creation");
|
||||
assert_eq!(receipt_create.source_version_id.as_deref(), Some(completed_version.as_str()));
|
||||
let receipt_complete = receipt_records
|
||||
.iter()
|
||||
.find(|record| {
|
||||
record.operation == Operation::CompleteMultipartUpload
|
||||
&& record.upload_id.as_deref() == Some(receipt_upload.as_str())
|
||||
})
|
||||
.expect("the source mtime must be retained on completion");
|
||||
assert_eq!(receipt_complete.source_mtime.as_deref(), Some("1970-01-01T00:00:00Z"));
|
||||
assert!(receipt_complete.source_replication_request);
|
||||
assert_eq!(
|
||||
receipt_complete
|
||||
.prepared_response
|
||||
.as_ref()
|
||||
.expect("the receipt-time response was prepared")
|
||||
.version_id
|
||||
.as_deref(),
|
||||
completed.version_id()
|
||||
);
|
||||
assert_eq!(
|
||||
get_bytes(&client, "target-bucket", "multipart-receipt-time", None).await?,
|
||||
Bytes::from_static(b"completed"),
|
||||
@@ -4648,6 +4905,23 @@ mod tests {
|
||||
.expect_err("disconnected request must not store an object");
|
||||
assert_sdk_error!(disconnected, 404, "NotFound");
|
||||
|
||||
target.inject_for_key(Operation::PutObject, "lost-response", FaultAction::DisconnectAfterResponse, 1);
|
||||
client
|
||||
.put_object()
|
||||
.bucket("target-bucket")
|
||||
.key("lost-response")
|
||||
.body(ByteStream::from_static(b"stored"))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a prepared response does not prove client delivery");
|
||||
let lost_response_head = client
|
||||
.head_object()
|
||||
.bucket("target-bucket")
|
||||
.key("lost-response")
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(lost_response_head.content_length(), Some(6));
|
||||
|
||||
assert_eq!(
|
||||
parse_request(&Method::PUT, &"/target-bucket/key?uploadId=upload&partNumber=bad".parse::<Uri>()?).operation,
|
||||
Operation::Unknown
|
||||
@@ -4661,6 +4935,28 @@ mod tests {
|
||||
.count(),
|
||||
4
|
||||
);
|
||||
assert!(
|
||||
requests
|
||||
.iter()
|
||||
.filter(|request| request.fault == Some(FaultAction::Status(StatusCode::SERVICE_UNAVAILABLE)))
|
||||
.all(|request| {
|
||||
request
|
||||
.prepared_response
|
||||
.as_ref()
|
||||
.is_some_and(|response| response.status == 503)
|
||||
})
|
||||
);
|
||||
let lost_response = requests
|
||||
.iter()
|
||||
.find(|request| request.operation == Operation::PutObject && request.key.as_deref() == Some("lost-response"))
|
||||
.expect("the disconnected response must be journaled");
|
||||
assert_eq!(lost_response.fault, Some(FaultAction::DisconnectAfterResponse));
|
||||
let prepared_lost_response = lost_response
|
||||
.prepared_response
|
||||
.as_ref()
|
||||
.expect("the target prepared the successful response");
|
||||
assert_eq!(prepared_lost_response.status, 200);
|
||||
assert_eq!(prepared_lost_response.version_id.as_deref(), lost_response_head.version_id());
|
||||
assert!(
|
||||
requests
|
||||
.iter()
|
||||
@@ -4851,8 +5147,44 @@ mod tests {
|
||||
assert_eq!(records.len(), MAX_REQUEST_RECORDS);
|
||||
assert_eq!(records.front().map(|record| record.sequence), Some(2));
|
||||
assert_eq!(records.back().map(|record| record.sequence), Some((MAX_REQUEST_RECORDS + 1) as u64));
|
||||
let mut oversized_allocation = String::with_capacity(MAX_RETAINED_IDENTIFIER_BYTES * 16);
|
||||
oversized_allocation.push_str("short");
|
||||
let short_value = bounded_journal_value(oversized_allocation);
|
||||
assert_eq!(short_value, "short");
|
||||
assert!(short_value.capacity() <= MAX_RETAINED_IDENTIFIER_BYTES);
|
||||
let mut response = Response::new(Body::empty());
|
||||
for name in ["x-amz-version-id", "etag", "last-modified"] {
|
||||
response.headers_mut().insert(
|
||||
name,
|
||||
HeaderValue::from_str(&"r".repeat(MAX_RETAINED_IDENTIFIER_BYTES + 1)).expect("header"),
|
||||
);
|
||||
}
|
||||
response.headers_mut().insert(CONTENT_LENGTH, HeaderValue::from_static("17"));
|
||||
response
|
||||
.headers_mut()
|
||||
.insert("authorization", HeaderValue::from_static("unrecorded-response-authorization"));
|
||||
response.headers_mut().insert(
|
||||
"x-amz-server-side-encryption-customer-key",
|
||||
HeaderValue::from_static("unrecorded-response-customer-key"),
|
||||
);
|
||||
update_prepared_response(&control, 1, &response, Duration::ZERO);
|
||||
update_prepared_response(&control, u64::MAX, &response, Duration::ZERO);
|
||||
assert_eq!(lock(&control).requests, records, "misses and evicted records must not be resurrected");
|
||||
let bounded_control = Mutex::new(ControlState::default());
|
||||
let utf8_boundary = format!("{}é", "a".repeat(MAX_RETAINED_IDENTIFIER_BYTES - 1));
|
||||
let mut source_headers = HeaderMap::new();
|
||||
for name in ["x-rustfs-source-version-id", "x-minio-source-mtime", "x-rustfs-source-etag"] {
|
||||
source_headers.insert(
|
||||
name,
|
||||
HeaderValue::from_str(&"s".repeat(MAX_RETAINED_IDENTIFIER_BYTES + 1)).expect("header"),
|
||||
);
|
||||
}
|
||||
source_headers.insert("x-minio-source-replication-request", HeaderValue::from_static("true"));
|
||||
source_headers.insert("authorization", HeaderValue::from_static("unrecorded-request-authorization"));
|
||||
source_headers.insert(
|
||||
"x-amz-server-side-encryption-customer-key",
|
||||
HeaderValue::from_static("unrecorded-request-customer-key"),
|
||||
);
|
||||
record_request(
|
||||
&bounded_control,
|
||||
Operation::GetObject,
|
||||
@@ -4865,8 +5197,9 @@ mod tests {
|
||||
upload_id: Some("u".repeat(MAX_RETAINED_IDENTIFIER_BYTES + 1)),
|
||||
..Default::default()
|
||||
},
|
||||
JournaledHeaders::default(),
|
||||
JournaledHeaders::from_headers(&source_headers),
|
||||
);
|
||||
update_prepared_response(&bounded_control, 1, &response, Duration::from_millis(7));
|
||||
{
|
||||
let bounded_records = lock(&bounded_control);
|
||||
let bounded = &bounded_records.requests[0];
|
||||
@@ -4874,7 +5207,39 @@ mod tests {
|
||||
assert_eq!(bounded.key.as_ref().map(String::len), Some(MAX_RETAINED_IDENTIFIER_BYTES - 1));
|
||||
assert_eq!(bounded.version_id.as_ref().map(String::len), Some(MAX_RETAINED_IDENTIFIER_BYTES));
|
||||
assert_eq!(bounded.upload_id.as_ref().map(String::len), Some(MAX_RETAINED_IDENTIFIER_BYTES));
|
||||
assert_eq!(bounded.source_version_id.as_ref().map(String::len), Some(MAX_RETAINED_IDENTIFIER_BYTES));
|
||||
assert_eq!(bounded.source_mtime.as_ref().map(String::len), Some(MAX_RETAINED_IDENTIFIER_BYTES));
|
||||
assert_eq!(bounded.source_etag.as_ref().map(String::len), Some(MAX_RETAINED_IDENTIFIER_BYTES));
|
||||
for value in [&bounded.source_version_id, &bounded.source_mtime, &bounded.source_etag]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
{
|
||||
assert!(value.capacity() <= MAX_RETAINED_IDENTIFIER_BYTES);
|
||||
}
|
||||
assert!(bounded.source_replication_request);
|
||||
let prepared = bounded
|
||||
.prepared_response
|
||||
.as_ref()
|
||||
.expect("the retained record receives its response");
|
||||
assert_eq!(prepared.version_id.as_ref().map(String::len), Some(MAX_RETAINED_IDENTIFIER_BYTES));
|
||||
assert_eq!(prepared.etag.as_ref().map(String::len), Some(MAX_RETAINED_IDENTIFIER_BYTES));
|
||||
assert_eq!(prepared.last_modified.as_ref().map(String::len), Some(MAX_RETAINED_IDENTIFIER_BYTES));
|
||||
for value in [&prepared.version_id, &prepared.etag, &prepared.last_modified]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
{
|
||||
assert!(value.capacity() <= MAX_RETAINED_IDENTIFIER_BYTES);
|
||||
}
|
||||
assert_eq!(prepared.content_length, Some(17));
|
||||
assert_eq!(prepared.elapsed, Duration::from_millis(7));
|
||||
assert!(
|
||||
!format!("{bounded:?}").contains("unrecorded-"),
|
||||
"authorization and customer keys must not enter the journal"
|
||||
);
|
||||
}
|
||||
lock(&bounded_control).requests.clear();
|
||||
update_prepared_response(&bounded_control, 1, &response, Duration::ZERO);
|
||||
assert!(lock(&bounded_control).requests.is_empty(), "a drained record must not be resurrected");
|
||||
let target = FakeS3Target::start().await.expect("target");
|
||||
target.create_bucket("b".repeat(63));
|
||||
let empty_bucket = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
|
||||
|
||||
@@ -61,6 +61,8 @@
|
||||
|
||||
use std::io;
|
||||
use std::net::{Ipv4Addr, SocketAddr};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
@@ -108,19 +110,33 @@ pub struct FaultProxy {
|
||||
mode_tx: watch::Sender<FaultMode>,
|
||||
shutdown_tx: watch::Sender<bool>,
|
||||
accept_task: JoinHandle<()>,
|
||||
traffic: Arc<ProxyTraffic>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct ProxyTraffic {
|
||||
forwarded_bytes: AtomicUsize,
|
||||
dropped_bytes: AtomicUsize,
|
||||
}
|
||||
|
||||
impl FaultProxy {
|
||||
/// Bind a listener on `127.0.0.1:0` and start forwarding accepted
|
||||
/// connections to `target`. Starts in [`FaultMode::Pass`].
|
||||
pub async fn start(target: SocketAddr) -> io::Result<Self> {
|
||||
let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?;
|
||||
Self::start_on(SocketAddr::from((Ipv4Addr::LOCALHOST, 0)), target).await
|
||||
}
|
||||
|
||||
/// Bind a specific address so advertised volume endpoints can retain the
|
||||
/// server's port and authenticated RPC identity on another loopback IP.
|
||||
pub async fn start_on(listen: SocketAddr, target: SocketAddr) -> io::Result<Self> {
|
||||
let listener = TcpListener::bind(listen).await?;
|
||||
let listen_addr = listener.local_addr()?;
|
||||
|
||||
let (mode_tx, mode_rx) = watch::channel(FaultMode::Pass);
|
||||
let (shutdown_tx, shutdown_rx) = watch::channel(false);
|
||||
|
||||
let accept_task = tokio::spawn(accept_loop(listener, target, mode_rx, shutdown_rx));
|
||||
let traffic = Arc::new(ProxyTraffic::default());
|
||||
let accept_task = tokio::spawn(accept_loop(listener, target, mode_rx, shutdown_rx, traffic.clone()));
|
||||
|
||||
Ok(Self {
|
||||
listen_addr,
|
||||
@@ -128,9 +144,20 @@ impl FaultProxy {
|
||||
mode_tx,
|
||||
shutdown_tx,
|
||||
accept_task,
|
||||
traffic,
|
||||
})
|
||||
}
|
||||
|
||||
/// Bytes actually forwarded in either direction, excluding dropped traffic.
|
||||
pub fn forwarded_bytes(&self) -> usize {
|
||||
self.traffic.forwarded_bytes.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Bytes actually discarded by a blackhole or one-way partition.
|
||||
pub fn dropped_bytes(&self) -> usize {
|
||||
self.traffic.dropped_bytes.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Local address the proxy is listening on. Point clients here instead of
|
||||
/// the real target to route their traffic through the proxy.
|
||||
pub fn local_addr(&self) -> SocketAddr {
|
||||
@@ -171,6 +198,7 @@ async fn accept_loop(
|
||||
target: SocketAddr,
|
||||
mode_rx: watch::Receiver<FaultMode>,
|
||||
mut shutdown_rx: watch::Receiver<bool>,
|
||||
traffic: Arc<ProxyTraffic>,
|
||||
) {
|
||||
loop {
|
||||
tokio::select! {
|
||||
@@ -188,6 +216,7 @@ async fn accept_loop(
|
||||
target,
|
||||
mode_rx.clone(),
|
||||
shutdown_rx.clone(),
|
||||
traffic.clone(),
|
||||
));
|
||||
}
|
||||
// A transient accept error should not tear the proxy down.
|
||||
@@ -205,6 +234,7 @@ async fn handle_connection(
|
||||
target: SocketAddr,
|
||||
mode_rx: watch::Receiver<FaultMode>,
|
||||
shutdown_rx: watch::Receiver<bool>,
|
||||
traffic: Arc<ProxyTraffic>,
|
||||
) {
|
||||
let server = match TcpStream::connect(target).await {
|
||||
Ok(s) => s,
|
||||
@@ -221,8 +251,9 @@ async fn handle_connection(
|
||||
server_wr,
|
||||
mode_rx.clone(),
|
||||
shutdown_rx.clone(),
|
||||
traffic.clone(),
|
||||
));
|
||||
let down = tokio::spawn(pump(Direction::ServerToClient, server_rd, client_wr, mode_rx, shutdown_rx));
|
||||
let down = tokio::spawn(pump(Direction::ServerToClient, server_rd, client_wr, mode_rx, shutdown_rx, traffic));
|
||||
|
||||
let _ = up.await;
|
||||
let _ = down.await;
|
||||
@@ -237,6 +268,7 @@ async fn pump<R, W>(
|
||||
mut to: W,
|
||||
mut mode_rx: watch::Receiver<FaultMode>,
|
||||
mut shutdown_rx: watch::Receiver<bool>,
|
||||
traffic: Arc<ProxyTraffic>,
|
||||
) where
|
||||
R: AsyncReadExt + Unpin,
|
||||
W: AsyncWriteExt + Unpin,
|
||||
@@ -257,15 +289,22 @@ async fn pump<R, W>(
|
||||
let mode = *mode_rx.borrow_and_update();
|
||||
match mode {
|
||||
// Whole connection is a black hole: drain and drop.
|
||||
FaultMode::Blackhole => continue,
|
||||
FaultMode::Blackhole => {
|
||||
traffic.dropped_bytes.fetch_add(n, Ordering::Relaxed);
|
||||
continue;
|
||||
}
|
||||
// This direction is partitioned off: drop its bytes; the peer keeps
|
||||
// reading nothing while the other direction may still flow.
|
||||
FaultMode::Partition(blocked) if blocked == dir => continue,
|
||||
FaultMode::Partition(blocked) if blocked == dir => {
|
||||
traffic.dropped_bytes.fetch_add(n, Ordering::Relaxed);
|
||||
continue;
|
||||
}
|
||||
FaultMode::Latency(delay) => {
|
||||
tokio::time::sleep(delay).await;
|
||||
if to.write_all(&buf[..n]).await.is_err() {
|
||||
break;
|
||||
}
|
||||
traffic.forwarded_bytes.fetch_add(n, Ordering::Relaxed);
|
||||
if to.flush().await.is_err() {
|
||||
break;
|
||||
}
|
||||
@@ -274,6 +313,7 @@ async fn pump<R, W>(
|
||||
if to.write_all(&buf[..n]).await.is_err() {
|
||||
break;
|
||||
}
|
||||
traffic.forwarded_bytes.fetch_add(n, Ordering::Relaxed);
|
||||
if to.flush().await.is_err() {
|
||||
break;
|
||||
}
|
||||
@@ -285,7 +325,6 @@ async fn pump<R, W>(
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
@@ -375,6 +414,8 @@ mod tests {
|
||||
|
||||
assert_eq!(got, payload, "pass mode must forward bytes unchanged");
|
||||
assert_eq!(echo.received().await, payload, "server must have received the bytes");
|
||||
assert_eq!(proxy.forwarded_bytes(), payload.len() * 2);
|
||||
assert_eq!(proxy.dropped_bytes(), 0);
|
||||
|
||||
proxy.shutdown().await;
|
||||
}
|
||||
@@ -414,6 +455,8 @@ mod tests {
|
||||
|
||||
assert!(got.is_empty(), "blackhole mode must not return any bytes, got {got:?}");
|
||||
assert!(echo.received().await.is_empty(), "blackhole must not forward to the server");
|
||||
assert_eq!(proxy.forwarded_bytes(), 0);
|
||||
assert_eq!(proxy.dropped_bytes(), b"into-the-void".len());
|
||||
|
||||
// Proxy is still alive and steerable after a blackholed connection.
|
||||
proxy.set_mode(FaultMode::Pass);
|
||||
|
||||
@@ -562,17 +562,23 @@ mod tests {
|
||||
create_bucket(&client, bucket).await.expect("Failed to create bucket");
|
||||
|
||||
let object_count = 1002;
|
||||
for i in 0..object_count {
|
||||
let key = format!("object{:04}.txt", i);
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(&key)
|
||||
.body(ByteStream::from_static(b"test content"))
|
||||
.send()
|
||||
.await
|
||||
.expect("Failed to put object");
|
||||
}
|
||||
// Finish every durable PUT before listing, with the same bounded fanout as the delimiter fixtures.
|
||||
stream::iter(0..object_count)
|
||||
.for_each_concurrent(16, |i| {
|
||||
let client = &client;
|
||||
async move {
|
||||
let key = format!("object{i:04}.txt");
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(&key)
|
||||
.body(ByteStream::from_static(b"test content"))
|
||||
.send()
|
||||
.await
|
||||
.unwrap_or_else(|err| panic!("Failed to put fixture object {key}: {err}"));
|
||||
}
|
||||
})
|
||||
.await;
|
||||
|
||||
eprintln!("Seeded {object_count} objects in {bucket}; starting ListObjectsV2 pagination");
|
||||
let deadline = Instant::now() + PAGINATION_TIMEOUT;
|
||||
|
||||
@@ -300,17 +300,28 @@ async fn test_degraded_cluster_read_quorum_follows_erasure_layout() -> TestResul
|
||||
}
|
||||
|
||||
cluster.stop_node(read_quorum - 1)?;
|
||||
let lock_read_quorum = node_count - node_count / 2;
|
||||
// EC:2 on six nodes still has three shared lock votes after losing
|
||||
// its fourth data shard. Preserve the distinct data-quorum error.
|
||||
let expected_code = if read_quorum - 1 < lock_read_quorum {
|
||||
"ServiceUnavailable"
|
||||
} else {
|
||||
"SlowDownRead"
|
||||
};
|
||||
for client in clients.iter().take(read_quorum - 1) {
|
||||
match client.get_object().bucket(&bucket).key("below-quorum").send().await {
|
||||
Ok(response) => assert!(
|
||||
response.body.collect().await.is_err(),
|
||||
"fewer than {read_quorum} valid fragments must not reconstruct an uncached object"
|
||||
),
|
||||
Err(error) => assert_eq!(
|
||||
error.as_service_error().and_then(|error| error.meta().code()),
|
||||
Some("ServiceUnavailable"),
|
||||
"a quorum loss must not be mistaken for a missing object"
|
||||
),
|
||||
Err(error) => {
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(503));
|
||||
assert_eq!(
|
||||
error.as_service_error().and_then(|error| error.meta().code()),
|
||||
Some(expected_code),
|
||||
"a quorum loss must not be mistaken for a missing object"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -425,6 +436,60 @@ async fn test_concurrent_cluster_overwrites_do_not_fail_namespace_lock_quorum()
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn unavailable_readiness_drive_indices(payload: &serde_json::Value) -> Vec<usize> {
|
||||
let mut indices = payload["details"]["storage"]["unavailableDrives"]
|
||||
.as_array()
|
||||
.expect("node readiness must report unavailable drives")
|
||||
.iter()
|
||||
.map(|drive| {
|
||||
usize::try_from(drive["diskIndex"].as_u64().expect("drive index must be unsigned"))
|
||||
.expect("drive index must fit usize")
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
indices.sort_unstable();
|
||||
indices
|
||||
}
|
||||
|
||||
fn assert_node_readiness_contract(payload: &serde_json::Value, survivors: usize) {
|
||||
let read_ready = survivors >= 2;
|
||||
let write_ready = survivors >= 3;
|
||||
let storage = &payload["details"]["storage"];
|
||||
assert_eq!(payload["ready"], read_ready);
|
||||
assert_eq!(storage["ready"], read_ready);
|
||||
assert_eq!(storage["readQuorum"], read_ready);
|
||||
assert_eq!(storage["writeQuorum"], write_ready);
|
||||
assert_eq!(storage["readinessScope"], "read_quorum");
|
||||
assert_eq!(storage["source"], "local_runtime");
|
||||
assert_eq!(storage["status"], if read_ready { "connected" } else { "disconnected" });
|
||||
assert_eq!(payload["details"]["poolMetadata"]["ready"], true);
|
||||
assert_eq!(payload["details"]["iam"]["ready"], true);
|
||||
assert_eq!(payload["details"]["lock"]["ready"], read_ready);
|
||||
assert_eq!(unavailable_readiness_drive_indices(payload), (survivors..4).collect::<Vec<_>>());
|
||||
for drive in storage["unavailableDrives"].as_array().expect("unavailable drives") {
|
||||
assert_eq!(drive["poolIndex"], 0);
|
||||
assert_eq!(drive["setIndex"], 0);
|
||||
let state = drive["runtimeState"].as_str().expect("runtime state must be reported");
|
||||
let host_online = drive["hostOnline"].as_bool().expect("host reachability must be reported");
|
||||
assert!(
|
||||
matches!(state, "online" | "suspect" | "offline" | "returning"),
|
||||
"unknown runtime state: {state}"
|
||||
);
|
||||
assert!(
|
||||
state != "online" || !host_online,
|
||||
"a reachable online drive cannot be excluded from quorum"
|
||||
);
|
||||
}
|
||||
if !read_ready {
|
||||
assert!(
|
||||
payload["degradedReasons"]
|
||||
.as_array()
|
||||
.expect("degraded reasons")
|
||||
.iter()
|
||||
.any(|reason| reason == "storage_and_lock_unavailable")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async fn assert_node_readiness_tracks_quorum(cluster: &mut RustFSTestClusterEnvironment) -> TestResult {
|
||||
let clients: Vec<_> = cluster
|
||||
.create_all_clients()?
|
||||
@@ -483,27 +548,14 @@ async fn assert_node_readiness_tracks_quorum(cluster: &mut RustFSTestClusterEnvi
|
||||
&& payload["details"]["poolMetadata"]["ready"] == true
|
||||
&& payload["details"]["iam"]["ready"] == true
|
||||
&& payload["details"]["lock"]["ready"] == read_quorum
|
||||
&& unavailable_readiness_drive_indices(&payload) == (survivors..4).collect::<Vec<_>>()
|
||||
{
|
||||
break payload;
|
||||
}
|
||||
assert!(Instant::now() < deadline, "node {idx}, survivors={survivors}: HTTP {status}, {payload}");
|
||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
||||
};
|
||||
assert_eq!(payload["details"]["storage"]["readinessScope"], "read_quorum");
|
||||
assert_eq!(payload["details"]["storage"]["source"], "local_runtime");
|
||||
assert_eq!(
|
||||
payload["details"]["storage"]["status"],
|
||||
if node_ready { "connected" } else { "disconnected" }
|
||||
);
|
||||
if !node_ready {
|
||||
assert!(
|
||||
payload["degradedReasons"]
|
||||
.as_array()
|
||||
.expect("degraded reasons")
|
||||
.iter()
|
||||
.any(|reason| reason == "storage_and_lock_unavailable")
|
||||
);
|
||||
}
|
||||
assert_node_readiness_contract(&payload, survivors);
|
||||
|
||||
for path in ["/health/ready", "/minio/health/ready"] {
|
||||
let head = http.head(format!("{url}{path}")).send().await?;
|
||||
@@ -512,7 +564,9 @@ async fn assert_node_readiness_tracks_quorum(cluster: &mut RustFSTestClusterEnvi
|
||||
let response = http.get(format!("{url}{path}")).send().await?;
|
||||
assert_eq!(response.status().as_u16(), node_status);
|
||||
let body: serde_json::Value = response.json().await?;
|
||||
assert_eq!(body["details"]["storage"], payload["details"]["storage"]);
|
||||
// Each response samples live diagnostics; suspect -> offline
|
||||
// must not change the quorum contract or excluded topology.
|
||||
assert_node_readiness_contract(&body, survivors);
|
||||
assert_eq!(body["details"]["poolMetadata"], payload["details"]["poolMetadata"]);
|
||||
}
|
||||
let live = http.get(format!("{url}/health/live")).send().await?;
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::disk::{RUSTFS_META_BUCKET, VolumeInfo, WalkDirOptions};
|
||||
pub(crate) use rustfs_ecstore::api::disk::{Endpoint, RUSTFS_META_BUCKET, VolumeInfo, WalkDirOptions};
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{AuthenticatedChannel, TonicInterceptor, node_service_time_out_client_no_auth};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{
|
||||
|
||||
@@ -165,4 +165,210 @@ mod tests {
|
||||
info!("RT-13c PASS: scanner/config persists across restart");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// RT-14: a transition PUT must not carry reserved `x-minio-internal-*` headers.
|
||||
///
|
||||
/// Regression pattern: transition forwarded the object's internal metadata as raw
|
||||
/// request headers, so MinIO-derived remotes (Storj) refused every transition with
|
||||
/// `400 InvalidArgument` while `tier add` still succeeded (rustfs#8362). The fake
|
||||
/// target refuses any PutObject carrying such a header, like MinIO and Storj do.
|
||||
#[tokio::test]
|
||||
async fn test_transition_put_omits_reserved_internal_headers() -> TestResult {
|
||||
use crate::fake_s3_target::{BucketMode, FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target, Operation};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
BucketLifecycleConfiguration, ExpirationStatus, LifecycleRule, LifecycleRuleFilter, Transition,
|
||||
TransitionStorageClass,
|
||||
};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
init_logging();
|
||||
info!("RT-14: transition PUT omits reserved internal headers");
|
||||
|
||||
const TIER: &str = "RT14COLD";
|
||||
const TARGET_BUCKET: &str = "rt14-cold";
|
||||
const SOURCE_BUCKET: &str = "rt14-hot";
|
||||
|
||||
let target = FakeS3Target::start().await.expect("start fake target");
|
||||
target.create_bucket_with_mode(TARGET_BUCKET, BucketMode::Unversioned);
|
||||
target.reject_reserved_internal_headers(true);
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
|
||||
env.start_rustfs_server_with_env(
|
||||
vec![],
|
||||
&[
|
||||
("RUSTFS_CONSOLE_ENABLE", "false"),
|
||||
("RUSTFS_TIER_RUSTFS_ALLOW_LOOPBACK_ENDPOINT", "true"),
|
||||
],
|
||||
)
|
||||
.await
|
||||
.expect("start RustFS");
|
||||
|
||||
let tier_body = serde_json::json!({
|
||||
"type": "rustfs",
|
||||
"rustfs": {
|
||||
"name": TIER,
|
||||
"endpoint": target.endpoint(),
|
||||
"accessKey": FAKE_ACCESS_KEY,
|
||||
"secretKey": FAKE_SECRET_KEY,
|
||||
"bucket": TARGET_BUCKET,
|
||||
"prefix": "cold/",
|
||||
"region": "us-east-1",
|
||||
"storageClass": ""
|
||||
}
|
||||
})
|
||||
.to_string();
|
||||
let deadline = Instant::now() + Duration::from_secs(30);
|
||||
loop {
|
||||
match admin_ok(&env, http::Method::PUT, "/rustfs/admin/v3/tier", Some(tier_body.clone())).await {
|
||||
Ok(_) => break,
|
||||
Err(err) if Instant::now() < deadline => {
|
||||
info!(" add tier not ready yet: {err}");
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
Err(err) => panic!("RT-14 FAIL: add tier: {err}"),
|
||||
}
|
||||
}
|
||||
|
||||
let client = env.create_s3_client();
|
||||
client
|
||||
.create_bucket()
|
||||
.bucket(SOURCE_BUCKET)
|
||||
.send()
|
||||
.await
|
||||
.expect("create bucket");
|
||||
|
||||
let objects: Vec<(&str, Vec<u8>)> = vec![
|
||||
("small.bin", (0..1024u32).map(|i| (i % 251) as u8).collect()),
|
||||
("medium.bin", (0..200_000u32).map(|i| (i.wrapping_mul(31) % 253) as u8).collect()),
|
||||
];
|
||||
for (key, body) in &objects {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(SOURCE_BUCKET)
|
||||
.key(*key)
|
||||
.body(ByteStream::from(body.clone()))
|
||||
.send()
|
||||
.await
|
||||
.expect("put source object");
|
||||
}
|
||||
|
||||
let rule = LifecycleRule::builder()
|
||||
.id("to-cold")
|
||||
.status(ExpirationStatus::Enabled)
|
||||
.filter(LifecycleRuleFilter::builder().prefix("").build())
|
||||
.transitions(
|
||||
Transition::builder()
|
||||
.days(0)
|
||||
.storage_class(TransitionStorageClass::from(TIER))
|
||||
.build(),
|
||||
)
|
||||
.build()
|
||||
.expect("build rule");
|
||||
client
|
||||
.put_bucket_lifecycle_configuration()
|
||||
.bucket(SOURCE_BUCKET)
|
||||
.lifecycle_configuration(
|
||||
BucketLifecycleConfiguration::builder()
|
||||
.rules(rule)
|
||||
.build()
|
||||
.expect("build config"),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.expect("put lifecycle");
|
||||
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::POST,
|
||||
&format!("/rustfs/admin/v3/ilm/transition/run?bucket={SOURCE_BUCKET}&prefix=&tier={TIER}&dryRun=false"),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("run transition");
|
||||
|
||||
let deadline = Instant::now() + Duration::from_secs(60);
|
||||
while target.stored_keys(TARGET_BUCKET).len() < objects.len() {
|
||||
assert!(
|
||||
Instant::now() < deadline,
|
||||
"RT-14 FAIL: transition did not reach the target; stored keys: {:?}",
|
||||
target.stored_keys(TARGET_BUCKET)
|
||||
);
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
|
||||
let keys = target.stored_keys(TARGET_BUCKET);
|
||||
assert_eq!(keys.len(), objects.len(), "RT-14 FAIL: unexpected remote objects: {keys:?}");
|
||||
|
||||
let mut stored_bodies = Vec::new();
|
||||
for key in &keys {
|
||||
let (body, metadata) = target.stored_object(TARGET_BUCKET, key).expect("remote object");
|
||||
for name in [
|
||||
"x-rustfs-internal-transition-transaction-id",
|
||||
"x-minio-internal-transition-transaction-id",
|
||||
"x-rustfs-internal-transition-tier-destination-id",
|
||||
"x-minio-internal-transition-tier-destination-id",
|
||||
] {
|
||||
assert!(
|
||||
metadata.contains_key(name),
|
||||
"RT-14 FAIL: remote object {key} lost transition identity {name}: {:?}",
|
||||
metadata.keys().collect::<Vec<_>>()
|
||||
);
|
||||
}
|
||||
stored_bodies.push(body.to_vec());
|
||||
}
|
||||
let mut expected_bodies: Vec<Vec<u8>> = objects.iter().map(|(_, body)| body.clone()).collect();
|
||||
expected_bodies.sort();
|
||||
stored_bodies.sort();
|
||||
assert_eq!(stored_bodies, expected_bodies, "RT-14 FAIL: remote bytes differ from the source objects");
|
||||
|
||||
let transition_puts = target
|
||||
.requests()
|
||||
.iter()
|
||||
.filter(|record| {
|
||||
record.operation == Operation::PutObject
|
||||
&& record
|
||||
.key
|
||||
.as_deref()
|
||||
.is_some_and(|key| key.contains("transition-transactions"))
|
||||
})
|
||||
.count();
|
||||
assert_eq!(
|
||||
transition_puts,
|
||||
objects.len(),
|
||||
"RT-14 FAIL: transition PUTs were retried or refused by the target"
|
||||
);
|
||||
|
||||
for (key, body) in &objects {
|
||||
let head = client
|
||||
.head_object()
|
||||
.bucket(SOURCE_BUCKET)
|
||||
.key(*key)
|
||||
.send()
|
||||
.await
|
||||
.expect("head transitioned");
|
||||
assert_eq!(
|
||||
head.storage_class().map(|class| class.as_str()),
|
||||
Some(TIER),
|
||||
"RT-14 FAIL: {key} is not reported as transitioned"
|
||||
);
|
||||
let got = client
|
||||
.get_object()
|
||||
.bucket(SOURCE_BUCKET)
|
||||
.key(*key)
|
||||
.send()
|
||||
.await
|
||||
.expect("get transitioned")
|
||||
.body
|
||||
.collect()
|
||||
.await
|
||||
.expect("read body")
|
||||
.into_bytes();
|
||||
assert_eq!(got.as_ref(), body.as_slice(), "RT-14 FAIL: read-back of {key} differs");
|
||||
}
|
||||
|
||||
target.shutdown().await;
|
||||
info!("RT-14 PASS: transition PUT omits reserved internal headers");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -33,6 +33,7 @@ use aws_sdk_s3::types::{
|
||||
};
|
||||
use http::{Method, StatusCode};
|
||||
use serde_json::Value;
|
||||
use std::future::Future;
|
||||
use std::io::Write;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
@@ -57,6 +58,10 @@ const UPGRADE_READINESS_BODY: &[u8] = b"upgrade write readiness";
|
||||
const PREVIOUS_RELEASE_SEED_KEY: &str = ".upgrade-readiness/previous-seed/node-0";
|
||||
const MULTIPART_WORKERS: usize = 16;
|
||||
const MULTIPART_UPLOADS_PER_WORKER: usize = 16;
|
||||
// Bound each 64 KiB upload, including completion acknowledgement. A peer
|
||||
// that accepts the request but never responds must fail the compatibility
|
||||
// check with the writer and object, rather than consume the whole CI job.
|
||||
const MIXED_MULTIPART_UPLOAD_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
// Peers keep a restarted node's drive in Suspect/Returning for roughly
|
||||
// probe_interval (2s) x success_threshold (3) after it comes back; 30s
|
||||
// comfortably covers that window plus CI scheduling jitter.
|
||||
@@ -335,14 +340,21 @@ fn configure_cluster_logs(cluster: &mut RustFSTestClusterEnvironment) -> TestRes
|
||||
async fn write_multipart_load(clients: &[Client], phase: &str) -> Result<Vec<String>, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let mut tasks = JoinSet::new();
|
||||
for worker in 0..MULTIPART_WORKERS {
|
||||
let client = clients[worker % clients.len()].clone();
|
||||
let node = worker % clients.len();
|
||||
let client = clients[node].clone();
|
||||
let phase = phase.to_string();
|
||||
tasks.spawn(async move {
|
||||
let mut keys = Vec::with_capacity(MULTIPART_UPLOADS_PER_WORKER);
|
||||
for upload in 0..MULTIPART_UPLOADS_PER_WORKER {
|
||||
let key = format!("{phase}/multipart/{worker:02}/{upload:02}");
|
||||
let part = vec![u8::try_from(worker)?; 64 * 1024];
|
||||
write_multipart(&client, MIXED_BUCKET, &key, &[part]).await?;
|
||||
mixed_multipart_before(
|
||||
Instant::now() + MIXED_MULTIPART_UPLOAD_TIMEOUT,
|
||||
node,
|
||||
&key,
|
||||
write_multipart(&client, MIXED_BUCKET, &key, &[part]),
|
||||
)
|
||||
.await?;
|
||||
keys.push(key);
|
||||
}
|
||||
Ok::<_, Box<dyn std::error::Error + Send + Sync>>(keys)
|
||||
@@ -356,6 +368,172 @@ async fn write_multipart_load(clients: &[Client], phase: &str) -> Result<Vec<Str
|
||||
Ok(keys)
|
||||
}
|
||||
|
||||
async fn mixed_multipart_before(
|
||||
deadline: Instant,
|
||||
node: usize,
|
||||
key: &str,
|
||||
upload: impl Future<Output = TestResult>,
|
||||
) -> TestResult {
|
||||
tokio::time::timeout_at(deadline, upload)
|
||||
.await
|
||||
.map_err(|_| MultipartAcknowledgementDeadline {
|
||||
node,
|
||||
key: key.to_owned(),
|
||||
})?
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct MultipartAcknowledgementDeadline {
|
||||
node: usize,
|
||||
key: String,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for MultipartAcknowledgementDeadline {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(
|
||||
formatter,
|
||||
"node {}: multipart upload {MIXED_BUCKET}/{} acknowledgement deadline exceeded",
|
||||
self.node, self.key
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for MultipartAcknowledgementDeadline {}
|
||||
|
||||
// Decode only the public lock snapshot fields. Unknown fields, including
|
||||
// credential/configuration payloads, never enter the failure artifact.
|
||||
#[derive(serde::Deserialize, serde::Serialize)]
|
||||
struct MixedLockSnapshot {
|
||||
total: Option<u64>,
|
||||
truncated: Option<bool>,
|
||||
capability_note: Option<String>,
|
||||
locks: Vec<MixedLockEntry>,
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize, serde::Serialize)]
|
||||
struct MixedLockEntry {
|
||||
resource: Option<String>,
|
||||
bucket: Option<String>,
|
||||
object: Option<String>,
|
||||
version: Option<String>,
|
||||
owner: Option<String>,
|
||||
#[serde(rename = "type")]
|
||||
lock_type: Option<String>,
|
||||
priority: Option<String>,
|
||||
since: Option<String>,
|
||||
elapsed_secs: Option<u64>,
|
||||
ttl_secs: Option<u64>,
|
||||
}
|
||||
|
||||
async fn read_mixed_lock_snapshot(response: &mut reqwest::Response) -> Result<MixedLockSnapshot, &'static str> {
|
||||
const MAX_BODY_BYTES: usize = 1024 * 1024;
|
||||
let mut body = Vec::new();
|
||||
while let Some(chunk) = response.chunk().await.map_err(|_| "body_read_error")? {
|
||||
if chunk.len() > MAX_BODY_BYTES - body.len() {
|
||||
return Err("body_limit_exceeded");
|
||||
}
|
||||
body.extend_from_slice(&chunk);
|
||||
}
|
||||
serde_json::from_slice(&body).map_err(|_| "invalid_lock_snapshot")
|
||||
}
|
||||
|
||||
async fn probe_mixed_multipart_failure(
|
||||
cluster: &RustFSTestClusterEnvironment,
|
||||
node: usize,
|
||||
path: &str,
|
||||
head: bool,
|
||||
deadline: Instant,
|
||||
) -> Value {
|
||||
let started = Instant::now();
|
||||
let method = if head { Method::HEAD } else { Method::GET };
|
||||
let mut observation = serde_json::json!({
|
||||
"node": node, "method": method.as_str(), "path": path,
|
||||
"status": null, "observation": "unknown", "timeout": false,
|
||||
});
|
||||
// The shared absolute deadline bounds both headers and a slow/infinite
|
||||
// response body; a send-only timeout would leave the cluster alive forever.
|
||||
let result = tokio::time::timeout_at(deadline, async {
|
||||
let mut response = signed_request(
|
||||
method,
|
||||
&format!("{}{path}", cluster.nodes[node].url.trim_end_matches('/')),
|
||||
&cluster.access_key,
|
||||
&cluster.secret_key,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.map_err(|_| "request_error")?;
|
||||
let status = response.status();
|
||||
observation["status"] = serde_json::json!(status.as_u16());
|
||||
if head {
|
||||
observation["observation"] = serde_json::json!(match status {
|
||||
StatusCode::OK => "visible",
|
||||
StatusCode::NOT_FOUND => "not_visible",
|
||||
_ => "unknown",
|
||||
});
|
||||
for (field, header) in [
|
||||
("content_length", "content-length"),
|
||||
("etag", "etag"),
|
||||
("version", "x-amz-version-id"),
|
||||
] {
|
||||
observation[field] = serde_json::json!(response.headers().get(header).and_then(|value| value.to_str().ok()));
|
||||
}
|
||||
} else if status == StatusCode::OK {
|
||||
observation["snapshot"] =
|
||||
serde_json::to_value(read_mixed_lock_snapshot(&mut response).await?).map_err(|_| "snapshot_encode_error")?;
|
||||
observation["observation"] = serde_json::json!("captured");
|
||||
}
|
||||
Ok::<_, &'static str>(())
|
||||
})
|
||||
.await;
|
||||
match result {
|
||||
Ok(Ok(())) => {}
|
||||
Ok(Err(reason)) => observation["error"] = serde_json::json!(reason),
|
||||
Err(_) => observation["timeout"] = serde_json::json!(true),
|
||||
}
|
||||
observation["elapsed_secs"] = serde_json::json!(started.elapsed().as_secs_f64());
|
||||
observation
|
||||
}
|
||||
|
||||
async fn observe_mixed_multipart_failure(
|
||||
cluster: &RustFSTestClusterEnvironment,
|
||||
phase: &str,
|
||||
error: BoxError,
|
||||
log_dir: Option<&Path>,
|
||||
) -> TestResult {
|
||||
if let (Some(log_dir), Some(failure)) = (log_dir, error.downcast_ref::<MultipartAcknowledgementDeadline>()) {
|
||||
let started = Instant::now();
|
||||
let deadline = started + Duration::from_secs(5);
|
||||
// Reserve the last second for serialization and bounded artifact I/O.
|
||||
// All probes run directly in this future; no diagnostic task is spawned.
|
||||
let network_deadline = started + Duration::from_secs(4);
|
||||
let path = format!("/{MIXED_BUCKET}/{}", failure.key);
|
||||
let _ = tokio::time::timeout_at(deadline, async {
|
||||
let (heads, locks) = tokio::join!(
|
||||
futures::future::join_all((0..cluster.nodes.len()).map(|node| probe_mixed_multipart_failure(
|
||||
cluster,
|
||||
node,
|
||||
&path,
|
||||
true,
|
||||
network_deadline
|
||||
))),
|
||||
probe_mixed_multipart_failure(cluster, 0, "/rustfs/admin/v3/top/locks?count=1000", false, network_deadline),
|
||||
);
|
||||
let snapshot = serde_json::json!({
|
||||
"phase": phase, "bucket": MIXED_BUCKET, "key": failure.key,
|
||||
"writer_node": failure.node, "heads": heads, "top_locks": locks,
|
||||
});
|
||||
let body = serde_json::to_vec_pretty(&snapshot).map_err(|_| ())?;
|
||||
tokio::fs::write(log_dir.join("mixed-multipart-failure.json"), body)
|
||||
.await
|
||||
.map_err(|_| ())
|
||||
})
|
||||
.await;
|
||||
}
|
||||
// Visibility and diagnostic failures cannot acknowledge the original MPU.
|
||||
Err(error)
|
||||
}
|
||||
|
||||
/// Assert that `client` eventually lists exactly `expected` objects under
|
||||
/// `{phase}/`, polling until [`LISTING_CONVERGENCE_TIMEOUT`].
|
||||
///
|
||||
@@ -400,7 +578,7 @@ async fn exercise_mixed_cluster(
|
||||
previous_node: usize,
|
||||
) -> TestResult {
|
||||
let clients = cluster.create_all_clients()?;
|
||||
wait_for_upgrade_write_readiness(&clients, phase, LISTING_CONVERGENCE_TIMEOUT).await?;
|
||||
wait_for_upgrade_write_readiness(&clients, MIXED_BUCKET, phase, LISTING_CONVERGENCE_TIMEOUT).await?;
|
||||
let current_client = &clients[current_node];
|
||||
let previous_client = &clients[previous_node];
|
||||
|
||||
@@ -426,7 +604,13 @@ async fn exercise_mixed_cluster(
|
||||
.await?;
|
||||
assert_eq!(read_object(current_client, MIXED_BUCKET, &previous_key, None).await?.1, previous_body);
|
||||
|
||||
let multipart_keys = write_multipart_load(&clients, phase).await?;
|
||||
let multipart_keys = match write_multipart_load(&clients, phase).await {
|
||||
Ok(keys) => keys,
|
||||
Err(error) => {
|
||||
let log_dir = std::env::var_os("RUSTFS_E2E_LOG_DIR");
|
||||
return observe_mixed_multipart_failure(cluster, phase, error, log_dir.as_deref().map(Path::new)).await;
|
||||
}
|
||||
};
|
||||
let expected_count = multipart_keys.len() + 2;
|
||||
for (label, client) in [("current", current_client), ("previous", previous_client)] {
|
||||
assert_eq!(
|
||||
@@ -462,7 +646,12 @@ fn upgrade_probe_client(client: &Client) -> Client {
|
||||
)
|
||||
}
|
||||
|
||||
async fn wait_for_upgrade_write_readiness(clients: &[Client], phase: &str, budget: Duration) -> TestResult {
|
||||
pub(crate) async fn wait_for_upgrade_write_readiness(
|
||||
clients: &[Client],
|
||||
bucket: &str,
|
||||
phase: &str,
|
||||
budget: Duration,
|
||||
) -> TestResult {
|
||||
// ListBuckets can succeed before peers recover a restarted disk. Cluster
|
||||
// health also accepts Returning disks whose write health is still FAULTY.
|
||||
// Probe every writer outside the asserted phase prefix; compatibility
|
||||
@@ -482,7 +671,7 @@ async fn wait_for_upgrade_write_readiness(clients: &[Client], phase: &str, budge
|
||||
deadline,
|
||||
client
|
||||
.put_object()
|
||||
.bucket(MIXED_BUCKET)
|
||||
.bucket(bucket)
|
||||
.key(&key)
|
||||
.body(ByteStream::from_static(UPGRADE_READINESS_BODY))
|
||||
.send(),
|
||||
@@ -517,19 +706,20 @@ async fn prepare_previous_release_baseline(cluster: &mut RustFSTestClusterEnviro
|
||||
// waits for pool metadata (#7473). First prove the elected writer can
|
||||
// persist data, then restart each old process once with three peers still
|
||||
// readable. This preparation ends before any current binary is started.
|
||||
wait_for_upgrade_write_readiness(&clients[..1], "previous-seed", LISTING_CONVERGENCE_TIMEOUT).await?;
|
||||
wait_for_upgrade_write_readiness(&clients[..1], MIXED_BUCKET, "previous-seed", LISTING_CONVERGENCE_TIMEOUT).await?;
|
||||
for node in [1, 2, 3, 0] {
|
||||
cluster.stop_node(node)?;
|
||||
cluster.start_node_from_binary(node, previous_binary).await?;
|
||||
wait_for_upgrade_write_readiness(
|
||||
std::slice::from_ref(&clients[node]),
|
||||
MIXED_BUCKET,
|
||||
&format!("previous-restart-{node}"),
|
||||
LISTING_CONVERGENCE_TIMEOUT,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
wait_for_upgrade_write_readiness(&clients, "previous-baseline", LISTING_CONVERGENCE_TIMEOUT).await?;
|
||||
wait_for_upgrade_write_readiness(&clients, MIXED_BUCKET, "previous-baseline", LISTING_CONVERGENCE_TIMEOUT).await?;
|
||||
for (reader, client) in clients.iter().enumerate() {
|
||||
assert_eq!(
|
||||
read_object(client, MIXED_BUCKET, PREVIOUS_RELEASE_SEED_KEY, None).await?.1,
|
||||
@@ -556,9 +746,10 @@ mod upgrade_write_readiness_tests {
|
||||
#[tokio::test]
|
||||
async fn waits_for_each_writer_after_metadata_is_ready() -> TestResult {
|
||||
let target = FakeS3Target::start().await?;
|
||||
target.create_bucket(MIXED_BUCKET);
|
||||
let bucket = "distributed-upgrade-history";
|
||||
target.create_bucket(bucket);
|
||||
let client = fake_source_client(&target);
|
||||
client.head_bucket().bucket(MIXED_BUCKET).send().await?;
|
||||
client.head_bucket().bucket(bucket).send().await?;
|
||||
|
||||
let phase = "one-previous-node";
|
||||
let first_key = format!(".upgrade-readiness/{phase}/node-0");
|
||||
@@ -578,7 +769,7 @@ mod upgrade_write_readiness_tests {
|
||||
// Metadata readiness does not prove that a data write can succeed.
|
||||
let premature = client
|
||||
.put_object()
|
||||
.bucket(MIXED_BUCKET)
|
||||
.bucket(bucket)
|
||||
.key(&first_key)
|
||||
.body(ByteStream::from_static(b"upgrade write readiness"))
|
||||
.send()
|
||||
@@ -586,15 +777,15 @@ mod upgrade_write_readiness_tests {
|
||||
.expect_err("metadata readiness does not prove write readiness");
|
||||
assert_eq!(premature.raw_response().map(|response| response.status().as_u16()), Some(503));
|
||||
|
||||
wait_for_upgrade_write_readiness(&[client.clone(), client.clone()], phase, Duration::from_secs(5)).await?;
|
||||
wait_for_upgrade_write_readiness(&[client.clone(), client.clone()], bucket, phase, Duration::from_secs(5)).await?;
|
||||
assert_eq!(target.count_requests(FakeTargetOperation::PutObject, &first_key), 3);
|
||||
assert_eq!(target.count_requests(FakeTargetOperation::PutObject, &second_key), 2);
|
||||
assert!(target.has_object(MIXED_BUCKET, &first_key));
|
||||
assert!(target.has_object(MIXED_BUCKET, &second_key));
|
||||
assert!(target.has_object(bucket, &first_key));
|
||||
assert!(target.has_object(bucket, &second_key));
|
||||
assert!(
|
||||
client
|
||||
.list_objects_v2()
|
||||
.bucket(MIXED_BUCKET)
|
||||
.bucket(bucket)
|
||||
.prefix(format!("{phase}/"))
|
||||
.send()
|
||||
.await?
|
||||
@@ -624,9 +815,10 @@ mod upgrade_write_readiness_tests {
|
||||
let phase = format!("permanent-{}", status.as_u16());
|
||||
let key = format!(".upgrade-readiness/{phase}/node-0");
|
||||
target.inject_for_key(FakeTargetOperation::PutObject, &key, FaultAction::Status(status), 1);
|
||||
let error = wait_for_upgrade_write_readiness(std::slice::from_ref(&client), &phase, Duration::from_secs(5))
|
||||
.await
|
||||
.expect_err("a permanent error must not be retried into success");
|
||||
let error =
|
||||
wait_for_upgrade_write_readiness(std::slice::from_ref(&client), MIXED_BUCKET, &phase, Duration::from_secs(5))
|
||||
.await
|
||||
.expect_err("a permanent error must not be retried into success");
|
||||
assert!(error.to_string().contains("node 0 write readiness failed"), "{error}");
|
||||
assert_eq!(target.count_requests(FakeTargetOperation::PutObject, &key), 1);
|
||||
assert!(!target.has_object(MIXED_BUCKET, &key));
|
||||
@@ -649,7 +841,7 @@ mod upgrade_write_readiness_tests {
|
||||
FaultAction::Status(StatusCode::SERVICE_UNAVAILABLE),
|
||||
10,
|
||||
);
|
||||
let error = wait_for_upgrade_write_readiness(&[client], phase, Duration::from_secs(1))
|
||||
let error = wait_for_upgrade_write_readiness(&[client], MIXED_BUCKET, phase, Duration::from_secs(1))
|
||||
.await
|
||||
.expect_err("persistent unavailability must exhaust the shared deadline");
|
||||
// Transport scheduling consumes the same budget; do not require a
|
||||
@@ -668,10 +860,134 @@ mod upgrade_write_readiness_tests {
|
||||
let phase = "stalled";
|
||||
let key = format!(".upgrade-readiness/{phase}/node-0");
|
||||
target.inject_for_key(FakeTargetOperation::PutObject, &key, FaultAction::Stall(Duration::from_secs(30)), 1);
|
||||
let error = wait_for_upgrade_write_readiness(&[client], phase, Duration::from_secs(1))
|
||||
let error = wait_for_upgrade_write_readiness(std::slice::from_ref(&client), MIXED_BUCKET, phase, Duration::from_secs(1))
|
||||
.await
|
||||
.expect_err("a stalled request must not outlive the readiness deadline");
|
||||
assert!(error.to_string().contains("deadline exceeded during PutObject"), "{error}");
|
||||
|
||||
let part = vec![2_u8; 64 * 1024];
|
||||
let successful_key = "stalled/multipart/02/00";
|
||||
mixed_multipart_before(
|
||||
Instant::now() + MIXED_MULTIPART_UPLOAD_TIMEOUT,
|
||||
2,
|
||||
successful_key,
|
||||
write_multipart(&client, MIXED_BUCKET, successful_key, std::slice::from_ref(&part)),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(read_object(&client, MIXED_BUCKET, successful_key, None).await?.1, part);
|
||||
|
||||
let stalled_key = "stalled/multipart/02/01";
|
||||
target.inject_for_key(
|
||||
FakeTargetOperation::CompleteMultipartUpload,
|
||||
stalled_key,
|
||||
FaultAction::Stall(Duration::from_secs(30)),
|
||||
1,
|
||||
);
|
||||
let error = {
|
||||
let upload = write_multipart(&client, MIXED_BUCKET, stalled_key, std::slice::from_ref(&part));
|
||||
tokio::pin!(upload);
|
||||
tokio::select! {
|
||||
result = &mut upload => panic!("stalled completion unexpectedly returned: {result:?}"),
|
||||
observed = tokio::time::timeout(Duration::from_secs(10), async {
|
||||
while target.count_requests(FakeTargetOperation::CompleteMultipartUpload, stalled_key) == 0
|
||||
|| !target.has_object(MIXED_BUCKET, stalled_key)
|
||||
{
|
||||
sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
}) => observed.expect("multipart completion must reach and commit on the fake peer"),
|
||||
}
|
||||
// An already committed object is not an acknowledged client success.
|
||||
// Expire only after observing the real completion request, so host
|
||||
// scheduling cannot turn this into a timeout before admission.
|
||||
tokio::time::timeout(Duration::from_secs(1), mixed_multipart_before(Instant::now(), 2, stalled_key, upload))
|
||||
.await
|
||||
.expect("expired deadline must stop the in-flight completion")
|
||||
.expect_err("unacknowledged completion must fail even if the peer committed")
|
||||
};
|
||||
assert!(error.to_string().contains("node 2"), "{error}");
|
||||
assert!(error.to_string().contains(stalled_key), "{error}");
|
||||
assert!(error.to_string().contains("acknowledgement deadline exceeded"), "{error}");
|
||||
assert_eq!(target.count_requests(FakeTargetOperation::CompleteMultipartUpload, stalled_key), 1);
|
||||
|
||||
let failure = error
|
||||
.downcast_ref::<MultipartAcknowledgementDeadline>()
|
||||
.expect("the timeout must retain typed writer/object context");
|
||||
assert_eq!(failure.node, 2);
|
||||
assert_eq!(failure.key, stalled_key);
|
||||
let original_message = error.to_string();
|
||||
// These four probe addresses share an in-process peer. This verifies
|
||||
// bounded diagnostic fan-out, not a four-node RustFS cluster.
|
||||
let mut probes = RustFSTestClusterEnvironment::new(MIXED_NODE_COUNT).await?;
|
||||
probes.access_key = FAKE_ACCESS_KEY.to_owned();
|
||||
probes.secret_key = FAKE_SECRET_KEY.to_owned();
|
||||
for node in &mut probes.nodes {
|
||||
node.url = target.endpoint().to_owned();
|
||||
}
|
||||
let log_dir = tempfile::tempdir()?;
|
||||
target.inject_for_key(
|
||||
FakeTargetOperation::HeadObject,
|
||||
stalled_key,
|
||||
FaultAction::Stall(Duration::from_secs(30)),
|
||||
1,
|
||||
);
|
||||
let error = tokio::time::timeout(
|
||||
Duration::from_secs(7),
|
||||
observe_mixed_multipart_failure(&probes, phase, error, Some(log_dir.path())),
|
||||
)
|
||||
.await
|
||||
.expect("the shared diagnostic budget must include artifact writing")
|
||||
.expect_err("observing committed data cannot acknowledge the stalled upload");
|
||||
assert_eq!(error.to_string(), original_message);
|
||||
assert!(error.downcast_ref::<MultipartAcknowledgementDeadline>().is_some());
|
||||
let snapshot: Value =
|
||||
serde_json::from_slice(&tokio::fs::read(log_dir.path().join("mixed-multipart-failure.json")).await?)?;
|
||||
assert_eq!(snapshot["key"], stalled_key);
|
||||
assert_eq!(snapshot["writer_node"], 2);
|
||||
let heads = snapshot["heads"]
|
||||
.as_array()
|
||||
.expect("failure artifact must retain every probe");
|
||||
assert_eq!(heads.len(), MIXED_NODE_COUNT);
|
||||
assert_eq!(
|
||||
heads
|
||||
.iter()
|
||||
.filter(|head| head["timeout"] == true && head["observation"] == "unknown")
|
||||
.count(),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
heads
|
||||
.iter()
|
||||
.filter(|head| head["status"] == 200 && head["observation"] == "visible")
|
||||
.count(),
|
||||
3
|
||||
);
|
||||
assert_eq!(target.count_requests(FakeTargetOperation::HeadObject, stalled_key), MIXED_NODE_COUNT);
|
||||
assert_eq!(target.count_requests(FakeTargetOperation::CompleteMultipartUpload, stalled_key), 1);
|
||||
assert_eq!(snapshot["top_locks"]["observation"], "unknown", "the fake peer has no admin lock API");
|
||||
|
||||
let locks: MixedLockSnapshot = serde_json::from_value(serde_json::json!({
|
||||
"total": 1, "truncated": true, "capability_note": "partial snapshot",
|
||||
"credentials": { "secret_key": "must not persist" },
|
||||
"locks": [{
|
||||
"resource": format!("{MIXED_BUCKET}/{stalled_key}"), "owner": "writer", "type": "WRITE",
|
||||
"elapsed_secs": 30, "ttl_secs": 1, "merged_config": { "secret_key": "must not persist" },
|
||||
}],
|
||||
}))?;
|
||||
let safe_locks = serde_json::to_value(locks)?;
|
||||
assert_eq!(safe_locks["total"], 1);
|
||||
assert_eq!(safe_locks["truncated"], true);
|
||||
assert_eq!(safe_locks["locks"][0]["owner"], "writer");
|
||||
assert_eq!(safe_locks["locks"][0]["elapsed_secs"], 30);
|
||||
assert!(safe_locks.get("credentials").is_none());
|
||||
assert!(safe_locks["locks"][0].get("merged_config").is_none());
|
||||
|
||||
let mut body = br#"{"locks":[]}"#.to_vec();
|
||||
body.resize(1024 * 1024, b' ');
|
||||
let mut at_limit = reqwest::Response::from(http::Response::new(body.clone()));
|
||||
assert!(read_mixed_lock_snapshot(&mut at_limit).await?.locks.is_empty());
|
||||
body.push(b' ');
|
||||
let mut oversized = reqwest::Response::from(http::Response::new(body));
|
||||
assert_eq!(read_mixed_lock_snapshot(&mut oversized).await.err(), Some("body_limit_exceeded"));
|
||||
target.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
@@ -2181,9 +2497,9 @@ struct LayoutTransport {
|
||||
uploaded_parts: Vec<i32>,
|
||||
single_puts: usize,
|
||||
completes: usize,
|
||||
/// Raw per-key journal in target order: (sequence, operation, part number,
|
||||
/// upload id), so duplicate drives can be told apart from retries.
|
||||
journal: Vec<(u64, String, Option<i32>, Option<String>)>,
|
||||
/// Allowlisted request identity and prepared response metadata. A prepared
|
||||
/// response does not prove that the client received it.
|
||||
journal: Vec<Value>,
|
||||
}
|
||||
|
||||
/// Configure every layout bucket to replicate its existing objects to a fresh
|
||||
@@ -2245,12 +2561,28 @@ async fn replicate_layouts(
|
||||
journal: key_requests
|
||||
.iter()
|
||||
.map(|record| {
|
||||
(
|
||||
record.sequence,
|
||||
format!("{:?}", record.operation),
|
||||
record.part_number,
|
||||
record.upload_id.as_ref().map(|id| id.chars().take(12).collect()),
|
||||
)
|
||||
serde_json::json!({
|
||||
"sequence": record.sequence,
|
||||
"operation": format!("{:?}", record.operation),
|
||||
"part_number": record.part_number,
|
||||
"upload_id": record.upload_id,
|
||||
"requested_version_id": record.version_id,
|
||||
"source_version_id": record.source_version_id,
|
||||
"source_mtime": record.source_mtime,
|
||||
"source_etag": record.source_etag,
|
||||
"source_replication_request": record.source_replication_request,
|
||||
"request_content_length": record.content_length,
|
||||
"journaled_at_unix_millis": record.journaled_at_unix_millis,
|
||||
"prepared_response": record.prepared_response.as_ref().map(|response| serde_json::json!({
|
||||
"status": response.status,
|
||||
"version_id": response.version_id,
|
||||
"etag": response.etag,
|
||||
"last_modified": response.last_modified,
|
||||
"content_length": response.content_length,
|
||||
"prepared_at_unix_millis": response.prepared_at_unix_millis,
|
||||
"elapsed_secs": response.elapsed.as_secs_f64(),
|
||||
})),
|
||||
})
|
||||
})
|
||||
.collect(),
|
||||
};
|
||||
|
||||
@@ -176,7 +176,7 @@ regex = { workspace = true }
|
||||
path-absolutize = { workspace = true }
|
||||
rmp.workspace = true
|
||||
rmp-serde.workspace = true
|
||||
tokio-util = { workspace = true, features = ["io", "compat"] }
|
||||
tokio-util = { workspace = true, features = ["io", "compat", "rt"] }
|
||||
tokio-stream = { workspace = true, features = ["sync"] }
|
||||
hmac = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
|
||||
@@ -240,7 +240,10 @@ pub mod bucket {
|
||||
}
|
||||
|
||||
pub mod quota {
|
||||
pub use crate::bucket::quota::{BucketQuota, QuotaCheckResult, QuotaError, QuotaOperation};
|
||||
pub use crate::bucket::quota::{
|
||||
BucketQuota, QUOTA_RESERVATION_PROTOCOL_V1, QUOTA_RESERVATION_PROTOCOL_V2, QuotaCheckResult, QuotaError,
|
||||
QuotaOperation,
|
||||
};
|
||||
|
||||
pub mod checker {
|
||||
pub use crate::bucket::quota::checker::QuotaChecker;
|
||||
@@ -362,11 +365,30 @@ pub mod config {
|
||||
ServerConfigCorruptError, ServerConfigSaveResult, ServerConfigSnapshot, delete_config, delete_config_no_lock,
|
||||
is_server_config_corrupt_error, lookup_configs, read_config, read_config_no_lock, read_config_with_metadata,
|
||||
read_config_without_migrate, read_config_without_migrate_no_lock, read_existing_server_config_no_lock,
|
||||
read_server_config_snapshot, save_config, save_config_no_lock, save_config_with_opts, save_server_config,
|
||||
save_server_config_no_lock, save_server_config_snapshot, save_server_config_snapshot_with_generation,
|
||||
server_config_path, try_migrate_server_config, with_config_object_read_lock, with_config_object_write_lock,
|
||||
with_server_config_read_lock, with_server_config_write_lock,
|
||||
read_server_config_snapshot, save_config, save_config_no_lock, save_config_with_opts, save_config_with_opts_quiet,
|
||||
save_server_config, save_server_config_no_lock, save_server_config_snapshot,
|
||||
save_server_config_snapshot_with_generation, server_config_path, try_migrate_server_config,
|
||||
with_config_object_read_lock, with_config_object_write_lock, with_server_config_read_lock,
|
||||
with_server_config_write_lock,
|
||||
};
|
||||
|
||||
/// Read config bytes and metadata without treating an empty payload as absent.
|
||||
pub async fn read_config_preserve_empty_with_metadata_opts(
|
||||
api: std::sync::Arc<crate::store::ECStore>,
|
||||
file: &str,
|
||||
opts: &crate::object_api::ObjectOptions,
|
||||
) -> crate::error::Result<(Vec<u8>, crate::object_api::ObjectInfo)> {
|
||||
crate::config::com::read_config_preserve_empty_with_metadata_opts(api, file, opts).await
|
||||
}
|
||||
|
||||
pub async fn read_config_limited_preserve_empty_with_metadata_opts(
|
||||
api: std::sync::Arc<crate::store::ECStore>,
|
||||
file: &str,
|
||||
opts: &crate::object_api::ObjectOptions,
|
||||
max_bytes: usize,
|
||||
) -> crate::error::Result<(Vec<u8>, crate::object_api::ObjectInfo)> {
|
||||
crate::config::com::read_config_limited_preserve_empty_with_metadata_opts(api, file, opts, max_bytes).await
|
||||
}
|
||||
}
|
||||
|
||||
pub mod storageclass {
|
||||
@@ -468,7 +490,7 @@ pub mod event {
|
||||
pub mod global {
|
||||
pub use crate::runtime::global::{
|
||||
set_global_endpoints, set_global_region, set_global_rustfs_port, set_object_store_resolver, shutdown_background_services,
|
||||
update_erasure_type,
|
||||
update_erasure_type, wait_for_detached_mutations,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -519,7 +541,7 @@ pub mod object {
|
||||
ObjectInfo, ObjectLockConfigSnapshot, ObjectMutationHook, ObjectOptions, PutObjReader, QuotaAdmission,
|
||||
RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest,
|
||||
SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, ScannerPublicationCommitScope, ScannerPublicationCommitStartError,
|
||||
ScannerPublicationCommitState, ShardIntegrityWriteMode, StreamConsumer, WriteCompletion,
|
||||
ScannerPublicationCommitState, ShardIntegrityWriteMode, StreamConsumer, WriteCommitGuard, WriteCompletion,
|
||||
get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, register_get_object_body_cache_hook,
|
||||
register_object_mutation_hook, unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
|
||||
};
|
||||
|
||||
@@ -87,6 +87,7 @@ use rustfs_filemeta::{
|
||||
FileInfo, FileInfoOpts, NULL_VERSION_ID, RestoreStatus, RestoreStatusOps, TRANSITION_COMPLETE, get_file_info,
|
||||
is_restored_object_on_disk,
|
||||
};
|
||||
use rustfs_lock::{NamespaceLockGuard, error::LockError};
|
||||
use rustfs_scanner_metrics::metrics::{
|
||||
IlmAction, Metrics, ScannerLifecycleExpiryStateUpdate, ScannerLifecycleTransitionStateUpdate, global_metrics,
|
||||
};
|
||||
@@ -190,6 +191,8 @@ const TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL: StdDuration = StdDuration::from_
|
||||
// before another full sweep can discover work persisted by another node.
|
||||
const TIER_FREE_VERSION_RECOVERY_MAX_IDLE_INTERVAL: StdDuration = StdDuration::from_secs(10 * 60);
|
||||
const TIER_FREE_VERSION_RECOVERY_JITTER_PERCENT: u64 = 10;
|
||||
const TIER_FREE_VERSION_RECOVERY_LEADER_LOCK: &str = "tier-free-version-recovery/leader.lock";
|
||||
const TIER_FREE_VERSION_RECOVERY_LOCK_TIMEOUT: StdDuration = StdDuration::from_secs(1);
|
||||
const DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS: i64 = 5;
|
||||
const EXPIRY_WORKER_QUEUE_CAPACITY: usize = 1000;
|
||||
const DEFAULT_MANUAL_TRANSITION_JOB_RECOVERY_LIMIT: usize = 100;
|
||||
@@ -1441,13 +1444,24 @@ pub(crate) async fn enqueue_committed_free_versions(api: &ECStore, free_versions
|
||||
queued
|
||||
}
|
||||
|
||||
async fn enqueue_recovered_free_version_with_state(state: &Arc<RwLock<ExpiryState>>, oi: ObjectInfo) -> bool {
|
||||
async fn enqueue_recovered_free_version_with_state(
|
||||
state: &Arc<RwLock<ExpiryState>>,
|
||||
oi: ObjectInfo,
|
||||
cancel: &CancellationToken,
|
||||
) -> bool {
|
||||
let task = FreeVersionTask(oi);
|
||||
let hash = task.op_hash();
|
||||
let (wrkr, stats) = {
|
||||
let state = state.read().await;
|
||||
let state = select! {
|
||||
biased;
|
||||
_ = cancel.cancelled() => return false,
|
||||
state = state.read() => state,
|
||||
};
|
||||
(state.get_worker_ch(hash), Arc::clone(&state.stats))
|
||||
};
|
||||
if cancel.is_cancelled() {
|
||||
return false;
|
||||
}
|
||||
let Some(wrkr) = wrkr else {
|
||||
stats.increment_missed_freevers_tasks();
|
||||
stats.record_scanner_expiry_state();
|
||||
@@ -1497,9 +1511,13 @@ fn set_recovered_free_version_enqueue_observer(
|
||||
RecoveredFreeVersionEnqueueObserverGuard
|
||||
}
|
||||
|
||||
pub async fn enqueue_recovered_free_version(api: &ECStore, oi: ObjectInfo) -> bool {
|
||||
pub(super) async fn enqueue_recovered_free_version_with_cancel(
|
||||
api: &ECStore,
|
||||
oi: ObjectInfo,
|
||||
cancel: &CancellationToken,
|
||||
) -> bool {
|
||||
let expiry_state = api.ctx.expiry_state();
|
||||
let queued = enqueue_recovered_free_version_with_state(&expiry_state, oi).await;
|
||||
let queued = enqueue_recovered_free_version_with_state(&expiry_state, oi, cancel).await;
|
||||
|
||||
#[cfg(test)]
|
||||
if let Some(observer) = RECOVERED_FREE_VERSION_ENQUEUE_OBSERVER
|
||||
@@ -2921,6 +2939,20 @@ fn resolve_tier_free_version_recovery_enabled(value: Result<String, env::VarErro
|
||||
}
|
||||
|
||||
fn spawn_tier_free_version_recovery_once(api: Arc<ECStore>, started: &OnceLock<()>) -> Option<JoinHandle<()>> {
|
||||
spawn_tier_free_version_recovery_once_with_options(
|
||||
api,
|
||||
started,
|
||||
TIER_FREE_VERSION_RECOVERY_LEADER_LOCK.to_string(),
|
||||
tier_free_version_recovery_leader_retry_delay,
|
||||
)
|
||||
}
|
||||
|
||||
fn spawn_tier_free_version_recovery_once_with_options(
|
||||
api: Arc<ECStore>,
|
||||
started: &OnceLock<()>,
|
||||
leader_lock_name: String,
|
||||
retry_delay: fn() -> StdDuration,
|
||||
) -> Option<JoinHandle<()>> {
|
||||
if !tier_free_version_recovery_enabled() {
|
||||
warn!(
|
||||
event = EVENT_LIFECYCLE_WORKER_STATE,
|
||||
@@ -2937,20 +2969,43 @@ fn spawn_tier_free_version_recovery_once(api: Arc<ECStore>, started: &OnceLock<(
|
||||
|
||||
Some(tokio::spawn(async move {
|
||||
let cancel_token = api.ctx.background_cancel_token().unwrap_or_default();
|
||||
let expiry_state = api.ctx.expiry_state();
|
||||
run_tier_free_version_recovery_loop(
|
||||
let acquire_api = Arc::clone(&api);
|
||||
let acquire_lock_name = Arc::<str>::from(leader_lock_name);
|
||||
run_tier_free_version_recovery_coordinator(
|
||||
cancel_token,
|
||||
expiry_state,
|
||||
jitter_tier_free_version_recovery_delay,
|
||||
move |bucket_marker, object_marker, recovery_cancel| {
|
||||
retry_delay,
|
||||
move || {
|
||||
let api = Arc::clone(&acquire_api);
|
||||
let lock_name = Arc::clone(&acquire_lock_name);
|
||||
async move {
|
||||
let lock = api.new_ns_lock(RUSTFS_META_BUCKET, &lock_name).await?;
|
||||
match lock.get_write_lock_quiet(TIER_FREE_VERSION_RECOVERY_LOCK_TIMEOUT).await {
|
||||
Ok(guard) => Ok(Some(guard)),
|
||||
Err(LockError::Timeout { .. }) => Ok(None),
|
||||
Err(err) => Err(err.into()),
|
||||
}
|
||||
}
|
||||
},
|
||||
move |leader_cancel| {
|
||||
let api = Arc::clone(&api);
|
||||
async move {
|
||||
recover_tier_free_versions_with_cancel(
|
||||
api,
|
||||
DEFAULT_FREE_VERSION_RECOVERY_LIMIT,
|
||||
bucket_marker,
|
||||
object_marker,
|
||||
recovery_cancel,
|
||||
run_tier_free_version_recovery_loop(
|
||||
leader_cancel,
|
||||
api.ctx.expiry_state(),
|
||||
jitter_tier_free_version_recovery_delay,
|
||||
move |bucket_marker, object_marker, recovery_cancel| {
|
||||
let api = Arc::clone(&api);
|
||||
async move {
|
||||
recover_tier_free_versions_with_cancel(
|
||||
api,
|
||||
DEFAULT_FREE_VERSION_RECOVERY_LIMIT,
|
||||
bucket_marker,
|
||||
object_marker,
|
||||
recovery_cancel,
|
||||
)
|
||||
.await
|
||||
}
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -2960,6 +3015,86 @@ fn spawn_tier_free_version_recovery_once(api: Arc<ECStore>, started: &OnceLock<(
|
||||
}))
|
||||
}
|
||||
|
||||
fn tier_free_version_recovery_leader_retry_delay() -> StdDuration {
|
||||
StdDuration::from_secs(rand::rng().random_range(15..=30))
|
||||
}
|
||||
|
||||
async fn run_tier_free_version_recovery_coordinator<A, AF, R, RF>(
|
||||
cancel: CancellationToken,
|
||||
retry_delay: fn() -> StdDuration,
|
||||
mut acquire: A,
|
||||
mut recover: R,
|
||||
) where
|
||||
A: FnMut() -> AF,
|
||||
AF: Future<Output = crate::error::Result<Option<NamespaceLockGuard>>>,
|
||||
R: FnMut(CancellationToken) -> RF,
|
||||
RF: Future<Output = ()>,
|
||||
{
|
||||
loop {
|
||||
let acquired = select! {
|
||||
biased;
|
||||
_ = cancel.cancelled() => return,
|
||||
result = acquire() => result,
|
||||
};
|
||||
match acquired {
|
||||
Ok(Some(mut guard)) => {
|
||||
let leader_cancel = cancel.child_token();
|
||||
// Hold ownership across every page, cooldown and idle wait. A
|
||||
// follower taking over starts a fresh sweep of persisted markers.
|
||||
let recovery = recover(leader_cancel.clone());
|
||||
tokio::pin!(recovery);
|
||||
debug!(
|
||||
event = EVENT_LIFECYCLE_WORKER_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
state = "free_version_recovery_leader_acquired",
|
||||
lock_name = TIER_FREE_VERSION_RECOVERY_LEADER_LOCK,
|
||||
"Tier free-version recovery leader acquired"
|
||||
);
|
||||
select! {
|
||||
biased;
|
||||
_ = cancel.cancelled() => {
|
||||
leader_cancel.cancel();
|
||||
recovery.await;
|
||||
}
|
||||
_ = guard.lock_lost_notified() => {
|
||||
debug!(
|
||||
event = EVENT_LIFECYCLE_WORKER_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
state = "free_version_recovery_leader_lost",
|
||||
lock_name = TIER_FREE_VERSION_RECOVERY_LEADER_LOCK,
|
||||
"Tier free-version recovery leader lost"
|
||||
);
|
||||
// Dropping the recovery future would detach its active
|
||||
// walk. Cancel it and poll the same future through cleanup.
|
||||
leader_cancel.cancel();
|
||||
recovery.await;
|
||||
}
|
||||
_ = &mut recovery => {}
|
||||
}
|
||||
leader_cancel.cancel();
|
||||
guard.release();
|
||||
}
|
||||
Ok(None) => {}
|
||||
Err(err) => warn!(
|
||||
event = EVENT_LIFECYCLE_WORKER_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
state = "free_version_recovery_leader_acquire_failed",
|
||||
lock_name = TIER_FREE_VERSION_RECOVERY_LEADER_LOCK,
|
||||
error = %err,
|
||||
"Tier free-version recovery leader acquisition failed"
|
||||
),
|
||||
}
|
||||
select! {
|
||||
biased;
|
||||
_ = cancel.cancelled() => return,
|
||||
_ = tokio::time::sleep(retry_delay()) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_tier_free_version_recovery_loop<F, Fut>(
|
||||
cancel_token: CancellationToken,
|
||||
expiry_state: Arc<RwLock<ExpiryState>>,
|
||||
@@ -2969,7 +3104,11 @@ async fn run_tier_free_version_recovery_loop<F, Fut>(
|
||||
F: FnMut(Option<String>, Option<String>, CancellationToken) -> Fut,
|
||||
Fut: Future<Output = crate::error::Result<FreeVersionRecoveryStats>>,
|
||||
{
|
||||
let recovery_notify = Arc::clone(&expiry_state.read().await.recovery_notify);
|
||||
let recovery_notify = select! {
|
||||
biased;
|
||||
_ = cancel_token.cancelled() => return,
|
||||
state = expiry_state.read() => Arc::clone(&state.recovery_notify),
|
||||
};
|
||||
let mut schedule = TierFreeVersionRecoverySchedule::default();
|
||||
|
||||
loop {
|
||||
@@ -2992,7 +3131,11 @@ async fn run_tier_free_version_recovery_loop<F, Fut>(
|
||||
schedule.record_success(&stats);
|
||||
rustfs_io_metrics::record_stage_duration("lifecycle_free_version_recovery", elapsed.as_secs_f64() * 1000.0);
|
||||
let (pending_tasks, active_tasks) = {
|
||||
let state = expiry_state.read().await;
|
||||
let state = select! {
|
||||
biased;
|
||||
_ = cancel_token.cancelled() => return,
|
||||
state = expiry_state.read() => state,
|
||||
};
|
||||
(state.pending_tasks(), state.stats.active_tasks())
|
||||
};
|
||||
debug!(
|
||||
@@ -4748,7 +4891,7 @@ async fn expire_transitioned_object_with_lock_lost_signal(
|
||||
bucket_incarnation_id: Uuid,
|
||||
lock_lost_signal: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
|
||||
) -> Result<ObjectInfo, std::io::Error> {
|
||||
let publication_guard = lifecycle_expiry_publication_guard(&api, oi, bucket_incarnation_id)
|
||||
let publication_guard = lifecycle_expiry_publication_guard(&api, &oi.bucket, &oi.name, bucket_incarnation_id)
|
||||
.await
|
||||
.ok_or_else(|| std::io::Error::other("lifecycle expiry is not allowed for this bucket"))?;
|
||||
let snapshot = lifecycle_delete_config_snapshot(&api, oi)
|
||||
@@ -4838,10 +4981,14 @@ pub fn gen_transition_objname(bucket: &str) -> Result<String, Error> {
|
||||
pub async fn transition_object(api: Arc<ECStore>, oi: &ObjectInfo, lae: LcAuditEvent) -> Result<(), Error> {
|
||||
let time_ilm = Metrics::time_ilm(lae.event.action);
|
||||
|
||||
let (publication_guard, bucket_incarnation_id) = lifecycle_transition_publication_guard(&api, oi)
|
||||
.await
|
||||
.ok_or_else(|| Error::other("lifecycle transition is not allowed for this bucket"))?;
|
||||
|
||||
let etag = if let Some(etag) = &oi.etag { etag } else { "" };
|
||||
let etag = etag.to_string();
|
||||
|
||||
let opts = ObjectOptions {
|
||||
let mut opts = ObjectOptions {
|
||||
transition: TransitionOptions {
|
||||
status: lifecycle::TRANSITION_PENDING.to_string(),
|
||||
tier: lae.event.storage_class,
|
||||
@@ -4853,8 +5000,10 @@ pub async fn transition_object(api: Arc<ECStore>, oi: &ObjectInfo, lae: LcAuditE
|
||||
versioned: BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await,
|
||||
version_suspended: BucketVersioningSys::prefix_suspended(&oi.bucket, &oi.name).await,
|
||||
mod_time: oi.mod_time,
|
||||
expected_bucket_incarnation_id: Some(bucket_incarnation_id),
|
||||
..Default::default()
|
||||
};
|
||||
opts.add_namespace_lock_guard(&publication_guard);
|
||||
let result = api.transition_object(&oi.bucket, &oi.name, &opts).await;
|
||||
time_ilm(1)();
|
||||
result
|
||||
@@ -5458,20 +5607,26 @@ pub async fn apply_transition_rule(event: &lifecycle::Event, src: &LcEventSrc, o
|
||||
.await
|
||||
}
|
||||
|
||||
async fn lifecycle_expiry_publication_guard(
|
||||
async fn acquire_lifecycle_publication_guard(api: &ECStore, bucket: &str) -> Result<rustfs_lock::NamespaceLockGuard, Error> {
|
||||
let lock = api
|
||||
.new_ns_lock(bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
|
||||
.await?;
|
||||
let guard = lock.get_read_lock(get_lock_acquire_timeout()).await.map_err(Error::other)?;
|
||||
if guard.is_lock_lost() {
|
||||
return Err(Error::other("table-bucket publication lock was lost before lifecycle admission"));
|
||||
}
|
||||
Ok(guard)
|
||||
}
|
||||
|
||||
pub(crate) async fn lifecycle_expiry_publication_guard(
|
||||
api: &ECStore,
|
||||
oi: &ObjectInfo,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
bucket_incarnation_id: Uuid,
|
||||
) -> Option<rustfs_lock::NamespaceLockGuard> {
|
||||
let result = async {
|
||||
let lock = api
|
||||
.new_ns_lock(&oi.bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
|
||||
.await?;
|
||||
let guard = lock.get_read_lock(get_lock_acquire_timeout()).await.map_err(Error::other)?;
|
||||
if guard.is_lock_lost() {
|
||||
return Err(Error::other("table-bucket publication lock was lost before lifecycle delete admission"));
|
||||
}
|
||||
if !metadata_boundary::lifecycle_expiry_allowed(api, &oi.bucket, bucket_incarnation_id).await? {
|
||||
let result: std::result::Result<Option<rustfs_lock::NamespaceLockGuard>, Error> = async {
|
||||
let guard = acquire_lifecycle_publication_guard(api, bucket).await?;
|
||||
if !metadata_boundary::lifecycle_expiry_allowed(api, bucket, bucket_incarnation_id).await? {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(guard))
|
||||
@@ -5484,8 +5639,8 @@ async fn lifecycle_expiry_publication_guard(
|
||||
event = EVENT_LIFECYCLE_DELETE_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
bucket = %oi.bucket,
|
||||
object = %oi.name,
|
||||
bucket,
|
||||
object,
|
||||
operation = "authorize_lifecycle_expiry",
|
||||
error = %err,
|
||||
"Lifecycle delete admission failed"
|
||||
@@ -5495,6 +5650,34 @@ async fn lifecycle_expiry_publication_guard(
|
||||
}
|
||||
}
|
||||
|
||||
async fn lifecycle_transition_publication_guard(
|
||||
api: &ECStore,
|
||||
oi: &ObjectInfo,
|
||||
) -> Option<(rustfs_lock::NamespaceLockGuard, Uuid)> {
|
||||
let result: std::result::Result<(rustfs_lock::NamespaceLockGuard, Uuid), Error> = async {
|
||||
let guard = acquire_lifecycle_publication_guard(api, &oi.bucket).await?;
|
||||
let configs = metadata_boundary::get_expiry_configs(api, &oi.bucket).await?;
|
||||
Ok((guard, configs.bucket_incarnation_id))
|
||||
}
|
||||
.await;
|
||||
match result {
|
||||
Ok(state) => Some(state),
|
||||
Err(err) => {
|
||||
warn!(
|
||||
event = EVENT_LIFECYCLE_TIER_OPERATION_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
bucket = %oi.bucket,
|
||||
object = %oi.name,
|
||||
operation = "authorize_lifecycle_transition",
|
||||
error = %err,
|
||||
"Lifecycle transition admission failed"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn apply_expiry_on_transitioned_object(
|
||||
api: Arc<ECStore>,
|
||||
oi: &ObjectInfo,
|
||||
@@ -5551,7 +5734,8 @@ async fn apply_expiry_on_non_transitioned_objects_with_lock_lost_signal(
|
||||
bucket_incarnation_id: Uuid,
|
||||
lock_lost_signal: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
|
||||
) -> bool {
|
||||
let Some(publication_guard) = lifecycle_expiry_publication_guard(&api, oi, bucket_incarnation_id).await else {
|
||||
let Some(publication_guard) = lifecycle_expiry_publication_guard(&api, &oi.bucket, &oi.name, bucket_incarnation_id).await
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
let snapshot = match lifecycle_delete_config_snapshot(&api, oi).await {
|
||||
@@ -5611,8 +5795,10 @@ async fn apply_expiry_on_non_transitioned_objects_with_lock_lost_signal(
|
||||
} else {
|
||||
None
|
||||
};
|
||||
// The delete owns the publication guard from here on, so a cancelled
|
||||
// expiry worker cannot let table-bucket publication overtake it.
|
||||
let mut dobj = match api
|
||||
.delete_object_with_tier_delete_journal(&oi.bucket, &encode_dir_object(&oi.name), opts)
|
||||
.delete_object_with_tier_delete_journal_and_guards(&oi.bucket, &encode_dir_object(&oi.name), opts, [publication_guard])
|
||||
.await
|
||||
{
|
||||
Ok(dobj) => dobj,
|
||||
@@ -5844,6 +6030,7 @@ pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc,
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::expiry_worker_count;
|
||||
use super::lifecycle_transition_publication_guard;
|
||||
use super::{
|
||||
DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS, DEFAULT_TRANSITION_QUEUE_CAPACITY, DEFAULT_TRANSITION_WORKERS_ABSOLUTE_MAX,
|
||||
DEFAULT_TRANSITION_WORKERS_CAP, EVENT_LIFECYCLE_EVALUATION_FAILED, EVENT_LIFECYCLE_EXPIRED_DETECTED,
|
||||
@@ -5862,10 +6049,10 @@ mod tests {
|
||||
merge_stale_multipart_candidate, persist_manual_transition_job_progress_if_owned,
|
||||
persist_manual_transition_page_checkpoint, recover_manual_transition_job, recover_manual_transition_jobs,
|
||||
resolve_tier_free_version_recovery_enabled, resolve_transition_queue_capacity, resolve_transition_queue_send_timeout,
|
||||
resolve_transition_worker_count, resolve_transition_workers_absolute_max, run_tier_free_version_recovery_loop,
|
||||
select_restore_s3_location, set_lifecycle_observability_observer, set_recovered_free_version_enqueue_observer,
|
||||
should_defer_date_expiry_for_recent_config_update, transitioned_cleanup_tuple, transitioned_object_delete_opts,
|
||||
wait_for_tier_free_version_recovery,
|
||||
resolve_transition_worker_count, resolve_transition_workers_absolute_max, run_tier_free_version_recovery_coordinator,
|
||||
run_tier_free_version_recovery_loop, select_restore_s3_location, set_lifecycle_observability_observer,
|
||||
set_recovered_free_version_enqueue_observer, should_defer_date_expiry_for_recent_config_update,
|
||||
transitioned_cleanup_tuple, transitioned_object_delete_opts, wait_for_tier_free_version_recovery,
|
||||
};
|
||||
#[cfg(feature = "test-util")]
|
||||
use super::{delete_free_version_remote_object_then, encode_dir_object, get_transitioned_object_reader_with_tier_manager};
|
||||
@@ -5959,6 +6146,231 @@ mod tests {
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use uuid::Uuid;
|
||||
|
||||
fn recovery_test_lock() -> (Arc<rustfs_lock::NamespaceLock>, Vec<Arc<dyn rustfs_lock::LockClient>>) {
|
||||
let clients: Vec<Arc<dyn rustfs_lock::LockClient>> = (0..3)
|
||||
.map(|_| {
|
||||
Arc::new(rustfs_lock::LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())))
|
||||
as Arc<dyn rustfs_lock::LockClient>
|
||||
})
|
||||
.collect();
|
||||
let lock =
|
||||
rustfs_lock::NamespaceLock::with_clients_and_quorum(format!("recovery-test-{}", Uuid::new_v4()), clients.clone(), 2);
|
||||
(Arc::new(lock), clients)
|
||||
}
|
||||
|
||||
async fn acquire_recovery_test_lock(
|
||||
lock: Arc<rustfs_lock::NamespaceLock>,
|
||||
owner: String,
|
||||
) -> crate::error::Result<Option<rustfs_lock::NamespaceLockGuard>> {
|
||||
// Keep this lease longer than the virtual-time cooldown/idle advances;
|
||||
// production uses the namespace wrapper's renewable 30-second lease.
|
||||
lock.lock_guard(
|
||||
rustfs_lock::ObjectKey::new(crate::disk::RUSTFS_META_BUCKET, super::TIER_FREE_VERSION_RECOVERY_LEADER_LOCK),
|
||||
&owner,
|
||||
StdDuration::from_millis(5),
|
||||
StdDuration::from_secs(3600),
|
||||
)
|
||||
.await
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
fn recovery_test_retry_delay() -> StdDuration {
|
||||
StdDuration::from_secs(15)
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial]
|
||||
async fn tier_free_version_recovery_eight_coordinators_hold_one_leader_and_take_over() {
|
||||
let (lock, _clients) = recovery_test_lock();
|
||||
let (pages_tx, mut pages_rx) = tokio::sync::mpsc::unbounded_channel();
|
||||
let mut tasks = Vec::new();
|
||||
let mut cancels = Vec::new();
|
||||
for node in 0..8 {
|
||||
let cancel = CancellationToken::new();
|
||||
cancels.push(cancel.clone());
|
||||
let lock = Arc::clone(&lock);
|
||||
let pages_tx = pages_tx.clone();
|
||||
tasks.push(tokio::spawn(async move {
|
||||
run_tier_free_version_recovery_coordinator(
|
||||
cancel,
|
||||
recovery_test_retry_delay,
|
||||
move || acquire_recovery_test_lock(Arc::clone(&lock), format!("node-{node}")),
|
||||
move |leader_cancel| {
|
||||
let pages_tx = pages_tx.clone();
|
||||
async move {
|
||||
let mut pages = 0;
|
||||
run_tier_free_version_recovery_loop(
|
||||
leader_cancel,
|
||||
ExpiryState::new(),
|
||||
std::convert::identity,
|
||||
move |bucket, object, _| {
|
||||
pages += 1;
|
||||
pages_tx.send((node, bucket, object)).expect("page observer should be open");
|
||||
std::future::ready(Ok(free_version_recovery_stats(0, 0, pages == 1)))
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}));
|
||||
}
|
||||
let (leader, bucket, object) = pages_rx.recv().await.expect("one coordinator should start recovery");
|
||||
assert!(bucket.is_none() && object.is_none());
|
||||
// The same leader must survive both a truncated page's cooldown and
|
||||
// complete idle sweeps. All other coordinators keep probing the lock.
|
||||
let mut observed_pages = 1;
|
||||
for _ in 0..27 {
|
||||
tokio::time::advance(StdDuration::from_secs(60)).await;
|
||||
for _ in 0..100 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
while let Ok((node, bucket, object)) = pages_rx.try_recv() {
|
||||
assert_eq!(node, leader, "followers must not repeat a namespace sweep");
|
||||
if observed_pages == 1 {
|
||||
assert_eq!(bucket.as_deref(), Some("bucket"));
|
||||
assert_eq!(object.as_deref(), Some("object"));
|
||||
} else {
|
||||
assert!(bucket.is_none() && object.is_none(), "complete sweeps should reset their cursors");
|
||||
}
|
||||
observed_pages += 1;
|
||||
}
|
||||
}
|
||||
assert!(observed_pages >= 7, "the leader should finish a full ten-minute idle wait");
|
||||
cancels[leader].cancel();
|
||||
let (successor, bucket, object) = tokio::time::timeout(StdDuration::from_secs(60), pages_rx.recv())
|
||||
.await
|
||||
.expect("a follower should take over after leader shutdown")
|
||||
.expect("page observer should remain open");
|
||||
assert_ne!(successor, leader);
|
||||
assert!(bucket.is_none() && object.is_none(), "a new leader must restart discovery safely");
|
||||
for cancel in cancels {
|
||||
cancel.cancel();
|
||||
}
|
||||
for task in tasks {
|
||||
task.await.expect("all coordinators should shut down cleanly");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial]
|
||||
async fn tier_free_version_recovery_lock_loss_waits_for_active_page_cleanup() {
|
||||
let (lock, _clients) = recovery_test_lock();
|
||||
let cancel = CancellationToken::new();
|
||||
let loop_cancel = cancel.clone();
|
||||
let (page_tx, mut page_rx) = tokio::sync::mpsc::unbounded_channel();
|
||||
let cleanup_started = Arc::new(tokio::sync::Notify::new());
|
||||
let finish_cleanup = Arc::new(tokio::sync::Notify::new());
|
||||
let observed_cleanup = Arc::clone(&cleanup_started);
|
||||
let release_cleanup = Arc::clone(&finish_cleanup);
|
||||
let task = tokio::spawn(async move {
|
||||
run_tier_free_version_recovery_coordinator(
|
||||
loop_cancel,
|
||||
recovery_test_retry_delay,
|
||||
move || acquire_recovery_test_lock(Arc::clone(&lock), "old-leader".to_string()),
|
||||
move |leader_cancel| {
|
||||
let page_tx = page_tx.clone();
|
||||
let cleanup_started = Arc::clone(&cleanup_started);
|
||||
let finish_cleanup = Arc::clone(&finish_cleanup);
|
||||
async move {
|
||||
run_tier_free_version_recovery_loop(
|
||||
leader_cancel,
|
||||
ExpiryState::new(),
|
||||
std::convert::identity,
|
||||
move |_, _, page_cancel| {
|
||||
let cleanup_started = Arc::clone(&cleanup_started);
|
||||
let finish_cleanup = Arc::clone(&finish_cleanup);
|
||||
page_tx.send(()).expect("page observer should remain open");
|
||||
async move {
|
||||
page_cancel.cancelled().await;
|
||||
cleanup_started.notify_one();
|
||||
finish_cleanup.notified().await;
|
||||
Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "cancelled page").into())
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
});
|
||||
page_rx.recv().await.expect("leader should start an active page");
|
||||
// Let authoritative backend leases expire before the next heartbeat
|
||||
// can renew them. A failed refresh must cancel the active page.
|
||||
tokio::time::advance(StdDuration::from_secs(3600)).await;
|
||||
tokio::time::timeout(StdDuration::from_secs(1), observed_cleanup.notified())
|
||||
.await
|
||||
.expect("expired backend leases must trigger recovery cancellation");
|
||||
assert!(!task.is_finished(), "lock loss must wait for the active page's cleanup");
|
||||
assert!(page_rx.try_recv().is_err(), "the old leader must not start another page");
|
||||
cancel.cancel();
|
||||
release_cleanup.notify_one();
|
||||
task.await.expect("coordinator should finish cleanup before exiting");
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn tier_free_version_recovery_coordinator_retries_without_scanning_after_lock_error() {
|
||||
let cancel = CancellationToken::new();
|
||||
let loop_cancel = cancel.clone();
|
||||
let attempts = Arc::new(AtomicUsize::new(0));
|
||||
let observed_attempts = Arc::clone(&attempts);
|
||||
let task = tokio::spawn(async move {
|
||||
run_tier_free_version_recovery_coordinator(
|
||||
loop_cancel,
|
||||
recovery_test_retry_delay,
|
||||
move || {
|
||||
observed_attempts.fetch_add(1, Ordering::SeqCst);
|
||||
std::future::ready(Err(rustfs_lock::LockError::InsufficientNodes {
|
||||
required: 2,
|
||||
available: 1,
|
||||
}
|
||||
.into()))
|
||||
},
|
||||
|_| async { panic!("a node without ownership must never start discovery") },
|
||||
)
|
||||
.await;
|
||||
});
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 1);
|
||||
tokio::time::advance(StdDuration::from_secs(14)).await;
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 1, "lock errors must not cause a tight retry loop");
|
||||
tokio::time::advance(StdDuration::from_secs(1)).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 2);
|
||||
cancel.cancel();
|
||||
task.await.expect("a follower should respond to shutdown during retry wait");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tier_free_version_recovery_enqueue_cancels_while_waiting_for_state_lock() {
|
||||
let state = ExpiryState::new();
|
||||
let (tx, mut rx) = tokio::sync::mpsc::channel(1);
|
||||
let mut held = state.write().await;
|
||||
held.tasks_tx.push(tx);
|
||||
let cancel = CancellationToken::new();
|
||||
let enqueue_cancel = cancel.clone();
|
||||
let enqueue_state = Arc::clone(&state);
|
||||
let enqueue = tokio::spawn(async move {
|
||||
enqueue_recovered_free_version_with_state(&enqueue_state, ObjectInfo::default(), &enqueue_cancel).await
|
||||
});
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!enqueue.is_finished(), "enqueue should be blocked on the state lock");
|
||||
cancel.cancel();
|
||||
let queued = tokio::time::timeout(StdDuration::from_secs(1), enqueue)
|
||||
.await
|
||||
.expect("cancellation must not wait for the state write guard")
|
||||
.expect("enqueue should return cleanly");
|
||||
assert!(!queued);
|
||||
assert_eq!(held.pending_tasks(), 0);
|
||||
assert_eq!(held.stats.missed_free_vers_tasks(), 0);
|
||||
assert!(rx.try_recv().is_err(), "a cancelled recovery must not enqueue a task");
|
||||
drop(held);
|
||||
assert!(!enqueue_recovered_free_version_with_state(&state, ObjectInfo::default(), &cancel).await);
|
||||
assert!(rx.try_recv().is_err(), "cancellation must win even when the lock is available");
|
||||
}
|
||||
|
||||
fn free_version_recovery_stats(enqueued: usize, failed: usize, truncated: bool) -> FreeVersionRecoveryStats {
|
||||
FreeVersionRecoveryStats {
|
||||
scanned: enqueued.saturating_add(failed),
|
||||
@@ -7638,7 +8050,7 @@ mod tests {
|
||||
assert_eq!(state.read().await.stats.pending_tasks(), 1);
|
||||
|
||||
// The single-slot queue is full for both enqueue paths.
|
||||
assert!(!enqueue_recovered_free_version_with_state(&state, oi.clone()).await);
|
||||
assert!(!enqueue_recovered_free_version_with_state(&state, oi.clone(), &CancellationToken::new()).await);
|
||||
assert_eq!(state.read().await.stats.pending_tasks(), 1);
|
||||
assert_eq!(state.read().await.stats.missed_free_vers_tasks(), 1);
|
||||
|
||||
@@ -7670,7 +8082,7 @@ mod tests {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let queued = enqueue_recovered_free_version_with_state(&state, oi).await;
|
||||
let queued = enqueue_recovered_free_version_with_state(&state, oi, &CancellationToken::new()).await;
|
||||
let state = state.read().await;
|
||||
|
||||
assert!(!queued);
|
||||
@@ -7799,8 +8211,8 @@ mod tests {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let first = enqueue_recovered_free_version_with_state(&state, oi.clone()).await;
|
||||
let second = enqueue_recovered_free_version_with_state(&state, oi).await;
|
||||
let first = enqueue_recovered_free_version_with_state(&state, oi.clone(), &CancellationToken::new()).await;
|
||||
let second = enqueue_recovered_free_version_with_state(&state, oi, &CancellationToken::new()).await;
|
||||
let state = state.read().await;
|
||||
|
||||
assert!(first);
|
||||
@@ -7901,7 +8313,7 @@ mod tests {
|
||||
};
|
||||
|
||||
assert!(
|
||||
super::enqueue_recovered_free_version(&ecstore, oi).await,
|
||||
super::enqueue_recovered_free_version_with_cancel(&ecstore, oi, &CancellationToken::new()).await,
|
||||
"the resized production worker queue should accept the task"
|
||||
);
|
||||
stop_tx.send(None).await.expect("worker stop signal should be delivered");
|
||||
@@ -13028,6 +13440,171 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn queued_noncurrent_batch_expiry_does_not_delete_from_table_bucket() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let bucket = format!("table-noncurrent-lifecycle-{}", Uuid::new_v4().simple());
|
||||
let object = "tables/table-id/data/part-00001.parquet";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
metadata_sys::update_in(
|
||||
&ecstore.ctx,
|
||||
&bucket,
|
||||
BUCKET_VERSIONING_CONFIG,
|
||||
b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
|
||||
)
|
||||
.await
|
||||
.expect("bucket versioning should be enabled");
|
||||
|
||||
let mut old_reader = PutObjReader::from_vec(b"old table data".to_vec());
|
||||
let old = ecstore
|
||||
.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut old_reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("the historical table version should be created");
|
||||
let old_version_id = old.version_id.expect("the historical version should have an identity");
|
||||
let mut current_reader = PutObjReader::from_vec(b"current table data".to_vec());
|
||||
ecstore
|
||||
.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut current_reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("the current table version should be created");
|
||||
|
||||
let incarnation = ecstore
|
||||
.bucket_incarnation_id_from_disk(&bucket)
|
||||
.await
|
||||
.expect("bucket incarnation should be available");
|
||||
let publication_lock = ecstore
|
||||
.new_ns_lock(&bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
|
||||
.await
|
||||
.expect("table-bucket publication lock should be created");
|
||||
let enable_guard = publication_lock
|
||||
.get_write_lock(get_lock_acquire_timeout())
|
||||
.await
|
||||
.expect("table-bucket enablement should acquire the publication lock");
|
||||
|
||||
let delete_store = ecstore.clone();
|
||||
let delete_bucket = bucket.clone();
|
||||
let delete_object = object.to_string();
|
||||
let mut delete = tokio::spawn(async move {
|
||||
let target = ObjectToDelete {
|
||||
object_name: delete_object,
|
||||
version_id: Some(old_version_id),
|
||||
..Default::default()
|
||||
};
|
||||
crate::bucket::lifecycle::object_handlers_common::delete_object_versions(
|
||||
&delete_store,
|
||||
&delete_bucket,
|
||||
std::slice::from_ref(&target),
|
||||
lifecycle::Event::default(),
|
||||
incarnation,
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::task::yield_now().await;
|
||||
assert!(
|
||||
tokio::time::timeout(StdDuration::from_millis(100), &mut delete)
|
||||
.await
|
||||
.is_err(),
|
||||
"noncurrent lifecycle cleanup must wait for the table-bucket publication fence"
|
||||
);
|
||||
|
||||
let sys = metadata_sys::bucket_metadata_sys_of(&ecstore.ctx).expect("metadata system should be initialized");
|
||||
let sys = sys.read().await.clone();
|
||||
let mut metadata = (*sys.get(&bucket).await.expect("bucket metadata should exist")).clone();
|
||||
metadata.table_bucket_config_json = br#"{"enabled":true}"#.to_vec();
|
||||
sys.persist_and_set(metadata)
|
||||
.await
|
||||
.expect("table bucket marker should be persisted");
|
||||
sys.reload_from_store(&bucket)
|
||||
.await
|
||||
.expect("table bucket marker should become authoritative");
|
||||
drop(enable_guard);
|
||||
|
||||
let failed = tokio::time::timeout(StdDuration::from_secs(2), delete)
|
||||
.await
|
||||
.expect("noncurrent lifecycle cleanup should resume after the fence is released")
|
||||
.expect("noncurrent lifecycle cleanup task should join");
|
||||
assert_eq!(failed, 1, "table-bucket noncurrent cleanup must be rejected");
|
||||
assert!(
|
||||
ecstore
|
||||
.get_object_info(
|
||||
&bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
version_id: Some(old_version_id.to_string()),
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.is_ok(),
|
||||
"table data must remain after queued noncurrent cleanup is rejected"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn lifecycle_transition_admission_holds_publication_reader() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let bucket = format!("lifecycle-transition-{}", Uuid::new_v4().simple());
|
||||
let object = "tables/table-id/data/part-00001.parquet";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
let object_info = ObjectInfo {
|
||||
bucket: bucket.clone(),
|
||||
name: object.to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let publication_lock = ecstore
|
||||
.new_ns_lock(&bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
|
||||
.await
|
||||
.expect("table-bucket publication lock should be created");
|
||||
let enable_guard = publication_lock
|
||||
.get_write_lock(get_lock_acquire_timeout())
|
||||
.await
|
||||
.expect("publication writer should be acquired");
|
||||
let admission_store = ecstore.clone();
|
||||
let admission_object = object_info.clone();
|
||||
let mut admission =
|
||||
tokio::spawn(async move { lifecycle_transition_publication_guard(&admission_store, &admission_object).await });
|
||||
assert!(
|
||||
tokio::time::timeout(StdDuration::from_millis(100), &mut admission)
|
||||
.await
|
||||
.is_err(),
|
||||
"transition admission must wait for an in-progress table publication"
|
||||
);
|
||||
drop(enable_guard);
|
||||
|
||||
let (transition_guard, incarnation) = tokio::time::timeout(StdDuration::from_secs(2), admission)
|
||||
.await
|
||||
.expect("transition admission should resume after publication")
|
||||
.expect("transition admission task should join")
|
||||
.expect("transition admission should succeed");
|
||||
assert_ne!(incarnation, Uuid::nil(), "transition admission must bind a bucket incarnation");
|
||||
let blocked_writer = publication_lock.get_write_lock(StdDuration::from_millis(100)).await;
|
||||
assert!(blocked_writer.is_err(), "transition admission must retain the publication read lock");
|
||||
drop(transition_guard);
|
||||
publication_lock
|
||||
.get_write_lock(get_lock_acquire_timeout())
|
||||
.await
|
||||
.expect("publication writer should proceed after transition admission releases its guard");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn queued_delete_all_rechecks_a_same_id_rule_moved_into_the_future() {
|
||||
@@ -13630,6 +14207,10 @@ mod tests {
|
||||
.await;
|
||||
}
|
||||
|
||||
fn fast_tier_free_version_recovery_leader_retry_delay() -> StdDuration {
|
||||
StdDuration::from_millis(10)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn tier_free_version_recovery_production_entrypoint_enqueues_seeded_item() {
|
||||
@@ -13645,8 +14226,13 @@ mod tests {
|
||||
seed_recoverable_free_version(&disk_paths, &bucket, object, None, None).await;
|
||||
|
||||
let started = OnceLock::new();
|
||||
let recovery = super::spawn_tier_free_version_recovery_once(Arc::clone(&ecstore), &started)
|
||||
.expect("production recovery entrypoint should start once");
|
||||
let recovery = super::spawn_tier_free_version_recovery_once_with_options(
|
||||
Arc::clone(&ecstore),
|
||||
&started,
|
||||
format!("tier-free-version-recovery/test/{}", Uuid::new_v4()),
|
||||
fast_tier_free_version_recovery_leader_retry_delay,
|
||||
)
|
||||
.expect("production recovery entrypoint should start once");
|
||||
let task = tokio::time::timeout(StdDuration::from_secs(30), async { recovery_rx.lock().await.recv().await })
|
||||
.await
|
||||
.expect("production recovery entrypoint should enqueue the seeded item")
|
||||
@@ -13689,13 +14275,14 @@ mod tests {
|
||||
};
|
||||
let mut recovery_rx = recovery_rx.lock().await;
|
||||
assert!(
|
||||
super::enqueue_recovered_free_version(
|
||||
super::enqueue_recovered_free_version_with_cancel(
|
||||
&ecstore,
|
||||
ObjectInfo {
|
||||
bucket: "prefill".to_string(),
|
||||
name: "prefill".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
&CancellationToken::new(),
|
||||
)
|
||||
.await,
|
||||
"the production recovery queue should accept its first task"
|
||||
|
||||
@@ -36,6 +36,10 @@ pub async fn delete_object_versions(
|
||||
_lc_event: lifecycle::Event,
|
||||
bucket_incarnation_id: Uuid,
|
||||
) -> usize {
|
||||
if to_del.is_empty() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if to_del.iter().any(|target| {
|
||||
target.version_id.is_none()
|
||||
|| (target.version_id.is_some_and(|version_id| version_id.is_nil()) && target.expected_identity.is_none())
|
||||
@@ -52,6 +56,25 @@ pub async fn delete_object_versions(
|
||||
return to_del.len();
|
||||
}
|
||||
|
||||
let Some(publication_guard) = super::bucket_lifecycle_ops::lifecycle_expiry_publication_guard(
|
||||
api,
|
||||
bucket,
|
||||
to_del[0].object_name.as_str(),
|
||||
bucket_incarnation_id,
|
||||
)
|
||||
.await
|
||||
else {
|
||||
debug!(
|
||||
event = EVENT_LIFECYCLE_CLEANUP_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
bucket,
|
||||
target_count = to_del.len(),
|
||||
reason = "publication_admission_rejected",
|
||||
"Skipped lifecycle noncurrent version cleanup"
|
||||
);
|
||||
return to_del.len();
|
||||
};
|
||||
let delete_config_snapshot = match ReplicationObjectBridge::delete_request_config(api, bucket).await {
|
||||
Ok(snapshot) => Arc::new(snapshot),
|
||||
Err(err) => {
|
||||
@@ -78,17 +101,13 @@ pub async fn delete_object_versions(
|
||||
remaining = &[];
|
||||
}
|
||||
|
||||
let (mut deleted_objs, errors) = api
|
||||
.delete_objects(
|
||||
bucket,
|
||||
to_del.to_vec(),
|
||||
ObjectOptions {
|
||||
delete_replication_config_snapshot: Some(Arc::clone(&delete_config_snapshot)),
|
||||
expected_bucket_incarnation_id: Some(bucket_incarnation_id),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let mut delete_options = ObjectOptions {
|
||||
delete_replication_config_snapshot: Some(Arc::clone(&delete_config_snapshot)),
|
||||
expected_bucket_incarnation_id: Some(bucket_incarnation_id),
|
||||
..Default::default()
|
||||
};
|
||||
delete_options.add_namespace_lock_guard(&publication_guard);
|
||||
let (mut deleted_objs, errors) = api.delete_objects(bucket, to_del.to_vec(), delete_options).await;
|
||||
failed += errors.iter().filter(|err| err.is_some()).count();
|
||||
|
||||
for (i, deleted_obj) in deleted_objs.iter_mut().enumerate() {
|
||||
|
||||
@@ -1122,9 +1122,11 @@ pub(crate) async fn resume_recovery_disposition(
|
||||
// Lock order is control record, then legacy source. Disposition revisions
|
||||
// use ETag CAS and never acquire either namespace lock internally.
|
||||
let control_lock = api.new_ns_lock(RUSTFS_META_BUCKET, &control_object).await?;
|
||||
let control_guard = control_lock
|
||||
.get_read_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await?;
|
||||
let control_guard = Arc::new(
|
||||
control_lock
|
||||
.get_read_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await?,
|
||||
);
|
||||
let source_lock = api
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &initial.disposition.identity.canonical_source_path)
|
||||
.await?;
|
||||
@@ -1504,7 +1506,7 @@ fn control_is_abandoned_successor(disposition: &IlmRecoveryDisposition, control:
|
||||
async fn abandon_recovery_control_fenced(
|
||||
api: Arc<ECStore>,
|
||||
disposition: &IlmRecoveryDisposition,
|
||||
control_guard: &rustfs_lock::NamespaceLockGuard,
|
||||
control_guard: &Arc<rustfs_lock::NamespaceLockGuard>,
|
||||
source_guard: &rustfs_lock::NamespaceLockGuard,
|
||||
proof: &IlmRecoveryExportFleetProofToken,
|
||||
) -> EcstoreResult<()> {
|
||||
@@ -1557,7 +1559,7 @@ async fn abandon_recovery_control_fenced(
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
options.add_namespace_lock_guard(control_guard);
|
||||
options.add_owned_write_lock(Arc::clone(control_guard), RUSTFS_META_BUCKET, &object);
|
||||
options.add_namespace_lock_guard(source_guard);
|
||||
let write_result = config_boundary::save_config_with_opts(api.clone(), &object, encoded, &options).await;
|
||||
let observed = match load_recovery_control_no_lock(api.clone(), disposition).await {
|
||||
@@ -1630,7 +1632,7 @@ async fn delete_exact_local_recovery_copy(
|
||||
api: Arc<ECStore>,
|
||||
copy: &IlmRecoverySourceCopy,
|
||||
source_schema: &str,
|
||||
control_guard: &rustfs_lock::NamespaceLockGuard,
|
||||
control_guard: &Arc<rustfs_lock::NamespaceLockGuard>,
|
||||
source_guard: &rustfs_lock::NamespaceLockGuard,
|
||||
) -> EcstoreResult<()> {
|
||||
let set = recovery_copy_set(&api, ©.authority)?;
|
||||
@@ -1704,7 +1706,7 @@ async fn save_recovery_disposition_step_fenced(
|
||||
api: Arc<ECStore>,
|
||||
current: &ObservedIlmRecoveryDisposition,
|
||||
next: &IlmRecoveryDisposition,
|
||||
control_guard: &rustfs_lock::NamespaceLockGuard,
|
||||
control_guard: &Arc<rustfs_lock::NamespaceLockGuard>,
|
||||
source_guard: &rustfs_lock::NamespaceLockGuard,
|
||||
proof: &IlmRecoveryExportFleetProofToken,
|
||||
) -> EcstoreResult<ObservedIlmRecoveryDisposition> {
|
||||
|
||||
@@ -20,7 +20,7 @@ use std::time::Duration;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::bucket::lifecycle::bucket_lifecycle_ops::enqueue_recovered_free_version;
|
||||
use crate::bucket::lifecycle::bucket_lifecycle_ops::enqueue_recovered_free_version_with_cancel;
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::Result;
|
||||
use crate::object_api::ObjectInfo;
|
||||
@@ -201,7 +201,11 @@ pub(super) async fn recover_tier_free_versions_with_cancel(
|
||||
return Err(tier_free_version_recovery_cancelled());
|
||||
}
|
||||
retry_cursor.visit(&oi);
|
||||
if !record_recovered_free_version_enqueue(&mut stats, enqueue_recovered_free_version(&api, oi).await) {
|
||||
let queued = enqueue_recovered_free_version_with_cancel(&api, oi, &cancel_token).await;
|
||||
if cancel_token.is_cancelled() {
|
||||
return Err(tier_free_version_recovery_cancelled());
|
||||
}
|
||||
if !record_recovered_free_version_enqueue(&mut stats, queued) {
|
||||
let (bucket_marker, object_marker) = retry_cursor.retry_markers();
|
||||
stats.truncated = true;
|
||||
stats.next_bucket_marker = bucket_marker;
|
||||
|
||||
@@ -1210,6 +1210,7 @@ impl BucketMetadata {
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
}
|
||||
self.notification_config = None;
|
||||
if !self.notification_config_xml.is_empty()
|
||||
&& let Err(e) = deserialize::<NotificationConfiguration>(&self.notification_config_xml)
|
||||
.map(|c| self.notification_config = Some(c))
|
||||
@@ -1224,6 +1225,7 @@ impl BucketMetadata {
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
}
|
||||
self.lifecycle_config = None;
|
||||
if !self.lifecycle_config_xml.is_empty()
|
||||
&& let Err(e) =
|
||||
deserialize::<BucketLifecycleConfiguration>(&self.lifecycle_config_xml).map(|c| self.lifecycle_config = Some(c))
|
||||
@@ -1238,6 +1240,7 @@ impl BucketMetadata {
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
}
|
||||
self.object_lock_config = None;
|
||||
if !self.object_lock_config_xml.is_empty()
|
||||
&& let Err(e) =
|
||||
deserialize::<ObjectLockConfiguration>(&self.object_lock_config_xml).map(|c| self.object_lock_config = Some(c))
|
||||
@@ -1267,6 +1270,7 @@ impl BucketMetadata {
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
}
|
||||
self.sse_config = None;
|
||||
if !self.encryption_config_xml.is_empty()
|
||||
&& let Err(e) =
|
||||
deserialize::<ServerSideEncryptionConfiguration>(&self.encryption_config_xml).map(|c| self.sse_config = Some(c))
|
||||
@@ -1281,6 +1285,7 @@ impl BucketMetadata {
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
}
|
||||
self.tagging_config = None;
|
||||
if !self.tagging_config_xml.is_empty()
|
||||
&& let Err(e) = deserialize::<Tagging>(&self.tagging_config_xml).map(|c| self.tagging_config = Some(c))
|
||||
{
|
||||
@@ -1346,6 +1351,7 @@ impl BucketMetadata {
|
||||
} else {
|
||||
self.bucket_target_config = Some(BucketTargets::default());
|
||||
}
|
||||
self.cors_config = None;
|
||||
if !self.cors_config_xml.is_empty()
|
||||
&& let Err(e) = deserialize::<CORSConfiguration>(&self.cors_config_xml).map(|c| self.cors_config = Some(c))
|
||||
{
|
||||
@@ -1359,6 +1365,7 @@ impl BucketMetadata {
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
}
|
||||
self.logging_config = None;
|
||||
if !self.logging_config_xml.is_empty()
|
||||
&& let Err(e) = deserialize::<BucketLoggingStatus>(&self.logging_config_xml).map(|c| self.logging_config = Some(c))
|
||||
{
|
||||
@@ -1372,6 +1379,7 @@ impl BucketMetadata {
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
}
|
||||
self.website_config = None;
|
||||
if !self.website_config_xml.is_empty()
|
||||
&& let Err(e) = deserialize::<WebsiteConfiguration>(&self.website_config_xml).map(|c| self.website_config = Some(c))
|
||||
{
|
||||
@@ -1385,6 +1393,7 @@ impl BucketMetadata {
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
}
|
||||
self.accelerate_config = None;
|
||||
if !self.accelerate_config_xml.is_empty()
|
||||
&& let Err(e) =
|
||||
deserialize::<AccelerateConfiguration>(&self.accelerate_config_xml).map(|c| self.accelerate_config = Some(c))
|
||||
@@ -1399,6 +1408,7 @@ impl BucketMetadata {
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
}
|
||||
self.request_payment_config = None;
|
||||
if !self.request_payment_config_xml.is_empty()
|
||||
&& let Err(e) = deserialize::<RequestPaymentConfiguration>(&self.request_payment_config_xml)
|
||||
.map(|c| self.request_payment_config = Some(c))
|
||||
@@ -1413,6 +1423,7 @@ impl BucketMetadata {
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
}
|
||||
self.public_access_block_config = None;
|
||||
if !self.public_access_block_config_xml.is_empty()
|
||||
&& let Err(e) = deserialize::<PublicAccessBlockConfiguration>(&self.public_access_block_config_xml)
|
||||
.map(|c| self.public_access_block_config = Some(c))
|
||||
@@ -1973,6 +1984,156 @@ mod test {
|
||||
assert_eq!(bm.lifecycle_config_xml, malformed_xml, "raw bytes are retained");
|
||||
}
|
||||
|
||||
const REPARSE_XML_FIXTURES: [(&str, &[u8]); 13] = [
|
||||
(BUCKET_NOTIFICATION_CONFIG, br#"<NotificationConfiguration/>"#),
|
||||
(
|
||||
BUCKET_LIFECYCLE_CONFIG,
|
||||
br#"<LifecycleConfiguration><Rule><ID>keep</ID><Status>Enabled</Status><Expiration><Days>30</Days></Expiration></Rule></LifecycleConfiguration>"#,
|
||||
),
|
||||
(
|
||||
OBJECT_LOCK_CONFIG,
|
||||
br#"<ObjectLockConfiguration><ObjectLockEnabled>Enabled</ObjectLockEnabled><Rule><DefaultRetention><Mode>GOVERNANCE</Mode><Days>7</Days></DefaultRetention></Rule></ObjectLockConfiguration>"#,
|
||||
),
|
||||
(BUCKET_VERSIONING_CONFIG, br#"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>"#),
|
||||
(
|
||||
BUCKET_SSECONFIG,
|
||||
br#"<ServerSideEncryptionConfiguration><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>AES256</SSEAlgorithm></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>"#,
|
||||
),
|
||||
(BUCKET_TAGGING_CONFIG, br#"<Tagging><TagSet><Tag><Key>team</Key><Value>blue</Value></Tag></TagSet></Tagging>"#),
|
||||
(
|
||||
BUCKET_REPLICATION_CONFIG,
|
||||
br#"<ReplicationConfiguration><Role>arn:aws:iam::123456789012:role/replication-role</Role><Rule><ID>copy</ID><Status>Enabled</Status><Prefix>documents/</Prefix><Destination><Bucket>arn:aws:s3:::destination-bucket</Bucket></Destination></Rule></ReplicationConfiguration>"#,
|
||||
),
|
||||
(
|
||||
BUCKET_CORS_CONFIG,
|
||||
br#"<CORSConfiguration><CORSRule><AllowedOrigin>https://example.test</AllowedOrigin><AllowedMethod>GET</AllowedMethod></CORSRule></CORSConfiguration>"#,
|
||||
),
|
||||
(BUCKET_LOGGING_CONFIG, br#"<BucketLoggingStatus/>"#),
|
||||
(BUCKET_WEBSITE_CONFIG, br#"<WebsiteConfiguration><IndexDocument><Suffix>index.html</Suffix></IndexDocument></WebsiteConfiguration>"#),
|
||||
(BUCKET_ACCELERATE_CONFIG, br#"<AccelerateConfiguration><Status>Enabled</Status></AccelerateConfiguration>"#),
|
||||
(BUCKET_REQUEST_PAYMENT_CONFIG, br#"<RequestPaymentConfiguration><Payer>BucketOwner</Payer></RequestPaymentConfiguration>"#),
|
||||
(
|
||||
BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG,
|
||||
br#"<PublicAccessBlockConfiguration><BlockPublicAcls>true</BlockPublicAcls><IgnorePublicAcls>true</IgnorePublicAcls><BlockPublicPolicy>true</BlockPublicPolicy><RestrictPublicBuckets>true</RestrictPublicBuckets></PublicAccessBlockConfiguration>"#,
|
||||
),
|
||||
];
|
||||
|
||||
fn reparse_xml_state(bm: &BucketMetadata, config: &str) -> &'static str {
|
||||
fn state<T>(raw: &[u8], parsed: &Option<T>) -> &'static str {
|
||||
match ConfigState::of(raw, parsed) {
|
||||
ConfigState::Absent => "absent",
|
||||
ConfigState::Valid(_) => "valid",
|
||||
ConfigState::Unreadable { .. } => "unreadable",
|
||||
}
|
||||
}
|
||||
match config {
|
||||
BUCKET_NOTIFICATION_CONFIG => state(&bm.notification_config_xml, &bm.notification_config),
|
||||
BUCKET_LIFECYCLE_CONFIG => state(&bm.lifecycle_config_xml, &bm.lifecycle_config),
|
||||
OBJECT_LOCK_CONFIG => state(&bm.object_lock_config_xml, &bm.object_lock_config),
|
||||
BUCKET_VERSIONING_CONFIG => state(&bm.versioning_config_xml, &bm.versioning_config),
|
||||
BUCKET_SSECONFIG => state(&bm.encryption_config_xml, &bm.sse_config),
|
||||
BUCKET_TAGGING_CONFIG => state(&bm.tagging_config_xml, &bm.tagging_config),
|
||||
BUCKET_REPLICATION_CONFIG => state(&bm.replication_config_xml, &bm.replication_config),
|
||||
BUCKET_CORS_CONFIG => state(&bm.cors_config_xml, &bm.cors_config),
|
||||
BUCKET_LOGGING_CONFIG => state(&bm.logging_config_xml, &bm.logging_config),
|
||||
BUCKET_WEBSITE_CONFIG => state(&bm.website_config_xml, &bm.website_config),
|
||||
BUCKET_ACCELERATE_CONFIG => state(&bm.accelerate_config_xml, &bm.accelerate_config),
|
||||
BUCKET_REQUEST_PAYMENT_CONFIG => state(&bm.request_payment_config_xml, &bm.request_payment_config),
|
||||
BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG => state(&bm.public_access_block_config_xml, &bm.public_access_block_config),
|
||||
_ => panic!("unknown XML config: {config}"),
|
||||
}
|
||||
}
|
||||
|
||||
// A persisted payload can change without going through the request decoder.
|
||||
fn replace_xml_raw(bm: &mut BucketMetadata, config: &str, raw: Vec<u8>) {
|
||||
match config {
|
||||
BUCKET_NOTIFICATION_CONFIG => bm.notification_config_xml = raw,
|
||||
BUCKET_LIFECYCLE_CONFIG => bm.lifecycle_config_xml = raw,
|
||||
OBJECT_LOCK_CONFIG => bm.object_lock_config_xml = raw,
|
||||
BUCKET_VERSIONING_CONFIG => bm.versioning_config_xml = raw,
|
||||
BUCKET_SSECONFIG => bm.encryption_config_xml = raw,
|
||||
BUCKET_TAGGING_CONFIG => bm.tagging_config_xml = raw,
|
||||
BUCKET_REPLICATION_CONFIG => bm.replication_config_xml = raw,
|
||||
BUCKET_CORS_CONFIG => bm.cors_config_xml = raw,
|
||||
BUCKET_LOGGING_CONFIG => bm.logging_config_xml = raw,
|
||||
BUCKET_WEBSITE_CONFIG => bm.website_config_xml = raw,
|
||||
BUCKET_ACCELERATE_CONFIG => bm.accelerate_config_xml = raw,
|
||||
BUCKET_REQUEST_PAYMENT_CONFIG => bm.request_payment_config_xml = raw,
|
||||
BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG => bm.public_access_block_config_xml = raw,
|
||||
_ => panic!("unknown XML config: {config}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reparse_xml_rejects_stale_views_after_corruption() {
|
||||
let corrupt = b"<not-a-valid-document";
|
||||
for (config, valid) in REPARSE_XML_FIXTURES {
|
||||
let mut bm = BucketMetadata::new("reparse-corrupt");
|
||||
bm.update_config(config, valid.to_vec()).expect("store valid XML");
|
||||
bm.parse_all_configs().expect("parse valid XML");
|
||||
assert_eq!(reparse_xml_state(&bm, config), "valid", "{config}: valid setup");
|
||||
|
||||
replace_xml_raw(&mut bm, config, corrupt.to_vec());
|
||||
let before = bm.marshal_msg().expect("marshal corrupt raw metadata");
|
||||
bm.parse_all_configs()
|
||||
.expect("one corrupt config must not fail the metadata load");
|
||||
|
||||
assert_eq!(reparse_xml_state(&bm, config), "unreadable", "{config}: stale view accepted");
|
||||
assert_eq!(bm.xml_config_unreadable_len(config), Some(corrupt.len()), "{config}: missing refusal");
|
||||
assert_eq!(
|
||||
bm.marshal_msg().expect("marshal reparsed metadata"),
|
||||
before,
|
||||
"{config}: persisted bytes changed"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reparse_xml_clears_stale_views_after_raw_is_removed() {
|
||||
for (config, valid) in REPARSE_XML_FIXTURES {
|
||||
let mut bm = BucketMetadata::new("reparse-removed");
|
||||
bm.update_config(config, valid.to_vec()).expect("store valid XML");
|
||||
bm.parse_all_configs().expect("parse valid XML");
|
||||
assert_eq!(reparse_xml_state(&bm, config), "valid", "{config}: valid setup");
|
||||
|
||||
replace_xml_raw(&mut bm, config, Vec::new());
|
||||
let before = bm.marshal_msg().expect("marshal absent raw metadata");
|
||||
bm.parse_all_configs().expect("parse removed XML");
|
||||
|
||||
assert_eq!(reparse_xml_state(&bm, config), "absent", "{config}: stale view survived removal");
|
||||
assert!(!bm.xml_config_unreadable(config), "{config}: absence is not corruption");
|
||||
assert_eq!(
|
||||
bm.marshal_msg().expect("marshal reparsed metadata"),
|
||||
before,
|
||||
"{config}: persisted bytes changed"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reparse_xml_recovers_after_corrupt_raw_is_repaired() {
|
||||
for (config, valid) in REPARSE_XML_FIXTURES {
|
||||
let mut bm = BucketMetadata::new("reparse-repaired");
|
||||
bm.update_config(config, valid.to_vec()).expect("store valid XML");
|
||||
bm.parse_all_configs().expect("parse valid XML");
|
||||
replace_xml_raw(&mut bm, config, b"<not-a-valid-document".to_vec());
|
||||
bm.parse_all_configs().expect("parse corrupt XML");
|
||||
assert_eq!(reparse_xml_state(&bm, config), "unreadable", "{config}: corrupt setup");
|
||||
|
||||
replace_xml_raw(&mut bm, config, valid.to_vec());
|
||||
let before = bm.marshal_msg().expect("marshal repaired raw metadata");
|
||||
bm.parse_all_configs().expect("parse repaired XML");
|
||||
|
||||
assert_eq!(reparse_xml_state(&bm, config), "valid", "{config}: repaired config must become readable");
|
||||
assert!(!bm.xml_config_unreadable(config), "{config}: repaired config is not corrupt");
|
||||
assert_eq!(
|
||||
bm.marshal_msg().expect("marshal reparsed metadata"),
|
||||
before,
|
||||
"{config}: persisted bytes changed"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lifecycle_update_config_clears_parsed_config_on_delete() {
|
||||
let mut bm = BucketMetadata::new("test-bucket");
|
||||
|
||||
@@ -576,6 +576,27 @@ pub(crate) async fn update_in(
|
||||
Box::pin(update_with_sys(require_bucket_metadata_sys_in(ctx)?, bucket, config_file, data)).await
|
||||
}
|
||||
|
||||
/// Validate a caller's fence after acquiring the lifecycle and metadata
|
||||
/// transaction locks, before changing the configuration.
|
||||
pub(crate) async fn update_validated_in<F>(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
config_file: &str,
|
||||
data: Vec<u8>,
|
||||
validate: F,
|
||||
) -> Result<OffsetDateTime>
|
||||
where
|
||||
F: FnOnce() -> Result<()> + Send,
|
||||
{
|
||||
Box::pin(async {
|
||||
let sys = require_bucket_metadata_sys_in(ctx)?;
|
||||
let guard = acquire_config_write_guard(sys.clone(), bucket).await?;
|
||||
validate()?;
|
||||
update_under_config_write_guard(sys, &guard, config_file, data, None).await
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn delete(bucket: &str, config_file: &str) -> Result<OffsetDateTime> {
|
||||
delete_with_sys(get_bucket_metadata_sys()?, bucket, config_file).await
|
||||
}
|
||||
@@ -1132,7 +1153,7 @@ async fn acquire_transaction_lock_with_sys(
|
||||
/// agree on, so renaming it would leave a mixed-version cluster with two
|
||||
/// disjoint keys — and old and new nodes would stop excluding each other on
|
||||
/// the very writes that are serialized today.
|
||||
fn bucket_metadata_transaction_lock_key(bucket: &str) -> String {
|
||||
pub(crate) fn bucket_metadata_transaction_lock_key(bucket: &str) -> String {
|
||||
format!("bucket-targets/{bucket}/transaction.lock")
|
||||
}
|
||||
|
||||
@@ -1271,14 +1292,34 @@ pub(crate) async fn get_object_lock_config_state_in(
|
||||
pub(crate) async fn get_object_lock_config_and_incarnation_from_disk_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
) -> Result<(ObjectLockConfigState, Uuid, OffsetDateTime)> {
|
||||
get_object_lock_config_and_incarnation_from_disk_with_guard_in(ctx, bucket, None).await
|
||||
}
|
||||
|
||||
/// Bucket deletion already owns the namespace/lifecycle fences. Reuse its
|
||||
/// live guard rather than recursively acquiring the bucket namespace lock.
|
||||
/// The caller must also hold the metadata transaction read lock.
|
||||
pub(crate) async fn get_object_lock_config_and_incarnation_from_disk_with_guard_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
guard: Option<&rustfs_lock::NamespaceLockGuard>,
|
||||
) -> Result<(ObjectLockConfigState, Uuid, OffsetDateTime)> {
|
||||
let bucket_meta_sys_lock = bucket_metadata_sys_of(ctx)?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await.clone();
|
||||
|
||||
match bucket_meta_sys
|
||||
.read_authoritative_metadata_from_disk_under_transaction_lock(bucket)
|
||||
.await?
|
||||
{
|
||||
let authority = match guard {
|
||||
Some(guard) => {
|
||||
bucket_meta_sys
|
||||
.read_authoritative_metadata_from_disk_under_guard(bucket, guard)
|
||||
.await?
|
||||
}
|
||||
None => {
|
||||
bucket_meta_sys
|
||||
.read_authoritative_metadata_from_disk_under_transaction_lock(bucket)
|
||||
.await?
|
||||
}
|
||||
};
|
||||
match authority {
|
||||
BucketMetadataAuthority::Authoritative(metadata)
|
||||
if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() =>
|
||||
{
|
||||
@@ -1331,10 +1372,54 @@ pub(crate) async fn get_quota_config_and_incarnation_from_disk_in(
|
||||
let bucket_meta_sys_lock = bucket_metadata_sys_of(ctx)?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await.clone();
|
||||
|
||||
match bucket_meta_sys
|
||||
.read_authoritative_metadata_from_disk_under_transaction_lock(bucket)
|
||||
.await?
|
||||
quota_config_and_incarnation_from_authority(
|
||||
bucket,
|
||||
bucket_meta_sys
|
||||
.read_authoritative_metadata_from_disk_under_transaction_lock(bucket)
|
||||
.await?,
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) async fn get_quota_config_and_incarnation_from_disk_for_options_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
opts: &ObjectOptions,
|
||||
transaction_guard: &Arc<rustfs_lock::NamespaceLockGuard>,
|
||||
) -> Result<(Option<BucketQuota>, Uuid, OffsetDateTime)> {
|
||||
let Some(snapshot) = opts.object_lock_config_snapshot.as_ref() else {
|
||||
return get_quota_config_and_incarnation_from_disk_in(ctx, bucket).await;
|
||||
};
|
||||
let sys = bucket_metadata_sys_of(ctx)?.read().await.clone();
|
||||
let store = sys.object_store();
|
||||
let snapshot_guard = snapshot
|
||||
.metadata_transaction_guard_for(store.id, bucket, opts.expected_bucket_incarnation_id)
|
||||
.filter(|guard| Arc::ptr_eq(guard, transaction_guard))
|
||||
.ok_or_else(|| Error::other("quota snapshot does not hold this bucket's metadata transaction fence"))?;
|
||||
if snapshot_guard.is_released() {
|
||||
return Err(Error::other("quota snapshot metadata transaction fence was released"));
|
||||
}
|
||||
|
||||
// The snapshot carries the caller's lifecycle fence, excluding deletion.
|
||||
// Its transaction guard excludes config writes, so a same-name object
|
||||
// owner need not recursively acquire the bucket namespace here.
|
||||
let authority = sys
|
||||
.read_authoritative_metadata_from_disk_under_guard(bucket, transaction_guard)
|
||||
.await?;
|
||||
if snapshot_guard.is_lock_lost()
|
||||
|| snapshot_guard.is_released()
|
||||
|| matches!(&authority, BucketMetadataAuthority::Authoritative(metadata)
|
||||
if !snapshot.is_valid_for_destructive_put(store.id, bucket, metadata.bucket_incarnation_id))
|
||||
{
|
||||
return Err(Error::other("quota snapshot fence or bucket incarnation changed during the disk read"));
|
||||
}
|
||||
quota_config_and_incarnation_from_authority(bucket, authority)
|
||||
}
|
||||
|
||||
fn quota_config_and_incarnation_from_authority(
|
||||
bucket: &str,
|
||||
authority: BucketMetadataAuthority,
|
||||
) -> Result<(Option<BucketQuota>, Uuid, OffsetDateTime)> {
|
||||
match authority {
|
||||
BucketMetadataAuthority::Authoritative(metadata)
|
||||
if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() =>
|
||||
{
|
||||
@@ -2703,29 +2788,30 @@ impl BucketMetadataSys {
|
||||
async fn read_authoritative_metadata_from_disk_under_transaction_lock(
|
||||
&self,
|
||||
bucket: &str,
|
||||
) -> Result<BucketMetadataAuthority> {
|
||||
let namespace_lock = self.object_store().new_ns_lock(bucket, bucket).await?;
|
||||
let namespace_guard = namespace_lock
|
||||
.get_read_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await?;
|
||||
self.read_authoritative_metadata_from_disk_under_guard(bucket, &namespace_guard)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn read_authoritative_metadata_from_disk_under_guard(
|
||||
&self,
|
||||
bucket: &str,
|
||||
guard: &rustfs_lock::NamespaceLockGuard,
|
||||
) -> Result<BucketMetadataAuthority> {
|
||||
#[cfg(test)]
|
||||
if self.object_lock_disk_read_errors.write().await.remove(bucket) {
|
||||
return Err(Error::other(format!("injected Object Lock metadata disk read failure: {bucket}")));
|
||||
}
|
||||
|
||||
let namespace_lock = self.object_store().new_ns_lock(bucket, bucket).await?;
|
||||
let namespace_guard = namespace_lock
|
||||
.get_read_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await?;
|
||||
match await_bucket_namespace_operation(
|
||||
Some(&namespace_guard),
|
||||
bucket,
|
||||
"bucket metadata snapshot existence check",
|
||||
async {
|
||||
self.object_store()
|
||||
.get_bucket_info_from_sets_at_read_quorum(
|
||||
bucket,
|
||||
&crate::storage_api_contracts::bucket::BucketOptions::default(),
|
||||
)
|
||||
.await
|
||||
},
|
||||
)
|
||||
match await_bucket_namespace_operation(Some(guard), bucket, "bucket metadata snapshot existence check", async {
|
||||
self.object_store()
|
||||
.get_bucket_info_from_sets_at_read_quorum(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
|
||||
.await
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(_) => {}
|
||||
@@ -2734,7 +2820,7 @@ impl BucketMetadataSys {
|
||||
}
|
||||
|
||||
let (metadata, persisted) = await_bucket_namespace_operation(
|
||||
Some(&namespace_guard),
|
||||
Some(guard),
|
||||
bucket,
|
||||
"bucket metadata authoritative snapshot",
|
||||
load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true),
|
||||
@@ -4121,6 +4207,114 @@ mod tests {
|
||||
sys.persist_new_and_set(bm).await.expect("initial metadata should persist");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reparse_xml_rmw_corruption_refuses_cached_and_reloaded_encryption() {
|
||||
use crate::bucket::metadata::{BUCKET_SSECONFIG, is_unreadable_config_error};
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
let (dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||
let sys = BucketMetadataSys::new(ecstore);
|
||||
let bucket = "reparse-encryption-corrupt";
|
||||
let valid = br#"<ServerSideEncryptionConfiguration><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>AES256</SSEAlgorithm></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>"#;
|
||||
let corrupt = b"<ServerSideEncryptionConfiguration";
|
||||
persist_bucket_with_raw_config(&sys, &dirs, bucket, BUCKET_SSECONFIG, valid).await;
|
||||
sys.get_sse_config(bucket).await.expect("valid encryption must be readable");
|
||||
|
||||
let updated = sys
|
||||
.update_config_with(bucket, BUCKET_SSECONFIG, |bm| {
|
||||
assert!(bm.sse_config.is_some(), "rewrite must start with parsed encryption");
|
||||
Ok(corrupt.to_vec())
|
||||
})
|
||||
.await
|
||||
.expect("persist future or corrupt raw encryption without failing the whole bucket");
|
||||
|
||||
let err = sys
|
||||
.get_sse_config(bucket)
|
||||
.await
|
||||
.expect_err("cached old encryption must not hide corrupt raw");
|
||||
assert!(is_unreadable_config_error(&err), "cached refusal must retain its type: {err}");
|
||||
let (cached, _) = sys.get_config(bucket).await.expect("bucket remains readable");
|
||||
assert_eq!(cached.encryption_config_xml, corrupt);
|
||||
assert_eq!(cached.encryption_config_updated_at, updated);
|
||||
|
||||
sys.metadata_map.write().await.clear();
|
||||
let err = sys
|
||||
.get_sse_config(bucket)
|
||||
.await
|
||||
.expect_err("reloaded encryption must still refuse corrupt raw");
|
||||
assert!(is_unreadable_config_error(&err), "reloaded refusal must retain its type: {err}");
|
||||
|
||||
let calls = AtomicUsize::new(0);
|
||||
let err = sys
|
||||
.update_config_with(bucket, BUCKET_SSECONFIG, |_| {
|
||||
calls.fetch_add(1, Ordering::SeqCst);
|
||||
Ok(Vec::new())
|
||||
})
|
||||
.await
|
||||
.expect_err("another rewrite must refuse before it consumes corrupt encryption");
|
||||
assert!(is_unreadable_config_error(&err));
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0, "refused rewrite must not erase the original raw");
|
||||
let disk = sys
|
||||
.get_config_from_disk(bucket)
|
||||
.await
|
||||
.expect("read unchanged metadata from disk");
|
||||
assert_eq!(disk.encryption_config_xml, corrupt);
|
||||
assert_eq!(disk.encryption_config_updated_at, updated);
|
||||
|
||||
sys.update(bucket, BUCKET_SSECONFIG, valid.to_vec())
|
||||
.await
|
||||
.expect("explicit repair must remain possible");
|
||||
sys.get_sse_config(bucket)
|
||||
.await
|
||||
.expect("repaired cached encryption must be readable");
|
||||
sys.metadata_map.write().await.clear();
|
||||
sys.get_sse_config(bucket).await.expect("repair must survive a disk reload");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reparse_xml_rmw_removal_clears_cached_and_reloaded_encryption() {
|
||||
use crate::bucket::metadata::BUCKET_SSECONFIG;
|
||||
|
||||
let (dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||
let sys = BucketMetadataSys::new(ecstore);
|
||||
let bucket = "reparse-encryption-removed";
|
||||
let valid = br#"<ServerSideEncryptionConfiguration><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>AES256</SSEAlgorithm></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>"#;
|
||||
persist_bucket_with_raw_config(&sys, &dirs, bucket, BUCKET_SSECONFIG, valid).await;
|
||||
sys.get_sse_config(bucket).await.expect("valid encryption must be readable");
|
||||
|
||||
let updated = sys
|
||||
.update_config_with(bucket, BUCKET_SSECONFIG, |bm| {
|
||||
assert!(bm.sse_config.is_some(), "rewrite must start with parsed encryption");
|
||||
Ok(Vec::new())
|
||||
})
|
||||
.await
|
||||
.expect("removing encryption must persist");
|
||||
|
||||
assert_eq!(
|
||||
sys.get_sse_config(bucket)
|
||||
.await
|
||||
.expect_err("old cached encryption must be gone"),
|
||||
Error::ConfigNotFound
|
||||
);
|
||||
let (cached, _) = sys.get_config(bucket).await.expect("bucket remains readable");
|
||||
assert!(cached.encryption_config_xml.is_empty());
|
||||
assert_eq!(cached.encryption_config_updated_at, updated);
|
||||
|
||||
sys.metadata_map.write().await.clear();
|
||||
assert_eq!(
|
||||
sys.get_sse_config(bucket)
|
||||
.await
|
||||
.expect_err("removed encryption must stay absent after reload"),
|
||||
Error::ConfigNotFound
|
||||
);
|
||||
let disk = sys
|
||||
.get_config_from_disk(bucket)
|
||||
.await
|
||||
.expect("read removed encryption from disk");
|
||||
assert!(disk.encryption_config_xml.is_empty());
|
||||
assert_eq!(disk.encryption_config_updated_at, updated);
|
||||
}
|
||||
|
||||
/// rustfs/backlog#1734: a read-modify-write of a stored config that cannot
|
||||
/// be parsed must be refused before `mutate` runs. Otherwise `mutate` sees
|
||||
/// the unreadable config as absent, rebuilds it from nothing, and the
|
||||
@@ -4736,6 +4930,231 @@ mod tests {
|
||||
.expect("DeleteBucket should succeed");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_snapshot_reuses_same_name_owner_and_rereads_disk_authority() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = "quota-same-name-owner";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("quota fixture bucket");
|
||||
let snapshot = store
|
||||
.object_lock_config_snapshot(bucket)
|
||||
.await
|
||||
.expect("guarded Object Lock snapshot");
|
||||
let incarnation = store.bucket_incarnation_id(bucket).await.expect("fixture incarnation");
|
||||
let transaction_guard = snapshot
|
||||
.metadata_transaction_guard_for(store.id, bucket, Some(incarnation))
|
||||
.expect("snapshot's actual transaction guard");
|
||||
let expected = get_quota_config_and_incarnation_from_disk_for_options_in(
|
||||
&store.ctx,
|
||||
bucket,
|
||||
&ObjectOptions::default(),
|
||||
&transaction_guard,
|
||||
)
|
||||
.await
|
||||
.expect("a reader without a snapshot must retain the ordinary namespace path");
|
||||
|
||||
let sys = bucket_metadata_sys_of(&store.ctx).expect("fixture metadata system");
|
||||
let sys = sys.read().await.clone();
|
||||
let mut stale = (*sys.get(bucket).await.expect("cached fixture metadata")).clone();
|
||||
let metadata_path = stale.save_file_path();
|
||||
stale.quota_config = Some(BucketQuota {
|
||||
quota: Some(123),
|
||||
..Default::default()
|
||||
});
|
||||
sys.set(bucket.to_string(), Arc::new(stale)).await;
|
||||
let namespace = store.new_ns_lock(bucket, bucket).await.expect("same-name object namespace");
|
||||
let _owner = namespace
|
||||
.get_write_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await
|
||||
.expect("same-name object write owner");
|
||||
let opts = ObjectOptions {
|
||||
object_lock_config_snapshot: Some(snapshot),
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(
|
||||
get_quota_config_and_incarnation_from_disk_for_options_in(&store.ctx, bucket, &opts, &transaction_guard)
|
||||
.await
|
||||
.expect("quota disk read must reuse the held lifecycle and transaction fences"),
|
||||
expected,
|
||||
"the cached quota must not replace disk authority"
|
||||
);
|
||||
|
||||
crate::config::com::save_config_with_opts(
|
||||
store.clone(),
|
||||
&metadata_path,
|
||||
b"corrupt bucket metadata".to_vec(),
|
||||
&ObjectOptions {
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("inject committed metadata corruption");
|
||||
assert!(
|
||||
get_quota_config_and_incarnation_from_disk_for_options_in(&store.ctx, bucket, &opts, &transaction_guard)
|
||||
.await
|
||||
.is_err(),
|
||||
"a valid snapshot must not hide corrupt disk metadata"
|
||||
);
|
||||
crate::config::com::delete_config(store.clone(), &metadata_path)
|
||||
.await
|
||||
.expect("remove the persisted metadata while retaining its sidecar");
|
||||
assert!(
|
||||
get_quota_config_and_incarnation_from_disk_for_options_in(&store.ctx, bucket, &opts, &transaction_guard)
|
||||
.await
|
||||
.is_err(),
|
||||
"a valid snapshot must not promote fabricated metadata to authority"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_snapshot_rejects_wrong_scope_incarnation_and_transaction_guard() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = "quota-snapshot-scope";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("quota fixture bucket");
|
||||
let incarnation = store.bucket_incarnation_id(bucket).await.expect("fixture incarnation");
|
||||
let snapshot = store
|
||||
.object_lock_config_snapshot(bucket)
|
||||
.await
|
||||
.expect("guarded Object Lock snapshot");
|
||||
let transaction_guard = snapshot
|
||||
.metadata_transaction_guard_for(store.id, bucket, Some(incarnation))
|
||||
.expect("snapshot's actual transaction guard");
|
||||
let mut opts = ObjectOptions {
|
||||
object_lock_config_snapshot: Some(snapshot),
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(
|
||||
get_quota_config_and_incarnation_from_disk_for_options_in(&store.ctx, "another-bucket", &opts, &transaction_guard)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
opts.expected_bucket_incarnation_id = Some(Uuid::new_v4());
|
||||
assert!(
|
||||
get_quota_config_and_incarnation_from_disk_for_options_in(&store.ctx, bucket, &opts, &transaction_guard)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
opts.expected_bucket_incarnation_id = Some(incarnation);
|
||||
let other_guard = Arc::new(
|
||||
acquire_bucket_metadata_transaction_read_lock_in(&store.ctx, bucket)
|
||||
.await
|
||||
.expect("distinct real transaction read guard"),
|
||||
);
|
||||
assert!(
|
||||
get_quota_config_and_incarnation_from_disk_for_options_in(&store.ctx, bucket, &opts, &other_guard)
|
||||
.await
|
||||
.is_err(),
|
||||
"the same namespace key does not prove that this is the snapshot's actual guard"
|
||||
);
|
||||
let (_other_dirs, other_store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(other_store.clone(), Vec::new()).await;
|
||||
assert!(
|
||||
get_quota_config_and_incarnation_from_disk_for_options_in(&other_store.ctx, bucket, &opts, &transaction_guard)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
opts.object_lock_config_snapshot = Some(Arc::new(crate::object_api::ObjectLockConfigSnapshot::new(
|
||||
ObjectLockConfigState::ConfirmedAbsent,
|
||||
)));
|
||||
assert!(
|
||||
get_quota_config_and_incarnation_from_disk_for_options_in(&store.ctx, bucket, &opts, &transaction_guard)
|
||||
.await
|
||||
.is_err(),
|
||||
"an unguarded snapshot must not authorize namespace reuse"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_snapshot_rejects_lifecycle_loss_during_read_and_changed_disk_incarnation() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = "quota-snapshot-loss";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("quota fixture bucket");
|
||||
let sys = bucket_metadata_sys_of(&store.ctx).expect("fixture metadata system");
|
||||
let sys = sys.read().await.clone();
|
||||
let mut metadata = (*sys.get(bucket).await.expect("fixture metadata")).clone();
|
||||
let incarnation = metadata.bucket_incarnation_id;
|
||||
let _lifecycle_guard = store
|
||||
.acquire_bucket_lifecycle_read_lock(bucket)
|
||||
.await
|
||||
.expect("real lifecycle owner");
|
||||
let transaction_guard = Arc::new(
|
||||
acquire_bucket_metadata_transaction_read_lock_in(&store.ctx, bucket)
|
||||
.await
|
||||
.expect("real metadata transaction owner"),
|
||||
);
|
||||
let (lifecycle_fence, lost) = crate::object_api::NamespaceLockFence::loss_handle_for_test();
|
||||
let snapshot = crate::object_api::ObjectLockConfigSnapshot::for_store_bucket_under_lifecycle_fence(
|
||||
store.id,
|
||||
bucket,
|
||||
incarnation,
|
||||
metadata.object_lock_config_updated_at,
|
||||
ObjectLockConfigState::ConfirmedAbsent,
|
||||
lifecycle_fence,
|
||||
Arc::clone(&transaction_guard),
|
||||
);
|
||||
let opts = ObjectOptions {
|
||||
object_lock_config_snapshot: Some(Arc::new(snapshot)),
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
..Default::default()
|
||||
};
|
||||
let mut read = Box::pin(get_quota_config_and_incarnation_from_disk_for_options_in(
|
||||
&store.ctx,
|
||||
bucket,
|
||||
&opts,
|
||||
&transaction_guard,
|
||||
));
|
||||
assert!(
|
||||
futures::poll!(read.as_mut()).is_pending(),
|
||||
"the authoritative disk read must be in flight"
|
||||
);
|
||||
lost.store(true, std::sync::atomic::Ordering::Release);
|
||||
assert!(read.await.is_err(), "a lifecycle fence lost during the disk read must reject its result");
|
||||
|
||||
let snapshot = store
|
||||
.object_lock_config_snapshot(bucket)
|
||||
.await
|
||||
.expect("fresh guarded snapshot");
|
||||
let transaction_guard = snapshot
|
||||
.metadata_transaction_guard_for(store.id, bucket, Some(incarnation))
|
||||
.expect("snapshot's actual transaction guard");
|
||||
let opts = ObjectOptions {
|
||||
object_lock_config_snapshot: Some(snapshot),
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
..Default::default()
|
||||
};
|
||||
metadata.bucket_incarnation_id = Uuid::new_v4();
|
||||
save_bucket_incarnation(store.clone(), bucket, metadata.bucket_incarnation_id)
|
||||
.await
|
||||
.expect("inject a new disk incarnation");
|
||||
metadata
|
||||
.save_with_store_committed(store.clone())
|
||||
.await
|
||||
.expect("persist its matching metadata blob");
|
||||
assert!(
|
||||
get_quota_config_and_incarnation_from_disk_for_options_in(&store.ctx, bucket, &opts, &transaction_guard)
|
||||
.await
|
||||
.is_err(),
|
||||
"a matching sidecar and blob from another incarnation must not reuse the old snapshot"
|
||||
);
|
||||
}
|
||||
|
||||
/// Writers on different nodes updating *different* config files of one
|
||||
/// bucket must both survive. Each rewrites the whole metadata blob, so
|
||||
/// without a lock spanning the read-modify-write the later save carries
|
||||
|
||||
@@ -280,11 +280,11 @@ where
|
||||
|
||||
debug!("Found {} migrating bucket metadata, migrating...", buckets.len());
|
||||
|
||||
let opts = ObjectOptions {
|
||||
let mut opts = ObjectOptions {
|
||||
max_parity: true,
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
opts.use_offline_recovery_write();
|
||||
let h = HeaderMap::new();
|
||||
|
||||
for bucket in buckets {
|
||||
@@ -397,11 +397,11 @@ where
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let opts = ObjectOptions {
|
||||
let mut opts = ObjectOptions {
|
||||
max_parity: true,
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
opts.use_offline_recovery_write();
|
||||
let h = HeaderMap::new();
|
||||
let prefix = format!("{IAM_CONFIG_PREFIX}/");
|
||||
let mut continuation: Option<String> = None;
|
||||
|
||||
@@ -32,7 +32,10 @@ pub enum QuotaType {
|
||||
Hard,
|
||||
}
|
||||
|
||||
pub(crate) const QUOTA_RESERVATION_PROTOCOL_V1: u32 = 1;
|
||||
pub const QUOTA_RESERVATION_PROTOCOL_V1: u32 = 1;
|
||||
/// Sharded durable reservations. Older nodes reject this protocol instead of
|
||||
/// silently falling back to the single-ledger protocol.
|
||||
pub const QUOTA_RESERVATION_PROTOCOL_V2: u32 = 2;
|
||||
|
||||
/// Bucket quota configuration. quota_type defaults to Hard when omitted.
|
||||
#[derive(Debug, Default, Clone, PartialEq)]
|
||||
@@ -89,7 +92,10 @@ impl<'de> Deserialize<'de> for BucketQuota {
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
let wire = BucketQuotaWire::deserialize(deserializer)?;
|
||||
let quota = if wire.reservation_protocol == Some(QUOTA_RESERVATION_PROTOCOL_V1) {
|
||||
let quota = if matches!(
|
||||
wire.reservation_protocol,
|
||||
Some(QUOTA_RESERVATION_PROTOCOL_V1) | Some(QUOTA_RESERVATION_PROTOCOL_V2)
|
||||
) {
|
||||
Some(
|
||||
wire.reservation_quota
|
||||
.ok_or_else(|| D::Error::custom("reservation_quota is required for reservation protocol v1"))?,
|
||||
@@ -129,17 +135,33 @@ impl BucketQuota {
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates a quota using the sharded durable reservation protocol.
|
||||
/// Deployment must complete the mixed-version rollout before enabling it;
|
||||
/// older nodes fail closed when they see protocol v2.
|
||||
pub fn new_sharded(quota: u64) -> Self {
|
||||
let mut config = Self::new(Some(quota));
|
||||
config.reservation_protocol = Some(QUOTA_RESERVATION_PROTOCOL_V2);
|
||||
config
|
||||
}
|
||||
|
||||
pub fn get_quota_limit(&self) -> Option<u64> {
|
||||
self.quota
|
||||
}
|
||||
|
||||
pub fn uses_durable_reservations(&self) -> bool {
|
||||
self.reservation_protocol == Some(QUOTA_RESERVATION_PROTOCOL_V1)
|
||||
matches!(
|
||||
self.reservation_protocol,
|
||||
Some(QUOTA_RESERVATION_PROTOCOL_V1) | Some(QUOTA_RESERVATION_PROTOCOL_V2)
|
||||
)
|
||||
}
|
||||
|
||||
pub fn uses_sharded_reservations(&self) -> bool {
|
||||
self.reservation_protocol == Some(QUOTA_RESERVATION_PROTOCOL_V2)
|
||||
}
|
||||
|
||||
pub fn has_unsupported_reservation_protocol(&self) -> bool {
|
||||
self.reservation_protocol
|
||||
.is_some_and(|version| version != QUOTA_RESERVATION_PROTOCOL_V1)
|
||||
.is_some_and(|version| !matches!(version, QUOTA_RESERVATION_PROTOCOL_V1 | QUOTA_RESERVATION_PROTOCOL_V2))
|
||||
}
|
||||
|
||||
pub fn check_operation_allowed(&self, current_usage: u64, operation_size: u64) -> bool {
|
||||
@@ -322,9 +344,21 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_reservation_protocol_does_not_activate_v1() {
|
||||
fn sharded_quota_roundtrips_with_authoritative_reservation_quota() {
|
||||
let quota = BucketQuota::new_sharded(4096);
|
||||
let json = serde_json::to_value("a).expect("sharded quota should serialize");
|
||||
assert_eq!(json["quota"], 0);
|
||||
assert_eq!(json["reservation_protocol"], QUOTA_RESERVATION_PROTOCOL_V2);
|
||||
assert_eq!(json["reservation_quota"], 4096);
|
||||
let restored: BucketQuota = serde_json::from_value(json).expect("sharded quota should parse");
|
||||
assert_eq!(restored.quota, Some(4096));
|
||||
assert!(restored.uses_sharded_reservations());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_reservation_protocol_does_not_activate_any_known_protocol() {
|
||||
let quota: BucketQuota =
|
||||
serde_json::from_str(r#"{"quota":0,"quota_type":"Hard","reservation_protocol":2,"reservation_quota":2048}"#)
|
||||
serde_json::from_str(r#"{"quota":0,"quota_type":"Hard","reservation_protocol":3,"reservation_quota":2048}"#)
|
||||
.expect("future protocol should remain parseable");
|
||||
|
||||
assert!(!quota.uses_durable_reservations());
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -13,7 +13,9 @@
|
||||
// limitations under the License.
|
||||
|
||||
use super::replication_error_boundary::Result;
|
||||
use super::replication_storage_boundary::{HTTPPreconditions, ObjectInfo, ObjectOptions, ReplicationObjectIO};
|
||||
use super::replication_storage_boundary::{
|
||||
HTTPPreconditions, ObjectInfo, ObjectOptions, ReplicationObjectIO, WriteCommitGuard, WriteCompletion,
|
||||
};
|
||||
use crate::config::{com, storageclass};
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -77,13 +79,6 @@ impl ReplicationConfigStore {
|
||||
com::save_config(api, file, data).await
|
||||
}
|
||||
|
||||
pub(crate) async fn save_no_lock<S>(api: Arc<S>, file: &str, data: Vec<u8>) -> Result<()>
|
||||
where
|
||||
S: ReplicationObjectIO,
|
||||
{
|
||||
com::save_config_no_lock(api, file, data).await
|
||||
}
|
||||
|
||||
pub(crate) async fn save_conditional<S>(
|
||||
api: Arc<S>,
|
||||
file: &str,
|
||||
@@ -111,21 +106,18 @@ impl ReplicationConfigStore {
|
||||
file: &str,
|
||||
data: Vec<u8>,
|
||||
http_preconditions: HTTPPreconditions,
|
||||
guard: &WriteCommitGuard,
|
||||
) -> Result<()>
|
||||
where
|
||||
S: ReplicationObjectIO,
|
||||
{
|
||||
com::save_config_with_opts_quiet(
|
||||
api,
|
||||
file,
|
||||
data,
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
no_lock: true,
|
||||
http_preconditions: Some(http_preconditions),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
let mut opts = ObjectOptions {
|
||||
max_parity: true,
|
||||
http_preconditions: Some(http_preconditions),
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
};
|
||||
opts.add_write_commit_guard(guard);
|
||||
com::save_config_with_opts_quiet(api, file, data, &opts).await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,7 +44,7 @@ use super::replication_resyncer::{
|
||||
use super::replication_state::ReplicationStats;
|
||||
use super::replication_storage_boundary::{
|
||||
HTTPPreconditions, ObjectInfo, ObjectOptions, ObjectToDelete, ReplicationDeletedObject, ReplicationObjectIO,
|
||||
ReplicationStorage,
|
||||
ReplicationStorage, WriteCommitGuard,
|
||||
};
|
||||
use super::replication_target_boundary::{BucketTargetError, ReplicationTargetStore, replication_object_is_ssec_encrypted};
|
||||
use super::replication_versioning_boundary::ReplicationVersioningStore;
|
||||
@@ -513,7 +513,7 @@ where
|
||||
// Lock order is transaction namespace lock -> force-delete journal object lock.
|
||||
// Keep the transaction guard alive through the conditional write so legacy
|
||||
// writers cannot interleave a read-modify-write transition within this process.
|
||||
let guard = lock.get_write_lock(ReplicationLockTiming::acquire_timeout()).await?;
|
||||
let guard = WriteCommitGuard::acquire(&lock, ReplicationLockTiming::acquire_timeout()).await?;
|
||||
let (mut entries, preconditions, exists) = read_force_delete_intents(storage.clone(), file).await?;
|
||||
if !update(&mut entries, exists)? {
|
||||
return Ok(());
|
||||
@@ -565,7 +565,7 @@ async fn read_force_delete_intents<S: ReplicationObjectIO>(
|
||||
async fn save_force_delete_intents<S: ReplicationStorage>(
|
||||
storage: Arc<S>,
|
||||
file: &str,
|
||||
guard: &rustfs_lock::NamespaceLockGuard,
|
||||
guard: &WriteCommitGuard,
|
||||
entries: Vec<MrfReplicateEntry>,
|
||||
preconditions: HTTPPreconditions,
|
||||
) -> Result<(), EcstoreError> {
|
||||
@@ -628,7 +628,7 @@ async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
|
||||
let lock = storage
|
||||
.new_ns_lock(ReplicationMetadataStore::rustfs_meta_bucket(), file)
|
||||
.await?;
|
||||
let guard = lock.get_write_lock(ReplicationLockTiming::acquire_timeout()).await?;
|
||||
let guard = WriteCommitGuard::acquire(&lock, ReplicationLockTiming::acquire_timeout()).await?;
|
||||
let current = ReplicationConfigStore::read_no_lock_with_metadata_preserve_empty(storage.clone(), file).await;
|
||||
let (current_data, current_etag, current_exists) = match current {
|
||||
Ok((data, object_info)) => (data, object_info.etag, true),
|
||||
@@ -658,7 +658,7 @@ async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
|
||||
} else {
|
||||
encode_mrf_file(&retained)?
|
||||
};
|
||||
match ReplicationConfigStore::save_conditional_no_lock(storage.clone(), file, data, preconditions).await {
|
||||
match ReplicationConfigStore::save_conditional_no_lock(storage.clone(), file, data, preconditions, &guard).await {
|
||||
Ok(()) => return Ok(retained),
|
||||
Err(EcstoreError::PreconditionFailed) => continue,
|
||||
Err(error) => return Err(error),
|
||||
@@ -2522,7 +2522,7 @@ async fn quarantine_mrf_file<S: ReplicationStorage>(storage: &Arc<S>, data: &[u8
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let guard = match lock.get_write_lock(ReplicationLockTiming::acquire_timeout()).await {
|
||||
let guard = match WriteCommitGuard::acquire(&lock, ReplicationLockTiming::acquire_timeout()).await {
|
||||
Ok(guard) => guard,
|
||||
Err(error) => {
|
||||
warn!(
|
||||
@@ -2573,6 +2573,7 @@ async fn quarantine_mrf_file<S: ReplicationStorage>(storage: &Arc<S>, data: &[u8
|
||||
ReplicationMetadataStore::MRF_REPLICATION_FILE,
|
||||
Vec::new(),
|
||||
preconditions,
|
||||
&guard,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -2627,13 +2628,12 @@ async fn recover_corrupt_mrf_generation<S: ReplicationStorage>(
|
||||
entries: &[MrfReplicateEntry],
|
||||
preconditions: Option<HTTPPreconditions>,
|
||||
storage: &Arc<S>,
|
||||
guard: &rustfs_lock::NamespaceLockGuard,
|
||||
guard: &WriteCommitGuard,
|
||||
pending_payload: &mut Option<PendingMrfAppend>,
|
||||
started: Instant,
|
||||
) -> Option<MrfAppendResult> {
|
||||
let quarantine_file = format!("{MRF_CORRUPT_FILE_PREFIX}.{}.bin", OffsetDateTime::now_utc().unix_timestamp_nanos());
|
||||
if let Err(error) = ReplicationConfigStore::save_no_lock(storage.clone(), &quarantine_file, corrupt_generation.to_vec()).await
|
||||
{
|
||||
if let Err(error) = ReplicationConfigStore::save(storage.clone(), &quarantine_file, corrupt_generation.to_vec()).await {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
@@ -2688,6 +2688,7 @@ async fn recover_corrupt_mrf_generation<S: ReplicationStorage>(
|
||||
ReplicationMetadataStore::MRF_REPLICATION_FILE,
|
||||
data,
|
||||
preconditions,
|
||||
guard,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -2735,7 +2736,7 @@ async fn append_mrf_entries_to_disk<S: ReplicationStorage>(
|
||||
return None;
|
||||
}
|
||||
};
|
||||
let guard = match lock.get_write_lock(ReplicationLockTiming::acquire_timeout()).await {
|
||||
let guard = match WriteCommitGuard::acquire(&lock, ReplicationLockTiming::acquire_timeout()).await {
|
||||
Ok(guard) => guard,
|
||||
Err(error) => {
|
||||
warn!(
|
||||
@@ -2862,6 +2863,7 @@ async fn append_mrf_entries_to_disk<S: ReplicationStorage>(
|
||||
ReplicationMetadataStore::MRF_REPLICATION_FILE,
|
||||
data,
|
||||
preconditions,
|
||||
&guard,
|
||||
)
|
||||
.await
|
||||
{
|
||||
|
||||
@@ -19,7 +19,8 @@ use super::replication_error_boundary::Error;
|
||||
use super::replication_filemeta_boundary::{replication_state_from_filemeta, version_purge_status_from_filemeta};
|
||||
pub(crate) type ReplicationObjectStore = crate::store::ECStore;
|
||||
pub(crate) use crate::object_api::{
|
||||
GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader, ReplicationStatusWritebackCondition, ReplicationStatusWritebackMode,
|
||||
GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader, ReplicationStatusWritebackCondition,
|
||||
ReplicationStatusWritebackMode, WriteCommitGuard, WriteCompletion,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::object_api::{NamespaceLockFence, NamespaceLockSignalTestFence};
|
||||
|
||||
@@ -12,9 +12,11 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#[cfg(test)]
|
||||
use crate::disk::DiskIoClass;
|
||||
use crate::disk::disk_store::{get_drive_walkdir_peek_timeout, get_drive_walkdir_stall_timeout};
|
||||
use crate::disk::error::DiskError;
|
||||
use crate::disk::{self, DiskAPI, DiskStore, WalkDirOptions};
|
||||
use crate::disk::{self, DiskAPI, DiskStore, WalkDirOptions, current_disk_io_context, with_optional_disk_io_context};
|
||||
use futures::future::join_all;
|
||||
use metrics::counter;
|
||||
use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetacacheReader, is_io_eof};
|
||||
@@ -245,6 +247,8 @@ pub struct ListPathRawOptions {
|
||||
pub(crate) test_fallback_reader_behaviors: Vec<TestReaderBehavior>,
|
||||
#[cfg(test)]
|
||||
pub(crate) peek_timeout: Option<Duration>,
|
||||
#[cfg(test)]
|
||||
pub(crate) test_disk_io_context_observed: Option<Arc<AtomicBool>>,
|
||||
// pub agreed: Option<Arc<dyn Fn(MetaCacheEntry) + Send + Sync>>,
|
||||
// pub partial: Option<Arc<dyn Fn(MetaCacheEntries, &[Option<Error>]) + Send + Sync>>,
|
||||
// pub finished: Option<Arc<dyn Fn(&[Option<Error>]) + Send + Sync>>,
|
||||
@@ -276,6 +280,8 @@ impl Clone for ListPathRawOptions {
|
||||
test_fallback_reader_behaviors: self.test_fallback_reader_behaviors.clone(),
|
||||
#[cfg(test)]
|
||||
peek_timeout: self.peek_timeout,
|
||||
#[cfg(test)]
|
||||
test_disk_io_context_observed: self.test_disk_io_context_observed.clone(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
@@ -344,6 +350,7 @@ async fn list_path_raw_inner(
|
||||
let log_path = opts.path.clone();
|
||||
|
||||
let mut jobs = JoinSet::new();
|
||||
let disk_io_context = current_disk_io_context();
|
||||
let mut readers = Vec::with_capacity(opts.disks.len());
|
||||
let fds = Arc::new(TokioMutex::new(opts.fallback_disks.iter().flatten().cloned().collect::<VecDeque<_>>()));
|
||||
#[cfg(test)]
|
||||
@@ -364,11 +371,19 @@ async fn list_path_raw_inner(
|
||||
let test_fallbacks_clone = test_fallbacks.clone();
|
||||
let cancel_rx_clone = cancel_rx.clone();
|
||||
let producer_errs_clone = producer_errs.clone();
|
||||
let disk_io_context_clone = disk_io_context.clone();
|
||||
let (rd, wr) = tokio::io::duplex(64);
|
||||
readers.push(MetacacheReader::new(rd));
|
||||
jobs.spawn(async move {
|
||||
jobs.spawn(with_optional_disk_io_context(disk_io_context_clone, async move {
|
||||
let mut wr = PublishedBytesWriter::new(wr);
|
||||
#[cfg(test)]
|
||||
if let Some(observer) = &opts_clone.test_disk_io_context_observed {
|
||||
observer.store(
|
||||
crate::disk::current_disk_io_context().is_some_and(|context| context.class == DiskIoClass::Background),
|
||||
std::sync::atomic::Ordering::Release,
|
||||
);
|
||||
}
|
||||
#[cfg(test)]
|
||||
let test_primary_error = if let Some(behavior) = opts_clone.test_reader_behaviors.get(disk_idx).cloned() {
|
||||
match behavior {
|
||||
TestReaderBehavior::Eof => return Ok(()),
|
||||
@@ -652,7 +667,7 @@ async fn list_path_raw_inner(
|
||||
|
||||
// warn!("list_path_raw: while need_fallback done");
|
||||
Ok(())
|
||||
});
|
||||
}));
|
||||
}
|
||||
|
||||
let revjob_rx = rx.clone();
|
||||
@@ -1168,6 +1183,68 @@ mod tests {
|
||||
assert_eq!(err, DiskError::ErasureReadQuorum);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_path_raw_joinset_producers_inherit_scanner_disk_io_class() {
|
||||
let observed = Arc::new(AtomicBool::new(false));
|
||||
crate::disk::with_background_disk_io(
|
||||
CancellationToken::new(),
|
||||
list_path_raw(
|
||||
CancellationToken::new(),
|
||||
ListPathRawOptions {
|
||||
disks: vec![None],
|
||||
min_disks: 1,
|
||||
test_reader_behaviors: vec![TestReaderBehavior::Eof],
|
||||
test_disk_io_context_observed: Some(observed.clone()),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("empty test producer should complete the listing");
|
||||
|
||||
assert!(
|
||||
observed.load(std::sync::atomic::Ordering::Acquire),
|
||||
"the per-disk JoinSet producer must retain the scanner background class"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_joinset_walk_uses_the_local_disk_background_gate() {
|
||||
let root = tempfile::tempdir().expect("local disk root should be created");
|
||||
let endpoint = crate::disk::endpoint::Endpoint::try_from(root.path().to_str().expect("disk path should be UTF-8"))
|
||||
.expect("endpoint should parse");
|
||||
let local = Arc::new(
|
||||
crate::disk::local::LocalDisk::new(&endpoint, false)
|
||||
.await
|
||||
.expect("local disk should open"),
|
||||
);
|
||||
let wrapper = crate::disk::disk_store::LocalDiskWrapper::new(local, false);
|
||||
wrapper.make_volume("bucket").await.expect("bucket volume should be created");
|
||||
let disk = Arc::new(crate::disk::Disk::Local(Box::new(wrapper.clone())));
|
||||
let cancellation = CancellationToken::new();
|
||||
|
||||
crate::disk::with_background_disk_io(
|
||||
cancellation.clone(),
|
||||
list_path_raw(
|
||||
cancellation,
|
||||
ListPathRawOptions {
|
||||
disks: vec![Some(disk)],
|
||||
bucket: "bucket".to_string(),
|
||||
min_disks: 1,
|
||||
recursive: true,
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("scanner metacache walk should complete");
|
||||
|
||||
assert!(
|
||||
wrapper.background_io_admissions_for_tests() > 0,
|
||||
"the LocalDiskWrapper walk should acquire background admission inside its JoinSet producer"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_path_raw_returns_typed_quorum_error_for_multiple_drive_failures() {
|
||||
let err = list_path_raw(
|
||||
|
||||
@@ -640,6 +640,7 @@ pub struct PeerLiveEventsBatch {
|
||||
pub struct ScannerPublicationLease {
|
||||
pub token: Uuid,
|
||||
pub movement_generation: u64,
|
||||
pub observational_only: bool,
|
||||
/// Stable storage owner identity. This is distinct from the activity
|
||||
/// session and is bound into both acquire and release proofs.
|
||||
pub owner_id: String,
|
||||
@@ -658,6 +659,7 @@ fn validate_scanner_publication_lease_response_fields(
|
||||
response: &ScannerPublicationLeaseResponse,
|
||||
expected_session_id: &str,
|
||||
expected_generation: u64,
|
||||
expected_observational_only: bool,
|
||||
) -> Result<(Uuid, String)> {
|
||||
if !response.success {
|
||||
return Err(Error::other(
|
||||
@@ -674,6 +676,9 @@ fn validate_scanner_publication_lease_response_fields(
|
||||
if response.session_id != expected_session_id {
|
||||
return Err(Error::other("peer returned a different scanner publication lease session"));
|
||||
}
|
||||
if response.observational_only != expected_observational_only {
|
||||
return Err(Error::other("peer returned a different scanner publication lease purpose"));
|
||||
}
|
||||
let owner_id = Uuid::parse_str(&response.owner_id)
|
||||
.ok()
|
||||
.filter(|owner_id| !owner_id.is_nil())
|
||||
@@ -2418,6 +2423,7 @@ impl PeerRestClient {
|
||||
&self,
|
||||
expected_session_id: &str,
|
||||
expected_generation: u64,
|
||||
observational_only: bool,
|
||||
) -> Result<ScannerPublicationLease> {
|
||||
let request_started = std::time::Instant::now();
|
||||
self.finalize_result(
|
||||
@@ -2434,6 +2440,7 @@ impl PeerRestClient {
|
||||
ttl_ms: crate::store::SCANNER_PUBLICATION_LEASE_TTL_MS,
|
||||
expected_session_id: expected_session_id.to_string(),
|
||||
token: Bytes::new(),
|
||||
observational_only,
|
||||
});
|
||||
let canonical = rustfs_protos::canonical_scanner_publication_lease_request_body(request.get_ref())
|
||||
.map_err(|_| Error::other("scanner publication lease request is too large to authenticate"))?;
|
||||
@@ -2444,11 +2451,16 @@ impl PeerRestClient {
|
||||
.map_err(|_| Error::other("scanner publication lease response is too large to authenticate"))?;
|
||||
verify_tonic_rpc_response_proof(&response_body, &response.response_proof)
|
||||
.map_err(|_| Error::other("peer returned an invalid scanner publication lease proof"))?;
|
||||
let (token, owner_id) =
|
||||
validate_scanner_publication_lease_response_fields(&response, expected_session_id, expected_generation)?;
|
||||
let (token, owner_id) = validate_scanner_publication_lease_response_fields(
|
||||
&response,
|
||||
expected_session_id,
|
||||
expected_generation,
|
||||
observational_only,
|
||||
)?;
|
||||
Ok(ScannerPublicationLease {
|
||||
token,
|
||||
movement_generation: response.movement_generation,
|
||||
observational_only,
|
||||
owner_id,
|
||||
session_id: response.session_id,
|
||||
expires_at: scanner_publication_lease_deadline(
|
||||
@@ -2483,6 +2495,7 @@ impl PeerRestClient {
|
||||
ttl_ms: crate::store::SCANNER_PUBLICATION_LEASE_TTL_MS,
|
||||
expected_session_id: lease.session_id.clone(),
|
||||
token: lease.token.as_bytes().to_vec().into(),
|
||||
observational_only: lease.observational_only,
|
||||
});
|
||||
let canonical = rustfs_protos::canonical_scanner_publication_lease_request_body(request.get_ref())
|
||||
.map_err(|_| Error::other("scanner publication lease validation request is too large to authenticate"))?;
|
||||
@@ -2494,8 +2507,12 @@ impl PeerRestClient {
|
||||
)?;
|
||||
verify_tonic_rpc_response_proof(&response_body, &response.response_proof)
|
||||
.map_err(|_| Error::other("peer returned an invalid scanner publication lease validation proof"))?;
|
||||
let (token, owner_id) =
|
||||
validate_scanner_publication_lease_response_fields(&response, &lease.session_id, lease.movement_generation)?;
|
||||
let (token, owner_id) = validate_scanner_publication_lease_response_fields(
|
||||
&response,
|
||||
&lease.session_id,
|
||||
lease.movement_generation,
|
||||
lease.observational_only,
|
||||
)?;
|
||||
if token != lease.token {
|
||||
return Err(Error::other("peer returned a different scanner publication lease token"));
|
||||
}
|
||||
@@ -3001,15 +3018,16 @@ mod tests {
|
||||
response_proof: Bytes::new(),
|
||||
owner_id: Uuid::new_v4().to_string(),
|
||||
session_id: "session-a".to_string(),
|
||||
observational_only: false,
|
||||
};
|
||||
|
||||
assert!(validate_scanner_publication_lease_response_fields(&response, "session-a", 7).is_ok());
|
||||
assert!(validate_scanner_publication_lease_response_fields(&response, "session-a", 7, false).is_ok());
|
||||
|
||||
let stale_generation = ScannerPublicationLeaseResponse {
|
||||
movement_generation: 6,
|
||||
..response.clone()
|
||||
};
|
||||
let error = validate_scanner_publication_lease_response_fields(&stale_generation, "session-a", 7)
|
||||
let error = validate_scanner_publication_lease_response_fields(&stale_generation, "session-a", 7, false)
|
||||
.expect_err("a response from an older movement generation must be rejected");
|
||||
assert!(error.to_string().contains("different scanner publication lease generation"));
|
||||
|
||||
@@ -3017,9 +3035,21 @@ mod tests {
|
||||
session_id: "session-b".to_string(),
|
||||
..response
|
||||
};
|
||||
let error = validate_scanner_publication_lease_response_fields(&stale_session, "session-a", 7)
|
||||
let error = validate_scanner_publication_lease_response_fields(&stale_session, "session-a", 7, false)
|
||||
.expect_err("a response from an older scanner session must be rejected");
|
||||
assert!(error.to_string().contains("different scanner publication lease session"));
|
||||
|
||||
let observational = ScannerPublicationLeaseResponse {
|
||||
observational_only: true,
|
||||
..stale_session
|
||||
};
|
||||
assert!(
|
||||
validate_scanner_publication_lease_response_fields(&observational, "session-b", 7, true).is_ok(),
|
||||
"the signed purpose echo should validate an observation lease"
|
||||
);
|
||||
let error = validate_scanner_publication_lease_response_fields(&observational, "session-b", 7, false)
|
||||
.expect_err("a peer must not upgrade an observational lease to authoritative");
|
||||
assert!(error.to_string().contains("different scanner publication lease purpose"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -74,7 +74,7 @@ fn config_task_join_error(operation: &'static str, error: tokio::task::JoinError
|
||||
/// The operation must use the corresponding no-lock read/save functions.
|
||||
pub async fn with_server_config_write_lock<F, Fut, T>(store: Arc<ECStore>, operation: F) -> Result<T>
|
||||
where
|
||||
F: FnOnce() -> Fut + Send + 'static,
|
||||
F: FnOnce(crate::object_api::WriteCommitGuard) -> Fut + Send + 'static,
|
||||
Fut: std::future::Future<Output = T> + Send + 'static,
|
||||
T: Send + 'static,
|
||||
{
|
||||
@@ -85,8 +85,8 @@ where
|
||||
let transaction_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &transaction_lock).await?;
|
||||
let _transaction_guard = transaction_lock.get_write_lock(get_lock_acquire_timeout()).await?;
|
||||
let namespace_lock = store.new_ns_lock(RUSTFS_META_BUCKET, SERVER_CONFIG_OBJECT).await?;
|
||||
let _write_guard = namespace_lock.get_write_lock(get_lock_acquire_timeout()).await?;
|
||||
Ok(operation().await)
|
||||
let write_guard = crate::object_api::WriteCommitGuard::acquire(&namespace_lock, get_lock_acquire_timeout()).await?;
|
||||
Ok(operation(write_guard.clone()).await)
|
||||
})
|
||||
.await
|
||||
.map_err(|error| config_task_join_error("server config write", error))?
|
||||
@@ -119,14 +119,14 @@ where
|
||||
/// one metadata config object. The operation must use no-lock object I/O.
|
||||
pub async fn with_config_object_write_lock<F, Fut, T>(store: Arc<ECStore>, object: String, operation: F) -> Result<T>
|
||||
where
|
||||
F: FnOnce() -> Fut + Send + 'static,
|
||||
F: FnOnce(crate::object_api::WriteCommitGuard) -> Fut + Send + 'static,
|
||||
Fut: std::future::Future<Output = T> + Send + 'static,
|
||||
T: Send + 'static,
|
||||
{
|
||||
tokio::spawn(async move {
|
||||
let namespace_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
|
||||
let _write_guard = namespace_lock.get_write_lock(get_lock_acquire_timeout()).await?;
|
||||
Ok(operation().await)
|
||||
let write_guard = crate::object_api::WriteCommitGuard::acquire(&namespace_lock, get_lock_acquire_timeout()).await?;
|
||||
Ok(operation(write_guard.clone()).await)
|
||||
})
|
||||
.await
|
||||
.map_err(|error| config_task_join_error("config object write", error))?
|
||||
@@ -495,10 +495,22 @@ pub(crate) async fn read_config_preserve_empty<S>(api: Arc<S>, file: &str) -> Re
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
let (data, _obj) = read_config_with_metadata_inner(api, file, &ObjectOptions::default(), true, None).await?;
|
||||
let (data, _obj) = read_config_preserve_empty_with_metadata_opts(api, file, &ObjectOptions::default()).await?;
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
/// Preserve an empty config payload, its metadata, and caller-supplied read options.
|
||||
pub(crate) async fn read_config_preserve_empty_with_metadata_opts<S>(
|
||||
api: Arc<S>,
|
||||
file: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<(Vec<u8>, ObjectInfo)>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
read_config_with_metadata_inner(api, file, opts, true, None).await
|
||||
}
|
||||
|
||||
pub async fn read_config_no_lock<S>(api: Arc<S>, file: &str) -> Result<Vec<u8>>
|
||||
where
|
||||
S: ObjectIO<
|
||||
@@ -527,15 +539,13 @@ pub(crate) async fn read_config_no_lock_preserve_empty_with_metadata<S>(api: Arc
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
read_config_with_metadata_inner(
|
||||
read_config_preserve_empty_with_metadata_opts(
|
||||
api,
|
||||
file,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
true,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -638,7 +648,12 @@ where
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn save_config_no_lock<S>(api: Arc<S>, file: &str, data: Vec<u8>) -> Result<()>
|
||||
pub async fn save_config_no_lock<S>(
|
||||
api: Arc<S>,
|
||||
file: &str,
|
||||
data: Vec<u8>,
|
||||
guard: &crate::object_api::WriteCommitGuard,
|
||||
) -> Result<()>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
@@ -650,17 +665,13 @@ where
|
||||
PutObjectReader = PutObjReader,
|
||||
>,
|
||||
{
|
||||
save_config_with_opts(
|
||||
api,
|
||||
file,
|
||||
data,
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
let mut opts = ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
};
|
||||
opts.add_write_commit_guard(guard);
|
||||
save_config_with_opts(api, file, data, &opts).await
|
||||
}
|
||||
|
||||
/// `delete_config` with `no_lock` set — for callers already holding the
|
||||
@@ -846,12 +857,12 @@ where
|
||||
Ok(cfg)
|
||||
}
|
||||
|
||||
async fn new_and_save_server_config_no_lock<S>(api: Arc<S>) -> Result<Config>
|
||||
async fn new_and_save_server_config_no_lock<S>(api: Arc<S>, guard: &crate::object_api::WriteCommitGuard) -> Result<Config>
|
||||
where
|
||||
S: EcstoreObjectIO + StorageAdminApi,
|
||||
{
|
||||
let cfg = new_server_config();
|
||||
save_server_config_no_lock(api, &cfg).await?;
|
||||
save_server_config_no_lock(api, &cfg, guard).await?;
|
||||
Ok(cfg)
|
||||
}
|
||||
|
||||
@@ -2138,14 +2149,18 @@ where
|
||||
}
|
||||
|
||||
/// Handle the situation where the configuration file does not exist, create and save a new configuration
|
||||
async fn handle_missing_config<S>(api: Arc<S>, context: &str, namespace_lock_held: bool) -> Result<Config>
|
||||
async fn handle_missing_config<S>(
|
||||
api: Arc<S>,
|
||||
context: &str,
|
||||
write_guard: Option<&crate::object_api::WriteCommitGuard>,
|
||||
) -> Result<Config>
|
||||
where
|
||||
S: EcstoreObjectIO + StorageAdminApi + NamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
|
||||
{
|
||||
warn!("Configuration not found ({}): Start initializing new configuration", context);
|
||||
let cfg = if runtime_sources::first_cluster_node_is_local().await {
|
||||
if namespace_lock_held {
|
||||
new_and_save_server_config_no_lock(api.clone()).await?
|
||||
if let Some(guard) = write_guard {
|
||||
new_and_save_server_config_no_lock(api.clone(), guard).await?
|
||||
} else {
|
||||
new_and_save_server_config(api.clone()).await?
|
||||
}
|
||||
@@ -2175,13 +2190,13 @@ where
|
||||
> + StorageAdminApi,
|
||||
S: NamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
|
||||
{
|
||||
read_config_without_migrate_inner(api, false).await
|
||||
read_config_without_migrate_inner(api, None).await
|
||||
}
|
||||
|
||||
/// Reads the server config while an upper layer holds the namespace write lock
|
||||
/// for [`SERVER_CONFIG_OBJECT`]. Missing-config initialization uses the matching
|
||||
/// no-lock save path and therefore cannot recursively acquire the same lock.
|
||||
pub async fn read_config_without_migrate_no_lock<S>(api: Arc<S>) -> Result<Config>
|
||||
pub async fn read_config_without_migrate_no_lock<S>(api: Arc<S>, guard: &crate::object_api::WriteCommitGuard) -> Result<Config>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
@@ -2194,7 +2209,7 @@ where
|
||||
> + StorageAdminApi
|
||||
+ NamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
|
||||
{
|
||||
read_config_without_migrate_inner(api, true).await
|
||||
read_config_without_migrate_inner(api, Some(guard)).await
|
||||
}
|
||||
|
||||
/// Reads an already-initialized server config while a caller owns a namespace
|
||||
@@ -2204,7 +2219,10 @@ pub async fn read_existing_server_config_no_lock(api: Arc<ECStore>) -> Result<Co
|
||||
Ok(decode_persisted_server_config(&data)?.merge())
|
||||
}
|
||||
|
||||
async fn read_config_without_migrate_inner<S>(api: Arc<S>, namespace_lock_held: bool) -> Result<Config>
|
||||
async fn read_config_without_migrate_inner<S>(
|
||||
api: Arc<S>,
|
||||
write_guard: Option<&crate::object_api::WriteCommitGuard>,
|
||||
) -> Result<Config>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
@@ -2220,19 +2238,23 @@ where
|
||||
let config_file = server_config_path();
|
||||
|
||||
// Try to read the configuration file.
|
||||
let data = if namespace_lock_held {
|
||||
let data = if write_guard.is_some() {
|
||||
read_config_no_lock(api.clone(), &config_file).await
|
||||
} else {
|
||||
read_config(api.clone(), &config_file).await
|
||||
};
|
||||
match data {
|
||||
Ok(data) => read_server_config(api, &data, namespace_lock_held).await,
|
||||
Err(Error::ConfigNotFound) => handle_missing_config(api, "Read the main configuration", namespace_lock_held).await,
|
||||
Ok(data) => read_server_config(api, &data, write_guard).await,
|
||||
Err(Error::ConfigNotFound) => handle_missing_config(api, "Read the main configuration", write_guard).await,
|
||||
Err(err) => handle_config_read_error(err, &config_file),
|
||||
}
|
||||
}
|
||||
|
||||
async fn read_server_config<S>(api: Arc<S>, data: &[u8], namespace_lock_held: bool) -> Result<Config>
|
||||
async fn read_server_config<S>(
|
||||
api: Arc<S>,
|
||||
data: &[u8],
|
||||
write_guard: Option<&crate::object_api::WriteCommitGuard>,
|
||||
) -> Result<Config>
|
||||
where
|
||||
S: EcstoreObjectIO + StorageAdminApi + NamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
|
||||
{
|
||||
@@ -2242,7 +2264,7 @@ where
|
||||
warn!("Received empty configuration data, try to reread from '{}'", config_file);
|
||||
|
||||
// Try to read the configuration again
|
||||
let data = if namespace_lock_held {
|
||||
let data = if write_guard.is_some() {
|
||||
read_config_no_lock(api.clone(), &config_file).await
|
||||
} else {
|
||||
read_config(api.clone(), &config_file).await
|
||||
@@ -2253,7 +2275,7 @@ where
|
||||
return Ok(cfg.merge());
|
||||
}
|
||||
Err(Error::ConfigNotFound) => {
|
||||
return handle_missing_config(api, "Read alternate configuration", namespace_lock_held).await;
|
||||
return handle_missing_config(api, "Read alternate configuration", write_guard).await;
|
||||
}
|
||||
Err(err) => return handle_config_read_error(err, &config_file),
|
||||
}
|
||||
@@ -2687,7 +2709,7 @@ where
|
||||
|
||||
/// Saves the server config while an upper layer holds the namespace write
|
||||
/// lock for [`SERVER_CONFIG_OBJECT`].
|
||||
pub async fn save_server_config_no_lock<S>(api: Arc<S>, cfg: &Config) -> Result<()>
|
||||
pub async fn save_server_config_no_lock<S>(api: Arc<S>, cfg: &Config, guard: &crate::object_api::WriteCommitGuard) -> Result<()>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
@@ -2699,10 +2721,10 @@ where
|
||||
PutObjectReader = PutObjReader,
|
||||
>,
|
||||
{
|
||||
save_server_config_inner(api, cfg, true).await
|
||||
save_server_config_inner(api, cfg, guard).await
|
||||
}
|
||||
|
||||
async fn save_server_config_inner<S>(api: Arc<S>, cfg: &Config, no_lock: bool) -> Result<()>
|
||||
async fn save_server_config_inner<S>(api: Arc<S>, cfg: &Config, guard: &crate::object_api::WriteCommitGuard) -> Result<()>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
@@ -2715,11 +2737,7 @@ where
|
||||
>,
|
||||
{
|
||||
let config_file = get_config_file();
|
||||
let existing = match if no_lock {
|
||||
read_config_no_lock(api.clone(), &config_file).await
|
||||
} else {
|
||||
read_config(api.clone(), &config_file).await
|
||||
} {
|
||||
let existing = match read_config_no_lock(api.clone(), &config_file).await {
|
||||
Ok(v) => Some(v),
|
||||
Err(Error::ConfigNotFound) => None,
|
||||
Err(err) => {
|
||||
@@ -2743,21 +2761,7 @@ where
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if no_lock {
|
||||
save_config_with_opts(
|
||||
api,
|
||||
&config_file,
|
||||
data,
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
save_config(api, &config_file, data).await
|
||||
}
|
||||
save_config_no_lock(api, &config_file, data, guard).await
|
||||
}
|
||||
|
||||
pub async fn lookup_configs<S>(cfg: &mut Config, api: Arc<S>) -> Result<()>
|
||||
@@ -2821,9 +2825,10 @@ mod tests {
|
||||
config_task_join_error, configs_semantically_equal, decode_server_config_blob, encode_server_config_blob,
|
||||
heal_config_descriptor, is_standard_object_server_config, lookup_configs, map_system_metadata_write_error,
|
||||
new_and_save_server_config, read_config, read_config_no_lock_preserve_empty_with_metadata, read_config_preserve_empty,
|
||||
read_config_with_metadata, read_config_without_migrate, read_server_config_snapshot, save_config_with_opts_inner,
|
||||
save_server_config, save_server_config_snapshot, save_server_config_snapshot_with_generation,
|
||||
server_config_transaction_lock_path, should_warn_ignored_scalar_section, storage_class_kvs_mut,
|
||||
read_config_preserve_empty_with_metadata_opts, read_config_with_metadata, read_config_without_migrate,
|
||||
read_server_config_snapshot, save_config_with_opts_inner, save_server_config, save_server_config_snapshot,
|
||||
save_server_config_snapshot_with_generation, server_config_transaction_lock_path, should_warn_ignored_scalar_section,
|
||||
storage_class_kvs_mut,
|
||||
};
|
||||
use crate::config::{audit, heal, notify, oidc, scanner};
|
||||
use crate::disk::{RUSTFS_META_BUCKET, endpoint::Endpoint};
|
||||
@@ -5071,6 +5076,7 @@ mod tests {
|
||||
heal_replacement: Option<Vec<u8>>,
|
||||
heal_calls: AtomicUsize,
|
||||
write_calls: AtomicUsize,
|
||||
last_get_no_lock: AtomicBool,
|
||||
last_put_no_lock: AtomicBool,
|
||||
last_put_preconditions: Mutex<Option<HTTPPreconditions>>,
|
||||
revision: AtomicUsize,
|
||||
@@ -5086,6 +5092,7 @@ mod tests {
|
||||
heal_replacement,
|
||||
heal_calls: AtomicUsize::new(0),
|
||||
write_calls: AtomicUsize::new(0),
|
||||
last_get_no_lock: AtomicBool::new(false),
|
||||
last_put_no_lock: AtomicBool::new(false),
|
||||
last_put_preconditions: Mutex::new(None),
|
||||
revision: AtomicUsize::new(1),
|
||||
@@ -5141,8 +5148,9 @@ mod tests {
|
||||
_object: &str,
|
||||
_range: Option<HTTPRangeSpec>,
|
||||
_h: HeaderMap,
|
||||
_opts: &ObjectOptions,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<GetObjectReader> {
|
||||
self.last_get_no_lock.store(opts.no_lock, Ordering::SeqCst);
|
||||
let data = match &*self.state.lock().expect("state lock poisoned") {
|
||||
RecoveryReadState::Missing => return Err(Error::ConfigNotFound),
|
||||
RecoveryReadState::Blob(data) => data.clone(),
|
||||
@@ -5437,6 +5445,60 @@ mod tests {
|
||||
assert!(data.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_config_preserve_empty_metadata_keeps_empty_and_options() {
|
||||
let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Blob(Vec::new()), None));
|
||||
for no_lock in [false, true] {
|
||||
let opts = ObjectOptions {
|
||||
no_lock,
|
||||
..Default::default()
|
||||
};
|
||||
let (data, info) = read_config_preserve_empty_with_metadata_opts(store.clone(), "config/empty.json", &opts)
|
||||
.await
|
||||
.expect("an existing empty config must retain its payload and metadata");
|
||||
assert!(data.is_empty());
|
||||
assert_eq!(info.size, 0);
|
||||
assert_eq!(info.actual_size, 0);
|
||||
assert_eq!(info.etag.as_deref(), Some("config-1"));
|
||||
assert_eq!(info.data_dir, Some(uuid::Uuid::from_u128(1)));
|
||||
assert_eq!(store.last_get_no_lock.load(Ordering::SeqCst), no_lock);
|
||||
}
|
||||
let err = read_config(store, "config/empty.json")
|
||||
.await
|
||||
.expect_err("the existing reader must still treat empty configs as missing");
|
||||
assert!(matches!(err, Error::ConfigNotFound));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_config_preserve_empty_metadata_keeps_payload_and_metadata() {
|
||||
let payload = b"original config bytes".to_vec();
|
||||
let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Blob(payload.clone()), None));
|
||||
let (data, info) =
|
||||
read_config_preserve_empty_with_metadata_opts(store, "config/nonempty.json", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("a nonempty config must retain its payload and metadata");
|
||||
assert_eq!(data, payload);
|
||||
let size = i64::try_from(payload.len()).expect("test payload size fits i64");
|
||||
assert_eq!(info.size, size);
|
||||
assert_eq!(info.actual_size, size);
|
||||
assert_eq!(info.etag.as_deref(), Some("config-1"));
|
||||
assert_eq!(info.data_dir, Some(uuid::Uuid::from_u128(1)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_config_preserve_empty_metadata_preserves_read_errors() {
|
||||
let missing = Arc::new(RecoveryMockStore::new(RecoveryReadState::Missing, None));
|
||||
let err = read_config_preserve_empty_with_metadata_opts(missing, "config/missing.json", &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("physical absence must remain ConfigNotFound");
|
||||
assert!(matches!(err, Error::ConfigNotFound));
|
||||
let quorum = Arc::new(RecoveryMockStore::new(RecoveryReadState::QuorumError, None));
|
||||
let err = read_config_preserve_empty_with_metadata_opts(quorum, "config/quorum.json", &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("a quorum failure must retain its typed read error");
|
||||
assert!(matches!(err, Error::ErasureReadQuorum));
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl StorageAdminApi for RecoveryMockStore {
|
||||
type BackendInfo = rustfs_madmin::BackendInfo;
|
||||
|
||||
@@ -3692,8 +3692,8 @@ fn activation_pool_meta_lock_error(err: rustfs_lock::LockError) -> Error {
|
||||
}
|
||||
|
||||
pub(crate) struct PoolRebalanceActivationFence {
|
||||
pool_meta_guard: rustfs_lock::NamespaceLockGuard,
|
||||
rebalance_meta_guard: rustfs_lock::NamespaceLockGuard,
|
||||
pool_meta_guard: Arc<rustfs_lock::NamespaceLockGuard>,
|
||||
rebalance_meta_guard: Arc<rustfs_lock::NamespaceLockGuard>,
|
||||
fleet_proof: Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>,
|
||||
decommission_target_fence_proof: Option<crate::services::notification_sys::DecommissionTargetFenceFleetProofToken>,
|
||||
#[cfg(test)]
|
||||
@@ -3720,7 +3720,12 @@ impl PoolRebalanceActivationFence {
|
||||
let forced_lost = self.forced_lost.load(Ordering::Acquire);
|
||||
#[cfg(not(test))]
|
||||
let forced_lost = false;
|
||||
if forced_lost || self.pool_meta_guard.is_lock_lost() || self.rebalance_meta_guard.is_lock_lost() {
|
||||
if forced_lost
|
||||
|| self.pool_meta_guard.is_lock_lost()
|
||||
|| self.pool_meta_guard.is_released()
|
||||
|| self.rebalance_meta_guard.is_lock_lost()
|
||||
|| self.rebalance_meta_guard.is_released()
|
||||
{
|
||||
return Err(Error::other("activation lock lost before metadata commit or worker admission"));
|
||||
}
|
||||
if self
|
||||
@@ -3742,8 +3747,8 @@ impl PoolRebalanceActivationFence {
|
||||
}
|
||||
|
||||
pub(crate) fn add_namespace_lock_fence(&self, opts: &mut ObjectOptions) {
|
||||
opts.add_namespace_lock_guard(&self.pool_meta_guard);
|
||||
opts.add_namespace_lock_guard(&self.rebalance_meta_guard);
|
||||
opts.add_owned_write_lock(Arc::clone(&self.pool_meta_guard), RUSTFS_META_BUCKET, POOL_META_NAME);
|
||||
opts.add_owned_write_lock(Arc::clone(&self.rebalance_meta_guard), RUSTFS_META_BUCKET, REBAL_META_NAME);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -3784,8 +3789,8 @@ where
|
||||
.map_err(activation_pool_meta_lock_error)?;
|
||||
|
||||
Ok(PoolRebalanceActivationFence {
|
||||
pool_meta_guard,
|
||||
rebalance_meta_guard,
|
||||
pool_meta_guard: Arc::new(pool_meta_guard),
|
||||
rebalance_meta_guard: Arc::new(rebalance_meta_guard),
|
||||
fleet_proof,
|
||||
decommission_target_fence_proof: None,
|
||||
#[cfg(test)]
|
||||
@@ -4067,7 +4072,7 @@ fn rollback_start_decommission_pool_meta(pool_meta: &mut PoolMeta, previous_pool
|
||||
}
|
||||
|
||||
fn ensure_pool_meta_write_fence(guard: &rustfs_lock::NamespaceLockGuard, operation: &str) -> Result<()> {
|
||||
if guard.is_lock_lost() {
|
||||
if guard.is_lock_lost() || guard.is_released() {
|
||||
return Err(Error::other(format!("{operation}: pool metadata distributed fence was lost")));
|
||||
}
|
||||
Ok(())
|
||||
@@ -6339,6 +6344,9 @@ fn encode_pool_meta_v3_envelope(
|
||||
}
|
||||
|
||||
enum PoolMetaPersistenceFence<'a> {
|
||||
Owned(Arc<rustfs_lock::NamespaceLockGuard>),
|
||||
OfflineRecovery,
|
||||
#[cfg(test)]
|
||||
Distributed(Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>),
|
||||
Activation(&'a PoolRebalanceActivationFence),
|
||||
}
|
||||
@@ -6346,6 +6354,10 @@ enum PoolMetaPersistenceFence<'a> {
|
||||
impl PoolMetaPersistenceFence<'_> {
|
||||
fn ensure_held(&self) -> Result<()> {
|
||||
match self {
|
||||
Self::Owned(guard) if guard.is_lock_lost() || guard.is_released() => {
|
||||
Err(Error::other("pool metadata distributed fence was lost before a replica write"))
|
||||
}
|
||||
#[cfg(test)]
|
||||
Self::Distributed(Some(signal)) if signal.is_lost() => {
|
||||
Err(Error::other("pool metadata distributed fence was lost before a replica write"))
|
||||
}
|
||||
@@ -6354,14 +6366,30 @@ impl PoolMetaPersistenceFence<'_> {
|
||||
}
|
||||
}
|
||||
|
||||
fn add_to_options(&self, opts: &mut ObjectOptions) {
|
||||
fn add_to_options(&self, opts: &mut ObjectOptions, object: &str) {
|
||||
match self {
|
||||
Self::Owned(guard) => opts.add_owned_write_lock(Arc::clone(guard), RUSTFS_META_BUCKET, object),
|
||||
Self::OfflineRecovery => opts.use_offline_recovery_write(),
|
||||
#[cfg(test)]
|
||||
Self::Distributed(Some(signal)) => opts.add_namespace_lock_lost_signal(Arc::clone(signal)),
|
||||
Self::Activation(fence) => fence.add_namespace_lock_fence(opts),
|
||||
Self::Activation(fence) => {
|
||||
fence.add_namespace_lock_fence(opts);
|
||||
opts.add_owned_write_lock(Arc::clone(&fence.pool_meta_guard), RUSTFS_META_BUCKET, object);
|
||||
}
|
||||
#[cfg(test)]
|
||||
Self::Distributed(None) => {}
|
||||
}
|
||||
}
|
||||
|
||||
fn allows_concurrent_phases(&self) -> bool {
|
||||
match self {
|
||||
Self::Owned(guard) => guard.lock_lost_signal().is_some(),
|
||||
#[cfg(test)]
|
||||
Self::Distributed(signal) => signal.is_some(),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_activation(&self) -> bool {
|
||||
matches!(self, Self::Activation(_))
|
||||
}
|
||||
@@ -6662,7 +6690,7 @@ fn pool_meta_cas_options(object: &str, token: &PoolMetaCasToken, fence: &PoolMet
|
||||
http_preconditions: Some(pool_meta_cas_preconditions(token, object)?),
|
||||
..Default::default()
|
||||
};
|
||||
fence.add_to_options(&mut opts);
|
||||
fence.add_to_options(&mut opts, object);
|
||||
Ok(opts)
|
||||
}
|
||||
|
||||
@@ -7049,7 +7077,7 @@ where
|
||||
write_state,
|
||||
initialized,
|
||||
PoolMetaIdentityWriteScope::All,
|
||||
&PoolMetaPersistenceFence::Distributed(None),
|
||||
&PoolMetaPersistenceFence::OfflineRecovery,
|
||||
&mut transaction_arm,
|
||||
)
|
||||
.await?;
|
||||
@@ -7076,7 +7104,7 @@ where
|
||||
write_state,
|
||||
false,
|
||||
PoolMetaIdentityWriteScope::Pools(pool_indices),
|
||||
&PoolMetaPersistenceFence::Distributed(None),
|
||||
&PoolMetaPersistenceFence::OfflineRecovery,
|
||||
&mut transaction_arm,
|
||||
)
|
||||
.await?;
|
||||
@@ -8154,11 +8182,16 @@ impl PoolMeta {
|
||||
.cloned()
|
||||
.ok_or_else(|| Error::other("pool metadata save failed: no storage pools available"))?;
|
||||
let pool_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
|
||||
let pool_meta_guard = pool_meta_lock.get_write_lock(get_lock_acquire_timeout()).await?;
|
||||
let pool_meta_guard = Arc::new(pool_meta_lock.get_write_lock(get_lock_acquire_timeout()).await?);
|
||||
let mut write_state = PoolMetaWriteState::default();
|
||||
let indices = (0..self.pools.len()).collect::<Vec<_>>();
|
||||
let outcome = self
|
||||
.save_no_lock_armed_scoped(pools, &mut write_state, pool_meta_guard.lock_lost_signal(), Some(&indices))
|
||||
.save_no_lock_armed_scoped(
|
||||
pools,
|
||||
&mut write_state,
|
||||
PoolMetaPersistenceFence::Owned(Arc::clone(&pool_meta_guard)),
|
||||
Some(&indices),
|
||||
)
|
||||
.await?;
|
||||
outcome.disarm();
|
||||
Ok(())
|
||||
@@ -8182,14 +8215,16 @@ impl PoolMeta {
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
let outcome = self.save_no_lock_armed_scoped(pools, write_state, None, None).await?;
|
||||
let outcome = self
|
||||
.save_no_lock_armed_scoped(pools, write_state, PoolMetaPersistenceFence::OfflineRecovery, None)
|
||||
.await?;
|
||||
Ok(outcome.into_committed())
|
||||
}
|
||||
|
||||
async fn save_no_lock_with_fence<S>(
|
||||
&self,
|
||||
pools: Vec<Arc<S>>,
|
||||
lock_lost: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
|
||||
fence: PoolMetaPersistenceFence<'_>,
|
||||
indices: &[usize],
|
||||
) -> Result<()>
|
||||
where
|
||||
@@ -8197,7 +8232,7 @@ impl PoolMeta {
|
||||
{
|
||||
let mut write_state = PoolMetaWriteState::default();
|
||||
let outcome = self
|
||||
.save_no_lock_armed_scoped(pools, &mut write_state, lock_lost, Some(indices))
|
||||
.save_no_lock_armed_scoped(pools, &mut write_state, fence, Some(indices))
|
||||
.await?;
|
||||
outcome.disarm();
|
||||
Ok(())
|
||||
@@ -8233,21 +8268,20 @@ impl PoolMeta {
|
||||
&self,
|
||||
pools: Vec<Arc<S>>,
|
||||
write_state: &mut PoolMetaWriteState,
|
||||
lock_lost: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
|
||||
fence: PoolMetaPersistenceFence<'_>,
|
||||
indices: &[usize],
|
||||
) -> Result<PoolMetaSaveOutcome>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
self.save_no_lock_armed_scoped(pools, write_state, lock_lost, Some(indices))
|
||||
.await
|
||||
self.save_no_lock_armed_scoped(pools, write_state, fence, Some(indices)).await
|
||||
}
|
||||
|
||||
async fn save_no_lock_armed_scoped<S>(
|
||||
&self,
|
||||
pools: Vec<Arc<S>>,
|
||||
write_state: &mut PoolMetaWriteState,
|
||||
lock_lost: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
|
||||
fence: PoolMetaPersistenceFence<'_>,
|
||||
indices: Option<&[usize]>,
|
||||
) -> Result<PoolMetaSaveOutcome>
|
||||
where
|
||||
@@ -8258,7 +8292,6 @@ impl PoolMeta {
|
||||
// returned outcome is dropped before publication, Drop latches the
|
||||
// sticky recovery gate.
|
||||
let mut transaction_arm = write_state.arm_transaction();
|
||||
let fence = PoolMetaPersistenceFence::Distributed(lock_lost);
|
||||
let committed = self
|
||||
.save_no_lock_transaction(pools, write_state, &fence, indices, &mut transaction_arm)
|
||||
.await
|
||||
@@ -8489,7 +8522,7 @@ impl PoolMeta {
|
||||
&& selection.revision.is_generation_protocol()
|
||||
&& !selection.replica_state.needs_repair
|
||||
&& !bootstrap_generation_required
|
||||
&& matches!(fence, PoolMetaPersistenceFence::Distributed(Some(_)));
|
||||
&& fence.allows_concurrent_phases();
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
let phase_barrier = if concurrent_phases && pools.len() == 4 {
|
||||
PoolMetaPhaseBarrier::bind(&previous, &committed, revision, &durable).await?
|
||||
@@ -8635,7 +8668,9 @@ impl PoolMeta {
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
let indices = (0..self.pools.len()).collect::<Vec<_>>();
|
||||
let outcome = self.save_no_lock_armed(pools, write_state, None, &indices).await?;
|
||||
let outcome = self
|
||||
.save_no_lock_armed(pools, write_state, PoolMetaPersistenceFence::OfflineRecovery, &indices)
|
||||
.await?;
|
||||
outcome.disarm();
|
||||
Ok(())
|
||||
}
|
||||
@@ -10831,7 +10866,7 @@ impl ECStore {
|
||||
&self,
|
||||
write_state: &mut PoolMetaWriteState,
|
||||
operation: &str,
|
||||
) -> Result<(rustfs_lock::NamespaceLockGuard, PoolMeta)> {
|
||||
) -> Result<(Arc<rustfs_lock::NamespaceLockGuard>, PoolMeta)> {
|
||||
self.acquire_pool_meta_write_guard_with_lock_error(write_state, operation, activation_pool_meta_lock_error)
|
||||
.await
|
||||
}
|
||||
@@ -10841,7 +10876,7 @@ impl ECStore {
|
||||
write_state: &mut PoolMetaWriteState,
|
||||
operation: &str,
|
||||
map_lock_error: F,
|
||||
) -> Result<(rustfs_lock::NamespaceLockGuard, PoolMeta)>
|
||||
) -> Result<(Arc<rustfs_lock::NamespaceLockGuard>, PoolMeta)>
|
||||
where
|
||||
F: FnOnce(rustfs_lock::LockError) -> Error,
|
||||
{
|
||||
@@ -10862,7 +10897,7 @@ impl ECStore {
|
||||
let selection = load_pool_meta_replicas_observing(self.pools.clone(), true, write_state).await?;
|
||||
write_state.observe_replicas(selection.replica_state);
|
||||
write_state.ensure_write_safe(operation)?;
|
||||
Ok((pool_meta_guard, selection.meta))
|
||||
Ok((Arc::new(pool_meta_guard), selection.meta))
|
||||
}
|
||||
|
||||
async fn acquire_pool_meta_read_guard(
|
||||
@@ -11132,7 +11167,7 @@ impl ECStore {
|
||||
pub(crate) async fn acquire_pool_meta_object_heal_fence(
|
||||
&self,
|
||||
target_pool_indices: &[usize],
|
||||
) -> Result<(rustfs_lock::NamespaceLockGuard, Vec<Result<()>>)> {
|
||||
) -> Result<(Arc<rustfs_lock::NamespaceLockGuard>, Vec<Result<()>>)> {
|
||||
let mut save_guard = self.pool_meta_save_gate.lock().await;
|
||||
let (pool_meta_guard, snapshot) = self
|
||||
.acquire_pool_meta_write_guard_with_lock_error(&mut save_guard, "pool metadata heal admission failed", Error::from)
|
||||
@@ -11361,7 +11396,12 @@ impl ECStore {
|
||||
ensure_exact_delete_capacity_namespace_fences(opts, bucket, object)?;
|
||||
|
||||
let outcome = snapshot
|
||||
.save_no_lock_armed(self.pools.clone(), &mut save_guard, write_guard.lock_lost_signal(), &source_pool_indices)
|
||||
.save_no_lock_armed(
|
||||
self.pools.clone(),
|
||||
&mut save_guard,
|
||||
PoolMetaPersistenceFence::Owned(Arc::clone(&write_guard)),
|
||||
&source_pool_indices,
|
||||
)
|
||||
.await?;
|
||||
ensure_pool_meta_write_fence(&write_guard, "exact delete capacity reconciliation save failed")?;
|
||||
for (target_pool_index, target_guard) in &target_guards {
|
||||
@@ -11470,7 +11510,7 @@ impl ECStore {
|
||||
.save_no_lock_armed(
|
||||
self.pools.clone(),
|
||||
&mut save_guard,
|
||||
pool_meta_guard.lock_lost_signal(),
|
||||
PoolMetaPersistenceFence::Owned(Arc::clone(&pool_meta_guard)),
|
||||
&[source_pool_index],
|
||||
)
|
||||
.await?;
|
||||
@@ -11659,7 +11699,7 @@ impl ECStore {
|
||||
.save_no_lock_armed(
|
||||
self.pools.clone(),
|
||||
&mut save_guard,
|
||||
pool_meta_guard.lock_lost_signal(),
|
||||
PoolMetaPersistenceFence::Owned(Arc::clone(&pool_meta_guard)),
|
||||
&[source_pool_index],
|
||||
)
|
||||
.await?;
|
||||
@@ -11965,7 +12005,12 @@ impl ECStore {
|
||||
};
|
||||
if reservation_grew || pending_added > 0 {
|
||||
let outcome = snapshot
|
||||
.save_no_lock_armed(self.pools.clone(), &mut save_guard, write_guard.lock_lost_signal(), &[source_pool_index])
|
||||
.save_no_lock_armed(
|
||||
self.pools.clone(),
|
||||
&mut save_guard,
|
||||
PoolMetaPersistenceFence::Owned(Arc::clone(&write_guard)),
|
||||
&[source_pool_index],
|
||||
)
|
||||
.await?;
|
||||
ensure_pool_meta_write_fence(&write_guard, "decommission target capacity intent save failed")?;
|
||||
snapshot = outcome.committed.clone();
|
||||
@@ -12137,7 +12182,12 @@ impl ECStore {
|
||||
return result;
|
||||
}
|
||||
let outcome = snapshot
|
||||
.save_no_lock_armed(self.pools.clone(), save_guard, write_guard.lock_lost_signal(), &[source_pool_index])
|
||||
.save_no_lock_armed(
|
||||
self.pools.clone(),
|
||||
save_guard,
|
||||
PoolMetaPersistenceFence::Owned(Arc::clone(write_guard)),
|
||||
&[source_pool_index],
|
||||
)
|
||||
.await?;
|
||||
ensure_pool_meta_write_fence(write_guard, "decommission target capacity progress save failed")?;
|
||||
if let Some(target_guard) = target_guard.as_ref() {
|
||||
@@ -12272,8 +12322,8 @@ impl ECStore {
|
||||
.first()
|
||||
.ok_or_else(|| Error::other("pool metadata recovery has no storage pools"))?;
|
||||
let lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
|
||||
let guard = lock.get_write_lock(get_lock_acquire_timeout()).await?;
|
||||
let fence = PoolMetaPersistenceFence::Distributed(guard.lock_lost_signal());
|
||||
let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await?);
|
||||
let fence = PoolMetaPersistenceFence::Owned(Arc::clone(&guard));
|
||||
let mut candidate = PoolMetaWriteState {
|
||||
aborted_transaction: Arc::new(AtomicBool::new(false)),
|
||||
transaction_failure: Arc::default(),
|
||||
@@ -12449,7 +12499,12 @@ impl ECStore {
|
||||
merge_pool_meta_updates_for_save(&mut snapshot, &pool_meta, indices, "pool metadata save failed")?;
|
||||
}
|
||||
let outcome = snapshot
|
||||
.save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), indices)
|
||||
.save_no_lock_armed(
|
||||
self.pools.clone(),
|
||||
&mut save_guard,
|
||||
PoolMetaPersistenceFence::Owned(Arc::clone(&pool_meta_guard)),
|
||||
indices,
|
||||
)
|
||||
.await?;
|
||||
let mut pool_meta = self.pool_meta.write().await;
|
||||
ensure_pool_meta_write_fence(&pool_meta_guard, "pool metadata save failed")?;
|
||||
@@ -12544,7 +12599,12 @@ impl ECStore {
|
||||
};
|
||||
|
||||
let outcome = match snapshot
|
||||
.save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), &[idx])
|
||||
.save_no_lock_armed(
|
||||
self.pools.clone(),
|
||||
&mut save_guard,
|
||||
PoolMetaPersistenceFence::Owned(Arc::clone(&pool_meta_guard)),
|
||||
&[idx],
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(outcome) => outcome,
|
||||
@@ -12610,7 +12670,12 @@ impl ECStore {
|
||||
#[cfg(test)]
|
||||
decommission_test_wrap_result("bucket_completion_before_save", &bucket.name, &bucket.prefix, 0, Ok(()))?;
|
||||
let outcome = snapshot
|
||||
.save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), &[idx])
|
||||
.save_no_lock_armed(
|
||||
self.pools.clone(),
|
||||
&mut save_guard,
|
||||
PoolMetaPersistenceFence::Owned(Arc::clone(&pool_meta_guard)),
|
||||
&[idx],
|
||||
)
|
||||
.await?;
|
||||
#[cfg(test)]
|
||||
decommission_test_wrap_result("bucket_completion_after_save", &bucket.name, &bucket.prefix, 0, Ok(()))?;
|
||||
@@ -12744,7 +12809,12 @@ impl ECStore {
|
||||
.await?;
|
||||
rollback_start_decommission_pool_meta(&mut snapshot, previous_pool_meta, indices);
|
||||
let outcome = snapshot
|
||||
.save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), indices)
|
||||
.save_no_lock_armed(
|
||||
self.pools.clone(),
|
||||
&mut save_guard,
|
||||
PoolMetaPersistenceFence::Owned(Arc::clone(&pool_meta_guard)),
|
||||
indices,
|
||||
)
|
||||
.await?;
|
||||
let mut pool_meta = self.pool_meta.write().await;
|
||||
ensure_pool_meta_write_fence(&pool_meta_guard, "decommission start rollback failed")?;
|
||||
@@ -13044,7 +13114,7 @@ impl ECStore {
|
||||
save_pool_meta: Save,
|
||||
) -> Result<()>
|
||||
where
|
||||
Save: FnOnce(PoolMeta, Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>) -> SaveFuture + Send + 'static,
|
||||
Save: FnOnce(PoolMeta, PoolMetaPersistenceFence<'static>) -> SaveFuture + Send + 'static,
|
||||
SaveFuture: Future<Output = Result<()>> + Send + 'static,
|
||||
{
|
||||
let store = self.clone();
|
||||
@@ -13064,7 +13134,7 @@ impl ECStore {
|
||||
save_pool_meta: Save,
|
||||
) -> Result<()>
|
||||
where
|
||||
Save: FnOnce(PoolMeta, Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>) -> SaveFuture,
|
||||
Save: FnOnce(PoolMeta, PoolMetaPersistenceFence<'static>) -> SaveFuture,
|
||||
SaveFuture: Future<Output = Result<()>>,
|
||||
{
|
||||
let owner = owner.as_ref();
|
||||
@@ -13113,9 +13183,17 @@ impl ECStore {
|
||||
));
|
||||
}
|
||||
}
|
||||
let pool_meta_fence = _pool_meta_guard
|
||||
.as_ref()
|
||||
.and_then(rustfs_lock::NamespaceLockGuard::lock_lost_signal);
|
||||
let pool_meta_fence = match _pool_meta_guard.as_ref() {
|
||||
Some(guard) => PoolMetaPersistenceFence::Owned(Arc::clone(guard)),
|
||||
None => {
|
||||
#[cfg(test)]
|
||||
{
|
||||
PoolMetaPersistenceFence::OfflineRecovery
|
||||
}
|
||||
#[cfg(not(test))]
|
||||
return Err(Error::other("runtime pool metadata cancellation requires its write-lock owner"));
|
||||
}
|
||||
};
|
||||
|
||||
// Lock order: start gate, target gates, save gate, distributed pool
|
||||
// metadata fence, rebalance_meta, decommission_cancelers, then
|
||||
@@ -13554,7 +13632,12 @@ impl ECStore {
|
||||
let (save_outcome, save_error) = if changed {
|
||||
ensure_decommission_target_fence_fleet_proof(target_fence_proof.as_ref(), target_fence_proof_required)?;
|
||||
match snapshot
|
||||
.save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), &changed_indices)
|
||||
.save_no_lock_armed(
|
||||
self.pools.clone(),
|
||||
&mut save_guard,
|
||||
PoolMetaPersistenceFence::Owned(Arc::clone(&pool_meta_guard)),
|
||||
&changed_indices,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(outcome) => (Some(outcome), None),
|
||||
@@ -13646,7 +13729,12 @@ impl ECStore {
|
||||
.await?;
|
||||
snapshot.mark_decommission_capacity_blocked(idx, err.to_string(), OffsetDateTime::now_utc())?;
|
||||
let outcome = snapshot
|
||||
.save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), &[idx])
|
||||
.save_no_lock_armed(
|
||||
self.pools.clone(),
|
||||
&mut save_guard,
|
||||
PoolMetaPersistenceFence::Owned(Arc::clone(&pool_meta_guard)),
|
||||
&[idx],
|
||||
)
|
||||
.await?;
|
||||
ensure_pool_meta_write_fence(&pool_meta_guard, "decommission capacity pause failed")?;
|
||||
{
|
||||
@@ -22023,7 +22111,7 @@ mod tests {
|
||||
let mut state = store.pool_meta_save_gate.lock().await;
|
||||
let (fence, _) = store.acquire_pool_meta_write_guard(&mut state, "test save").await.unwrap();
|
||||
let outcome = requested
|
||||
.save_no_lock_armed(store.pools.clone(), &mut state, fence.lock_lost_signal(), &[0])
|
||||
.save_no_lock_armed(store.pools.clone(), &mut state, PoolMetaPersistenceFence::Owned(Arc::clone(&fence)), &[0])
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(outcome.committed.pools[0].last_update, requested.pools[0].last_update);
|
||||
@@ -24825,11 +24913,12 @@ mod pools_tests {
|
||||
config[3][0].gate = Some(gate.clone());
|
||||
let fixture = Fixture::new(config);
|
||||
let mut state = fixture.state();
|
||||
let mut save = Box::pin(
|
||||
fixture
|
||||
.desired
|
||||
.save_no_lock_armed(fixture.pools.clone(), &mut state, signal(), &[0]),
|
||||
);
|
||||
let mut save = Box::pin(fixture.desired.save_no_lock_armed(
|
||||
fixture.pools.clone(),
|
||||
&mut state,
|
||||
PoolMetaPersistenceFence::Distributed(signal()),
|
||||
&[0],
|
||||
));
|
||||
tokio::select! { _ = fixture.trace.wait_for(4, 3) => {}, _ = &mut save => panic!("commit crossed the last prepare") }
|
||||
assert_eq!(fixture.trace.commit_count(), 0);
|
||||
gate.add_permits(1);
|
||||
@@ -24849,7 +24938,7 @@ mod pools_tests {
|
||||
let mut state = fixture.state();
|
||||
let err = fixture
|
||||
.desired
|
||||
.save_no_lock_armed(fixture.pools.clone(), &mut state, signal(), &[0])
|
||||
.save_no_lock_armed(fixture.pools.clone(), &mut state, PoolMetaPersistenceFence::Distributed(signal()), &[0])
|
||||
.await
|
||||
.expect_err("a missing conditional-write revision must fail");
|
||||
assert!(err.to_string().contains("without a conditional-write revision"));
|
||||
@@ -24866,11 +24955,12 @@ mod pools_tests {
|
||||
config[3][0].gate = Some(gate.clone());
|
||||
let fixture = Fixture::new(config);
|
||||
let mut state = fixture.state();
|
||||
let mut save = Box::pin(
|
||||
fixture
|
||||
.desired
|
||||
.save_no_lock_armed(fixture.pools.clone(), &mut state, signal(), &[0]),
|
||||
);
|
||||
let mut save = Box::pin(fixture.desired.save_no_lock_armed(
|
||||
fixture.pools.clone(),
|
||||
&mut state,
|
||||
PoolMetaPersistenceFence::Distributed(signal()),
|
||||
&[0],
|
||||
));
|
||||
tokio::select! { _ = fixture.trace.wait_for(4, 3) => {}, _ = &mut save => panic!("phase returned with a live replica") }
|
||||
assert_eq!(fixture.trace.commit_count(), 0);
|
||||
gate.add_permits(1);
|
||||
@@ -25045,11 +25135,12 @@ mod pools_tests {
|
||||
}
|
||||
let fixture = Fixture::new(config);
|
||||
let mut state = fixture.state();
|
||||
let mut save = Box::pin(
|
||||
fixture
|
||||
.desired
|
||||
.save_no_lock_armed(fixture.pools.clone(), &mut state, signal(), &[0]),
|
||||
);
|
||||
let mut save = Box::pin(fixture.desired.save_no_lock_armed(
|
||||
fixture.pools.clone(),
|
||||
&mut state,
|
||||
PoolMetaPersistenceFence::Distributed(signal()),
|
||||
&[0],
|
||||
));
|
||||
tokio::select! { _ = fixture.trace.wait_for(4 * (phase + 1), 4 * phase) => {}, _ = &mut save => panic!("blocked phase finished") }
|
||||
drop(save);
|
||||
assert!(state.ensure_write_safe("cancelled phase").is_err());
|
||||
@@ -25060,7 +25151,7 @@ mod pools_tests {
|
||||
let mut state = fixture.state();
|
||||
let outcome = fixture
|
||||
.desired
|
||||
.save_no_lock_armed(fixture.pools.clone(), &mut state, signal(), &[0])
|
||||
.save_no_lock_armed(fixture.pools.clone(), &mut state, PoolMetaPersistenceFence::Distributed(signal()), &[0])
|
||||
.await
|
||||
.expect("durable save before publication");
|
||||
drop(outcome);
|
||||
@@ -25088,6 +25179,7 @@ mod pools_tests {
|
||||
.await
|
||||
.expect("lease request")
|
||||
.expect("lease acquired");
|
||||
let guard = Arc::new(guard);
|
||||
let gate = Arc::new(tokio::sync::Semaphore::new(0));
|
||||
let mut config = steps(6);
|
||||
for step in &mut config {
|
||||
@@ -25096,7 +25188,7 @@ mod pools_tests {
|
||||
let fixture = Fixture::new(config);
|
||||
let state = fixture.state();
|
||||
let mut arm = state.arm_transaction();
|
||||
let fence = PoolMetaPersistenceFence::Distributed(guard.lock_lost_signal());
|
||||
let fence = PoolMetaPersistenceFence::Owned(Arc::clone(&guard));
|
||||
let mut pending = Box::pin(prepare(&fixture, &fence, &mut arm));
|
||||
tokio::select! { _ = fixture.trace.wait_for(4, 0) => {}, _ = &mut pending => panic!("gate ignored") }
|
||||
tokio::time::timeout(StdDuration::from_secs(3), guard.lock_lost_notified())
|
||||
@@ -25122,7 +25214,7 @@ mod pools_tests {
|
||||
let mut state = fixture.state();
|
||||
let result = fixture
|
||||
.desired
|
||||
.save_no_lock_armed(fixture.pools.clone(), &mut state, signal(), &[0])
|
||||
.save_no_lock_armed(fixture.pools.clone(), &mut state, PoolMetaPersistenceFence::Distributed(signal()), &[0])
|
||||
.await;
|
||||
assert_eq!(fixture.trace.commit_count(), 4);
|
||||
if matches!(fault, WriteFault::TimeoutAfterWrite) {
|
||||
@@ -25172,7 +25264,7 @@ mod pools_tests {
|
||||
let mut save = Box::pin(fixture.desired.save_no_lock_armed(
|
||||
fixture.pools.clone(),
|
||||
&mut state,
|
||||
if legacy || needs_repair { signal() } else { None },
|
||||
PoolMetaPersistenceFence::Distributed(if legacy || needs_repair { signal() } else { None }),
|
||||
&[0],
|
||||
));
|
||||
tokio::select! { _ = fixture.trace.wait_for(1, 0) => {}, _ = &mut save => panic!("serial write gate ignored") }
|
||||
@@ -25732,7 +25824,7 @@ mod pools_tests {
|
||||
let save_task = tokio::spawn(async move {
|
||||
let mut save_guard = save_store.pool_meta_save_gate.lock().await;
|
||||
let outcome = snapshot
|
||||
.save_no_lock_armed(vec![save_committed], &mut save_guard, None, &[0])
|
||||
.save_no_lock_armed(vec![save_committed], &mut save_guard, PoolMetaPersistenceFence::OfflineRecovery, &[0])
|
||||
.await?;
|
||||
task_publish_started.notify_one();
|
||||
task_publish_release.notified().await;
|
||||
@@ -25759,6 +25851,73 @@ mod pools_tests {
|
||||
.expect_err("cancellation after replica save but before publication must keep writes blocked");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pool_meta_cas_context_retains_actual_owner_for_exact_target() {
|
||||
use crate::object_api::write_commit_context::WriteCommitContext;
|
||||
|
||||
let lock = NamespaceLock::with_local_manager("pool-meta-cas-owner".to_string(), Arc::new(GlobalLockManager::new()));
|
||||
let request = LockRequest::new(
|
||||
ObjectKey::new(super::RUSTFS_META_BUCKET, POOL_META_NAME),
|
||||
LockType::Exclusive,
|
||||
"cas-writer",
|
||||
)
|
||||
.with_acquire_timeout(StdDuration::from_secs(1));
|
||||
let owner = Arc::new(
|
||||
lock.acquire_guard(&request)
|
||||
.await
|
||||
.expect("pool metadata lock acquisition should succeed")
|
||||
.expect("pool metadata lock should be available"),
|
||||
);
|
||||
let weak_owner = Arc::downgrade(&owner);
|
||||
let fence = PoolMetaPersistenceFence::Owned(Arc::clone(&owner));
|
||||
let mut contexts = Vec::new();
|
||||
for object in [POOL_META_NAME, POOL_META_IDENTITY_NAME] {
|
||||
let options = super::pool_meta_cas_options(object, &PoolMetaCasToken::Missing, &fence)
|
||||
.expect("a held pool metadata lock should authorize its CAS target");
|
||||
let cloned_options = options.clone();
|
||||
let context = WriteCommitContext::from_options(&cloned_options, super::RUSTFS_META_BUCKET, object, false)
|
||||
.expect("the actual CAS target should retain its borrowed namespace owner");
|
||||
assert!(context.has_borrowed_owner(), "CAS must borrow the real protocol lock owner");
|
||||
assert!(
|
||||
WriteCommitContext::from_options(&cloned_options, super::RUSTFS_META_BUCKET, "unrelated-object", false).is_err(),
|
||||
"a pool metadata CAS capability must not authorize an unrelated object"
|
||||
);
|
||||
drop(options);
|
||||
drop(cloned_options);
|
||||
contexts.push(context);
|
||||
}
|
||||
drop(fence);
|
||||
drop(owner);
|
||||
assert!(
|
||||
weak_owner.upgrade().is_some(),
|
||||
"queued commit contexts must keep the actual lock guard alive"
|
||||
);
|
||||
drop(contexts);
|
||||
assert!(weak_owner.upgrade().is_none(), "the last commit context must release its lock owner");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pool_meta_cas_rejects_released_owner() {
|
||||
let lock = NamespaceLock::with_local_manager("pool-meta-released-owner".to_string(), Arc::new(GlobalLockManager::new()));
|
||||
let request = LockRequest::new(
|
||||
ObjectKey::new(super::RUSTFS_META_BUCKET, POOL_META_NAME),
|
||||
LockType::Exclusive,
|
||||
"released-cas-writer",
|
||||
)
|
||||
.with_acquire_timeout(StdDuration::from_secs(1));
|
||||
let mut owner = lock
|
||||
.acquire_guard(&request)
|
||||
.await
|
||||
.expect("pool metadata lock acquisition should succeed")
|
||||
.expect("pool metadata lock should be available");
|
||||
assert!(owner.release(), "the test must release a real namespace owner");
|
||||
let fence = PoolMetaPersistenceFence::Owned(Arc::new(owner));
|
||||
assert!(
|
||||
super::pool_meta_cas_options(POOL_META_NAME, &PoolMetaCasToken::Missing, &fence).is_err(),
|
||||
"a released namespace owner must not dispatch another CAS"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_lost_pool_meta_fence_rejects_replica_write() {
|
||||
let client = Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new())));
|
||||
@@ -25776,6 +25935,7 @@ mod pools_tests {
|
||||
.await
|
||||
.expect("pool metadata fence acquisition should not error")
|
||||
.expect("the stale writer should acquire the pool metadata fence");
|
||||
let guard = Arc::new(guard);
|
||||
tokio::time::timeout(StdDuration::from_secs(2), guard.lock_lost_notified())
|
||||
.await
|
||||
.expect("the stale writer lease should expire");
|
||||
@@ -25797,7 +25957,12 @@ mod pools_tests {
|
||||
};
|
||||
let mut write_state = super::PoolMetaWriteState::default();
|
||||
let err = snapshot
|
||||
.save_no_lock_armed(vec![storage.clone()], &mut write_state, guard.lock_lost_signal(), &[0])
|
||||
.save_no_lock_armed(
|
||||
vec![storage.clone()],
|
||||
&mut write_state,
|
||||
PoolMetaPersistenceFence::Owned(Arc::clone(&guard)),
|
||||
&[0],
|
||||
)
|
||||
.await
|
||||
.expect_err("a writer must not persist after losing the distributed pool metadata fence");
|
||||
|
||||
@@ -25825,6 +25990,7 @@ mod pools_tests {
|
||||
.await
|
||||
.expect("pool metadata fence acquisition should not error")
|
||||
.expect("the publishing writer should acquire the pool metadata fence");
|
||||
let guard = Arc::new(guard);
|
||||
let storage = Arc::new(PartialPoolMetaWriteStorage {
|
||||
fail_write: false,
|
||||
fail_after_first_write: false,
|
||||
@@ -25845,7 +26011,7 @@ mod pools_tests {
|
||||
current.pools[0].last_update = OffsetDateTime::UNIX_EPOCH;
|
||||
let mut write_state = super::PoolMetaWriteState::for_test_bootstrap();
|
||||
let outcome = saved
|
||||
.save_no_lock_armed(vec![storage], &mut write_state, guard.lock_lost_signal(), &[0])
|
||||
.save_no_lock_armed(vec![storage], &mut write_state, PoolMetaPersistenceFence::Owned(Arc::clone(&guard)), &[0])
|
||||
.await
|
||||
.expect("the replica save should finish while the fence is valid");
|
||||
|
||||
|
||||
@@ -738,7 +738,12 @@ where
|
||||
.filter(|etag| !etag.is_empty())
|
||||
.map(str::to_owned)
|
||||
.ok_or_else(|| Error::other("data usage snapshot has no ETag"))?;
|
||||
let mut data_usage_info = parse_usage_snapshot(&reader.read_all().await?)?;
|
||||
let mut data_usage_info = parse_usage_snapshot(
|
||||
&reader
|
||||
.read_all()
|
||||
.await
|
||||
.map_err(|err| map_data_usage_metadata_read_error(err, object))?,
|
||||
)?;
|
||||
populate_backward_compatible_usage_maps(&mut data_usage_info);
|
||||
validate_complete_usage_snapshot(&mut data_usage_info);
|
||||
Ok(Some((data_usage_info, revision)))
|
||||
@@ -2997,6 +3002,61 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct UsageSnapshotReadFailureStore;
|
||||
|
||||
struct UsageSnapshotFailureReader;
|
||||
|
||||
impl tokio::io::AsyncRead for UsageSnapshotFailureReader {
|
||||
fn poll_read(
|
||||
self: std::pin::Pin<&mut Self>,
|
||||
_cx: &mut std::task::Context<'_>,
|
||||
_buf: &mut tokio::io::ReadBuf<'_>,
|
||||
) -> std::task::Poll<std::io::Result<()>> {
|
||||
std::task::Poll::Ready(Err(std::io::Error::other(Error::VolumeNotFound)))
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::storage_api_contracts::object::ObjectIO for UsageSnapshotReadFailureStore {
|
||||
type Error = Error;
|
||||
type RangeSpec = crate::storage_api_contracts::range::HTTPRangeSpec;
|
||||
type HeaderMap = http::HeaderMap;
|
||||
type ObjectOptions = ObjectOptions;
|
||||
type ObjectInfo = ObjectInfo;
|
||||
type GetObjectReader = crate::object_api::GetObjectReader;
|
||||
type PutObjectReader = PutObjReader;
|
||||
|
||||
async fn get_object_reader(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_object: &str,
|
||||
_range: Option<Self::RangeSpec>,
|
||||
_headers: Self::HeaderMap,
|
||||
_opts: &Self::ObjectOptions,
|
||||
) -> Result<Self::GetObjectReader, Self::Error> {
|
||||
Ok(crate::object_api::GetObjectReader {
|
||||
stream: Box::new(UsageSnapshotFailureReader),
|
||||
object_info: ObjectInfo {
|
||||
etag: Some("usage-revision".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
buffered_body: None,
|
||||
body_source: Default::default(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn put_object(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_object: &str,
|
||||
_data: &mut Self::PutObjectReader,
|
||||
_opts: &Self::ObjectOptions,
|
||||
) -> Result<Self::ObjectInfo, Self::Error> {
|
||||
unimplemented!("the read-failure fixture does not write")
|
||||
}
|
||||
}
|
||||
|
||||
/// Minimal ObjectIO backing `load_data_usage_cache` tests: records the keys
|
||||
/// read and fails the first N reads with a transient (non-absence) error.
|
||||
#[derive(Debug, Default)]
|
||||
@@ -3161,6 +3221,20 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn data_usage_removal_maps_missing_system_volume_during_snapshot_stream_read() {
|
||||
let store = UsageSnapshotReadFailureStore;
|
||||
|
||||
let error = load_data_usage_for_bucket_removal(&store, "bucket-metadata/.usage.json")
|
||||
.await
|
||||
.expect_err("a system volume disappearing during body read must remain a retryable storage error");
|
||||
|
||||
assert_eq!(
|
||||
error,
|
||||
Error::InsufficientReadQuorum(RUSTFS_META_BUCKET.to_string(), "bucket-metadata/.usage.json".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn load_data_usage_cache_retries_a_transient_failure() {
|
||||
let name = "usage-cache";
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use rustfs_io_metrics::record_object_lock_acquire_failure;
|
||||
use rustfs_lock::{LockError, NamespaceLockGuard, NamespaceLockWrapper};
|
||||
use std::time::{Duration, Instant};
|
||||
use tracing::Span;
|
||||
@@ -21,6 +22,26 @@ const EVENT_OBJECT_NAMESPACE_LOCK: &str = "object_namespace_lock";
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_OBJECT_LOCK: &str = "object_lock";
|
||||
|
||||
pub(crate) fn record_lock_acquire_failure(op: &'static str, bucket: &str, object: &str, mode: &'static str, error: &LockError) {
|
||||
let resource_class = if bucket == crate::disk::RUSTFS_META_MULTIPART_BUCKET {
|
||||
"multipart_metadata"
|
||||
} else if bucket == crate::disk::RUSTFS_META_BUCKET || bucket == ".minio.sys" {
|
||||
"system_metadata"
|
||||
} else if object.is_empty() || object == bucket {
|
||||
"bucket_namespace"
|
||||
} else {
|
||||
"object_namespace"
|
||||
};
|
||||
let reason = match error {
|
||||
LockError::Timeout { .. } => "timeout",
|
||||
LockError::AlreadyLocked { .. } => "contention",
|
||||
LockError::QuorumNotReached { .. } | LockError::InsufficientNodes { .. } => "quorum",
|
||||
LockError::Network { .. } | LockError::Internal { .. } => "transport",
|
||||
_ => "other",
|
||||
};
|
||||
record_object_lock_acquire_failure(op, mode, resource_class, reason);
|
||||
}
|
||||
|
||||
/// Diagnostic lifetime only; it never owns or changes a namespace lock.
|
||||
/// Keep it alongside the real guard when ownership moves to a commit tail.
|
||||
#[derive(Default)]
|
||||
|
||||
@@ -12,6 +12,10 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::disk::io_admission::{
|
||||
AdmissionReader, BoxedChunkReader, DiskIoAdmission, DiskIoClass, DiskIoContext, current_disk_io_context,
|
||||
with_disk_io_context, with_disk_io_permit,
|
||||
};
|
||||
use crate::disk::{
|
||||
CheckPartsResp, DataDirDeleteStatus, DeleteOptions, DiskAPI, DiskError, DiskInfo, DiskInfoOptions, DiskLocation, Endpoint,
|
||||
Error, FileInfoVersions, MmapCopyStageMetrics, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, Result,
|
||||
@@ -35,13 +39,15 @@ use std::sync::atomic::AtomicBool;
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
path::PathBuf,
|
||||
pin::Pin,
|
||||
sync::{
|
||||
Arc, LazyLock, RwLock as StdRwLock,
|
||||
atomic::{AtomicI64, AtomicU32, AtomicU64, Ordering},
|
||||
},
|
||||
task::{Context, Poll},
|
||||
time::{Duration, Instant, SystemTime, UNIX_EPOCH},
|
||||
};
|
||||
use tokio::{sync::RwLock, time};
|
||||
use tokio::{io::AsyncWrite, sync::RwLock, time};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{info, warn};
|
||||
use uuid::Uuid;
|
||||
@@ -54,6 +60,121 @@ const LOG_SUBSYSTEM_DISK: &str = "disk";
|
||||
const EVENT_DISK_HEALTH_CHECK_FAILED: &str = "disk_health_check_failed";
|
||||
const EVENT_DISK_RECOVERY_PROBE_STATE: &str = "disk_recovery_probe_state";
|
||||
const EVENT_DISK_TIMEOUT_POLICY_FALLBACK: &str = "disk_timeout_policy_fallback";
|
||||
const EVENT_DISK_OPERATION_SLOW: &str = "disk_operation_slow";
|
||||
const SLOW_DISK_OPERATION_THRESHOLD: Duration = Duration::from_secs(5);
|
||||
|
||||
fn warn_slow_disk_operation(endpoint: &Endpoint, op: &'static str, elapsed: Duration) {
|
||||
warn!(
|
||||
event = EVENT_DISK_OPERATION_SLOW,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DISK,
|
||||
state = "pending",
|
||||
endpoint = %endpoint,
|
||||
op,
|
||||
elapsed_ms = elapsed.as_millis(),
|
||||
threshold_ms = SLOW_DISK_OPERATION_THRESHOLD.as_millis(),
|
||||
"Disk operation is still pending"
|
||||
);
|
||||
}
|
||||
|
||||
async fn observe_disk_operation<F: std::future::Future>(
|
||||
endpoint: impl FnOnce() -> Endpoint,
|
||||
op: &'static str,
|
||||
operation: F,
|
||||
) -> F::Output {
|
||||
if !tracing::enabled!(tracing::Level::WARN) {
|
||||
return operation.await;
|
||||
}
|
||||
let started = time::Instant::now();
|
||||
tokio::pin!(operation);
|
||||
tokio::select! {
|
||||
biased;
|
||||
result = &mut operation => result,
|
||||
_ = time::sleep(SLOW_DISK_OPERATION_THRESHOLD) => {
|
||||
warn_slow_disk_operation(&endpoint(), op, started.elapsed());
|
||||
operation.await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct SlowDiskWriter {
|
||||
inner: crate::disk::FileWriter,
|
||||
endpoint: Endpoint,
|
||||
pending: Option<(&'static str, time::Instant)>,
|
||||
timer: Option<Pin<Box<time::Sleep>>>,
|
||||
warned: bool,
|
||||
}
|
||||
|
||||
impl SlowDiskWriter {
|
||||
fn new(inner: crate::disk::FileWriter, endpoint: Endpoint) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
endpoint,
|
||||
pending: None,
|
||||
timer: None,
|
||||
warned: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn observe<T>(&mut self, cx: &mut Context<'_>, op: &'static str, result: Poll<T>) -> Poll<T> {
|
||||
if result.is_ready() {
|
||||
self.pending = None;
|
||||
self.warned = false;
|
||||
} else if tracing::enabled!(tracing::Level::WARN) {
|
||||
if self.pending.is_none_or(|(pending_op, _)| pending_op != op) {
|
||||
let started = time::Instant::now();
|
||||
self.pending = Some((op, started));
|
||||
self.warned = false;
|
||||
let deadline = started + SLOW_DISK_OPERATION_THRESHOLD;
|
||||
// Reuse one timer per stream; ready writes need no timer allocation.
|
||||
if let Some(timer) = &mut self.timer {
|
||||
timer.as_mut().reset(deadline);
|
||||
} else {
|
||||
self.timer = Some(Box::pin(time::sleep_until(deadline)));
|
||||
}
|
||||
}
|
||||
if !self.warned
|
||||
&& let Some(timer) = &mut self.timer
|
||||
&& std::future::Future::poll(timer.as_mut(), cx).is_ready()
|
||||
&& let Some((_, started)) = self.pending
|
||||
{
|
||||
warn_slow_disk_operation(&self.endpoint, op, started.elapsed());
|
||||
self.warned = true;
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncWrite for SlowDiskWriter {
|
||||
fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8]) -> Poll<std::io::Result<usize>> {
|
||||
let result = Pin::new(&mut self.inner).poll_write(cx, buf);
|
||||
self.observe(cx, "write", result)
|
||||
}
|
||||
|
||||
fn poll_write_vectored(
|
||||
mut self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
bufs: &[std::io::IoSlice<'_>],
|
||||
) -> Poll<std::io::Result<usize>> {
|
||||
let result = Pin::new(&mut self.inner).poll_write_vectored(cx, bufs);
|
||||
self.observe(cx, "write_vectored", result)
|
||||
}
|
||||
|
||||
fn is_write_vectored(&self) -> bool {
|
||||
self.inner.is_write_vectored()
|
||||
}
|
||||
|
||||
fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
|
||||
let result = Pin::new(&mut self.inner).poll_flush(cx);
|
||||
self.observe(cx, "flush", result)
|
||||
}
|
||||
|
||||
fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
|
||||
let result = Pin::new(&mut self.inner).poll_shutdown(cx);
|
||||
self.observe(cx, "shutdown", result)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
enum TimeoutHealthAction {
|
||||
@@ -813,6 +934,7 @@ impl Drop for DiskMetricWaitingGuard<'_> {
|
||||
pub(crate) struct ReconnectDiskHealthState {
|
||||
pub(crate) health: Arc<DiskHealthTracker>,
|
||||
pub(crate) metrics: Arc<DiskHealthMetricEpoch>,
|
||||
pub(crate) io_admission: Arc<DiskIoAdmission>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
@@ -1329,6 +1451,8 @@ pub struct LocalDiskWrapper {
|
||||
health: Arc<DiskHealthTracker>,
|
||||
/// Internal metrics epoch preserved across local disk reconnects.
|
||||
metrics: Arc<DiskHealthMetricEpoch>,
|
||||
/// Per-drive foreground/background admission shared across wrapper reconnects.
|
||||
io_admission: Arc<DiskIoAdmission>,
|
||||
/// Whether health checking is enabled
|
||||
health_check: bool,
|
||||
/// Cancellation token for monitoring tasks
|
||||
@@ -1342,11 +1466,13 @@ pub struct LocalDiskWrapper {
|
||||
impl LocalDiskWrapper {
|
||||
/// Create a new LocalDiskWrapper
|
||||
pub fn new(disk: Arc<LocalDisk>, health_check: bool) -> Self {
|
||||
let admission = DiskIoAdmission::new(disk.endpoint().to_string());
|
||||
Self::new_with_health_and_metrics(
|
||||
disk,
|
||||
health_check,
|
||||
Arc::new(DiskHealthTracker::new()),
|
||||
Arc::new(DiskHealthMetricEpoch::default()),
|
||||
admission,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1386,11 +1512,13 @@ impl LocalDiskWrapper {
|
||||
health_check: bool,
|
||||
reconnect: Option<ReconnectDiskHealthState>,
|
||||
) -> Self {
|
||||
let endpoint = disk.endpoint().to_string();
|
||||
let reconnect = reconnect.unwrap_or_else(|| ReconnectDiskHealthState {
|
||||
health: Arc::new(DiskHealthTracker::new()),
|
||||
metrics: Arc::new(DiskHealthMetricEpoch::default()),
|
||||
io_admission: DiskIoAdmission::new(endpoint),
|
||||
});
|
||||
Self::new_with_health_and_metrics(disk, health_check, reconnect.health, reconnect.metrics)
|
||||
Self::new_with_health_and_metrics(disk, health_check, reconnect.health, reconnect.metrics, reconnect.io_admission)
|
||||
}
|
||||
|
||||
fn new_with_health_and_metrics(
|
||||
@@ -1398,6 +1526,7 @@ impl LocalDiskWrapper {
|
||||
health_check: bool,
|
||||
health: Arc<DiskHealthTracker>,
|
||||
metrics: Arc<DiskHealthMetricEpoch>,
|
||||
io_admission: Arc<DiskIoAdmission>,
|
||||
) -> Self {
|
||||
// Check environment variable for health check override.
|
||||
// Only enable if both param and env are true.
|
||||
@@ -1408,6 +1537,7 @@ impl LocalDiskWrapper {
|
||||
disk,
|
||||
health,
|
||||
metrics,
|
||||
io_admission,
|
||||
health_check: health_check && env_health_check,
|
||||
cancel_token: CancellationToken::new(),
|
||||
disk_id: Arc::new(RwLock::new(None)),
|
||||
@@ -1421,6 +1551,7 @@ impl LocalDiskWrapper {
|
||||
ReconnectDiskHealthState {
|
||||
health: Arc::new(self.health.metric_epoch_for_reconnect()),
|
||||
metrics: self.metrics.clone(),
|
||||
io_admission: self.io_admission.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1428,6 +1559,58 @@ impl LocalDiskWrapper {
|
||||
self.disk.clone()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn background_io_admissions_for_tests(&self) -> usize {
|
||||
self.io_admission.background_admissions_for_tests()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn background_io_active_for_tests(&self) -> bool {
|
||||
self.io_admission.background_active_for_tests()
|
||||
}
|
||||
|
||||
fn disk_io_context(&self) -> DiskIoContext {
|
||||
current_disk_io_context().unwrap_or_else(|| DiskIoContext {
|
||||
class: DiskIoClass::Foreground,
|
||||
cancellation: CancellationToken::new(),
|
||||
admission: None,
|
||||
permit: None,
|
||||
})
|
||||
}
|
||||
|
||||
fn disk_walk_io_context(&self) -> DiskIoContext {
|
||||
let mut context = self.disk_io_context();
|
||||
context.admission = Some(self.io_admission.clone());
|
||||
context.permit = None;
|
||||
context
|
||||
}
|
||||
|
||||
async fn acquire_disk_io_permit(&self) -> Result<Option<crate::disk::io_admission::DiskIoPermit>> {
|
||||
if let Some(context) = current_disk_io_context() {
|
||||
if let Some(permit) = context.permit
|
||||
&& permit.is_for(&self.io_admission)
|
||||
{
|
||||
return Ok(Some(permit));
|
||||
}
|
||||
if context
|
||||
.admission
|
||||
.as_ref()
|
||||
.is_some_and(|admission| Arc::ptr_eq(admission, &self.io_admission))
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
if context.class == DiskIoClass::Background {
|
||||
return self
|
||||
.io_admission
|
||||
.background(&context.cancellation)
|
||||
.await
|
||||
.map(Some)
|
||||
.map_err(DiskError::Io);
|
||||
}
|
||||
}
|
||||
Ok(Some(self.io_admission.foreground()))
|
||||
}
|
||||
|
||||
pub fn get_object_path_if_local(&self, volume: &str, path: &str) -> crate::disk::error::Result<std::path::PathBuf> {
|
||||
self.disk.get_object_path(volume, path)
|
||||
}
|
||||
@@ -1899,6 +2082,31 @@ impl LocalDiskWrapper {
|
||||
where
|
||||
F: FnOnce() -> Fut,
|
||||
Fut: std::future::Future<Output = Result<T>>,
|
||||
{
|
||||
self.track_disk_health_with_op_timeout_action_and_mutation_with_io_permit(
|
||||
op,
|
||||
move |io_permit| async move {
|
||||
let _io_permit = io_permit;
|
||||
operation().await
|
||||
},
|
||||
timeout_duration,
|
||||
timeout_health_action,
|
||||
mutation,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn track_disk_health_with_op_timeout_action_and_mutation_with_io_permit<T, F, Fut>(
|
||||
&self,
|
||||
op: &'static str,
|
||||
operation: F,
|
||||
timeout_duration: Duration,
|
||||
timeout_health_action: TimeoutHealthAction,
|
||||
mutation: DiskMetricMutation,
|
||||
) -> Result<T>
|
||||
where
|
||||
F: FnOnce(Option<crate::disk::io_admission::DiskIoPermit>) -> Fut,
|
||||
Fut: std::future::Future<Output = Result<T>>,
|
||||
{
|
||||
self.metrics.record_operation_call(op);
|
||||
// Check if disk is faulty
|
||||
@@ -1921,14 +2129,17 @@ impl LocalDiskWrapper {
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
let io_permit = self.acquire_disk_io_permit().await?;
|
||||
|
||||
// Record operation start
|
||||
self.health.last_started.store(current_unix_nanos(), Ordering::Relaxed);
|
||||
let _waiting_guard = self.health.waiting_guard();
|
||||
let _metric_waiting_guard = self.metrics.waiting_guard();
|
||||
let started = Instant::now();
|
||||
let operation = observe_disk_operation(|| self.endpoint(), op, operation(io_permit));
|
||||
|
||||
if timeout_duration == Duration::ZERO {
|
||||
let result = operation().await;
|
||||
let result = operation.await;
|
||||
self.metrics.record_operation_latency(op, started.elapsed());
|
||||
self.record_result_error_metrics(&result);
|
||||
if result.is_ok() {
|
||||
@@ -1938,7 +2149,7 @@ impl LocalDiskWrapper {
|
||||
return result;
|
||||
}
|
||||
// Execute the operation with timeout
|
||||
let result = tokio::time::timeout(timeout_duration, operation()).await;
|
||||
let result = tokio::time::timeout(timeout_duration, operation).await;
|
||||
|
||||
match result {
|
||||
Ok(operation_result) => {
|
||||
@@ -1989,11 +2200,12 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
}
|
||||
|
||||
async fn read_metadata(&self, volume: &str, path: &str) -> Result<Bytes> {
|
||||
self.track_disk_health_with_op_and_timeout_action(
|
||||
self.track_disk_health_with_op_timeout_action_and_mutation_with_io_permit(
|
||||
"read_metadata",
|
||||
|| async { self.disk.read_metadata(volume, path).await },
|
||||
|io_permit| async { self.disk.read_metadata_with_permit(volume, path, io_permit).await },
|
||||
get_drive_metadata_timeout(),
|
||||
self.scanner_timeout_health_action(),
|
||||
DiskMetricMutation::None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -2165,12 +2377,15 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
opts.timeout_duration().unwrap_or_else(get_drive_walkdir_timeout)
|
||||
};
|
||||
|
||||
self.track_disk_health_with_op_and_timeout_action(
|
||||
"walk_dir",
|
||||
|| async { self.disk.walk_dir(opts, wr).await },
|
||||
timeout_duration,
|
||||
// Listing/scanner backpressure should fail only the current walk, not poison drive health.
|
||||
TimeoutHealthAction::IgnoreFailure,
|
||||
with_disk_io_context(
|
||||
self.disk_walk_io_context(),
|
||||
self.track_disk_health_with_op_and_timeout_action(
|
||||
"walk_dir",
|
||||
|| async { self.disk.walk_dir(opts, wr).await },
|
||||
timeout_duration,
|
||||
// Listing/scanner backpressure should fail only the current walk, not poison drive health.
|
||||
TimeoutHealthAction::IgnoreFailure,
|
||||
),
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -2193,6 +2408,7 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
}
|
||||
|
||||
async fn delete_versions(&self, volume: &str, versions: Vec<FileInfoVersions>, opts: DeleteOptions) -> Vec<Option<Error>> {
|
||||
let version_count = versions.len();
|
||||
self.metrics.record_operation_call("delete_versions");
|
||||
// Check if disk is faulty before proceeding
|
||||
if self.health.is_faulty() {
|
||||
@@ -2206,9 +2422,14 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
return vec![Some(e); versions.len()];
|
||||
}
|
||||
|
||||
let _io_permit = match self.acquire_disk_io_permit().await {
|
||||
Ok(permit) => permit,
|
||||
Err(error) => return vec![Some(error); version_count],
|
||||
};
|
||||
|
||||
// Record operation start
|
||||
self.health.last_started.store(current_unix_nanos(), Ordering::Relaxed);
|
||||
self.health.increment_waiting();
|
||||
let waiting_guard = self.health.waiting_guard();
|
||||
let metric_waiting_guard = self.metrics.waiting_guard();
|
||||
let started = Instant::now();
|
||||
|
||||
@@ -2217,7 +2438,7 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
self.metrics.record_operation_latency("delete_versions", started.elapsed());
|
||||
self.record_batch_delete_error_metrics(&result);
|
||||
|
||||
self.health.decrement_waiting();
|
||||
drop(waiting_guard);
|
||||
drop(metric_waiting_guard);
|
||||
let has_err = result.iter().any(|e| e.is_some());
|
||||
if !has_err {
|
||||
@@ -2322,19 +2543,25 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
version_id: &str,
|
||||
opts: &ReadOptions,
|
||||
) -> Result<FileInfo> {
|
||||
self.track_disk_health_with_op(
|
||||
self.track_disk_health_with_op_timeout_action_and_mutation_with_io_permit(
|
||||
"read_version",
|
||||
|| async { self.disk.read_version(org_volume, volume, path, version_id, opts).await },
|
||||
|io_permit| async move {
|
||||
with_disk_io_permit(io_permit, self.disk.read_version(org_volume, volume, path, version_id, opts)).await
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
TimeoutHealthAction::MarkFailure,
|
||||
DiskMetricMutation::None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo> {
|
||||
self.track_disk_health_with_op(
|
||||
self.track_disk_health_with_op_timeout_action_and_mutation_with_io_permit(
|
||||
"read_xl",
|
||||
|| async { self.disk.read_xl(volume, path, read_data).await },
|
||||
|io_permit| async move { with_disk_io_permit(io_permit, self.disk.read_xl(volume, path, read_data)).await },
|
||||
get_max_timeout_duration(),
|
||||
TimeoutHealthAction::MarkFailure,
|
||||
DiskMetricMutation::None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -2362,21 +2589,25 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
}
|
||||
|
||||
async fn read_file(&self, volume: &str, path: &str) -> Result<crate::disk::FileReader> {
|
||||
self.track_disk_health_with_op(
|
||||
"read_file",
|
||||
|| async { self.disk.read_file(volume, path).await },
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
let reader = self
|
||||
.track_disk_health_with_op(
|
||||
"read_file",
|
||||
|| async { self.disk.read_file(volume, path).await },
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await?;
|
||||
Ok(Box::new(AdmissionReader::new(reader, self.io_admission.clone(), self.disk_io_context())))
|
||||
}
|
||||
|
||||
async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<crate::disk::FileReader> {
|
||||
self.track_disk_health_with_op(
|
||||
"read_file_stream",
|
||||
|| async { self.disk.read_file_stream(volume, path, offset, length).await },
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
let reader = self
|
||||
.track_disk_health_with_op(
|
||||
"read_file_stream",
|
||||
|| async { self.disk.read_file_stream(volume, path, offset, length).await },
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await?;
|
||||
Ok(Box::new(AdmissionReader::new(reader, self.io_admission.clone(), self.disk_io_context())))
|
||||
}
|
||||
|
||||
async fn read_file_stream_chunks(
|
||||
@@ -2386,12 +2617,20 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
offset: usize,
|
||||
length: usize,
|
||||
) -> Result<Option<rustfs_rio::ChunkReaderBox>> {
|
||||
self.track_disk_health_with_op(
|
||||
"read_file_stream_chunks",
|
||||
|| async { self.disk.read_file_stream_chunks(volume, path, offset, length).await },
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
let reader = self
|
||||
.track_disk_health_with_op(
|
||||
"read_file_stream_chunks",
|
||||
|| async { self.disk.read_file_stream_chunks(volume, path, offset, length).await },
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await?;
|
||||
Ok(reader.map(|reader| {
|
||||
Box::new(AdmissionReader::new(
|
||||
BoxedChunkReader(reader),
|
||||
self.io_admission.clone(),
|
||||
self.disk_io_context(),
|
||||
)) as rustfs_rio::ChunkReaderBox
|
||||
}))
|
||||
}
|
||||
|
||||
async fn read_file_mmap_copy(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<bytes::Bytes> {
|
||||
@@ -2424,18 +2663,22 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
}
|
||||
|
||||
async fn append_file(&self, volume: &str, path: &str) -> Result<crate::disk::FileWriter> {
|
||||
self.track_disk_health_with_op("append_file", || async { self.disk.append_file(volume, path).await }, Duration::ZERO)
|
||||
.await
|
||||
let writer = self
|
||||
.track_disk_health_with_op("append_file", || async { self.disk.append_file(volume, path).await }, Duration::ZERO)
|
||||
.await?;
|
||||
Ok(Box::new(SlowDiskWriter::new(writer, self.endpoint())))
|
||||
}
|
||||
|
||||
async fn create_file(&self, origvolume: &str, volume: &str, path: &str, file_size: i64) -> Result<crate::disk::FileWriter> {
|
||||
self.track_disk_health_mutation(
|
||||
"create_file",
|
||||
DiskMetricMutation::Write,
|
||||
|| async { self.disk.create_file(origvolume, volume, path, file_size).await },
|
||||
Duration::ZERO,
|
||||
)
|
||||
.await
|
||||
let writer = self
|
||||
.track_disk_health_mutation(
|
||||
"create_file",
|
||||
DiskMetricMutation::Write,
|
||||
|| async { self.disk.create_file(origvolume, volume, path, file_size).await },
|
||||
Duration::ZERO,
|
||||
)
|
||||
.await?;
|
||||
Ok(Box::new(SlowDiskWriter::new(writer, self.endpoint())))
|
||||
}
|
||||
|
||||
async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> {
|
||||
@@ -2524,8 +2767,14 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
}
|
||||
|
||||
async fn read_parts(&self, bucket: &str, paths: &[String]) -> Result<Vec<ObjectPartInfo>> {
|
||||
self.track_disk_health_with_op("read_parts", || async { self.disk.read_parts(bucket, paths).await }, Duration::ZERO)
|
||||
.await
|
||||
self.track_disk_health_with_op_timeout_action_and_mutation_with_io_permit(
|
||||
"read_parts",
|
||||
|io_permit| async move { with_disk_io_permit(io_permit, self.disk.read_parts(bucket, paths)).await },
|
||||
Duration::ZERO,
|
||||
TimeoutHealthAction::MarkFailure,
|
||||
DiskMetricMutation::None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn read_multiple(&self, req: ReadMultipleReq) -> Result<Vec<ReadMultipleResp>> {
|
||||
@@ -2582,7 +2831,268 @@ mod tests {
|
||||
pin::Pin,
|
||||
task::{Context, Poll},
|
||||
};
|
||||
use tokio::io::AsyncWrite;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
use tracing_subscriber::fmt::MakeWriter;
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct SlowOperationLogs(Arc<std::sync::Mutex<Vec<u8>>>);
|
||||
|
||||
impl SlowOperationLogs {
|
||||
fn contents(&self) -> String {
|
||||
String::from_utf8(self.0.lock().expect("log buffer should not be poisoned").clone()).expect("logs should be UTF-8")
|
||||
}
|
||||
}
|
||||
|
||||
impl io::Write for SlowOperationLogs {
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
self.0
|
||||
.lock()
|
||||
.expect("log buffer should not be poisoned")
|
||||
.extend_from_slice(buf);
|
||||
Ok(buf.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> MakeWriter<'a> for SlowOperationLogs {
|
||||
type Writer = Self;
|
||||
|
||||
fn make_writer(&'a self) -> Self::Writer {
|
||||
self.clone()
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn slow_disk_operation_warns_once_while_pending_and_preserves_result() {
|
||||
let dir = tempfile::tempdir().expect("temp dir should be created");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("path should be UTF-8")).expect("endpoint should parse");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created"));
|
||||
let wrapper = LocalDiskWrapper::new(disk, false);
|
||||
let logs = SlowOperationLogs::default();
|
||||
let subscriber = tracing_subscriber::fmt()
|
||||
.with_writer(logs.clone())
|
||||
.with_ansi(false)
|
||||
.without_time()
|
||||
.finish();
|
||||
let _subscriber = tracing::subscriber::set_default(subscriber);
|
||||
let _interest = crate::test_tracing::pin_callsite_interest_for_test();
|
||||
let (release, wait) = tokio::sync::oneshot::channel();
|
||||
let operation = wrapper.track_disk_health_with_op(
|
||||
"rename_data",
|
||||
|| async { wait.await.expect("operation should be released") },
|
||||
Duration::ZERO,
|
||||
);
|
||||
tokio::pin!(operation);
|
||||
assert!(futures::poll!(&mut operation).is_pending());
|
||||
time::advance(SLOW_DISK_OPERATION_THRESHOLD - Duration::from_millis(1)).await;
|
||||
assert!(futures::poll!(&mut operation).is_pending());
|
||||
assert!(logs.contents().is_empty(), "normal disk latency should not warn");
|
||||
time::advance(Duration::from_millis(1)).await;
|
||||
assert!(futures::poll!(&mut operation).is_pending());
|
||||
let warning = logs.contents();
|
||||
assert!(warning.contains("WARN"));
|
||||
assert!(warning.contains("event=\"disk_operation_slow\""));
|
||||
assert!(warning.contains("state=\"pending\""));
|
||||
assert!(warning.contains(&format!("endpoint={endpoint}")));
|
||||
assert!(warning.contains("op=\"rename_data\""));
|
||||
assert!(warning.contains("elapsed_ms=5000"));
|
||||
time::advance(Duration::from_secs(120)).await;
|
||||
assert!(futures::poll!(&mut operation).is_pending(), "warning must not cancel the disk operation");
|
||||
assert_eq!(logs.contents(), warning, "a pending operation should warn only once");
|
||||
release
|
||||
.send(Err::<(), _>(DiskError::FileAccessDenied))
|
||||
.expect("operation should still be alive");
|
||||
assert_eq!(operation.await, Err(DiskError::FileAccessDenied));
|
||||
assert_eq!(wrapper.runtime_state(), RuntimeDriveHealthState::Online);
|
||||
assert_eq!(wrapper.metrics_snapshot().total_waiting, 0);
|
||||
|
||||
let timed = wrapper.track_disk_health_with_op(
|
||||
"write_all",
|
||||
std::future::pending::<Result<()>>,
|
||||
SLOW_DISK_OPERATION_THRESHOLD + Duration::from_secs(1),
|
||||
);
|
||||
tokio::pin!(timed);
|
||||
assert!(futures::poll!(&mut timed).is_pending());
|
||||
time::advance(SLOW_DISK_OPERATION_THRESHOLD).await;
|
||||
assert!(futures::poll!(&mut timed).is_pending());
|
||||
assert_eq!(logs.contents().matches(EVENT_DISK_OPERATION_SLOW).count(), 2);
|
||||
assert_eq!(wrapper.runtime_state(), RuntimeDriveHealthState::Online);
|
||||
time::advance(Duration::from_secs(1)).await;
|
||||
assert_eq!(
|
||||
timed.await,
|
||||
Err(DiskError::Timeout),
|
||||
"existing timeout must still apply after the warning"
|
||||
);
|
||||
assert_eq!(wrapper.metrics_snapshot().total_errors_timeout, 1);
|
||||
assert_eq!(wrapper.metrics_snapshot().total_waiting, 0);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn slow_disk_operation_fast_path_is_silent_and_does_not_resolve_endpoint() {
|
||||
let logs = SlowOperationLogs::default();
|
||||
let subscriber = tracing_subscriber::fmt()
|
||||
.with_writer(logs.clone())
|
||||
.with_ansi(false)
|
||||
.without_time()
|
||||
.finish();
|
||||
let _subscriber = tracing::subscriber::set_default(subscriber);
|
||||
let _interest = crate::test_tracing::pin_callsite_interest_for_test();
|
||||
for delay in [Duration::ZERO, SLOW_DISK_OPERATION_THRESHOLD] {
|
||||
let result =
|
||||
observe_disk_operation(|| panic!("completed operation should not clone its endpoint"), "write_all", async {
|
||||
if !delay.is_zero() {
|
||||
time::sleep(delay).await;
|
||||
}
|
||||
Ok::<_, DiskError>(17)
|
||||
})
|
||||
.await;
|
||||
assert_eq!(result, Ok(17));
|
||||
}
|
||||
assert!(logs.contents().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn slow_disk_diagnostics_are_bypassed_when_warnings_are_disabled() {
|
||||
let _subscriber = tracing::subscriber::set_default(tracing::subscriber::NoSubscriber::default());
|
||||
let endpoint = Endpoint::try_from("/slow-disk").expect("endpoint should parse");
|
||||
let mut writer = SlowDiskWriter::new(Box::new(PendingWriter), endpoint);
|
||||
{
|
||||
let write = writer.write(b"bytes");
|
||||
tokio::pin!(write);
|
||||
assert!(futures::poll!(&mut write).is_pending());
|
||||
}
|
||||
assert!(writer.timer.is_none(), "disabled warnings should not allocate a stream timer");
|
||||
let result = observe_disk_operation(|| panic!("disabled warnings should not resolve endpoints"), "write_all", async {
|
||||
time::sleep(SLOW_DISK_OPERATION_THRESHOLD * 2).await;
|
||||
17
|
||||
})
|
||||
.await;
|
||||
assert_eq!(result, 17);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn slow_disk_writer_preserves_vectored_bytes_and_warns_once_while_blocked() {
|
||||
let endpoint = Endpoint::try_from("/slow-disk").expect("endpoint should parse");
|
||||
let logs = SlowOperationLogs::default();
|
||||
let subscriber = tracing_subscriber::fmt()
|
||||
.with_writer(logs.clone())
|
||||
.with_ansi(false)
|
||||
.without_time()
|
||||
.finish();
|
||||
let _subscriber = tracing::subscriber::set_default(subscriber);
|
||||
let _interest = crate::test_tracing::pin_callsite_interest_for_test();
|
||||
let (stream, mut receiver) = tokio::io::duplex(1);
|
||||
let mut writer = SlowDiskWriter::new(Box::new(stream), endpoint.clone());
|
||||
writer.write_all(b"a").await.expect("first byte should fit in the buffer");
|
||||
assert!(writer.timer.is_none(), "ready writes should not allocate a timer");
|
||||
assert!(logs.contents().is_empty());
|
||||
let mut received = [0; 1];
|
||||
{
|
||||
let bufs = [io::IoSlice::new(b"b"), io::IoSlice::new(b"c")];
|
||||
let write = writer.write_vectored(&bufs);
|
||||
tokio::pin!(write);
|
||||
assert!(futures::poll!(&mut write).is_pending());
|
||||
time::advance(SLOW_DISK_OPERATION_THRESHOLD).await;
|
||||
assert!(futures::poll!(&mut write).is_pending());
|
||||
assert!(logs.contents().contains("op=\"write_vectored\""));
|
||||
time::advance(SLOW_DISK_OPERATION_THRESHOLD).await;
|
||||
assert!(futures::poll!(&mut write).is_pending());
|
||||
assert_eq!(logs.contents().matches(EVENT_DISK_OPERATION_SLOW).count(), 1);
|
||||
tokio::io::AsyncReadExt::read_exact(&mut receiver, &mut received)
|
||||
.await
|
||||
.expect("buffered byte should be readable");
|
||||
assert_eq!(received, *b"a");
|
||||
assert_eq!(write.await.expect("blocked write should finish when capacity returns"), 1);
|
||||
}
|
||||
tokio::io::AsyncReadExt::read_exact(&mut receiver, &mut received)
|
||||
.await
|
||||
.expect("written byte should be readable");
|
||||
assert_eq!(received, *b"b");
|
||||
|
||||
drop(receiver);
|
||||
let err = writer.write_all(b"d").await.expect_err("closed receiver should fail writes");
|
||||
assert_eq!(err.kind(), io::ErrorKind::BrokenPipe);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn slow_disk_writer_preserves_full_vectored_write_and_capability() {
|
||||
let endpoint = Endpoint::try_from("/slow-disk").expect("endpoint should parse");
|
||||
let (stream, mut receiver) = tokio::io::duplex(64);
|
||||
let vectored = stream.is_write_vectored();
|
||||
let mut writer = SlowDiskWriter::new(Box::new(stream), endpoint);
|
||||
assert_eq!(writer.is_write_vectored(), vectored);
|
||||
let bufs = [io::IoSlice::new(b"hash"), io::IoSlice::new(b"data")];
|
||||
assert_eq!(writer.write_vectored(&bufs).await.expect("vectored write should succeed"), 8);
|
||||
writer.shutdown().await.expect("shutdown should succeed");
|
||||
let mut received = Vec::new();
|
||||
tokio::io::AsyncReadExt::read_to_end(&mut receiver, &mut received)
|
||||
.await
|
||||
.expect("all bytes should be readable");
|
||||
assert_eq!(received, b"hashdata");
|
||||
assert!(writer.timer.is_none());
|
||||
}
|
||||
|
||||
struct StalledWriter(Arc<AtomicBool>);
|
||||
|
||||
impl AsyncWrite for StalledWriter {
|
||||
fn poll_write(self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &[u8]) -> Poll<io::Result<usize>> {
|
||||
if self.0.load(Ordering::Relaxed) {
|
||||
Poll::Ready(Ok(buf.len()))
|
||||
} else {
|
||||
Poll::Pending
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
if self.0.load(Ordering::Relaxed) {
|
||||
Poll::Ready(Ok(()))
|
||||
} else {
|
||||
Poll::Pending
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
self.poll_flush(cx)
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn slow_disk_writer_warns_for_write_flush_and_shutdown_and_rearms_after_progress() {
|
||||
let endpoint = Endpoint::try_from("/slow-disk").expect("endpoint should parse");
|
||||
let logs = SlowOperationLogs::default();
|
||||
let subscriber = tracing_subscriber::fmt()
|
||||
.with_writer(logs.clone())
|
||||
.with_ansi(false)
|
||||
.without_time()
|
||||
.finish();
|
||||
let _subscriber = tracing::subscriber::set_default(subscriber);
|
||||
let _interest = crate::test_tracing::pin_callsite_interest_for_test();
|
||||
let ready = Arc::new(AtomicBool::new(false));
|
||||
let mut writer = SlowDiskWriter::new(Box::new(StalledWriter(Arc::clone(&ready))), endpoint);
|
||||
for (index, op) in ["write", "flush", "shutdown"].into_iter().enumerate() {
|
||||
ready.store(false, Ordering::Relaxed);
|
||||
let operation = async {
|
||||
match op {
|
||||
"write" => writer.write_all(b"bytes").await,
|
||||
"flush" => writer.flush().await,
|
||||
"shutdown" => writer.shutdown().await,
|
||||
_ => unreachable!("only write operations are tested"),
|
||||
}
|
||||
};
|
||||
tokio::pin!(operation);
|
||||
assert!(futures::poll!(&mut operation).is_pending());
|
||||
time::advance(SLOW_DISK_OPERATION_THRESHOLD).await;
|
||||
assert!(futures::poll!(&mut operation).is_pending());
|
||||
assert!(logs.contents().contains(&format!("op=\"{op}\"")));
|
||||
assert_eq!(logs.contents().matches(EVENT_DISK_OPERATION_SLOW).count(), index + 1);
|
||||
ready.store(true, Ordering::Relaxed);
|
||||
operation.await.expect("warning should not prevent operation completion");
|
||||
}
|
||||
assert!(writer.pending.is_none());
|
||||
}
|
||||
|
||||
struct DropProbe(Arc<std::sync::atomic::AtomicUsize>);
|
||||
|
||||
@@ -2592,6 +3102,84 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
struct StalledMetacacheWriter {
|
||||
entered: Arc<tokio::sync::Notify>,
|
||||
}
|
||||
|
||||
impl AsyncWrite for StalledMetacacheWriter {
|
||||
fn poll_write(self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &[u8]) -> Poll<io::Result<usize>> {
|
||||
self.entered.notify_one();
|
||||
Poll::Pending
|
||||
}
|
||||
|
||||
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
|
||||
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_walk_releases_disk_admission_while_output_is_backpressured() {
|
||||
let root = tempfile::tempdir().expect("disk root should be created");
|
||||
let endpoint =
|
||||
Endpoint::try_from(root.path().to_str().expect("disk root should be UTF-8")).expect("endpoint should parse");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should open"));
|
||||
let wrapper = Arc::new(LocalDiskWrapper::new(disk, false));
|
||||
wrapper.make_volume("bucket").await.expect("bucket should be created");
|
||||
let object_path = root.path().join("bucket/object");
|
||||
tokio::fs::create_dir_all(&object_path)
|
||||
.await
|
||||
.expect("object directory should be created");
|
||||
tokio::fs::write(object_path.join("xl.meta"), b"scanner metadata")
|
||||
.await
|
||||
.expect("object metadata should be created");
|
||||
|
||||
let entered = Arc::new(tokio::sync::Notify::new());
|
||||
let writer = StalledMetacacheWriter {
|
||||
entered: entered.clone(),
|
||||
};
|
||||
let walk_wrapper = wrapper.clone();
|
||||
let walk = tokio::spawn(async move {
|
||||
crate::disk::with_background_disk_io(CancellationToken::new(), async move {
|
||||
let mut writer = writer;
|
||||
walk_wrapper
|
||||
.walk_dir(
|
||||
WalkDirOptions {
|
||||
bucket: "bucket".to_string(),
|
||||
recursive: true,
|
||||
skip_total_timeout: true,
|
||||
..Default::default()
|
||||
},
|
||||
&mut writer,
|
||||
)
|
||||
.await
|
||||
})
|
||||
.await
|
||||
});
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(2), entered.notified())
|
||||
.await
|
||||
.expect("walk should reach the blocked output writer");
|
||||
assert!(
|
||||
!wrapper.background_io_active_for_tests(),
|
||||
"filesystem read permits must be released before metacache output backpressure"
|
||||
);
|
||||
|
||||
let checkpoint_permit = tokio::time::timeout(
|
||||
Duration::from_secs(1),
|
||||
crate::disk::with_background_disk_io(CancellationToken::new(), wrapper.acquire_disk_io_permit()),
|
||||
)
|
||||
.await
|
||||
.expect("checkpoint admission should progress while the scanner consumer is paused")
|
||||
.expect("checkpoint should acquire the disk gate");
|
||||
drop(checkpoint_permit);
|
||||
walk.abort();
|
||||
let _ = walk.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn owned_mutation_keeps_publication_guard_after_waiter_cancellation() {
|
||||
let drops = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
@@ -2742,6 +3330,7 @@ mod tests {
|
||||
let reconnect = ReconnectDiskHealthState {
|
||||
health: Arc::new(health.metric_epoch_for_reconnect()),
|
||||
metrics: metrics.clone(),
|
||||
io_admission: DiskIoAdmission::new("test-disk"),
|
||||
};
|
||||
metrics.record_operation_call("read_all");
|
||||
reconnect.metrics.record_timeout_error();
|
||||
@@ -3023,6 +3612,40 @@ mod tests {
|
||||
assert_eq!(snapshot.total_deletes, 1);
|
||||
}
|
||||
|
||||
// A batch delete dropped while the disk call is in flight (for example by a
|
||||
// cancelled caller) must not leave the drive looking busy forever.
|
||||
#[tokio::test]
|
||||
async fn delete_versions_releases_waiting_slot_when_dropped() {
|
||||
let dir = tempfile::tempdir().expect("temp dir should be created");
|
||||
let endpoint =
|
||||
Endpoint::try_from(dir.path().to_str().expect("temp dir should be valid UTF-8")).expect("endpoint should parse");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created"));
|
||||
let wrapper = LocalDiskWrapper::new(Arc::clone(&disk), false);
|
||||
wrapper.make_volume("bucket").await.expect("volume should be created");
|
||||
let object_path = disk.get_object_path("bucket", "object").expect("object path");
|
||||
// Holding the object's metadata lease parks the disk call before it mutates anything.
|
||||
let metadata_lease = crate::disk::os::acquire_metadata_mutation_lease(&object_path, None).await;
|
||||
|
||||
let mut delete = Box::pin(wrapper.delete_versions(
|
||||
"bucket",
|
||||
vec![FileInfoVersions {
|
||||
name: "object".to_string(),
|
||||
versions: vec![FileInfo::default()],
|
||||
..Default::default()
|
||||
}],
|
||||
DeleteOptions::default(),
|
||||
));
|
||||
assert!(
|
||||
futures::poll!(delete.as_mut()).is_pending(),
|
||||
"the disk call should wait for the metadata lease"
|
||||
);
|
||||
assert_eq!(wrapper.health.waiting_count(), 1);
|
||||
|
||||
drop(delete);
|
||||
assert_eq!(wrapper.health.waiting_count(), 0, "a dropped delete must release its waiting slot");
|
||||
drop(metadata_lease);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_versions_counts_faulty_drive_availability_rejection() {
|
||||
let dir = tempfile::tempdir().expect("temp dir should be created");
|
||||
|
||||
@@ -0,0 +1,706 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Per-local-disk foreground-priority admission for scanner I/O.
|
||||
|
||||
use std::collections::VecDeque;
|
||||
use std::future::Future;
|
||||
use std::io;
|
||||
use std::pin::Pin;
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::task::{Context, Poll};
|
||||
use std::time::Duration;
|
||||
|
||||
use metrics::{counter, gauge, histogram};
|
||||
use rustfs_rio::ChunkReader;
|
||||
use tokio::io::{AsyncRead, ReadBuf};
|
||||
use tokio::sync::Notify;
|
||||
use tokio::time::Instant;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
const BACKGROUND_FOREGROUND_BURST: u32 = 8;
|
||||
const BACKGROUND_MAX_WAIT: Duration = Duration::from_millis(100);
|
||||
|
||||
tokio::task_local! {
|
||||
static DISK_IO_CONTEXT: DiskIoContext;
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub(crate) enum DiskIoClass {
|
||||
Foreground,
|
||||
Background,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct DiskIoContext {
|
||||
pub(crate) class: DiskIoClass,
|
||||
pub(crate) cancellation: CancellationToken,
|
||||
pub(crate) admission: Option<Arc<DiskIoAdmission>>,
|
||||
pub(crate) permit: Option<DiskIoPermit>,
|
||||
}
|
||||
|
||||
/// Mark disk I/O issued by `future` as background work for local-disk admission.
|
||||
#[doc(hidden)]
|
||||
pub async fn with_background_disk_io<F: Future>(cancellation: CancellationToken, future: F) -> F::Output {
|
||||
with_disk_io_context(
|
||||
DiskIoContext {
|
||||
class: DiskIoClass::Background,
|
||||
cancellation,
|
||||
admission: None,
|
||||
permit: None,
|
||||
},
|
||||
future,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn with_disk_io_context<F: Future>(context: DiskIoContext, future: F) -> F::Output {
|
||||
DISK_IO_CONTEXT.scope(context, future).await
|
||||
}
|
||||
|
||||
pub(crate) async fn with_disk_io_permit<F: Future>(permit: Option<DiskIoPermit>, future: F) -> F::Output {
|
||||
let Some(permit) = permit else {
|
||||
return future.await;
|
||||
};
|
||||
let mut context = current_disk_io_context().unwrap_or_else(|| DiskIoContext {
|
||||
class: DiskIoClass::Foreground,
|
||||
cancellation: CancellationToken::new(),
|
||||
admission: None,
|
||||
permit: None,
|
||||
});
|
||||
context.admission = Some(permit.admission());
|
||||
with_disk_io_context(
|
||||
DiskIoContext {
|
||||
permit: Some(permit),
|
||||
..context
|
||||
},
|
||||
future,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn with_optional_disk_io_context<F: Future>(context: Option<DiskIoContext>, future: F) -> F::Output {
|
||||
match context {
|
||||
Some(context) => with_disk_io_context(context, future).await,
|
||||
None => future.await,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn current_disk_io_context() -> Option<DiskIoContext> {
|
||||
DISK_IO_CONTEXT.try_with(Clone::clone).ok()
|
||||
}
|
||||
|
||||
/// Acquire one low-level permit only when the current walker explicitly owns a gate.
|
||||
/// Ordinary wrapper calls use their outer operation permit instead.
|
||||
pub(crate) async fn acquire_context_disk_io_permit() -> io::Result<Option<DiskIoPermit>> {
|
||||
let Some(context) = current_disk_io_context() else {
|
||||
return Ok(None);
|
||||
};
|
||||
let Some(admission) = context.admission else {
|
||||
return Ok(None);
|
||||
};
|
||||
if let Some(permit) = context.permit
|
||||
&& permit.is_for(&admission)
|
||||
{
|
||||
return Ok(Some(permit));
|
||||
}
|
||||
match context.class {
|
||||
DiskIoClass::Foreground => Ok(Some(admission.foreground())),
|
||||
DiskIoClass::Background => Ok(Some(admission.background(&context.cancellation).await?)),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct AdmissionState {
|
||||
foreground_completions: u32,
|
||||
next_background_ticket: u64,
|
||||
background_queue: VecDeque<u64>,
|
||||
background_front_since: Option<Instant>,
|
||||
}
|
||||
|
||||
/// One gate is shared by all handles to a local disk, including reconnects.
|
||||
/// Foreground I/O never waits on this gate. Background I/O runs exclusively
|
||||
/// while the disk is idle, or after a bounded foreground burst/wait.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct DiskIoAdmission {
|
||||
state: Mutex<AdmissionState>,
|
||||
changed: Notify,
|
||||
disk: String,
|
||||
foreground_active: AtomicUsize,
|
||||
background_active: AtomicBool,
|
||||
background_waiters: AtomicUsize,
|
||||
#[cfg(test)]
|
||||
background_admissions: AtomicUsize,
|
||||
}
|
||||
|
||||
impl DiskIoAdmission {
|
||||
pub(crate) fn new(disk: impl Into<String>) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
state: Mutex::new(AdmissionState::default()),
|
||||
changed: Notify::new(),
|
||||
disk: disk.into(),
|
||||
foreground_active: AtomicUsize::new(0),
|
||||
background_active: AtomicBool::new(false),
|
||||
background_waiters: AtomicUsize::new(0),
|
||||
#[cfg(test)]
|
||||
background_admissions: AtomicUsize::new(0),
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn background_admissions_for_tests(&self) -> usize {
|
||||
self.background_admissions.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn background_active_for_tests(&self) -> bool {
|
||||
self.background_active.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn foreground_active_for_tests(&self) -> usize {
|
||||
self.foreground_active.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub(crate) fn foreground(self: &Arc<Self>) -> DiskIoPermit {
|
||||
self.foreground_active.fetch_add(1, Ordering::AcqRel);
|
||||
if self.background_waiters.load(Ordering::Acquire) > 0 {
|
||||
self.record_state();
|
||||
}
|
||||
DiskIoPermit {
|
||||
lease: Arc::new(DiskIoPermitLease {
|
||||
admission: self.clone(),
|
||||
class: DiskIoClass::Foreground,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn background(self: &Arc<Self>, cancellation: &CancellationToken) -> std::io::Result<DiskIoPermit> {
|
||||
let started_at = Instant::now();
|
||||
let ticket = {
|
||||
let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner());
|
||||
let ticket = state.next_background_ticket;
|
||||
state.next_background_ticket = state.next_background_ticket.wrapping_add(1);
|
||||
if state.background_queue.is_empty() && !self.background_active.load(Ordering::Acquire) {
|
||||
state.background_front_since = Some(Instant::now());
|
||||
}
|
||||
state.background_queue.push_back(ticket);
|
||||
self.background_waiters.fetch_add(1, Ordering::Release);
|
||||
ticket
|
||||
};
|
||||
self.record_state();
|
||||
|
||||
let mut waiter = BackgroundWaiter {
|
||||
admission: self.clone(),
|
||||
ticket,
|
||||
waiting: true,
|
||||
};
|
||||
loop {
|
||||
let notified = self.changed.notified();
|
||||
tokio::pin!(notified);
|
||||
notified.as_mut().enable();
|
||||
|
||||
let granted = {
|
||||
let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner());
|
||||
let is_front = state.background_queue.front() == Some(&ticket);
|
||||
let bounded_wait_elapsed = is_front
|
||||
&& state
|
||||
.background_front_since
|
||||
.is_some_and(|front_since| front_since.elapsed() >= BACKGROUND_MAX_WAIT);
|
||||
if is_front
|
||||
&& !self.background_active.load(Ordering::Acquire)
|
||||
&& (self.foreground_active.load(Ordering::Acquire) == 0
|
||||
|| state.foreground_completions >= BACKGROUND_FOREGROUND_BURST
|
||||
|| bounded_wait_elapsed)
|
||||
{
|
||||
self.background_active.store(true, Ordering::Release);
|
||||
state.background_queue.pop_front();
|
||||
state.background_front_since = None;
|
||||
self.background_waiters.fetch_sub(1, Ordering::AcqRel);
|
||||
state.foreground_completions = 0;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
};
|
||||
|
||||
if granted {
|
||||
waiter.waiting = false;
|
||||
#[cfg(test)]
|
||||
self.background_admissions.fetch_add(1, Ordering::Relaxed);
|
||||
histogram!("rustfs_disk_io_admission_wait_seconds", "disk" => self.disk.clone(), "class" => "background")
|
||||
.record(started_at.elapsed().as_secs_f64());
|
||||
counter!("rustfs_disk_io_admission_total", "disk" => self.disk.clone(), "class" => "background").increment(1);
|
||||
self.record_state();
|
||||
return Ok(DiskIoPermit {
|
||||
lease: Arc::new(DiskIoPermitLease {
|
||||
admission: self.clone(),
|
||||
class: DiskIoClass::Background,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
let remaining = {
|
||||
let state = self.state.lock().unwrap_or_else(|error| error.into_inner());
|
||||
if state.background_queue.front() == Some(&ticket) {
|
||||
state
|
||||
.background_front_since
|
||||
.map(|front_since| BACKGROUND_MAX_WAIT.saturating_sub(front_since.elapsed()))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
};
|
||||
if let Some(remaining) = remaining.filter(|remaining| !remaining.is_zero()) {
|
||||
tokio::select! {
|
||||
_ = cancellation.cancelled() => {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "background disk I/O admission cancelled"));
|
||||
}
|
||||
_ = tokio::time::sleep(remaining) => {
|
||||
self.changed.notify_waiters();
|
||||
}
|
||||
_ = &mut notified => {}
|
||||
}
|
||||
} else {
|
||||
tokio::select! {
|
||||
_ = cancellation.cancelled() => {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "background disk I/O admission cancelled"));
|
||||
}
|
||||
_ = &mut notified => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn record_state(&self) {
|
||||
let (foreground_active, background_waiters) = {
|
||||
let state = self.state.lock().unwrap_or_else(|error| error.into_inner());
|
||||
(
|
||||
if state.background_queue.is_empty() {
|
||||
0
|
||||
} else {
|
||||
self.foreground_active.load(Ordering::Acquire)
|
||||
},
|
||||
state.background_queue.len(),
|
||||
)
|
||||
};
|
||||
let foreground_active = u32::try_from(foreground_active).unwrap_or(u32::MAX);
|
||||
let background_waiters = u32::try_from(background_waiters).unwrap_or(u32::MAX);
|
||||
gauge!("rustfs_disk_io_admission_active", "disk" => self.disk.clone(), "class" => "foreground")
|
||||
.set(f64::from(foreground_active));
|
||||
gauge!("rustfs_disk_io_admission_active", "disk" => self.disk.clone(), "class" => "background").set(
|
||||
if self.background_active.load(Ordering::Acquire) {
|
||||
1.0
|
||||
} else {
|
||||
0.0
|
||||
},
|
||||
);
|
||||
gauge!("rustfs_disk_io_admission_waiters", "disk" => self.disk.clone(), "class" => "background")
|
||||
.set(f64::from(background_waiters));
|
||||
}
|
||||
}
|
||||
|
||||
struct BackgroundWaiter {
|
||||
admission: Arc<DiskIoAdmission>,
|
||||
ticket: u64,
|
||||
waiting: bool,
|
||||
}
|
||||
|
||||
impl Drop for BackgroundWaiter {
|
||||
fn drop(&mut self) {
|
||||
if !self.waiting {
|
||||
return;
|
||||
}
|
||||
let mut state = self.admission.state.lock().unwrap_or_else(|error| error.into_inner());
|
||||
if let Some(index) = state.background_queue.iter().position(|ticket| *ticket == self.ticket) {
|
||||
state.background_queue.remove(index);
|
||||
if index == 0 {
|
||||
state.background_front_since =
|
||||
if state.background_queue.is_empty() || self.admission.background_active.load(Ordering::Acquire) {
|
||||
None
|
||||
} else {
|
||||
Some(Instant::now())
|
||||
};
|
||||
}
|
||||
self.admission.background_waiters.fetch_sub(1, Ordering::AcqRel);
|
||||
}
|
||||
drop(state);
|
||||
self.admission.record_state();
|
||||
self.admission.changed.notify_waiters();
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct DiskIoPermit {
|
||||
lease: Arc<DiskIoPermitLease>,
|
||||
}
|
||||
|
||||
impl DiskIoPermit {
|
||||
pub(crate) fn is_for(&self, admission: &Arc<DiskIoAdmission>) -> bool {
|
||||
Arc::ptr_eq(&self.lease.admission, admission)
|
||||
}
|
||||
|
||||
fn admission(&self) -> Arc<DiskIoAdmission> {
|
||||
self.lease.admission.clone()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct DiskIoPermitLease {
|
||||
admission: Arc<DiskIoAdmission>,
|
||||
class: DiskIoClass,
|
||||
}
|
||||
|
||||
pub(crate) fn spawn_blocking_with_disk_io_permit<T, F>(
|
||||
io_permit: Option<DiskIoPermit>,
|
||||
operation: F,
|
||||
) -> tokio::task::JoinHandle<T>
|
||||
where
|
||||
T: Send + 'static,
|
||||
F: FnOnce() -> T + Send + 'static,
|
||||
{
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let _io_permit = io_permit;
|
||||
operation()
|
||||
})
|
||||
}
|
||||
|
||||
type AdmissionWait = Pin<Box<dyn Future<Output = io::Result<DiskIoPermit>> + Send>>;
|
||||
|
||||
impl Drop for DiskIoPermitLease {
|
||||
fn drop(&mut self) {
|
||||
match self.class {
|
||||
DiskIoClass::Foreground => {
|
||||
self.admission.foreground_active.fetch_sub(1, Ordering::AcqRel);
|
||||
let record_state = self.admission.background_waiters.load(Ordering::Acquire) > 0;
|
||||
if record_state {
|
||||
let mut state = self.admission.state.lock().unwrap_or_else(|error| error.into_inner());
|
||||
if !state.background_queue.is_empty() {
|
||||
state.foreground_completions = state.foreground_completions.saturating_add(1);
|
||||
}
|
||||
}
|
||||
if record_state {
|
||||
self.admission.record_state();
|
||||
self.admission.changed.notify_waiters();
|
||||
}
|
||||
}
|
||||
DiskIoClass::Background => {
|
||||
let mut state = self.admission.state.lock().unwrap_or_else(|error| error.into_inner());
|
||||
self.admission.background_active.store(false, Ordering::Release);
|
||||
state.background_front_since = if state.background_queue.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(Instant::now())
|
||||
};
|
||||
drop(state);
|
||||
self.admission.record_state();
|
||||
self.admission.changed.notify_waiters();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct AdmissionReader<R> {
|
||||
inner: R,
|
||||
admission: Arc<DiskIoAdmission>,
|
||||
context: DiskIoContext,
|
||||
wait: Mutex<Option<AdmissionWait>>,
|
||||
permit: Option<DiskIoPermit>,
|
||||
}
|
||||
|
||||
impl<R> AdmissionReader<R> {
|
||||
pub(crate) fn new(inner: R, admission: Arc<DiskIoAdmission>, context: DiskIoContext) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
admission,
|
||||
context,
|
||||
wait: Mutex::new(None),
|
||||
permit: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_permit(&mut self, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
if self.permit.is_some() {
|
||||
return Poll::Ready(Ok(()));
|
||||
}
|
||||
|
||||
if self.context.class == DiskIoClass::Foreground {
|
||||
self.permit = Some(self.admission.foreground());
|
||||
return Poll::Ready(Ok(()));
|
||||
}
|
||||
|
||||
let mut wait = self.wait.lock().unwrap_or_else(|error| error.into_inner());
|
||||
if wait.is_none() {
|
||||
let admission = self.admission.clone();
|
||||
let cancellation = self.context.cancellation.clone();
|
||||
*wait = Some(Box::pin(async move { admission.background(&cancellation).await }));
|
||||
}
|
||||
let Some(wait_future) = wait.as_mut() else {
|
||||
return Poll::Ready(Err(io::Error::other("background disk I/O admission waiter is missing")));
|
||||
};
|
||||
let result = wait_future.as_mut().poll(cx);
|
||||
match result {
|
||||
Poll::Ready(Ok(permit)) => {
|
||||
wait.take();
|
||||
drop(wait);
|
||||
self.permit = Some(permit);
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
Poll::Ready(Err(error)) => {
|
||||
wait.take();
|
||||
Poll::Ready(Err(error))
|
||||
}
|
||||
Poll::Pending => Poll::Pending,
|
||||
}
|
||||
}
|
||||
|
||||
fn finish_poll<T>(&mut self, result: Poll<io::Result<T>>) -> Poll<io::Result<T>> {
|
||||
if result.is_ready() {
|
||||
self.permit = None;
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
impl<R: AsyncRead + Unpin> AsyncRead for AdmissionReader<R> {
|
||||
fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
|
||||
match self.poll_permit(cx) {
|
||||
Poll::Ready(Ok(())) => {}
|
||||
Poll::Ready(Err(error)) => return Poll::Ready(Err(error)),
|
||||
Poll::Pending => return Poll::Pending,
|
||||
}
|
||||
let result = Pin::new(&mut self.inner).poll_read(cx, buf);
|
||||
self.finish_poll(result)
|
||||
}
|
||||
}
|
||||
|
||||
impl<R: ChunkReader + Unpin> ChunkReader for AdmissionReader<R> {
|
||||
fn poll_read_chunk(mut self: Pin<&mut Self>, cx: &mut Context<'_>, max: usize) -> Poll<io::Result<Option<bytes::Bytes>>> {
|
||||
match self.poll_permit(cx) {
|
||||
Poll::Ready(Ok(())) => {}
|
||||
Poll::Ready(Err(error)) => return Poll::Ready(Err(error)),
|
||||
Poll::Pending => return Poll::Pending,
|
||||
}
|
||||
let result = Pin::new(&mut self.inner).poll_read_chunk(cx, max);
|
||||
self.finish_poll(result)
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct BoxedChunkReader(pub(crate) rustfs_rio::ChunkReaderBox);
|
||||
|
||||
impl AsyncRead for BoxedChunkReader {
|
||||
fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
|
||||
Pin::new(&mut self.0).poll_read(cx, buf)
|
||||
}
|
||||
}
|
||||
|
||||
impl ChunkReader for BoxedChunkReader {
|
||||
fn poll_read_chunk(self: Pin<&mut Self>, cx: &mut Context<'_>, max: usize) -> Poll<io::Result<Option<bytes::Bytes>>> {
|
||||
Pin::new(self.get_mut().0.as_mut()).poll_read_chunk(cx, max)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
|
||||
struct BlockingWorker(tokio::task::JoinHandle<()>);
|
||||
|
||||
async fn wait_for_background_waiters(admission: &DiskIoAdmission, expected: usize) {
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
loop {
|
||||
let waiters = admission
|
||||
.state
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner())
|
||||
.background_queue
|
||||
.len();
|
||||
if waiters == expected {
|
||||
return;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("background admission waiter should reach expected count");
|
||||
}
|
||||
|
||||
async fn wait_for_foreground_active(admission: &DiskIoAdmission, expected: usize) {
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
loop {
|
||||
let active = admission.foreground_active.load(Ordering::Acquire);
|
||||
if active == expected {
|
||||
return;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("foreground disk read should retain its permit while pending");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn foreground_bypasses_background_and_background_progress_is_bounded() {
|
||||
let admission = DiskIoAdmission::new("disk-a");
|
||||
let foreground = admission.foreground();
|
||||
let background_admission = admission.clone();
|
||||
let cancellation = CancellationToken::new();
|
||||
let background = tokio::spawn(async move { background_admission.background(&cancellation).await });
|
||||
|
||||
wait_for_background_waiters(&admission, 1).await;
|
||||
for _ in 0..BACKGROUND_FOREGROUND_BURST {
|
||||
drop(admission.foreground());
|
||||
}
|
||||
|
||||
let background_permit = tokio::time::timeout(Duration::from_secs(1), background)
|
||||
.await
|
||||
.expect("background admission should progress after the foreground burst")
|
||||
.expect("background waiter task should complete")
|
||||
.expect("background admission should not be cancelled");
|
||||
assert_eq!(
|
||||
admission.foreground_active.load(Ordering::Acquire),
|
||||
1,
|
||||
"foreground operations remain admitted while one background operation runs"
|
||||
);
|
||||
drop(background_permit);
|
||||
drop(foreground);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancellation_removes_a_queued_background_waiter() {
|
||||
let admission = DiskIoAdmission::new("disk-a");
|
||||
let background = admission
|
||||
.background(&CancellationToken::new())
|
||||
.await
|
||||
.expect("first background operation should be admitted");
|
||||
let cancellation = CancellationToken::new();
|
||||
let waiter_admission = admission.clone();
|
||||
let waiter_cancellation = cancellation.clone();
|
||||
let waiter = tokio::spawn(async move { waiter_admission.background(&waiter_cancellation).await });
|
||||
wait_for_background_waiters(&admission, 1).await;
|
||||
|
||||
cancellation.cancel();
|
||||
let error = tokio::time::timeout(Duration::from_secs(1), waiter)
|
||||
.await
|
||||
.expect("cancelled background waiter should stop")
|
||||
.expect("background waiter task should complete")
|
||||
.expect_err("cancelled admission must not start disk I/O");
|
||||
assert_eq!(error.kind(), io::ErrorKind::Interrupted);
|
||||
assert_eq!(
|
||||
admission
|
||||
.state
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner())
|
||||
.background_queue
|
||||
.len(),
|
||||
0
|
||||
);
|
||||
drop(background);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn background_admission_is_isolated_per_disk() {
|
||||
let disk_a = DiskIoAdmission::new("disk-a");
|
||||
let disk_b = DiskIoAdmission::new("disk-b");
|
||||
let _disk_a_permit = disk_a
|
||||
.background(&CancellationToken::new())
|
||||
.await
|
||||
.expect("disk A background operation should be admitted");
|
||||
let _disk_b_permit = tokio::time::timeout(Duration::from_secs(1), disk_b.background(&CancellationToken::new()))
|
||||
.await
|
||||
.expect("disk A must not block disk B")
|
||||
.expect("disk B background operation should be admitted");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streamed_read_holds_admission_until_disk_read_resolves() {
|
||||
let admission = DiskIoAdmission::new("disk-a");
|
||||
let (mut writer, reader) = tokio::io::duplex(8);
|
||||
let mut reader = AdmissionReader::new(
|
||||
reader,
|
||||
admission.clone(),
|
||||
DiskIoContext {
|
||||
class: DiskIoClass::Foreground,
|
||||
cancellation: CancellationToken::new(),
|
||||
admission: None,
|
||||
permit: None,
|
||||
},
|
||||
);
|
||||
let read = tokio::spawn(async move {
|
||||
let mut bytes = [0; 4];
|
||||
reader
|
||||
.read_exact(&mut bytes)
|
||||
.await
|
||||
.expect("streamed disk read should complete");
|
||||
bytes
|
||||
});
|
||||
|
||||
wait_for_foreground_active(&admission, 1).await;
|
||||
writer.write_all(b"rust").await.expect("test bytes should be written");
|
||||
assert_eq!(read.await.expect("reader task should finish"), *b"rust");
|
||||
wait_for_foreground_active(&admission, 0).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn blocking_worker_keeps_permit_after_caller_timeout() {
|
||||
let admission = DiskIoAdmission::new("disk-a");
|
||||
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
|
||||
let worker = spawn_blocking_with_disk_io_permit(Some(admission.foreground()), move || {
|
||||
started_tx.send(()).expect("worker start should be observed");
|
||||
std::thread::sleep(Duration::from_millis(100));
|
||||
});
|
||||
|
||||
started_rx.await.expect("blocking worker should start");
|
||||
assert!(tokio::time::timeout(Duration::from_millis(10), worker).await.is_err());
|
||||
assert_eq!(admission.foreground_active_for_tests(), 1);
|
||||
wait_for_foreground_active(&admission, 0).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cloned_permit_remains_active_until_all_workers_finish() {
|
||||
let admission = DiskIoAdmission::new("disk-a");
|
||||
let permit = admission.foreground();
|
||||
let worker_permit = permit.clone();
|
||||
drop(permit);
|
||||
assert_eq!(admission.foreground_active_for_tests(), 1);
|
||||
|
||||
drop(worker_permit);
|
||||
assert_eq!(admission.foreground_active_for_tests(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scoped_foreground_permit_is_inherited_by_blocking_worker() {
|
||||
let admission = DiskIoAdmission::new("disk-a");
|
||||
let permit = admission.foreground();
|
||||
let worker = with_disk_io_permit(Some(permit), async {
|
||||
let worker_permit = acquire_context_disk_io_permit()
|
||||
.await
|
||||
.expect("context permit should be available")
|
||||
.expect("scoped permit should be inherited");
|
||||
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
|
||||
let worker = spawn_blocking_with_disk_io_permit(Some(worker_permit), move || {
|
||||
started_tx.send(()).expect("worker start should be observed");
|
||||
std::thread::sleep(Duration::from_millis(100));
|
||||
});
|
||||
started_rx.await.expect("blocking worker should start");
|
||||
BlockingWorker(worker)
|
||||
})
|
||||
.await;
|
||||
|
||||
assert_eq!(admission.foreground_active_for_tests(), 1);
|
||||
assert!(tokio::time::timeout(Duration::from_millis(10), worker.0).await.is_err());
|
||||
wait_for_foreground_active(&admission, 0).await;
|
||||
}
|
||||
}
|
||||
+1156
-158
File diff suppressed because it is too large
Load Diff
@@ -84,46 +84,107 @@ async fn wait_for_heal_commit_test_barrier(root: &Path, bucket: &str, object: &s
|
||||
}
|
||||
}
|
||||
|
||||
fn rollback_committed_rename_std(
|
||||
dst_file_path: &Path,
|
||||
new_data_path: Option<&Path>,
|
||||
rollback_data_dir: Option<Uuid>,
|
||||
) -> std::io::Result<()> {
|
||||
if let Some(old_data_dir) = rollback_data_dir {
|
||||
let Some(dst_parent) = dst_file_path.parent() else {
|
||||
return Err(std::io::Error::new(ErrorKind::InvalidInput, "missing object metadata parent"));
|
||||
};
|
||||
let backup_path = dst_parent.join(old_data_dir.to_string()).join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
std::fs::rename(backup_path, dst_file_path)?;
|
||||
} else {
|
||||
remove_file_if_exists(dst_file_path)?;
|
||||
}
|
||||
|
||||
if let Some(new_data_path) = new_data_path {
|
||||
remove_dir_all_if_exists(new_data_path)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
struct RenameCommitContext<'a> {
|
||||
// Drop the directory identity guard before releasing mutation ownership.
|
||||
directory_guard: os::RenameCommitGuard,
|
||||
publication_root: &'a os::PublicationRoot,
|
||||
mutation_lease: Arc<os::NamespaceMutationLease>,
|
||||
source_metadata: &'a Path,
|
||||
destination_metadata: &'a Path,
|
||||
source_volume_dir: &'a Path,
|
||||
data_paths: Option<&'a (PathBuf, PathBuf)>,
|
||||
}
|
||||
|
||||
fn rollback_inline_metadata_commit_std(
|
||||
dst_file_path: &Path,
|
||||
rollback_data_dir: Option<Uuid>,
|
||||
local_rollback_path: Option<&Path>,
|
||||
) -> std::io::Result<()> {
|
||||
#[cfg(all(test, not(windows)))]
|
||||
os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Rollback, dst_file_path);
|
||||
if let Some(backup_path) = local_rollback_path {
|
||||
// The commit immediately before this rollback renamed the staged
|
||||
// xl.meta from the same directory as `backup_path` onto
|
||||
// `dst_file_path`, proving both paths are on the same filesystem.
|
||||
// Unix rename atomically replaces the committed destination; never
|
||||
// unlink it first or an interrupted rollback could lose xl.meta.
|
||||
std::fs::rename(backup_path, dst_file_path)?;
|
||||
} else {
|
||||
rollback_committed_rename_std(dst_file_path, None, rollback_data_dir)?;
|
||||
struct StagedRollbackBackup(PathBuf);
|
||||
|
||||
impl StagedRollbackBackup {
|
||||
fn path(&self) -> &Path {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl RenameCommitContext<'_> {
|
||||
fn rollback_metadata_std(&self, rollback_data_dir: Option<Uuid>) -> std::io::Result<()> {
|
||||
if let Some(old_data_dir) = rollback_data_dir {
|
||||
let Some(dst_parent) = self.destination_metadata.parent() else {
|
||||
return Err(std::io::Error::new(ErrorKind::InvalidInput, "missing object metadata parent"));
|
||||
};
|
||||
let backup_path = dst_parent.join(old_data_dir.to_string()).join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
std::fs::rename(backup_path, self.destination_metadata)?;
|
||||
} else {
|
||||
remove_file_if_exists(self.destination_metadata)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn rollback_non_inline_std(&self, rollback_data_dir: Option<Uuid>) -> std::io::Result<()> {
|
||||
self.rollback_metadata_std(rollback_data_dir)?;
|
||||
if let Some((_, new_data_path)) = self.data_paths {
|
||||
remove_dir_all_if_exists(new_data_path)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn rollback_inline_std(
|
||||
&self,
|
||||
rollback_data_dir: Option<Uuid>,
|
||||
local_backup: Option<&StagedRollbackBackup>,
|
||||
) -> std::io::Result<()> {
|
||||
#[cfg(all(test, not(windows)))]
|
||||
os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Rollback, self.destination_metadata);
|
||||
if let Some(backup) = local_backup {
|
||||
// The preceding commit used this staged directory on the same
|
||||
// filesystem. Never unlink xl.meta before its atomic replacement.
|
||||
std::fs::rename(backup.path(), self.destination_metadata)?;
|
||||
} else {
|
||||
self.rollback_metadata_std(rollback_data_dir)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn restore_published_data_source(&self) -> Result<()> {
|
||||
let Some((src_data_path, dst_data_path)) = self.data_paths else {
|
||||
return Ok(());
|
||||
};
|
||||
let src_volume_dir = self.source_volume_dir;
|
||||
let publication_root = self.publication_root;
|
||||
let mutation_lease = self.mutation_lease.clone();
|
||||
if fs::symlink_metadata(src_data_path).await.is_ok() {
|
||||
return Ok(());
|
||||
}
|
||||
let result =
|
||||
match os::rename_all_with_lease(dst_data_path, src_data_path, src_volume_dir, publication_root, mutation_lease).await
|
||||
{
|
||||
Ok(()) => Ok(()),
|
||||
Err(DiskError::FileNotFound) => {
|
||||
let source_exists = fs::symlink_metadata(src_data_path).await.is_ok();
|
||||
let destination_missing = matches!(
|
||||
fs::symlink_metadata(dst_data_path).await,
|
||||
Err(err) if err.kind() == ErrorKind::NotFound
|
||||
);
|
||||
if source_exists && destination_missing {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(DiskError::FileNotFound)
|
||||
}
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
};
|
||||
if let Err(err) = &result {
|
||||
warn!(
|
||||
target: "rustfs_ecstore::disk::local",
|
||||
event = EVENT_DISK_LOCAL_RENAME_REJECTED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
|
||||
reason = "restore_staged_data_source_failed",
|
||||
src_path = ?src_data_path,
|
||||
dst_path = ?dst_data_path,
|
||||
error = ?err,
|
||||
"Failed to restore staged data after a metadata commit was rejected"
|
||||
);
|
||||
}
|
||||
result
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(any(not(windows), test))]
|
||||
@@ -212,61 +273,6 @@ async fn read_rename_destination_metadata(
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
|
||||
async fn restore_renamed_data_source(
|
||||
src_volume_dir: &Path,
|
||||
src_data_path: &Path,
|
||||
dst_data_path: &Path,
|
||||
publication_root: &os::PublicationRoot,
|
||||
mutation_lease: Arc<os::NamespaceMutationLease>,
|
||||
) -> Result<()> {
|
||||
if fs::symlink_metadata(src_data_path).await.is_ok() {
|
||||
return Ok(());
|
||||
}
|
||||
let result =
|
||||
match os::rename_all_with_lease(dst_data_path, src_data_path, src_volume_dir, publication_root, mutation_lease).await {
|
||||
Ok(()) => Ok(()),
|
||||
Err(DiskError::FileNotFound) => {
|
||||
let source_exists = fs::symlink_metadata(src_data_path).await.is_ok();
|
||||
let destination_missing = matches!(
|
||||
fs::symlink_metadata(dst_data_path).await,
|
||||
Err(err) if err.kind() == ErrorKind::NotFound
|
||||
);
|
||||
if source_exists && destination_missing {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(DiskError::FileNotFound)
|
||||
}
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
};
|
||||
if let Err(err) = &result {
|
||||
warn!(
|
||||
target: "rustfs_ecstore::disk::local",
|
||||
event = EVENT_DISK_LOCAL_RENAME_REJECTED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
|
||||
reason = "restore_staged_data_source_failed",
|
||||
src_path = ?src_data_path,
|
||||
dst_path = ?dst_data_path,
|
||||
error = ?err,
|
||||
"Failed to restore staged data after a metadata commit was rejected"
|
||||
);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
async fn restore_published_data_source(
|
||||
data_paths: Option<&(PathBuf, PathBuf)>,
|
||||
src_volume_dir: &Path,
|
||||
publication_root: &os::PublicationRoot,
|
||||
mutation_lease: Arc<os::NamespaceMutationLease>,
|
||||
) -> Result<()> {
|
||||
let Some((src_data_path, dst_data_path)) = data_paths else {
|
||||
return Ok(());
|
||||
};
|
||||
restore_renamed_data_source(src_volume_dir, src_data_path, dst_data_path, publication_root, mutation_lease).await
|
||||
}
|
||||
|
||||
/// Proof produced only when the local rename returns at an existing access
|
||||
/// preflight, before metadata, backups, or object data can be published.
|
||||
#[derive(Debug)]
|
||||
@@ -457,7 +463,18 @@ impl LocalDisk {
|
||||
mutation_lease.clone(),
|
||||
)
|
||||
.await?;
|
||||
let has_dst_buf = read_rename_destination_metadata(&dst_file_path, &rename_commit_guard, mutation_lease.clone()).await?;
|
||||
let mut commit = RenameCommitContext {
|
||||
directory_guard: rename_commit_guard,
|
||||
publication_root: &self.publication_root,
|
||||
mutation_lease,
|
||||
source_metadata: &src_file_path,
|
||||
destination_metadata: &dst_file_path,
|
||||
source_volume_dir: &src_volume_dir,
|
||||
data_paths: has_data_dir_path.as_ref(),
|
||||
};
|
||||
let has_dst_buf =
|
||||
read_rename_destination_metadata(commit.destination_metadata, &commit.directory_guard, commit.mutation_lease.clone())
|
||||
.await?;
|
||||
|
||||
if no_inline {
|
||||
// Non-inline: read xl.meta, parse, write, rename data dir, rename xl.meta
|
||||
@@ -533,18 +550,18 @@ impl LocalDisk {
|
||||
// rename below to report through the existing rollback path. Payload
|
||||
// durability is kept by both strict and relaxed.
|
||||
let tmp_meta_write = {
|
||||
let src_file_path = src_file_path.clone();
|
||||
let src_file_path = commit.source_metadata.to_path_buf();
|
||||
let dst_file_path = dst_file_path.clone();
|
||||
let rename_commit_guard = rename_commit_guard.clone();
|
||||
let mutation_lease = mutation_lease.clone();
|
||||
let owned_directory_guard = commit.directory_guard.clone();
|
||||
let owned_mutation_lease = commit.mutation_lease.clone();
|
||||
async move {
|
||||
os::run_blocking_namespace_operation(mutation_lease, move || {
|
||||
os::run_blocking_namespace_operation(owned_mutation_lease, move || {
|
||||
#[cfg(test)]
|
||||
run_owned_file_write_before_open(&src_file_path);
|
||||
let mut prepared_metadata_source = os::create_prepared_rename_source_with_commit_guard(
|
||||
&src_file_path,
|
||||
&dst_file_path,
|
||||
&rename_commit_guard,
|
||||
&owned_directory_guard,
|
||||
)?;
|
||||
prepared_metadata_source.write_all(&new_dst_buf, tmp_meta_sync != SyncMode::None)?;
|
||||
Ok(prepared_metadata_source)
|
||||
@@ -570,14 +587,14 @@ impl LocalDisk {
|
||||
// sequential version did.
|
||||
let prepared_metadata_source = tmp_meta_res?;
|
||||
shard_sync_res?;
|
||||
let rename_commit_guard = remove_dst_base_before_commit(
|
||||
commit.directory_guard = remove_dst_base_before_commit(
|
||||
dst_path,
|
||||
rename_commit_guard,
|
||||
commit.directory_guard,
|
||||
src_file_parent,
|
||||
dst_file_parent,
|
||||
&dst_volume_dir,
|
||||
&self.publication_root,
|
||||
mutation_lease.clone(),
|
||||
commit.publication_root,
|
||||
commit.mutation_lease.clone(),
|
||||
)
|
||||
.await?;
|
||||
if should_remove_staged_meta_before_commit(dst_path) {
|
||||
@@ -595,7 +612,7 @@ impl LocalDisk {
|
||||
if fi_healing
|
||||
&& let Some((_, dst_data_path)) = has_data_dir_path.as_ref()
|
||||
&& let Err(err) = self
|
||||
.move_to_trash_with_namespace_owner(dst_data_path, true, false, Some(mutation_lease.clone()))
|
||||
.move_to_trash_with_namespace_owner(dst_data_path, true, false, Some(commit.mutation_lease.clone()))
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
@@ -613,9 +630,9 @@ impl LocalDisk {
|
||||
src_data_path,
|
||||
dst_data_path,
|
||||
&skip_parent,
|
||||
&self.publication_root,
|
||||
&rename_commit_guard,
|
||||
mutation_lease.clone(),
|
||||
commit.publication_root,
|
||||
&commit.directory_guard,
|
||||
commit.mutation_lease.clone(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -630,13 +647,7 @@ impl LocalDisk {
|
||||
error = ?err,
|
||||
"Disk local rename flow failed"
|
||||
);
|
||||
restore_published_data_source(
|
||||
has_data_dir_path.as_ref(),
|
||||
&src_volume_dir,
|
||||
&self.publication_root,
|
||||
mutation_lease.clone(),
|
||||
)
|
||||
.await?;
|
||||
commit.restore_published_data_source().await?;
|
||||
return Err(err);
|
||||
}
|
||||
#[cfg(test)]
|
||||
@@ -661,13 +672,7 @@ impl LocalDisk {
|
||||
reason = "test_fail_before_old_metadata_backup",
|
||||
"Disk local rename flow failed before metadata commit"
|
||||
);
|
||||
restore_published_data_source(
|
||||
has_data_dir_path.as_ref(),
|
||||
&src_volume_dir,
|
||||
&self.publication_root,
|
||||
mutation_lease.clone(),
|
||||
)
|
||||
.await?;
|
||||
commit.restore_published_data_source().await?;
|
||||
return Err(DiskError::Unexpected);
|
||||
}
|
||||
|
||||
@@ -685,25 +690,16 @@ impl LocalDisk {
|
||||
let backup_parent = dst_file_parent.join(old_data_dir.to_string());
|
||||
#[cfg(not(windows))]
|
||||
if let Err(err) = os::make_dir_all(&backup_parent, &skip_parent).await {
|
||||
restore_published_data_source(
|
||||
has_data_dir_path.as_ref(),
|
||||
&src_volume_dir,
|
||||
&self.publication_root,
|
||||
mutation_lease.clone(),
|
||||
)
|
||||
.await?;
|
||||
commit.restore_published_data_source().await?;
|
||||
return Err(err);
|
||||
}
|
||||
let backup_path_guard = match rename_commit_guard.create_destination_directory_for_path_access(&backup_parent) {
|
||||
let backup_path_guard = match commit
|
||||
.directory_guard
|
||||
.create_destination_directory_for_path_access(&backup_parent)
|
||||
{
|
||||
Ok(guard) => guard,
|
||||
Err(err) => {
|
||||
restore_published_data_source(
|
||||
has_data_dir_path.as_ref(),
|
||||
&src_volume_dir,
|
||||
&self.publication_root,
|
||||
mutation_lease.clone(),
|
||||
)
|
||||
.await?;
|
||||
commit.restore_published_data_source().await?;
|
||||
return Err(DiskError::from(to_file_error(err)));
|
||||
}
|
||||
};
|
||||
@@ -711,13 +707,7 @@ impl LocalDisk {
|
||||
if let Err(err) = check_path_length(backup_path.to_string_lossy().as_ref()) {
|
||||
#[cfg(windows)]
|
||||
drop(backup_path_guard);
|
||||
restore_published_data_source(
|
||||
has_data_dir_path.as_ref(),
|
||||
&src_volume_dir,
|
||||
&self.publication_root,
|
||||
mutation_lease.clone(),
|
||||
)
|
||||
.await?;
|
||||
commit.restore_published_data_source().await?;
|
||||
return Err(err);
|
||||
}
|
||||
let backup_bytes = dst_buf.clone();
|
||||
@@ -726,7 +716,7 @@ impl LocalDisk {
|
||||
// sync finish. A detached spawn_blocking writer could survive
|
||||
// cancellation and later truncate a newer transaction's
|
||||
// deterministic rollback backup.
|
||||
let write_result = os::run_blocking_namespace_operation(mutation_lease.clone(), move || {
|
||||
let write_result = os::run_blocking_namespace_operation(commit.mutation_lease.clone(), move || {
|
||||
#[cfg(test)]
|
||||
run_owned_file_write_before_open(&backup_path);
|
||||
backup_path_guard.write_file_for_path_access(
|
||||
@@ -749,13 +739,7 @@ impl LocalDisk {
|
||||
error = ?err,
|
||||
"Disk local rename flow failed"
|
||||
);
|
||||
restore_published_data_source(
|
||||
has_data_dir_path.as_ref(),
|
||||
&src_volume_dir,
|
||||
&self.publication_root,
|
||||
mutation_lease.clone(),
|
||||
)
|
||||
.await?;
|
||||
commit.restore_published_data_source().await?;
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
@@ -773,9 +757,9 @@ impl LocalDisk {
|
||||
&src_file_path,
|
||||
&dst_file_path,
|
||||
&skip_parent,
|
||||
&self.publication_root,
|
||||
&rename_commit_guard,
|
||||
mutation_lease.clone(),
|
||||
commit.publication_root,
|
||||
&commit.directory_guard,
|
||||
commit.mutation_lease.clone(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -790,20 +774,11 @@ impl LocalDisk {
|
||||
error = ?err,
|
||||
"Disk local rename flow failed"
|
||||
);
|
||||
restore_published_data_source(
|
||||
has_data_dir_path.as_ref(),
|
||||
&src_volume_dir,
|
||||
&self.publication_root,
|
||||
mutation_lease.clone(),
|
||||
)
|
||||
.await?;
|
||||
commit.restore_published_data_source().await?;
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
let committed_new_data_path = has_data_dir_path.as_ref().map(|(_, dst_data_path)| dst_data_path.as_path());
|
||||
if should_fail_after_metadata_commit(dst_path) {
|
||||
rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir)
|
||||
.map_err(to_file_error)?;
|
||||
commit.rollback_non_inline_std(rollback_data_dir).map_err(to_file_error)?;
|
||||
return Err(DiskError::Unexpected);
|
||||
}
|
||||
|
||||
@@ -823,13 +798,12 @@ impl LocalDisk {
|
||||
&& let Some(parent) = dst_file_path.parent()
|
||||
{
|
||||
let fsync_started = rustfs_io_metrics::put_stage_timer();
|
||||
if let Err(err) = os::fsync_dst_dir_group_commit(parent, Some(mutation_lease.clone())).await {
|
||||
if let Err(err) = os::fsync_dst_dir_group_commit(parent, Some(commit.mutation_lease.clone())).await {
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC,
|
||||
fsync_started,
|
||||
);
|
||||
rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir)
|
||||
.map_err(to_file_error)?;
|
||||
commit.rollback_non_inline_std(rollback_data_dir).map_err(to_file_error)?;
|
||||
// The commit rename changed the dst part inodes before this fsync
|
||||
// failed and rolled them back; drop any fd cached during that
|
||||
// window so readers re-open the restored inode (rustfs/backlog#1177).
|
||||
@@ -861,13 +835,12 @@ impl LocalDisk {
|
||||
break;
|
||||
}
|
||||
let fsync_started = rustfs_io_metrics::put_stage_timer();
|
||||
if let Err(err) = os::fsync_dir_with_owner(dir, Some(mutation_lease.clone())).await {
|
||||
if let Err(err) = os::fsync_dir_with_owner(dir, Some(commit.mutation_lease.clone())).await {
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC,
|
||||
fsync_started,
|
||||
);
|
||||
rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir)
|
||||
.map_err(to_file_error)?;
|
||||
commit.rollback_non_inline_std(rollback_data_dir).map_err(to_file_error)?;
|
||||
// Same post-commit rollback window as above — drop cached
|
||||
// dst part fds so readers re-open the restored inode
|
||||
// (rustfs/backlog#1177).
|
||||
@@ -891,7 +864,7 @@ impl LocalDisk {
|
||||
// complete. Do not retain the Windows object identity guard while
|
||||
// cleaning staging paths or invalidating cached descriptors.
|
||||
#[cfg(windows)]
|
||||
drop(rename_commit_guard);
|
||||
drop(commit.directory_guard);
|
||||
|
||||
if let Some(src_file_path_parent) = src_file_path.parent() {
|
||||
if src_volume != super::super::RUSTFS_META_MULTIPART_BUCKET {
|
||||
@@ -939,7 +912,7 @@ impl LocalDisk {
|
||||
let dst_path_for_failpoint = dst_path.to_string();
|
||||
#[cfg(windows)]
|
||||
let source_parent = src_file_parent.to_path_buf();
|
||||
let rename_commit_guard_for_preparation = rename_commit_guard.clone();
|
||||
let rename_commit_guard_for_preparation = commit.directory_guard.clone();
|
||||
let sync = durability.syncs_commit_metadata();
|
||||
#[cfg(test)]
|
||||
run_inline_before_file_sync_admission(dst_path);
|
||||
@@ -1022,7 +995,7 @@ impl LocalDisk {
|
||||
if sync {
|
||||
std::fs::File::open(&backup_path)?.sync_data()?;
|
||||
}
|
||||
staged_rollback_path = Some(backup_path);
|
||||
staged_rollback_path = Some(StagedRollbackBackup(backup_path));
|
||||
}
|
||||
|
||||
Ok::<_, std::io::Error>((
|
||||
@@ -1036,9 +1009,10 @@ impl LocalDisk {
|
||||
))
|
||||
};
|
||||
let inline_preparation = if let Some(admission) = file_sync_admission.as_ref() {
|
||||
os::run_blocking_namespace_file_sync_operation(mutation_lease.clone(), admission, prepare_inline_metadata).await
|
||||
os::run_blocking_namespace_file_sync_operation(commit.mutation_lease.clone(), admission, prepare_inline_metadata)
|
||||
.await
|
||||
} else {
|
||||
os::run_blocking_namespace_operation(mutation_lease.clone(), prepare_inline_metadata).await
|
||||
os::run_blocking_namespace_operation(commit.mutation_lease.clone(), prepare_inline_metadata).await
|
||||
}
|
||||
.map_err(to_file_error)
|
||||
.map_err(DiskError::from);
|
||||
@@ -1061,28 +1035,30 @@ impl LocalDisk {
|
||||
}
|
||||
};
|
||||
|
||||
let rename_commit_guard = remove_dst_base_before_commit(
|
||||
commit.directory_guard = remove_dst_base_before_commit(
|
||||
dst_path,
|
||||
rename_commit_guard,
|
||||
commit.directory_guard,
|
||||
src_file_parent,
|
||||
dst_file_parent,
|
||||
&dst_volume_dir,
|
||||
&self.publication_root,
|
||||
mutation_lease.clone(),
|
||||
commit.publication_root,
|
||||
commit.mutation_lease.clone(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
if should_remove_staged_meta_before_commit(dst_path) {
|
||||
drop(prepared_metadata_source);
|
||||
let remove_result = std::fs::remove_file(&src_file_path);
|
||||
if let Some(backup_path) = local_rollback_path.as_deref() {
|
||||
if let Some(backup_path) = local_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
remove_result.map_err(to_file_error)?;
|
||||
return Err(DiskError::FileNotFound);
|
||||
}
|
||||
|
||||
if let (Some(rollback_data_dir), Some(staged_backup)) = (rollback_data_dir, local_rollback_path.as_deref()) {
|
||||
if let (Some(rollback_data_dir), Some(staged_backup)) =
|
||||
(rollback_data_dir, local_rollback_path.as_ref().map(StagedRollbackBackup::path))
|
||||
{
|
||||
let Some(dst_parent) = dst_file_path.parent() else {
|
||||
return Err(DiskError::other("missing object metadata parent"));
|
||||
};
|
||||
@@ -1096,8 +1072,8 @@ impl LocalDisk {
|
||||
staged_backup,
|
||||
&backup_path,
|
||||
&dst_volume_dir,
|
||||
&self.publication_root,
|
||||
Some(mutation_lease.clone()),
|
||||
commit.publication_root,
|
||||
Some(commit.mutation_lease.clone()),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -1119,7 +1095,8 @@ impl LocalDisk {
|
||||
{
|
||||
let fsync_started = rustfs_io_metrics::put_stage_timer();
|
||||
if let Err(err) =
|
||||
os::fsync_dir_with_namespace_file_sync_limit(backup_parent, mutation_lease.clone(), admission).await
|
||||
os::fsync_dir_with_namespace_file_sync_limit(backup_parent, commit.mutation_lease.clone(), admission)
|
||||
.await
|
||||
{
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_BACKUP_DIR_FSYNC,
|
||||
@@ -1143,14 +1120,14 @@ impl LocalDisk {
|
||||
&src_file_path,
|
||||
&dst_file_path,
|
||||
&dst_volume_dir,
|
||||
&self.publication_root,
|
||||
&rename_commit_guard,
|
||||
mutation_lease.clone(),
|
||||
commit.publication_root,
|
||||
&commit.directory_guard,
|
||||
commit.mutation_lease.clone(),
|
||||
)
|
||||
.await
|
||||
};
|
||||
if let Err(err) = commit_result {
|
||||
if let Some(backup_path) = local_rollback_path.as_deref() {
|
||||
if let Some(backup_path) = local_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
for part_path in &invalidate_part_paths {
|
||||
@@ -1161,7 +1138,7 @@ impl LocalDisk {
|
||||
|
||||
let post_commit = async {
|
||||
if should_fail_after_metadata_commit(dst_path) {
|
||||
rollback_inline_metadata_commit_std(&dst_file_path, rollback_data_dir, local_rollback_path.as_deref())?;
|
||||
commit.rollback_inline_std(rollback_data_dir, local_rollback_path.as_ref())?;
|
||||
return Err(std::io::Error::other("test fail after metadata commit"));
|
||||
}
|
||||
|
||||
@@ -1170,15 +1147,18 @@ impl LocalDisk {
|
||||
&& let Some(dst_parent) = dst_file_path.parent()
|
||||
{
|
||||
let fsync_started = rustfs_io_metrics::put_stage_timer();
|
||||
if let Err(err) =
|
||||
os::fsync_dst_dir_group_commit_or_namespace_file_sync_limit(dst_parent, mutation_lease.clone(), admission)
|
||||
.await
|
||||
if let Err(err) = os::fsync_dst_dir_group_commit_or_namespace_file_sync_limit(
|
||||
dst_parent,
|
||||
commit.mutation_lease.clone(),
|
||||
admission,
|
||||
)
|
||||
.await
|
||||
{
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC,
|
||||
fsync_started,
|
||||
);
|
||||
rollback_inline_metadata_commit_std(&dst_file_path, rollback_data_dir, local_rollback_path.as_deref())?;
|
||||
commit.rollback_inline_std(rollback_data_dir, local_rollback_path.as_ref())?;
|
||||
return Err(err);
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
@@ -1204,17 +1184,14 @@ impl LocalDisk {
|
||||
}
|
||||
let fsync_started = rustfs_io_metrics::put_stage_timer();
|
||||
if let Err(err) =
|
||||
os::fsync_dir_with_namespace_file_sync_limit(ancestor_dir, mutation_lease.clone(), admission).await
|
||||
os::fsync_dir_with_namespace_file_sync_limit(ancestor_dir, commit.mutation_lease.clone(), admission)
|
||||
.await
|
||||
{
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC,
|
||||
fsync_started,
|
||||
);
|
||||
rollback_inline_metadata_commit_std(
|
||||
&dst_file_path,
|
||||
rollback_data_dir,
|
||||
local_rollback_path.as_deref(),
|
||||
)?;
|
||||
commit.rollback_inline_std(rollback_data_dir, local_rollback_path.as_ref())?;
|
||||
return Err(err);
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
@@ -1251,9 +1228,9 @@ impl LocalDisk {
|
||||
// The commit no longer has a rollback path. Release the Windows
|
||||
// object identity guard before best-effort staging cleanup.
|
||||
#[cfg(windows)]
|
||||
drop(rename_commit_guard);
|
||||
drop(commit.directory_guard);
|
||||
|
||||
if let Some(backup_path) = local_rollback_path.as_deref() {
|
||||
if let Some(backup_path) = local_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
|
||||
|
||||
@@ -22,9 +22,15 @@ pub mod error_reduce;
|
||||
pub mod format;
|
||||
pub mod fs;
|
||||
pub mod health_state;
|
||||
mod io_admission;
|
||||
pub mod local;
|
||||
pub mod os;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) use io_admission::DiskIoClass;
|
||||
pub use io_admission::with_background_disk_io;
|
||||
pub(crate) use io_admission::{acquire_context_disk_io_permit, current_disk_io_context, with_optional_disk_io_context};
|
||||
|
||||
pub const RUSTFS_META_BUCKET: &str = ".rustfs.sys";
|
||||
pub const MIGRATING_META_BUCKET: &str = ".minio.sys";
|
||||
pub const RUSTFS_META_MULTIPART_BUCKET: &str = ".rustfs.sys/multipart";
|
||||
|
||||
@@ -259,7 +259,7 @@ async fn blackbox_put_unknown_actual_size_restores_body_and_records_written_size
|
||||
.map(|idx| ((idx * 17) % 251) as u8)
|
||||
.collect::<Vec<_>>();
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -300,7 +300,7 @@ async fn blackbox_get_restores_body_after_one_shard_file_is_removed() {
|
||||
.map(|idx| ((idx * 19) % 251) as u8)
|
||||
.collect::<Vec<_>>();
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -406,7 +406,7 @@ async fn blackbox_heal_requests_preserve_repair_scope() {
|
||||
put_object,
|
||||
&mut put_reader,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
versioned: true,
|
||||
expected_bucket_incarnation_id: Some(put_incarnation),
|
||||
..Default::default()
|
||||
@@ -469,7 +469,7 @@ async fn blackbox_heal_requests_preserve_repair_scope() {
|
||||
put_object,
|
||||
&mut healthy_reader,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
@@ -509,7 +509,7 @@ async fn blackbox_heal_requests_preserve_repair_scope() {
|
||||
.map(|idx| ((idx * 29) % 251) as u8)
|
||||
.collect::<Vec<_>>();
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -567,7 +567,7 @@ async fn blackbox_heal_requests_preserve_repair_scope() {
|
||||
let suspended_object = "suspended.bin";
|
||||
let payload = vec![0x5a; 1 << 20];
|
||||
let mpu_opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
};
|
||||
@@ -675,7 +675,7 @@ async fn blackbox_heal_requests_preserve_repair_scope() {
|
||||
.expect("fourth disk should be online before suspended completion")
|
||||
};
|
||||
let suspended_opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
version_suspended: true,
|
||||
..Default::default()
|
||||
};
|
||||
@@ -712,7 +712,7 @@ async fn blackbox_range_read_restores_exact_slice_with_one_offline_disk() {
|
||||
let range_start = 513usize;
|
||||
let range_len = 8192usize;
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -756,7 +756,7 @@ async fn blackbox_delete_marker_hides_object_body_without_erasing_prior_version_
|
||||
let bucket = "bb-delete-marker-read-negative";
|
||||
let object = "object.bin";
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
version_suspended: true,
|
||||
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))),
|
||||
..Default::default()
|
||||
@@ -853,7 +853,7 @@ async fn blackbox_issue3031_diag_covers_put_success_cleanup_and_error_summary()
|
||||
let first_payload = b"first diagnostic body".to_vec();
|
||||
let second_payload = b"second diagnostic body that replaces the first".to_vec();
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -959,7 +959,7 @@ mod old_current_size_backfill {
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -993,7 +993,7 @@ mod old_current_size_backfill {
|
||||
// latest's size, not the incoming version's.
|
||||
let versioned = "versioned.bin";
|
||||
let first_version_opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
versioned: true,
|
||||
version_id: Some(uuid::Uuid::new_v4().to_string()),
|
||||
..Default::default()
|
||||
@@ -1005,7 +1005,7 @@ mod old_current_size_backfill {
|
||||
let expected = prelookup_expectation(&set_disks, bucket, versioned).await;
|
||||
assert_eq!(expected, OldCurrentSize::Present(111));
|
||||
let second_version_opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
versioned: true,
|
||||
version_id: Some(uuid::Uuid::new_v4().to_string()),
|
||||
..Default::default()
|
||||
@@ -1037,7 +1037,7 @@ mod old_current_size_backfill {
|
||||
let expected = prelookup_expectation(&set_disks, bucket, versioned).await;
|
||||
assert_eq!(expected, OldCurrentSize::Present(0));
|
||||
let third_version_opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
versioned: true,
|
||||
version_id: Some(uuid::Uuid::new_v4().to_string()),
|
||||
..Default::default()
|
||||
@@ -1061,7 +1061,7 @@ mod old_current_size_backfill {
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
|
||||
@@ -497,7 +497,7 @@ where
|
||||
|
||||
/// Write a (hash+data) block. Returns the number of data bytes written.
|
||||
/// Returns an error if called after a short write or if data exceeds shard_size.
|
||||
#[hotpath::measure(label = "BitrotWriter::write", impl_type = "BitrotWriter")]
|
||||
#[hotpath::measure(impl_type = "BitrotWriter", future = true)]
|
||||
pub async fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
|
||||
if buf.is_empty() {
|
||||
return Ok(0);
|
||||
|
||||
@@ -791,7 +791,7 @@ impl Erasure {
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip_all, fields(data_len=data.len()))]
|
||||
#[hotpath::measure(label = "Erasure::encode_data", impl_type = "Erasure")]
|
||||
#[hotpath::measure(impl_type = "Erasure")]
|
||||
pub(crate) fn encode_data_block(&self, data: &[u8]) -> io::Result<EncodedBlock> {
|
||||
self.encode_data_block_inner(data)
|
||||
}
|
||||
@@ -810,7 +810,7 @@ impl Erasure {
|
||||
.map(|block| block.into_shards(self.total_shard_count()))
|
||||
}
|
||||
|
||||
#[hotpath::measure(label = "Erasure::encode_data_owned", impl_type = "Erasure")]
|
||||
#[hotpath::measure(impl_type = "Erasure")]
|
||||
pub(crate) fn encode_data_owned_block(&self, data: Vec<u8>) -> io::Result<EncodedBlock> {
|
||||
self.encode_data_owned_block_inner(data)
|
||||
}
|
||||
@@ -844,7 +844,7 @@ impl Erasure {
|
||||
.map(|block| block.into_shards(self.total_shard_count()))
|
||||
}
|
||||
|
||||
#[hotpath::measure(label = "Erasure::encode_data_bytes_mut", impl_type = "Erasure")]
|
||||
#[hotpath::measure(impl_type = "Erasure")]
|
||||
pub(crate) fn encode_data_bytes_mut_block(&self, data_buffer: BytesMut, data_len: usize) -> io::Result<EncodedBlock> {
|
||||
self.encode_buffer(data_buffer, data_len)
|
||||
}
|
||||
|
||||
@@ -442,12 +442,19 @@ impl WritePlan {
|
||||
}
|
||||
|
||||
if let Some(encryption) = self.encryption {
|
||||
let etag_nonce = match encryption.mode {
|
||||
WriteEncryptionMode::Singlepart { base_nonce } | WriteEncryptionMode::MultipartLegacy { base_nonce, .. } => {
|
||||
base_nonce
|
||||
}
|
||||
WriteEncryptionMode::SinglepartObjectKey | WriteEncryptionMode::MultipartObjectKey { .. } => [0u8; 12],
|
||||
};
|
||||
let protected_reader = rustfs_rio::EtagReader::new_encrypted(reader, encryption.key_bytes, etag_nonce)?;
|
||||
reader = match encryption.mode {
|
||||
WriteEncryptionMode::SinglepartObjectKey => HashReader::from_reader(
|
||||
#[cfg(feature = "rio-v2")]
|
||||
EncryptReader::new_with_object_key(reader, encryption.key_bytes),
|
||||
EncryptReader::new_with_object_key(protected_reader, encryption.key_bytes),
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
EncryptReader::new(reader, encryption.key_bytes, [0u8; 12]),
|
||||
EncryptReader::new(protected_reader, encryption.key_bytes, [0u8; 12]),
|
||||
HashReader::SIZE_PRESERVE_LAYER,
|
||||
actual_size,
|
||||
None,
|
||||
@@ -457,12 +464,12 @@ impl WritePlan {
|
||||
WriteEncryptionMode::Singlepart { base_nonce } => {
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
let encrypt_reader = if encryption_frame_v2_enabled() {
|
||||
EncryptReader::new_v2(reader, encryption.key_bytes, base_nonce)
|
||||
EncryptReader::new_v2(protected_reader, encryption.key_bytes, base_nonce)
|
||||
} else {
|
||||
EncryptReader::new(reader, encryption.key_bytes, base_nonce)
|
||||
EncryptReader::new(protected_reader, encryption.key_bytes, base_nonce)
|
||||
};
|
||||
#[cfg(feature = "rio-v2")]
|
||||
let encrypt_reader = EncryptReader::new(reader, encryption.key_bytes, base_nonce);
|
||||
let encrypt_reader = EncryptReader::new(protected_reader, encryption.key_bytes, base_nonce);
|
||||
HashReader::from_reader(encrypt_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false)?
|
||||
}
|
||||
WriteEncryptionMode::MultipartLegacy {
|
||||
@@ -471,20 +478,25 @@ impl WritePlan {
|
||||
} => {
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
let encrypt_reader = if encryption_frame_v2_enabled() {
|
||||
EncryptReader::new_multipart_v2(reader, encryption.key_bytes, base_nonce, multipart_part_number)
|
||||
EncryptReader::new_multipart_v2(protected_reader, encryption.key_bytes, base_nonce, multipart_part_number)
|
||||
} else {
|
||||
EncryptReader::new_multipart(reader, encryption.key_bytes, base_nonce, multipart_part_number)
|
||||
EncryptReader::new_multipart(protected_reader, encryption.key_bytes, base_nonce, multipart_part_number)
|
||||
};
|
||||
#[cfg(feature = "rio-v2")]
|
||||
let encrypt_reader =
|
||||
EncryptReader::new_multipart(reader, encryption.key_bytes, base_nonce, multipart_part_number);
|
||||
EncryptReader::new_multipart(protected_reader, encryption.key_bytes, base_nonce, multipart_part_number);
|
||||
HashReader::from_reader(encrypt_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false)?
|
||||
}
|
||||
WriteEncryptionMode::MultipartObjectKey { multipart_part_number } => HashReader::from_reader(
|
||||
#[cfg(feature = "rio-v2")]
|
||||
EncryptReader::new_multipart_with_object_key(reader, encryption.key_bytes, multipart_part_number),
|
||||
EncryptReader::new_multipart_with_object_key(protected_reader, encryption.key_bytes, multipart_part_number),
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
EncryptReader::new_multipart(reader, encryption.key_bytes, [0u8; 12], multipart_part_number as usize),
|
||||
EncryptReader::new_multipart(
|
||||
protected_reader,
|
||||
encryption.key_bytes,
|
||||
[0u8; 12],
|
||||
multipart_part_number as usize,
|
||||
),
|
||||
HashReader::SIZE_PRESERVE_LAYER,
|
||||
actual_size,
|
||||
None,
|
||||
@@ -497,6 +509,7 @@ impl WritePlan {
|
||||
// `ignore_value` deliberately avoids a second hasher over compressed or
|
||||
// encrypted bytes. The inner reader still validates the plaintext request
|
||||
// checksum while this outer reader exposes the request checksum context.
|
||||
reader.encrypted_etag = encrypted;
|
||||
reader.add_non_trailing_checksum(checksum, true)?;
|
||||
reader.set_trailer(trailer);
|
||||
|
||||
@@ -513,6 +526,36 @@ mod tests {
|
||||
use std::io::Cursor;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
#[tokio::test]
|
||||
async fn write_plan_encryption_persists_protected_etags_for_every_mode() {
|
||||
let data = b"Salary 2026: 85000 EUR";
|
||||
let size = i64::try_from(data.len()).expect("test data size");
|
||||
let mut control =
|
||||
HashReader::from_stream(Cursor::new(data), size, size, None, None, false).expect("create control reader");
|
||||
control.read_to_end(&mut Vec::new()).await.expect("read control plaintext");
|
||||
let md5 = control.try_resolve_etag().expect("control MD5");
|
||||
for encryption in [
|
||||
WriteEncryption::singlepart([1; 32], [1; 12]),
|
||||
WriteEncryption::singlepart_object_key([1; 32]),
|
||||
WriteEncryption::multipart([1; 32], [1; 12], 1),
|
||||
WriteEncryption::multipart_object_key([1; 32], 1),
|
||||
] {
|
||||
let plaintext = HashReader::from_stream(Cursor::new(data), size, size, Some(md5.clone()), None, false)
|
||||
.expect("create checked reader");
|
||||
let mut reader = WritePlan::new()
|
||||
.with_encryption(encryption)
|
||||
.apply(plaintext, size)
|
||||
.expect("apply encryption");
|
||||
let mut ciphertext = Vec::new();
|
||||
reader.read_to_end(&mut ciphertext).await.expect("stream ciphertext");
|
||||
assert_ne!(ciphertext, data);
|
||||
let tag = reader.try_resolve_etag().expect("resolve persisted encrypted ETag");
|
||||
assert_ne!(tag, md5, "stored ETag must not reveal the plaintext MD5");
|
||||
assert_eq!(tag.len(), 32, "multipart ETags require a 16-byte hexadecimal part tag");
|
||||
assert_eq!(reader.try_resolve_etag().as_ref(), Some(&tag));
|
||||
}
|
||||
}
|
||||
|
||||
async fn assert_non_trailing_checksum_survives(plan: WritePlan) {
|
||||
let plaintext = b"checksum-context-through-write-plan".repeat(256);
|
||||
let actual_size = plaintext.len() as i64;
|
||||
|
||||
@@ -55,6 +55,7 @@ mod set_disk;
|
||||
mod storage_api_contracts;
|
||||
mod store;
|
||||
|
||||
pub use disk::with_background_disk_io;
|
||||
pub use store::PoolMetaWriteGateStatus;
|
||||
|
||||
// pub mod checksum;
|
||||
|
||||
@@ -104,6 +104,7 @@ mod hook_slot;
|
||||
mod object_mutation_hook;
|
||||
mod readers;
|
||||
mod types;
|
||||
pub(crate) mod write_commit_context;
|
||||
|
||||
#[cfg(test)]
|
||||
mod persisted_metadata_keys_tests;
|
||||
@@ -125,3 +126,4 @@ pub(crate) use object_mutation_hook::notify_object_mutation;
|
||||
pub use object_mutation_hook::{ObjectMutationHook, register_object_mutation_hook, unregister_object_mutation_hook};
|
||||
pub use readers::*;
|
||||
pub use types::*;
|
||||
pub use write_commit_context::{WriteCommitGuard, WriteLockContext};
|
||||
|
||||
@@ -935,6 +935,9 @@ pub struct ObjectOptions {
|
||||
/// Persisted bucket incarnation observed before authorization.
|
||||
pub expected_bucket_incarnation_id: Option<Uuid>,
|
||||
pub no_lock: bool,
|
||||
/// A genuine target write guard; loss fences alone do not authorize a borrowed write.
|
||||
#[doc(hidden)]
|
||||
pub write_lock_context: Option<WriteLockContext>,
|
||||
/// Internal read-only inspection must not enqueue metadata or payload repairs.
|
||||
pub suppress_read_repair: bool,
|
||||
/// Control-plane writers that immediately read or CAS the same namespace
|
||||
@@ -1234,7 +1237,8 @@ impl ObjectOptions {
|
||||
self.version_id.is_some() || !self.versioned || self.version_suspended
|
||||
}
|
||||
|
||||
pub(crate) fn add_namespace_lock_lost_signal(&mut self, signal: Arc<rustfs_lock::distributed_lock::LockLostSignal>) {
|
||||
#[doc(hidden)]
|
||||
pub fn add_namespace_lock_lost_signal(&mut self, signal: Arc<rustfs_lock::distributed_lock::LockLostSignal>) {
|
||||
#[cfg(test)]
|
||||
let test_fence = NAMESPACE_LOCK_SIGNAL_TEST_FENCES
|
||||
.get_or_init(|| std::sync::Mutex::new(Vec::new()))
|
||||
|
||||
@@ -0,0 +1,192 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::{ObjectOptions, WriteCompletion};
|
||||
use crate::error::{Error, Result};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
/// A real write guard shared with an inner commit, never a lock-loss signal alone.
|
||||
///
|
||||
/// Callers cannot fabricate a guard without acquiring its namespace lock.
|
||||
///
|
||||
/// ```compile_fail
|
||||
/// use rustfs_ecstore::api::object::WriteCommitGuard;
|
||||
/// use std::sync::Arc;
|
||||
///
|
||||
/// let guard = WriteCommitGuard {
|
||||
/// bucket: "bucket".into(),
|
||||
/// object: "object".into(),
|
||||
/// guards: Arc::new(Vec::new()),
|
||||
/// };
|
||||
/// ```
|
||||
#[doc(hidden)]
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct WriteCommitGuard {
|
||||
bucket: Arc<str>,
|
||||
object: Arc<str>,
|
||||
guards: Arc<Vec<Arc<rustfs_lock::NamespaceLockGuard>>>,
|
||||
}
|
||||
|
||||
impl WriteCommitGuard {
|
||||
/// Acquire the key recorded by the namespace wrapper and retain its owner.
|
||||
pub async fn acquire(
|
||||
lock: &rustfs_lock::NamespaceLockWrapper,
|
||||
timeout: Duration,
|
||||
) -> std::result::Result<Self, rustfs_lock::LockError> {
|
||||
let guard = Arc::new(lock.get_write_lock(timeout).await?);
|
||||
let key = lock.resource();
|
||||
Ok(Self::from_guard(guard, &key.bucket, &key.object))
|
||||
}
|
||||
|
||||
pub(crate) fn from_guard(guard: Arc<rustfs_lock::NamespaceLockGuard>, bucket: &str, object: &str) -> Self {
|
||||
Self {
|
||||
bucket: bucket.into(),
|
||||
object: object.into(),
|
||||
guards: Arc::new(vec![guard]),
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the loss signal of the namespace owner retained by acquisition.
|
||||
#[doc(hidden)]
|
||||
pub fn lock_lost_signal(&self) -> Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>> {
|
||||
self.guards.first().and_then(|guard| guard.lock_lost_signal())
|
||||
}
|
||||
|
||||
pub fn is_lock_lost(&self) -> bool {
|
||||
self.guards.iter().any(|guard| guard.is_lock_lost() || guard.is_released())
|
||||
}
|
||||
|
||||
fn matches(&self, bucket: &str, object: &str) -> bool {
|
||||
self.bucket.as_ref() == bucket && self.object.as_ref() == object
|
||||
}
|
||||
}
|
||||
|
||||
/// Opaque boundary capability; only real acquisitions or offline startup construct it.
|
||||
///
|
||||
/// ```compile_fail
|
||||
/// use rustfs_ecstore::object_api::WriteLockContext;
|
||||
///
|
||||
/// let context = WriteLockContext {};
|
||||
/// ```
|
||||
#[doc(hidden)]
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct WriteLockContext {
|
||||
mode: WriteLockMode,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
enum WriteLockMode {
|
||||
Borrowed(Vec<WriteCommitGuard>),
|
||||
OfflineRecovery,
|
||||
}
|
||||
|
||||
pub(crate) enum WriteCommitContext {
|
||||
Owned,
|
||||
Borrowed(Vec<WriteCommitGuard>),
|
||||
Publication,
|
||||
OfflineRecovery,
|
||||
#[cfg(test)]
|
||||
UnlockedFixture,
|
||||
}
|
||||
|
||||
impl WriteCommitContext {
|
||||
pub(crate) fn from_options(opts: &ObjectOptions, bucket: &str, object: &str, publication: bool) -> Result<Self> {
|
||||
if publication {
|
||||
if let Some(WriteLockContext {
|
||||
mode: WriteLockMode::Borrowed(guards),
|
||||
}) = opts.write_lock_context.as_ref()
|
||||
&& !guards.iter().any(|guard| guard.matches(bucket, object))
|
||||
{
|
||||
return Err(Error::InvalidArgument(
|
||||
bucket.to_owned(),
|
||||
object.to_owned(),
|
||||
"publication carries a different borrowed namespace".to_owned(),
|
||||
));
|
||||
}
|
||||
return Ok(Self::Publication);
|
||||
}
|
||||
match (opts.no_lock, opts.write_lock_context.as_ref().map(|context| &context.mode)) {
|
||||
(false, None | Some(WriteLockMode::Borrowed(_))) => Ok(Self::Owned),
|
||||
(true, Some(WriteLockMode::Borrowed(guards))) if guards.iter().any(|guard| guard.matches(bucket, object)) => {
|
||||
Ok(Self::Borrowed(guards.clone()))
|
||||
}
|
||||
(true, Some(WriteLockMode::OfflineRecovery)) => Ok(Self::OfflineRecovery),
|
||||
#[cfg(test)]
|
||||
(true, None) if !opts.metadata_cache_safe => Ok(Self::UnlockedFixture),
|
||||
_ => Err(Error::InvalidArgument(
|
||||
bucket.to_owned(),
|
||||
object.to_owned(),
|
||||
"write lock context does not prove the target namespace".to_owned(),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn acquires_namespace(&self) -> bool {
|
||||
matches!(self, Self::Owned)
|
||||
}
|
||||
|
||||
pub(crate) fn has_borrowed_owner(&self) -> bool {
|
||||
matches!(self, Self::Borrowed(_))
|
||||
}
|
||||
|
||||
pub(crate) fn is_lock_lost(&self) -> bool {
|
||||
matches!(self, Self::Borrowed(guards) if guards.iter().any(WriteCommitGuard::is_lock_lost))
|
||||
}
|
||||
|
||||
pub(crate) fn allows_early_ack(&self, completion: WriteCompletion) -> bool {
|
||||
completion == WriteCompletion::Quorum && matches!(self, Self::Owned | Self::Borrowed(_) | Self::Publication)
|
||||
}
|
||||
}
|
||||
|
||||
impl ObjectOptions {
|
||||
pub fn add_write_commit_guard(&mut self, guard: &WriteCommitGuard) {
|
||||
for owner in guard.guards.iter() {
|
||||
self.add_namespace_lock_guard(owner);
|
||||
}
|
||||
let mut guards = match self.write_lock_context.as_ref() {
|
||||
Some(WriteLockContext {
|
||||
mode: WriteLockMode::Borrowed(existing),
|
||||
}) => existing.clone(),
|
||||
_ => Vec::new(),
|
||||
};
|
||||
if let Some(existing) = guards.iter_mut().find(|held| held.matches(&guard.bucket, &guard.object)) {
|
||||
let owners = Arc::make_mut(&mut existing.guards);
|
||||
for owner in guard.guards.iter() {
|
||||
if !owners.iter().any(|held| Arc::ptr_eq(held, owner)) {
|
||||
owners.push(Arc::clone(owner));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
guards.push(guard.clone());
|
||||
}
|
||||
self.no_lock = true;
|
||||
self.write_lock_context = Some(WriteLockContext {
|
||||
mode: WriteLockMode::Borrowed(guards),
|
||||
});
|
||||
}
|
||||
|
||||
pub(crate) fn add_owned_write_lock(&mut self, guard: Arc<rustfs_lock::NamespaceLockGuard>, bucket: &str, object: &str) {
|
||||
self.add_write_commit_guard(&WriteCommitGuard::from_guard(guard, bucket, object));
|
||||
}
|
||||
|
||||
/// Only startup migration may publish without a running namespace owner.
|
||||
pub(crate) fn use_offline_recovery_write(&mut self) {
|
||||
self.no_lock = true;
|
||||
self.write_lock_context = Some(WriteLockContext {
|
||||
mode: WriteLockMode::OfflineRecovery,
|
||||
});
|
||||
self.write_completion = WriteCompletion::TailDrained;
|
||||
}
|
||||
}
|
||||
@@ -307,6 +307,13 @@ pub fn shutdown_background_services() {
|
||||
}
|
||||
}
|
||||
|
||||
/// Wait up to `timeout` for object mutations that outlived their callers
|
||||
/// (see `InstanceContext::wait_for_detached_mutations`). Returns `false` when
|
||||
/// some are still running at the deadline.
|
||||
pub async fn wait_for_detached_mutations(timeout: std::time::Duration) -> bool {
|
||||
current_ctx().wait_for_detached_mutations(timeout).await
|
||||
}
|
||||
|
||||
/// Set the global lock client (first LocalClient created)
|
||||
///
|
||||
/// # Arguments
|
||||
|
||||
@@ -59,6 +59,8 @@ use std::sync::{
|
||||
use tokio::sync::{Mutex, Notify, OnceCell, OwnedRwLockReadGuard, RwLock};
|
||||
use tokio::time::Instant;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tokio_util::task::TaskTracker;
|
||||
use tracing::Instrument as _;
|
||||
use uuid::Uuid;
|
||||
|
||||
const SCANNER_PUBLICATION_STATE_UNKNOWN: u8 = 0;
|
||||
@@ -72,10 +74,19 @@ pub(crate) const SCANNER_PUBLICATION_LEASE_MAX_ENTRIES: usize = 256;
|
||||
/// have started movement after the lease was abandoned.
|
||||
pub(crate) const SCANNER_PUBLICATION_LEASE_TTL: std::time::Duration = std::time::Duration::from_secs(60);
|
||||
|
||||
/// Observational leases hold data-movement fencing but may coexist with
|
||||
/// ordinary namespace commits; authoritative leases require a stable namespace.
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub(crate) enum ScannerPublicationLeasePurpose {
|
||||
Authoritative,
|
||||
Observational,
|
||||
}
|
||||
|
||||
pub(crate) struct ScannerPublicationLeaseEntry {
|
||||
pub(crate) expires_at: Instant,
|
||||
pub(crate) movement_generation: u64,
|
||||
pub(crate) namespace_generation: u64,
|
||||
pub(crate) namespace_generation: Option<u64>,
|
||||
pub(crate) purpose: ScannerPublicationLeasePurpose,
|
||||
pub(crate) _operation_guard: OwnedRwLockReadGuard<()>,
|
||||
}
|
||||
|
||||
@@ -234,6 +245,10 @@ pub struct InstanceContext {
|
||||
suppress_tier_delete_journal_recovery: bool,
|
||||
transition_transaction_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
|
||||
tier_delete_journal_recovery_wakeup: tokio::sync::Notify,
|
||||
/// Object mutations that run on their own task so that dropping the
|
||||
/// caller (for example, an S3 client disconnect) cannot stop them between
|
||||
/// their on-disk steps. Shutdown waits for them after the HTTP drain.
|
||||
detached_mutations: TaskTracker,
|
||||
}
|
||||
|
||||
impl InstanceContext {
|
||||
@@ -284,6 +299,7 @@ impl InstanceContext {
|
||||
suppress_tier_delete_journal_recovery: false,
|
||||
transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()),
|
||||
tier_delete_journal_recovery_wakeup: tokio::sync::Notify::new(),
|
||||
detached_mutations: TaskTracker::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -297,6 +313,31 @@ impl InstanceContext {
|
||||
self.lock_manager.clone()
|
||||
}
|
||||
|
||||
/// Run a multi-step mutation on its own task and wait for its output.
|
||||
///
|
||||
/// Dropping the returned future leaves the task running to completion, so
|
||||
/// the mutation, together with the locks and guards it owns, never stops
|
||||
/// between two of its on-disk steps because its caller went away.
|
||||
pub(crate) async fn run_detached_mutation<F>(&self, mutation: F) -> Result<F::Output, tokio::task::JoinError>
|
||||
where
|
||||
F: std::future::Future + Send + 'static,
|
||||
F::Output: Send + 'static,
|
||||
{
|
||||
self.detached_mutations.spawn(mutation.in_current_span()).await
|
||||
}
|
||||
|
||||
/// Wait up to `timeout` for every detached mutation to finish. Returns
|
||||
/// `false` when some are still running at the deadline.
|
||||
pub async fn wait_for_detached_mutations(&self, timeout: std::time::Duration) -> bool {
|
||||
self.detached_mutations.close();
|
||||
tokio::time::timeout(timeout, self.detached_mutations.wait()).await.is_ok()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn detached_mutation_count(&self) -> usize {
|
||||
self.detached_mutations.len()
|
||||
}
|
||||
|
||||
pub(crate) fn data_movement_operation_gate(&self) -> Arc<RwLock<()>> {
|
||||
Arc::clone(&self.data_movement_operation_gate)
|
||||
}
|
||||
@@ -306,7 +347,8 @@ impl InstanceContext {
|
||||
token: Uuid,
|
||||
expires_at: Instant,
|
||||
movement_generation: u64,
|
||||
namespace_generation: u64,
|
||||
namespace_generation: Option<u64>,
|
||||
purpose: ScannerPublicationLeasePurpose,
|
||||
operation_guard: OwnedRwLockReadGuard<()>,
|
||||
) -> bool {
|
||||
let mut leases = self.scanner_publication_leases.lock().await;
|
||||
@@ -319,6 +361,7 @@ impl InstanceContext {
|
||||
expires_at,
|
||||
movement_generation,
|
||||
namespace_generation,
|
||||
purpose,
|
||||
_operation_guard: operation_guard,
|
||||
},
|
||||
);
|
||||
@@ -329,21 +372,24 @@ impl InstanceContext {
|
||||
self.scanner_publication_leases.lock().await.remove(&token).is_some()
|
||||
}
|
||||
|
||||
/// Return both generations from the same live lease while the caller holds
|
||||
/// the movement read guard. Namespace commits do not take that guard, so
|
||||
/// the caller must compare the saved namespace generation after this await.
|
||||
/// Return the generations and purpose from the same live lease while the
|
||||
/// caller holds the movement read guard. Namespace commits do not take that
|
||||
/// guard, so strict leases compare their saved generation after this await.
|
||||
/// The process-owned table rejects tokens from a prior instance or expiry.
|
||||
pub(crate) async fn scanner_publication_lease_generations(&self, token: Uuid) -> Option<(u64, u64)> {
|
||||
pub(crate) async fn scanner_publication_lease_generations(
|
||||
&self,
|
||||
token: Uuid,
|
||||
) -> Option<(u64, Option<u64>, ScannerPublicationLeasePurpose)> {
|
||||
let mut leases = self.scanner_publication_leases.lock().await;
|
||||
let now = Instant::now();
|
||||
let (expires_at, movement_generation, namespace_generation) = leases
|
||||
let (expires_at, movement_generation, namespace_generation, purpose) = leases
|
||||
.get(&token)
|
||||
.map(|entry| (entry.expires_at, entry.movement_generation, entry.namespace_generation))?;
|
||||
.map(|entry| (entry.expires_at, entry.movement_generation, entry.namespace_generation, entry.purpose))?;
|
||||
if expires_at <= now {
|
||||
leases.remove(&token);
|
||||
return None;
|
||||
}
|
||||
Some((movement_generation, namespace_generation))
|
||||
Some((movement_generation, namespace_generation, purpose))
|
||||
}
|
||||
|
||||
pub(crate) async fn expire_scanner_publication_lease(&self, token: Uuid, expires_at: Instant) {
|
||||
@@ -859,12 +905,22 @@ mod tests {
|
||||
let gate = ctx.data_movement_operation_gate();
|
||||
let permit = gate.clone().read_owned().await;
|
||||
assert!(
|
||||
ctx.install_scanner_publication_lease(token, Instant::now() + SCANNER_PUBLICATION_LEASE_TTL, 7, 11, permit)
|
||||
.await
|
||||
ctx.install_scanner_publication_lease(
|
||||
token,
|
||||
Instant::now() + SCANNER_PUBLICATION_LEASE_TTL,
|
||||
7,
|
||||
Some(11),
|
||||
ScannerPublicationLeasePurpose::Authoritative,
|
||||
permit,
|
||||
)
|
||||
.await
|
||||
);
|
||||
drop(ctx.begin_namespace_commit());
|
||||
assert_eq!(ctx.namespace_commit_generation(), 2);
|
||||
assert_eq!(ctx.scanner_publication_lease_generations(token).await, Some((7, 11)));
|
||||
assert_eq!(
|
||||
ctx.scanner_publication_lease_generations(token).await,
|
||||
Some((7, Some(11), ScannerPublicationLeasePurpose::Authoritative))
|
||||
);
|
||||
assert!(gate.clone().try_write_owned().is_err(), "lookup must retain the stored permit");
|
||||
tokio::time::advance(SCANNER_PUBLICATION_LEASE_TTL).await;
|
||||
assert_eq!(ctx.scanner_publication_lease_generations(token).await, None);
|
||||
@@ -896,6 +952,33 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
// A detached mutation keeps running after its caller is dropped, and
|
||||
// shutdown waits for it instead of tearing it down mid-way.
|
||||
#[tokio::test]
|
||||
async fn detached_mutation_outlives_dropped_caller_and_is_awaited() {
|
||||
let ctx = InstanceContext::new();
|
||||
let finished = Arc::new(AtomicBool::new(false));
|
||||
let (release_tx, release_rx) = tokio::sync::oneshot::channel::<()>();
|
||||
let mut caller = Box::pin(ctx.run_detached_mutation({
|
||||
let finished = Arc::clone(&finished);
|
||||
async move {
|
||||
let _ = release_rx.await;
|
||||
finished.store(true, Ordering::Release);
|
||||
}
|
||||
}));
|
||||
assert!(futures::poll!(caller.as_mut()).is_pending());
|
||||
drop(caller);
|
||||
assert_eq!(ctx.detached_mutation_count(), 1, "dropping the caller must not cancel the mutation");
|
||||
|
||||
assert!(
|
||||
!ctx.wait_for_detached_mutations(std::time::Duration::from_millis(50)).await,
|
||||
"the wait must report a mutation that is still running"
|
||||
);
|
||||
release_tx.send(()).expect("the detached mutation should still be waiting");
|
||||
assert!(ctx.wait_for_detached_mutations(std::time::Duration::from_secs(5)).await);
|
||||
assert!(finished.load(Ordering::Acquire));
|
||||
}
|
||||
|
||||
// A fresh context (before any update) reflects the initial all-false state.
|
||||
#[tokio::test]
|
||||
async fn fresh_context_is_all_false() {
|
||||
|
||||
@@ -2888,8 +2888,23 @@ impl NotificationSys {
|
||||
/// leases are released before returning so a partial acquisition cannot
|
||||
/// pin movement on one peer.
|
||||
pub async fn acquire_scanner_publication_leases(
|
||||
&self,
|
||||
targets: Vec<(String, String, u64)>,
|
||||
) -> Result<Vec<ScannerPublicationLeaseGrant>> {
|
||||
self.acquire_scanner_publication_leases_with_purpose(targets, false).await
|
||||
}
|
||||
|
||||
pub async fn acquire_observational_scanner_publication_leases(
|
||||
&self,
|
||||
targets: Vec<(String, String, u64)>,
|
||||
) -> Result<Vec<ScannerPublicationLeaseGrant>> {
|
||||
self.acquire_scanner_publication_leases_with_purpose(targets, true).await
|
||||
}
|
||||
|
||||
async fn acquire_scanner_publication_leases_with_purpose(
|
||||
&self,
|
||||
mut targets: Vec<(String, String, u64)>,
|
||||
observational_only: bool,
|
||||
) -> Result<Vec<ScannerPublicationLeaseGrant>> {
|
||||
targets.sort_by(|left, right| left.0.cmp(&right.0));
|
||||
for pair in targets.windows(2) {
|
||||
@@ -2910,7 +2925,10 @@ impl NotificationSys {
|
||||
let _ = self.release_scanner_publication_leases(grants).await;
|
||||
return Err(Error::other(format!("scanner publication lease peer {host} is unavailable")));
|
||||
};
|
||||
match client.acquire_scanner_publication_lease(&session_id, generation).await {
|
||||
match client
|
||||
.acquire_scanner_publication_lease(&session_id, generation, observational_only)
|
||||
.await
|
||||
{
|
||||
Ok(lease) => grants.push(ScannerPublicationLeaseGrant { host, lease }),
|
||||
Err(err) => {
|
||||
let _ = self.release_scanner_publication_leases(grants).await;
|
||||
@@ -4949,6 +4967,7 @@ mod tests {
|
||||
lease: ScannerPublicationLease {
|
||||
token: Uuid::new_v4(),
|
||||
movement_generation: 3,
|
||||
observational_only: false,
|
||||
owner_id: Uuid::new_v4().to_string(),
|
||||
session_id: "session-a".to_string(),
|
||||
expires_at: Instant::now() + Duration::from_secs(30),
|
||||
|
||||
@@ -521,11 +521,12 @@ impl ECStore {
|
||||
.map_err(|err| rebalance_meta_lock_error(err, "write"))
|
||||
})
|
||||
.await?;
|
||||
let guard = Arc::new(guard);
|
||||
let mut opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
};
|
||||
opts.add_namespace_lock_guard(&guard);
|
||||
opts.add_owned_write_lock(Arc::clone(&guard), crate::disk::RUSTFS_META_BUCKET, REBAL_META_NAME);
|
||||
|
||||
merge_and_save_rebalance_meta_no_lock(pool, local_snapshot, stage, opts, None, expected_id).await?;
|
||||
if guard.is_lock_lost() {
|
||||
|
||||
@@ -298,6 +298,11 @@ pub fn build_transition_put_options(storage_class: String, mut metadata: HashMap
|
||||
}
|
||||
}
|
||||
|
||||
// x-minio-internal-* is reserved by MinIO and its derivatives; they reject
|
||||
// any request carrying it with 400 InvalidArgument. The transition ids above
|
||||
// are the only internal keys the remote needs, and they travel as x-amz-meta-*.
|
||||
metadata.retain(|key, _| !rustfs_utils::http::is_internal_key(key));
|
||||
|
||||
opts.user_metadata = metadata;
|
||||
opts
|
||||
}
|
||||
@@ -2244,6 +2249,38 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_transition_put_options_drops_reserved_internal_headers() {
|
||||
let transaction_suffix = rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID;
|
||||
let mut metadata = HashMap::new();
|
||||
for suffix in [
|
||||
rustfs_utils::http::SUFFIX_INLINE_DATA,
|
||||
rustfs_utils::http::SUFFIX_ACTUAL_SIZE,
|
||||
rustfs_utils::http::SUFFIX_PART_CHECKSUMS,
|
||||
transaction_suffix,
|
||||
] {
|
||||
rustfs_utils::http::metadata_compat::insert_str(&mut metadata, suffix, "x".to_string());
|
||||
}
|
||||
metadata.insert("X-Minio-Internal-Compression".to_string(), "klauspost/compress/s2".to_string());
|
||||
metadata.insert("x-amz-meta-foo".to_string(), "bar".to_string());
|
||||
metadata.insert("name".to_string(), "object".to_string());
|
||||
|
||||
let opts = build_transition_put_options("COLD".to_string(), metadata);
|
||||
|
||||
let leaked: Vec<_> = opts
|
||||
.user_metadata
|
||||
.keys()
|
||||
.filter(|key| rustfs_utils::http::is_internal_key(key))
|
||||
.collect();
|
||||
assert!(leaked.is_empty(), "reserved internal keys must not reach the remote: {leaked:?}");
|
||||
assert_eq!(opts.user_metadata.get("x-amz-meta-foo").map(String::as_str), Some("bar"));
|
||||
assert_eq!(opts.user_metadata.get("name").map(String::as_str), Some("object"));
|
||||
assert!(opts.user_metadata.contains_key(&format!(
|
||||
"x-amz-meta-{}",
|
||||
rustfs_utils::http::metadata_compat::internal_key_rustfs(transaction_suffix)
|
||||
)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_transition_put_options_requests_no_checksum_and_content_md5() {
|
||||
// Regression for rustfs/rustfs#4811: transition uploads must leave the
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -29,6 +29,27 @@ use crate::disk::error_reduce::OBJECT_OP_IGNORED_ERRS;
|
||||
use crate::set_disk::file_info_is_valid_for_metadata;
|
||||
use rustfs_filemeta::FileInfo;
|
||||
|
||||
/// One disk's observation. Pending is neither a successful vote nor an offline
|
||||
/// disk: the scheduler may not have issued this slot or may still be waiting.
|
||||
// Option<Result<...>> keeps FileInfo inline without a per-response Box.
|
||||
// None is pending; Some(Ok(_)) is success; Some(Err(_)) retains a disk error.
|
||||
pub(in crate::set_disk) type MetadataDiskResult = Option<crate::disk::error::Result<FileInfo>>;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(in crate::set_disk) struct MetadataDiskObservation {
|
||||
pub(in crate::set_disk) disk_index: usize,
|
||||
pub(in crate::set_disk) result: MetadataDiskResult,
|
||||
}
|
||||
|
||||
impl MetadataDiskObservation {
|
||||
pub(in crate::set_disk) fn file_info(&self) -> Option<&FileInfo> {
|
||||
match &self.result {
|
||||
Some(Ok(metadata)) => Some(metadata),
|
||||
None | Some(Err(_)) => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub(in crate::set_disk) struct MetadataEarlyStopDecision {
|
||||
pub(in crate::set_disk) reason: &'static str,
|
||||
@@ -59,6 +80,14 @@ pub(in crate::set_disk) struct MetadataQuorumAccumulator {
|
||||
}
|
||||
|
||||
impl MetadataQuorumAccumulator {
|
||||
pub(in crate::set_disk) fn observe(&mut self, observation: &MetadataDiskObservation) {
|
||||
match &observation.result {
|
||||
None => {}
|
||||
Some(Ok(metadata)) => self.observe_file_info_at(observation.disk_index, metadata),
|
||||
Some(Err(error)) => self.observe_error(error),
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn new(total_disks: usize, default_parity_count: usize, allow_early_stop: bool) -> Self {
|
||||
Self {
|
||||
total_disks,
|
||||
@@ -86,6 +115,7 @@ impl MetadataQuorumAccumulator {
|
||||
self
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(in crate::set_disk) fn observe_file_info(&mut self, file_info: &FileInfo) {
|
||||
self.observe_file_info_with_index(None, file_info);
|
||||
}
|
||||
@@ -383,3 +413,282 @@ pub(in crate::set_disk) fn metadata_early_stop_candidate_matches(left: &FileInfo
|
||||
pub(in crate::set_disk) fn is_metadata_fanout_ignored_error(err: &DiskError) -> bool {
|
||||
OBJECT_OP_IGNORED_ERRS.iter().any(|ignored| ignored == err)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::set_disk::SetDisks;
|
||||
use time::OffsetDateTime;
|
||||
use uuid::Uuid;
|
||||
|
||||
fn payload(disk_index: usize) -> FileInfo {
|
||||
let mut metadata = FileInfo::new("object", 2, 2);
|
||||
metadata.volume = "bucket".to_string();
|
||||
metadata.name = "object".to_string();
|
||||
metadata.size = 1;
|
||||
metadata.mod_time = Some(OffsetDateTime::UNIX_EPOCH);
|
||||
metadata.data_dir = Some(Uuid::from_u128(1));
|
||||
metadata.erasure.index = metadata.erasure.distribution[disk_index];
|
||||
metadata.metadata.insert("etag".to_string(), "etag".to_string());
|
||||
metadata.add_object_part(1, "etag".to_string(), 1, None, 1, None, None);
|
||||
assert!(file_info_is_valid_for_metadata(&metadata), "the observation corpus needs a valid payload");
|
||||
metadata
|
||||
}
|
||||
|
||||
fn observation(disk_index: usize, state: usize) -> MetadataDiskObservation {
|
||||
MetadataDiskObservation {
|
||||
disk_index,
|
||||
result: match state {
|
||||
0 => Some(Ok(payload(disk_index))),
|
||||
1 => Some(Err(DiskError::FileNotFound)),
|
||||
2 => Some(Err(DiskError::FileCorrupt)),
|
||||
3 => Some(Err(DiskError::DiskNotFound)),
|
||||
4 => None,
|
||||
_ => panic!("unexpected test state"),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_same_reduction(typed: &MetadataQuorumAccumulator, legacy: &MetadataQuorumAccumulator) {
|
||||
assert_eq!(typed.early_stop_decision(), legacy.early_stop_decision());
|
||||
assert_eq!(typed.version_early_stop_decision(), legacy.version_early_stop_decision());
|
||||
assert_eq!(typed.final_miss_reason(), legacy.final_miss_reason());
|
||||
assert_eq!(typed.valid_responses, legacy.valid_responses);
|
||||
assert_eq!(typed.not_found_responses, legacy.not_found_responses);
|
||||
assert_eq!(typed.version_not_found_responses, legacy.version_not_found_responses);
|
||||
assert_eq!(typed.ignored_errors, legacy.ignored_errors);
|
||||
assert_eq!(typed.hard_errors, legacy.hard_errors);
|
||||
assert_eq!(typed.candidate_votes, legacy.candidate_votes);
|
||||
assert_eq!(typed.candidate_shard_mask, legacy.candidate_shard_mask);
|
||||
assert_eq!(typed.matching_version_votes, legacy.matching_version_votes);
|
||||
assert_eq!(typed.delete_marker_votes, legacy.delete_marker_votes);
|
||||
assert_eq!(typed.conflicting_metadata, legacy.conflicting_metadata);
|
||||
assert_eq!(
|
||||
typed.candidate.as_ref().map(SetDisks::file_info_quorum_hash),
|
||||
legacy.candidate.as_ref().map(SetDisks::file_info_quorum_hash)
|
||||
);
|
||||
}
|
||||
|
||||
fn observe_legacy(accumulator: &mut MetadataQuorumAccumulator, observation: &MetadataDiskObservation) {
|
||||
match &observation.result {
|
||||
None => {}
|
||||
Some(Ok(metadata)) => accumulator.observe_file_info_at(observation.disk_index, metadata),
|
||||
Some(Err(error)) => accumulator.observe_error(error),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_observation_all_four_slot_states_and_arrival_orders_preserve_reduction() {
|
||||
let permutations = (0..4)
|
||||
.flat_map(|a| (0..4).filter(move |b| *b != a).map(move |b| (a, b)))
|
||||
.flat_map(|(a, b)| (0..4).filter(move |c| *c != a && *c != b).map(move |c| (a, b, c)))
|
||||
.map(|(a, b, c)| [a, b, c, 6 - a - b - c])
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(permutations.len(), 24);
|
||||
// 5^4 states and all 4! arrivals, both with early-stop enabled and disabled.
|
||||
for encoded in 0..625usize {
|
||||
let states = [encoded % 5, encoded / 5 % 5, encoded / 25 % 5, encoded / 125 % 5];
|
||||
let inputs = std::array::from_fn::<_, 4, _>(|index| observation(index, states[index]));
|
||||
for enabled in [false, true] {
|
||||
for order in &permutations {
|
||||
let mut typed = MetadataQuorumAccumulator::new(4, 2, enabled);
|
||||
let mut legacy = MetadataQuorumAccumulator::new(4, 2, enabled);
|
||||
for &index in order {
|
||||
typed.observe(&inputs[index]);
|
||||
observe_legacy(&mut legacy, &inputs[index]);
|
||||
assert_same_reduction(&typed, &legacy);
|
||||
}
|
||||
assert_eq!(typed.valid_responses, states.iter().filter(|&&state| state == 0).count());
|
||||
assert_eq!(typed.not_found_responses, states.iter().filter(|&&state| state == 1).count());
|
||||
assert_eq!(typed.hard_errors, states.iter().filter(|&&state| state == 2).count());
|
||||
assert_eq!(typed.ignored_errors, states.iter().filter(|&&state| state == 3).count());
|
||||
let expected_early_stop =
|
||||
enabled && typed.valid_responses >= 3 && !states.iter().any(|&state| state == 1 || state == 2);
|
||||
assert_eq!(
|
||||
typed.early_stop_decision().is_some(),
|
||||
expected_early_stop,
|
||||
"states={states:?}, order={order:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_observation_mixed_current_identity_arrivals_preserve_selection() {
|
||||
let permutations = (0..4)
|
||||
.flat_map(|a| (0..4).filter(move |b| *b != a).map(move |b| (a, b)))
|
||||
.flat_map(|(a, b)| (0..4).filter(move |c| *c != a && *c != b).map(move |c| (a, b, c)))
|
||||
.map(|(a, b, c)| [a, b, c, 6 - a - b - c])
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(permutations.len(), 24);
|
||||
|
||||
for (case, newer_version) in [("newer-current-version", true), ("same-time-different-directory", false)] {
|
||||
let metadata = std::array::from_fn::<_, 4, _>(|index| {
|
||||
let mut file_info = payload(index);
|
||||
file_info.version_id = Some(Uuid::from_u128(if index > 0 && newer_version { 11 } else { 10 }));
|
||||
file_info.is_latest = true;
|
||||
if index > 0 {
|
||||
file_info.data_dir = Some(Uuid::from_u128(2));
|
||||
if newer_version {
|
||||
file_info.mod_time = Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1));
|
||||
}
|
||||
}
|
||||
assert!(file_info_is_valid_for_metadata(&file_info), "case={case}, disk={index}");
|
||||
file_info
|
||||
});
|
||||
let expected_current = &metadata[1];
|
||||
let expected_hash = SetDisks::file_info_quorum_hash(expected_current);
|
||||
assert_ne!(SetDisks::file_info_quorum_hash(&metadata[0]), expected_hash);
|
||||
|
||||
for enabled in [false, true] {
|
||||
for order in &permutations {
|
||||
let mut slots = std::array::from_fn::<_, 4, _>(|index| observation(index, 4));
|
||||
let mut typed = MetadataQuorumAccumulator::new(4, 2, enabled);
|
||||
let mut legacy = MetadataQuorumAccumulator::new(4, 2, enabled);
|
||||
for slot in &slots {
|
||||
typed.observe(slot);
|
||||
observe_legacy(&mut legacy, slot);
|
||||
}
|
||||
assert_same_reduction(&typed, &legacy);
|
||||
assert_eq!(typed.valid_responses, 0, "pending slots cannot supply votes");
|
||||
|
||||
let mut old_seen = false;
|
||||
let mut current_votes = 0;
|
||||
for (prefix, &index) in order.iter().enumerate() {
|
||||
slots[index].result = Some(Ok(metadata[index].clone()));
|
||||
typed.observe(&slots[index]);
|
||||
observe_legacy(&mut legacy, &slots[index]);
|
||||
assert_same_reduction(&typed, &legacy);
|
||||
if index == 0 {
|
||||
old_seen = true;
|
||||
} else {
|
||||
current_votes += 1;
|
||||
}
|
||||
|
||||
// A+B+B must wait for pending B; three B responses may
|
||||
// finish early only before the conflicting A arrives.
|
||||
let expected_decision = if enabled && !old_seen && current_votes == 3 {
|
||||
Some(MetadataEarlyStopDecision {
|
||||
reason: GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM,
|
||||
})
|
||||
} else {
|
||||
None
|
||||
};
|
||||
assert_eq!(
|
||||
typed.early_stop_decision(),
|
||||
expected_decision,
|
||||
"case={case}, enabled={enabled}, order={order:?}, prefix={prefix}"
|
||||
);
|
||||
assert_eq!(typed.valid_responses, prefix + 1);
|
||||
assert_eq!(typed.conflicting_metadata, old_seen && current_votes > 0);
|
||||
assert_eq!(typed.version_early_stop_decision(), None, "current reads do not request a version ID");
|
||||
if expected_decision.is_some() {
|
||||
assert_eq!(
|
||||
SetDisks::file_info_quorum_hash(
|
||||
typed.candidate.as_ref().expect("three B votes need a candidate")
|
||||
),
|
||||
expected_hash
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let completed_metadata = slots
|
||||
.iter()
|
||||
.map(|slot| slot.file_info().expect("every disk must have arrived").clone())
|
||||
.collect::<Vec<_>>();
|
||||
let (_, selected, selection_quorum) =
|
||||
SetDisks::select_valid_fileinfo(&vec![None; 4], &completed_metadata, &vec![None; 4], "", 2, 3)
|
||||
.expect("three matching B disks must select the current identity");
|
||||
assert_eq!(selection_quorum, 3);
|
||||
assert_eq!(selected.version_id, expected_current.version_id);
|
||||
assert_eq!(selected.mod_time, expected_current.mod_time);
|
||||
assert_eq!(selected.data_dir, expected_current.data_dir);
|
||||
assert_eq!(SetDisks::file_info_quorum_hash(&selected), expected_hash);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_observation_pending_newer_version_and_same_time_different_directory_force_full_wait() {
|
||||
let mut accumulator = MetadataQuorumAccumulator::new(4, 2, true);
|
||||
for index in 0..2 {
|
||||
accumulator.observe(&observation(index, 0));
|
||||
}
|
||||
accumulator.observe(&observation(2, 4));
|
||||
assert_eq!(accumulator.candidate_votes, 2, "an outstanding response cannot supply the deciding vote");
|
||||
assert_eq!(accumulator.early_stop_decision(), None);
|
||||
|
||||
let mut newer = payload(2);
|
||||
newer.mod_time = Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1));
|
||||
accumulator.observe(&MetadataDiskObservation {
|
||||
disk_index: 2,
|
||||
result: Some(Ok(newer)),
|
||||
});
|
||||
assert!(accumulator.conflicting_metadata);
|
||||
assert_eq!(accumulator.early_stop_decision(), None);
|
||||
|
||||
let mut same_time_different_directory = payload(3);
|
||||
same_time_different_directory.data_dir = Some(Uuid::from_u128(2));
|
||||
accumulator.observe(&MetadataDiskObservation {
|
||||
disk_index: 3,
|
||||
result: Some(Ok(same_time_different_directory)),
|
||||
});
|
||||
assert_eq!(accumulator.early_stop_decision(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_observation_duplicate_shard_cannot_supply_a_read_reserve() {
|
||||
let mut accumulator = MetadataQuorumAccumulator::new(4, 2, true);
|
||||
let first = payload(0);
|
||||
for disk_index in 0..3 {
|
||||
accumulator.observe(&MetadataDiskObservation {
|
||||
disk_index,
|
||||
result: Some(Ok(first.clone())),
|
||||
});
|
||||
}
|
||||
assert_eq!(accumulator.candidate_votes, 3);
|
||||
assert!(
|
||||
!accumulator.candidate_has_read_reserve(),
|
||||
"copied erasure indexes cannot supply independent data shards"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_observation_null_versions_markers_and_invalid_success_keep_their_meaning() {
|
||||
let mut null_version = MetadataQuorumAccumulator::new(4, 2, true);
|
||||
for index in 0..3 {
|
||||
null_version.observe(&observation(index, 0));
|
||||
}
|
||||
assert!(null_version.early_stop_decision().is_some());
|
||||
assert_eq!(null_version.matching_version_votes, 0);
|
||||
|
||||
let mut marker = MetadataQuorumAccumulator::new(4, 2, true);
|
||||
for disk_index in 0..3 {
|
||||
let metadata = FileInfo {
|
||||
volume: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
deleted: true,
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1)),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(metadata.is_canonical_delete_marker(), "the null marker fixture must reach marker voting");
|
||||
marker.observe(&MetadataDiskObservation {
|
||||
disk_index,
|
||||
result: Some(Ok(metadata)),
|
||||
});
|
||||
}
|
||||
assert_eq!(marker.delete_marker_votes, 3);
|
||||
assert!(marker.early_stop_decision().is_some());
|
||||
|
||||
let mut invalid = MetadataQuorumAccumulator::new(4, 2, true);
|
||||
invalid.observe(&MetadataDiskObservation {
|
||||
disk_index: 0,
|
||||
result: Some(Ok(FileInfo::default())),
|
||||
});
|
||||
assert_eq!(invalid.valid_responses, 0);
|
||||
assert_eq!(invalid.hard_errors, 1);
|
||||
assert_eq!(invalid.early_stop_decision(), None);
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -18,4 +18,5 @@
|
||||
//! duplicating read/write/erasure logic.
|
||||
|
||||
pub(crate) mod io_primitives;
|
||||
mod metadata_quorum;
|
||||
pub(in crate::set_disk) mod metadata_quorum;
|
||||
pub(in crate::set_disk) mod metadata_read;
|
||||
|
||||
@@ -62,7 +62,7 @@ use crate::diagnostics::get::{
|
||||
classify_storage_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure,
|
||||
record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled,
|
||||
};
|
||||
use crate::diagnostics::object_lock::ObjectLockAttempt;
|
||||
use crate::diagnostics::object_lock::{ObjectLockAttempt, record_lock_acquire_failure};
|
||||
use crate::disk::error_reduce::{
|
||||
BUCKET_OP_IGNORED_ERRS, OBJECT_OP_IGNORED_ERRS, build_write_quorum_failure_summary, count_errs, reduce_read_quorum_errs,
|
||||
reduce_write_quorum_errs,
|
||||
@@ -868,7 +868,7 @@ static OBJECT_LOCK_DIAG_ENABLED: OnceLock<bool> = OnceLock::new();
|
||||
mod core;
|
||||
#[cfg(test)]
|
||||
pub(crate) use core::io_primitives::disk_call_counters;
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
#[cfg(test)]
|
||||
pub(crate) use core::io_primitives::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, rename_fanout_barrier};
|
||||
mod ctx;
|
||||
mod metadata;
|
||||
@@ -885,8 +885,6 @@ pub(crate) use ops::hermetic_set_disks_isolated;
|
||||
pub(crate) use ops::multipart::NewMultipartUploadCommitObservation;
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause};
|
||||
#[cfg(test)]
|
||||
pub(crate) use ops::object::DeleteObjectCommitBarrier;
|
||||
#[cfg(feature = "test-util")]
|
||||
pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier;
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
@@ -894,6 +892,8 @@ pub(crate) use ops::object::TransitionUploadedCommitBarrier as SetDiskTransition
|
||||
pub(crate) use ops::object::body_cache_plaintext_len;
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) use ops::object::cleanup_rejected_transition_upload_durably;
|
||||
#[cfg(test)]
|
||||
pub(crate) use ops::object::{DeleteCleanupBarrier, DeleteObjectCommitBarrier};
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub use ops::object::{PutObjectCommitBarrier, PutObjectCommitPause};
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
@@ -1441,6 +1441,172 @@ mod prepared_get_object_metadata_tests {
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn non_inline_two_phase_get_rejects_generation_change_before_late_refresh() {
|
||||
use crate::object_api::WriteCompletion;
|
||||
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let (dirs, set_disks) = crate::ecstore_validation_blackbox::make_local_set_disks_with_ctx(4, 2, ctx).await;
|
||||
let bucket = "non-inline-late-generation";
|
||||
let object = object_with_initial_data_shards(bucket, "late-generation-object");
|
||||
let generation_a_body = vec![0x5a; 1024 * 1024];
|
||||
let generation_b_body = vec![0xa5; generation_a_body.len()];
|
||||
// This unlocked, real-disk fixture exercises the internal generation
|
||||
// boundary, not concurrent HTTP writes through a namespace read lock.
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
suppress_read_repair: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
};
|
||||
set_disks
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create late generation bucket");
|
||||
set_disks
|
||||
.put_object(bucket, &object, &mut PutObjReader::from_vec(generation_a_body), &opts)
|
||||
.await
|
||||
.expect("commit every generation A shard before GET");
|
||||
let disks = set_disks.get_disks_internal().await;
|
||||
let original = SetDisks::read_metadata_observed(&disks, "", bucket, &object, "", true, false, false, false, 2)
|
||||
.await
|
||||
.expect("read every generation A disk slot");
|
||||
assert!(original.is_complete());
|
||||
let (original_metadata, original_errors, _) = original.into_legacy();
|
||||
assert!(original_errors.iter().all(Option::is_none));
|
||||
let (_, generation_a, _) = SetDisks::select_valid_fileinfo(&disks, &original_metadata, &original_errors, "", 2, 3)
|
||||
.expect("generation A must have real metadata quorum");
|
||||
assert!(!generation_a.inline_data(), "fixture must use non-inline shard files");
|
||||
let order = bounded_metadata_fanout_order(bucket, &object, 4, 2);
|
||||
for disk_index in order.iter().take(2) {
|
||||
let data_dir = original_metadata[*disk_index]
|
||||
.data_dir
|
||||
.expect("generation A metadata must name its real shard directory");
|
||||
let part_path = dirs[*disk_index]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join(&object)
|
||||
.join(data_dir.to_string())
|
||||
.join("part.1");
|
||||
let mut shard = std::fs::read(&part_path).expect("selected generation A shard must exist");
|
||||
*shard.first_mut().expect("real shard must contain a checksum frame") ^= 0xff;
|
||||
std::fs::write(part_path, shard).expect("corrupt selected shard without changing generation A metadata");
|
||||
}
|
||||
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
("RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE", Some("true")),
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
|
||||
],
|
||||
async {
|
||||
let _hedge_timer = rename_fanout_barrier::arm(&object, 0, rename_fanout_barrier::PHASE_NON_INLINE_HEDGE_TIMER);
|
||||
let refresh = rename_fanout_barrier::arm(&object, 0, rename_fanout_barrier::PHASE_LATE_METADATA_REFRESH);
|
||||
let calls = disk_call_counters::observe(&object);
|
||||
let get = set_disks.get_object_reader(bucket, &object, None, HeaderMap::new(), &opts);
|
||||
tokio::pin!(get);
|
||||
tokio::time::timeout(READ_VERSION_BARRIER_GUARD, async {
|
||||
tokio::select! {
|
||||
result = &mut get => panic!("GET returned before the late refresh boundary: {:?}", result.err()),
|
||||
() = refresh.wait_until_paused() => {}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("failed generation A decode must reach the real late refresh boundary");
|
||||
assert!(refresh.is_paused());
|
||||
assert_eq!(
|
||||
calls.total(disk_call_counters::KIND_READ_VERSION),
|
||||
3,
|
||||
"A must be an early metadata candidate"
|
||||
);
|
||||
assert_eq!(
|
||||
calls.for_disk(disk_call_counters::KIND_READ_VERSION, order[3]),
|
||||
0,
|
||||
"late refresh must not have read the omitted disk before replacement"
|
||||
);
|
||||
|
||||
set_disks
|
||||
.put_object(bucket, &object, &mut PutObjReader::from_vec(generation_b_body.clone()), &opts)
|
||||
.await
|
||||
.expect("commit every healthy generation B shard while the late GET is paused");
|
||||
let replacement = SetDisks::read_metadata_observed(&disks, "", bucket, &object, "", true, false, false, false, 2)
|
||||
.await
|
||||
.expect("read every committed generation B disk slot");
|
||||
assert!(replacement.is_complete());
|
||||
let (replacement_metadata, replacement_errors, _) = replacement.into_legacy();
|
||||
assert!(replacement_errors.iter().all(Option::is_none));
|
||||
let (online, generation_b, _) =
|
||||
SetDisks::select_valid_fileinfo(&disks, &replacement_metadata, &replacement_errors, "", 2, 3)
|
||||
.expect("replacement must have healthy real metadata quorum");
|
||||
assert_eq!(online.iter().flatten().count(), 4);
|
||||
assert_ne!(generation_a.data_dir, generation_b.data_dir);
|
||||
assert_ne!(
|
||||
SetDisks::file_info_quorum_hash(&generation_a),
|
||||
SetDisks::file_info_quorum_hash(&generation_b)
|
||||
);
|
||||
assert_eq!(generation_a.size, generation_b.size);
|
||||
assert_eq!(generation_a.erasure.distribution, generation_b.erasure.distribution);
|
||||
let before_late = (0..4)
|
||||
.map(|index| calls.for_disk(disk_call_counters::KIND_READ_VERSION, index))
|
||||
.collect::<Vec<_>>();
|
||||
refresh.release();
|
||||
let result = tokio::time::timeout(READ_VERSION_BARRIER_GUARD, &mut get)
|
||||
.await
|
||||
.expect("late GET must finish after its refresh boundary is released");
|
||||
for (index, before) in before_late.into_iter().enumerate() {
|
||||
assert_eq!(
|
||||
calls.for_disk(disk_call_counters::KIND_READ_VERSION, index) - before,
|
||||
1,
|
||||
"late refresh must read each real disk exactly once"
|
||||
);
|
||||
}
|
||||
match result {
|
||||
Err(Error::InsufficientReadQuorum(failed_bucket, failed_object)) => {
|
||||
assert_eq!(failed_bucket, bucket);
|
||||
assert_eq!(failed_object, object);
|
||||
}
|
||||
Err(error) => panic!("changed late generation must return InsufficientReadQuorum, got {error:?}"),
|
||||
Ok(mut reader) => {
|
||||
assert_eq!(reader.object_info.size, i64::try_from(generation_b_body.len()).expect("fixture size"));
|
||||
let mut body = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut body)
|
||||
.await
|
||||
.expect("unexpected replacement body must be readable");
|
||||
assert_eq!(body.len(), generation_b_body.len());
|
||||
assert_eq!(body, generation_b_body);
|
||||
println!(
|
||||
"late generation oracle exposed complete B: initial_reads=3, late_reads=4, bytes={}",
|
||||
body.len()
|
||||
);
|
||||
panic!("late GET must reject a changed object generation before exposing a body");
|
||||
}
|
||||
}
|
||||
drop(refresh);
|
||||
let mut reader = set_disks
|
||||
.get_object_reader(bucket, &object, None, HeaderMap::new(), &opts)
|
||||
.await
|
||||
.expect("fresh GET must open the healthy replacement generation");
|
||||
assert_eq!(reader.object_info.size, i64::try_from(generation_b_body.len()).expect("fixture size"));
|
||||
let mut body = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut body)
|
||||
.await
|
||||
.expect("fresh generation B must stream completely");
|
||||
assert_eq!(body.len(), generation_b_body.len());
|
||||
assert_eq!(body, generation_b_body);
|
||||
println!(
|
||||
"late generation oracle rejected crossing: initial_reads=3, late_per_disk=[1,1,1,1], fresh_bytes={}",
|
||||
body.len()
|
||||
);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn four_data_two_parity_two_phase_read_recovers_one_failed_data_shard() {
|
||||
@@ -3964,7 +4130,6 @@ impl SetDisks {
|
||||
b"rustfs-marker-purge-receipt-v1".to_vec(),
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
@@ -4550,6 +4715,9 @@ impl SetDisks {
|
||||
let timeout = get_lock_acquire_timeout();
|
||||
let mut attempt = ObjectLockAttempt::start(op, bucket, object, None, &ns_lock, "read", timeout);
|
||||
let result = ns_lock.get_read_lock(timeout).await;
|
||||
if let Err(error) = &result {
|
||||
record_lock_acquire_failure(op, bucket, object, "read", error);
|
||||
}
|
||||
attempt.observe(&result);
|
||||
let guard = result.map_err(|e| self.map_namespace_lock_error(bucket, object, "read", e))?;
|
||||
let owner = diag_enabled.then(|| ns_lock.owner().to_string());
|
||||
@@ -4608,6 +4776,9 @@ impl SetDisks {
|
||||
let acquire_timeout = get_put_object_commit_lock_acquire_timeout(op);
|
||||
let mut attempt = ObjectLockAttempt::start(op, bucket, object, None, &ns_lock, "write", acquire_timeout);
|
||||
let result = ns_lock.get_write_lock(acquire_timeout).await;
|
||||
if let Err(error) = &result {
|
||||
record_lock_acquire_failure(op, bucket, object, "write", error);
|
||||
}
|
||||
attempt.observe(&result);
|
||||
let guard = resolve_put_object_commit_lock_acquire_result(self, op, bucket, object, result)?;
|
||||
Self::record_put_object_commit_namespace_lock_wait(op, acquire_start);
|
||||
@@ -4660,6 +4831,9 @@ impl SetDisks {
|
||||
std::task::Poll::Ready(result) => std::task::Poll::Ready(result),
|
||||
})
|
||||
.await;
|
||||
if let Err(error) = &result {
|
||||
record_lock_acquire_failure(op, bucket, object, "write", error);
|
||||
}
|
||||
attempt.observe(&result);
|
||||
let guard = resolve_put_object_commit_lock_acquire_result(self, op, bucket, object, result)?;
|
||||
Self::record_put_object_commit_namespace_lock_wait(op, acquire_start);
|
||||
@@ -5527,12 +5701,29 @@ fn collect_inline_data_shard_fileinfos_by_index_or_reason<'a>(
|
||||
parts_metadata: &'a [FileInfo],
|
||||
fi: &FileInfo,
|
||||
data_shards: usize,
|
||||
disk_is_online: impl FnMut(usize) -> bool,
|
||||
) -> std::result::Result<Vec<&'a FileInfo>, &'static str> {
|
||||
collect_inline_data_shard_fileinfos_from_observations(
|
||||
parts_metadata
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, metadata)| (index, Some(metadata))),
|
||||
fi,
|
||||
data_shards,
|
||||
disk_is_online,
|
||||
)
|
||||
}
|
||||
|
||||
fn collect_inline_data_shard_fileinfos_from_observations<'a>(
|
||||
observations: impl IntoIterator<Item = (usize, Option<&'a FileInfo>)>,
|
||||
fi: &FileInfo,
|
||||
data_shards: usize,
|
||||
mut disk_is_online: impl FnMut(usize) -> bool,
|
||||
) -> std::result::Result<Vec<&'a FileInfo>, &'static str> {
|
||||
let distribution = &fi.erasure.distribution;
|
||||
let mut data_files = vec![None; data_shards];
|
||||
|
||||
for (disk_index, file_info) in parts_metadata.iter().enumerate() {
|
||||
for (disk_index, file_info) in observations {
|
||||
if !disk_is_online(disk_index) {
|
||||
continue;
|
||||
}
|
||||
@@ -5542,9 +5733,9 @@ fn collect_inline_data_shard_fileinfos_by_index_or_reason<'a>(
|
||||
if block_index == 0 || block_index > data_shards {
|
||||
continue;
|
||||
}
|
||||
if file_info.name.is_empty() {
|
||||
let Some(file_info) = file_info.filter(|metadata| !metadata.name.is_empty()) else {
|
||||
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD);
|
||||
}
|
||||
};
|
||||
if file_info.erasure.index != block_index {
|
||||
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH);
|
||||
}
|
||||
@@ -9639,6 +9830,141 @@ mod tests {
|
||||
assert!(nested.join(STORAGE_FORMAT_FILE).exists());
|
||||
}
|
||||
|
||||
/// The residue a delete leaves once it committed on a disk but before its
|
||||
/// cleanup pass: a marked data dir and the rollback backup of the same
|
||||
/// transaction. Returns `(data dir, rollback backup)`.
|
||||
async fn write_committed_delete_with_rollback_backup(
|
||||
object_dir: &std::path::Path,
|
||||
backup_transaction: Uuid,
|
||||
marker_transaction: Option<Uuid>,
|
||||
) -> (std::path::PathBuf, std::path::PathBuf) {
|
||||
let data_dir = object_dir.join(Uuid::new_v4().to_string());
|
||||
fs::create_dir_all(&data_dir).await.expect("data dir should be created");
|
||||
fs::write(data_dir.join("part.1"), b"stale")
|
||||
.await
|
||||
.expect("stale part should be written");
|
||||
if let Some(transaction) = marker_transaction {
|
||||
fs::write(
|
||||
data_dir.join(format!("{}{}", crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX, transaction)),
|
||||
[],
|
||||
)
|
||||
.await
|
||||
.expect("committed delete marker should be written");
|
||||
}
|
||||
let backup = object_dir
|
||||
.join(backup_transaction.to_string())
|
||||
.join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
fs::create_dir_all(backup.parent().expect("backup parent"))
|
||||
.await
|
||||
.expect("rollback dir should be created");
|
||||
fs::write(&backup, b"rollback metadata")
|
||||
.await
|
||||
.expect("rollback backup should be written");
|
||||
(data_dir, backup)
|
||||
}
|
||||
|
||||
// #6898: the rollback backup of a delete whose committed marker sits in a
|
||||
// data dir of the same object is residue too, so the empty listing can
|
||||
// reclaim the whole object directory.
|
||||
#[tokio::test]
|
||||
async fn purge_orphan_dir_object_removes_rollback_backup_of_committed_delete() {
|
||||
let (dir, disk) = make_single_local_disk().await;
|
||||
let transaction = Uuid::new_v4();
|
||||
let object_dir = dir.path().join("bucket/pfx/obj");
|
||||
write_committed_delete_with_rollback_backup(&object_dir, transaction, Some(transaction)).await;
|
||||
|
||||
let set = make_set_disks_with(vec![Some(disk)]).await;
|
||||
let purged = set
|
||||
.purge_orphan_dir_object("bucket", "pfx/")
|
||||
.await
|
||||
.expect("purge should succeed");
|
||||
|
||||
assert!(purged);
|
||||
assert!(!dir.path().join("bucket/pfx").exists(), "the whole committed residue tree must go");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn purge_orphan_dir_object_preserves_rollback_backup_of_other_transaction() {
|
||||
let (dir, disk) = make_single_local_disk().await;
|
||||
let object_dir = dir.path().join("bucket/pfx/obj");
|
||||
let (data_dir, backup) =
|
||||
write_committed_delete_with_rollback_backup(&object_dir, Uuid::new_v4(), Some(Uuid::new_v4())).await;
|
||||
|
||||
let set = make_set_disks_with(vec![Some(disk)]).await;
|
||||
set.purge_orphan_dir_object("bucket", "pfx/")
|
||||
.await
|
||||
.expect("scan should succeed");
|
||||
|
||||
assert!(!data_dir.exists(), "the committed data dir is still reclaimed");
|
||||
assert!(backup.exists(), "a backup whose own transaction is unproven must remain");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn purge_orphan_dir_object_preserves_rollback_backup_without_marker() {
|
||||
let (dir, disk) = make_single_local_disk().await;
|
||||
let object_dir = dir.path().join("bucket/pfx/obj");
|
||||
let backup = object_dir.join(Uuid::new_v4().to_string()).join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
fs::create_dir_all(backup.parent().expect("backup parent"))
|
||||
.await
|
||||
.expect("rollback dir should be created");
|
||||
fs::write(&backup, b"rollback metadata")
|
||||
.await
|
||||
.expect("rollback backup should be written");
|
||||
|
||||
let set = make_set_disks_with(vec![Some(disk)]).await;
|
||||
let purged = set
|
||||
.purge_orphan_dir_object("bucket", "pfx/")
|
||||
.await
|
||||
.expect("scan should succeed");
|
||||
|
||||
assert!(!purged, "a lone rollback backup may still be needed to roll a delete back");
|
||||
assert!(backup.exists());
|
||||
}
|
||||
|
||||
// Live metadata on one disk blocks the object there and keeps it intact;
|
||||
// on the other disks only what each disk's own committed marker proves is
|
||||
// reclaimed, the same rule that already applies to a marked data dir.
|
||||
#[tokio::test]
|
||||
async fn purge_orphan_dir_object_keeps_object_with_metadata_on_another_disk() {
|
||||
let mut dirs = Vec::new();
|
||||
let mut disks = Vec::new();
|
||||
for _ in 0..4 {
|
||||
let (dir, disk) = make_single_local_disk().await;
|
||||
dirs.push(dir);
|
||||
disks.push(Some(disk));
|
||||
}
|
||||
let transaction = Uuid::new_v4();
|
||||
let mut residue = Vec::new();
|
||||
for dir in &dirs[..3] {
|
||||
residue.push(
|
||||
write_committed_delete_with_rollback_backup(&dir.path().join("bucket/pfx/obj"), transaction, Some(transaction))
|
||||
.await,
|
||||
);
|
||||
}
|
||||
let live_object = dirs[3].path().join("bucket/pfx/obj");
|
||||
let live_part = live_object.join(Uuid::new_v4().to_string()).join("part.1");
|
||||
fs::create_dir_all(live_part.parent().expect("data dir"))
|
||||
.await
|
||||
.expect("live data dir should be created");
|
||||
fs::write(&live_part, b"live").await.expect("live part should be written");
|
||||
fs::write(live_object.join(STORAGE_FORMAT_FILE), b"meta")
|
||||
.await
|
||||
.expect("live metadata should be written");
|
||||
|
||||
let set = make_set_disks_with(disks).await;
|
||||
set.purge_orphan_dir_object("bucket", "pfx/")
|
||||
.await
|
||||
.expect("scan should succeed");
|
||||
|
||||
assert!(
|
||||
live_object.join(STORAGE_FORMAT_FILE).exists() && live_part.exists(),
|
||||
"the object with metadata on another disk must stay intact there"
|
||||
);
|
||||
for (data_dir, backup) in &residue {
|
||||
assert!(!data_dir.exists() && !backup.exists(), "this disk's committed residue is reclaimed");
|
||||
}
|
||||
}
|
||||
|
||||
// An unreadable directory anywhere under the prefix aborts the purge on
|
||||
// every disk before anything is deleted.
|
||||
#[cfg(unix)]
|
||||
|
||||
@@ -34,6 +34,41 @@ pub(crate) enum BucketInfoQuorum {
|
||||
}
|
||||
|
||||
impl SetDisks {
|
||||
/// Require proof from every configured disk before retiring a bucket
|
||||
/// generation whose visible volume was removed outside the storage API.
|
||||
pub(crate) async fn ensure_bucket_volume_absent_on_every_disk(&self, bucket: &str) -> Result<()> {
|
||||
let disks = self.disk_inventory().await;
|
||||
if disks.is_empty() {
|
||||
return Err(Error::ErasureReadQuorum);
|
||||
}
|
||||
|
||||
let futures = disks.into_iter().map(|disk| async move {
|
||||
match disk {
|
||||
Some(disk) => disk.stat_volume(bucket).await.map(|_| ()),
|
||||
None => Err(DiskError::DiskNotFound),
|
||||
}
|
||||
});
|
||||
let results = join_all(futures).await;
|
||||
let mut volume_present = false;
|
||||
let mut first_unknown = None;
|
||||
for result in results {
|
||||
match result {
|
||||
Ok(()) => volume_present = true,
|
||||
Err(DiskError::VolumeNotFound) => {}
|
||||
Err(error) if first_unknown.is_none() => first_unknown = Some(error),
|
||||
Err(_) => {}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(error) = first_unknown {
|
||||
return Err(error.into());
|
||||
}
|
||||
if volume_present {
|
||||
return Err(Error::BucketExists(bucket.to_owned()));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn stat_bucket_with_quorum(&self, bucket: &str, quorum: BucketInfoQuorum) -> Result<BucketInfo> {
|
||||
let disks = self.disk_inventory().await;
|
||||
let disk_count = disks.len();
|
||||
|
||||
@@ -598,17 +598,18 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
scope.check()?;
|
||||
let (_, errors) =
|
||||
Self::read_all_fileinfo(disks.as_slice(), "", bucket, object, &version.to_string(), false, false, false).await?;
|
||||
let metadata_read =
|
||||
Self::read_metadata(disks.as_slice(), "", bucket, object, &version.to_string(), false, false, false).await?;
|
||||
let all_absent = metadata_read
|
||||
.slots
|
||||
.iter()
|
||||
.all(|slot| matches!(&slot.result, Some(Err(DiskError::FileNotFound | DiskError::FileVersionNotFound))));
|
||||
|
||||
let selected = self.get_disks_internal().await;
|
||||
let same_targets = selected.len() == disks.len() && selected.iter().zip(&disks).all(|(current, original)| {
|
||||
matches!((current, original), (Some(current), Some(original)) if std::sync::Arc::ptr_eq(current, original))
|
||||
});
|
||||
if !same_targets
|
||||
|| !errors
|
||||
.iter()
|
||||
.all(|error| matches!(error, Some(DiskError::FileNotFound | DiskError::FileVersionNotFound)))
|
||||
{
|
||||
if !same_targets || !all_absent {
|
||||
return Err(StorageError::SlowDown);
|
||||
}
|
||||
scope.check()?;
|
||||
@@ -734,7 +735,7 @@ impl SetDisks {
|
||||
object: &str,
|
||||
version_id: &str,
|
||||
) -> disk::error::Result<ReadRepairCommitFingerprint> {
|
||||
let (parts_metadata, errs, _) = Self::read_all_fileinfo_observed(
|
||||
let metadata_read = Self::read_metadata_observed(
|
||||
disks,
|
||||
"",
|
||||
bucket,
|
||||
@@ -747,6 +748,7 @@ impl SetDisks {
|
||||
self.default_parity_count,
|
||||
)
|
||||
.await?;
|
||||
let (parts_metadata, errs, _) = metadata_read.into_legacy();
|
||||
let (read_quorum, _) = Self::object_quorum_from_meta(&parts_metadata, &errs, self.default_parity_count)?;
|
||||
let read_quorum = usize::try_from(read_quorum).map_err(|_| DiskError::ErasureReadQuorum)?;
|
||||
let (_, quorum_mod_time, quorum_etag) = Self::list_online_disks(disks, &parts_metadata, &errs, read_quorum);
|
||||
@@ -931,8 +933,9 @@ impl SetDisks {
|
||||
}
|
||||
};
|
||||
|
||||
let (mut parts_metadata, errs) =
|
||||
Self::read_all_fileinfo(&disks, "", bucket, object, version_id, true, true, false).await?;
|
||||
let (mut parts_metadata, errs, _) = Self::read_metadata(&disks, "", bucket, object, version_id, true, true, false)
|
||||
.await?
|
||||
.into_legacy();
|
||||
|
||||
trace!(
|
||||
event = EVENT_SET_DISK_HEAL,
|
||||
@@ -1284,8 +1287,10 @@ impl SetDisks {
|
||||
// in `parts_metadata` to defaults. Re-read only before
|
||||
// destructive cleanup so the guard sees every original
|
||||
// identity.
|
||||
let (delete_guard_metadata, delete_guard_errs) =
|
||||
Self::read_all_fileinfo(&disks, "", bucket, object, version_id, true, true, false).await?;
|
||||
let (delete_guard_metadata, delete_guard_errs, _) =
|
||||
Self::read_metadata(&disks, "", bucket, object, version_id, true, true, false)
|
||||
.await?
|
||||
.into_legacy();
|
||||
if self
|
||||
.dangling_delete_safety(bucket, object, &delete_guard_metadata, &delete_guard_errs, &disks)
|
||||
.await?
|
||||
@@ -2316,16 +2321,16 @@ impl SetDisks {
|
||||
return Err(DiskError::retired_marker_deferred(format!("conditional marker deletion failed: {error}")));
|
||||
}
|
||||
}
|
||||
let (_, errors) = Self::read_all_fileinfo(disks, "", bucket, object, &version.to_string(), false, false, false).await?;
|
||||
let metadata_read = Self::read_metadata(disks, "", bucket, object, &version.to_string(), false, false, false).await?;
|
||||
let all_absent = metadata_read
|
||||
.slots
|
||||
.iter()
|
||||
.all(|slot| matches!(&slot.result, Some(Err(DiskError::FileNotFound | DiskError::FileVersionNotFound))));
|
||||
let selected = self.get_disks_internal().await;
|
||||
let same_targets = selected.len() == disks.len() && selected.iter().zip(disks).all(|(current, original)| {
|
||||
matches!((current, original), (Some(current), Some(original)) if std::sync::Arc::ptr_eq(current, original))
|
||||
});
|
||||
if !same_targets
|
||||
|| !errors
|
||||
.iter()
|
||||
.all(|error| matches!(error, Some(DiskError::FileNotFound | DiskError::FileVersionNotFound)))
|
||||
{
|
||||
if !same_targets || !all_absent {
|
||||
return Err(DiskError::retired_marker_deferred(
|
||||
"complete marker absence could not be verified under the original target set",
|
||||
));
|
||||
@@ -3383,9 +3388,10 @@ impl SetDisks {
|
||||
// The inner heal and missing-object report read the registry again;
|
||||
// release this snapshot guard before a topology writer can queue between reads.
|
||||
let disks = self.get_disks_internal().await;
|
||||
let (_, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, version_id, false, false, false)
|
||||
let (_, errs, _) = Self::read_metadata(&disks, "", bucket, object, version_id, false, false, false)
|
||||
.await
|
||||
.map_err(|e| to_object_err(e.into(), vec![bucket, object]))?;
|
||||
.map_err(|e| to_object_err(e.into(), vec![bucket, object]))?
|
||||
.into_legacy();
|
||||
if DiskError::is_all_not_found(&errs) {
|
||||
debug!(
|
||||
event = EVENT_SET_DISK_HEAL,
|
||||
|
||||
@@ -21,6 +21,7 @@
|
||||
//! unchanged; method bodies are moved verbatim and runtime behavior is the same.
|
||||
|
||||
use crate::core::pools::DecommissionCapacityAdmission;
|
||||
use crate::object_api::write_commit_context::WriteCommitContext;
|
||||
|
||||
#[cfg(test)]
|
||||
use super::super::GetObjectMetadataCacheKey;
|
||||
@@ -2446,17 +2447,19 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<ObjectInfo> {
|
||||
crate::hp_guard!("SetDisks::complete_multipart_upload");
|
||||
let write_context = WriteCommitContext::from_options(opts, bucket, object, false)?;
|
||||
self.invalidate_get_object_metadata_cache(bucket, object).await;
|
||||
|
||||
let upload_id_path = Self::get_multipart_upload_dir(bucket, object, upload_id, opts.data_movement);
|
||||
let range_seek_rollout_enabled = crate::object_api::legacy_encrypted_range_seek_enabled() && !opts.no_lock;
|
||||
let range_seek_rollout_enabled =
|
||||
crate::object_api::legacy_encrypted_range_seek_enabled() && write_context.acquires_namespace();
|
||||
let mut object_lock_guard = None;
|
||||
let mut decommission_object_lock_guard = None;
|
||||
let mut decommission_target_lock_covered = false;
|
||||
let mut decommission_capacity_guard = None;
|
||||
|
||||
if opts.http_preconditions.is_some() {
|
||||
if !opts.no_lock {
|
||||
if write_context.acquires_namespace() {
|
||||
object_lock_guard = Some(
|
||||
self.acquire_write_lock_diag("complete_multipart_upload_precondition", bucket, object)
|
||||
.await?,
|
||||
@@ -2479,7 +2482,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
self.pool_index,
|
||||
bucket,
|
||||
object,
|
||||
opts.no_lock || object_lock_guard.is_some(),
|
||||
!write_context.acquires_namespace() || object_lock_guard.is_some(),
|
||||
DecommissionCapacityAdmission::ExistingMultipart,
|
||||
)
|
||||
.await?;
|
||||
@@ -2487,7 +2490,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
decommission_target_lock_covered = target_lock_covered;
|
||||
decommission_capacity_guard = capacity_guard;
|
||||
}
|
||||
if !opts.no_lock && object_lock_guard.is_none() && !decommission_target_lock_covered {
|
||||
if write_context.acquires_namespace() && object_lock_guard.is_none() && !decommission_target_lock_covered {
|
||||
object_lock_guard = Some(
|
||||
self.acquire_write_lock_diag("complete_multipart_upload_commit", bucket, object)
|
||||
.await?,
|
||||
@@ -3001,6 +3004,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
// etag
|
||||
let etag = resolve_complete_etag(opts, &uploaded_parts);
|
||||
|
||||
// An upload can contain parts from an older writer or a peer. Until
|
||||
// every part's format is attested, keep its ETag out of keyless lists.
|
||||
rustfs_utils::http::metadata_compat::remove_str(&mut fi.metadata, rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG);
|
||||
|
||||
fi.metadata.insert("etag".to_owned(), etag);
|
||||
|
||||
let persist_encryption_original_size = should_persist_encryption_original_size(&fi.metadata);
|
||||
@@ -3354,8 +3361,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
// usable as a CopyObject source. Do not return on rename quorum while
|
||||
// a tail owner may still hold the object guard and finish shard moves.
|
||||
let commit_allows_early_ack = false;
|
||||
let detach_commit_owner = commit_allows_early_ack || upload_guard.is_some() || quota_mutation_fence;
|
||||
let detach_commit_owner =
|
||||
write_context.has_borrowed_owner() || commit_allows_early_ack || upload_guard.is_some() || quota_mutation_fence;
|
||||
let commit = async move {
|
||||
let write_context = write_context;
|
||||
let mut _object_lock_guard = commit_object_lock_guard;
|
||||
let mut _decommission_object_lock_guard = commit_decommission_object_lock_guard;
|
||||
let mut _upload_guard = upload_guard;
|
||||
@@ -3377,6 +3386,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
pause_multipart_commit(&commit_bucket, &commit_object, MultipartCommitPause::BeforeQuotaRename).await;
|
||||
if quota_reservation.is_lock_lost()
|
||||
|| !quota_reservation.capability_proof_matches()
|
||||
|| write_context.is_lock_lost()
|
||||
|| _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| _decommission_object_lock_guard
|
||||
.as_ref()
|
||||
@@ -3431,6 +3441,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
}
|
||||
if quota_reservation.is_lock_lost()
|
||||
|| !quota_reservation.capability_proof_matches()
|
||||
|| write_context.is_lock_lost()
|
||||
|| _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| _decommission_object_lock_guard
|
||||
.as_ref()
|
||||
@@ -3761,6 +3772,7 @@ mod tests {
|
||||
use crate::layout::endpoints::SetupType;
|
||||
use crate::services::notification_sys::install_remote_version_state_fleet_proof_for_test;
|
||||
use crate::set_disk::core::io_primitives::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, rename_fanout_barrier};
|
||||
use crate::set_disk::get_lock_acquire_timeout;
|
||||
// No-locker helpers resolve to the isolated-context variants (see
|
||||
// `hermetic_set_disks_isolated`); the guard-based tests build through
|
||||
// `hermetic_set_disks_with_lockers`, which stays on the bootstrap context
|
||||
@@ -7122,6 +7134,54 @@ mod tests {
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn borrowed_complete_keeps_namespace_owner_after_caller_cancel() {
|
||||
let (_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-borrowed-cancel";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disks, bucket).await;
|
||||
let (upload_id, parts) =
|
||||
stage_upload_with_create_opts(&set, bucket, object, &[0x53; 4096], &ObjectOptions::default()).await;
|
||||
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::AfterRename);
|
||||
let writer = Arc::clone(&set);
|
||||
let complete = tokio::spawn(async move {
|
||||
let outer_guard = crate::object_api::WriteCommitGuard::acquire(
|
||||
&writer.new_ns_lock(bucket, object).await.expect("namespace wrapper"),
|
||||
get_lock_acquire_timeout(),
|
||||
)
|
||||
.await
|
||||
.expect("outer MPU namespace guard");
|
||||
let mut opts = ObjectOptions::default();
|
||||
opts.add_write_commit_guard(&outer_guard);
|
||||
writer
|
||||
.complete_multipart_upload(bucket, object, &upload_id, parts, &opts)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("borrowed completion reaches committed cleanup");
|
||||
complete.abort();
|
||||
assert!(complete.await.expect_err("completion caller canceled").is_cancelled());
|
||||
let mut probe = Box::pin(set.acquire_write_lock_diag("borrowed_complete_probe", bucket, object));
|
||||
assert!(
|
||||
futures::poll!(probe.as_mut()).is_pending(),
|
||||
"borrowed completion retains the real outer namespace owner"
|
||||
);
|
||||
barrier.release();
|
||||
drop(
|
||||
tokio::time::timeout(Duration::from_secs(30), probe)
|
||||
.await
|
||||
.expect("completion owner finishes cleanup")
|
||||
.expect("next writer obtains namespace"),
|
||||
);
|
||||
let info = set
|
||||
.get_object_info(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("borrowed MPU result remains readable");
|
||||
assert_eq!(info.size, 4096);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn cancelled_complete_keeps_upload_lock_through_tail_cleanup() {
|
||||
|
||||
@@ -281,7 +281,7 @@ use super::super::GetObjectMetadataCacheEntry;
|
||||
use super::super::GetObjectMetadataCacheKey;
|
||||
#[cfg(test)]
|
||||
use super::super::capacity_scope_from_disks;
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
#[cfg(test)]
|
||||
use super::super::get_lock_acquire_timeout;
|
||||
use crate::bucket::lifecycle::{
|
||||
tier_delete_journal::{TierDeleteDispatchAuthorization, record_tier_delete_journal_backend_identity},
|
||||
@@ -320,10 +320,13 @@ use crate::disk::format::FormatV3;
|
||||
use crate::disk::new_disk;
|
||||
use crate::disk::{DataDirDeleteStatus, OldCurrentSize};
|
||||
use crate::error::is_err_invalid_upload_id;
|
||||
#[cfg(test)]
|
||||
use crate::object_api::WriteCompletion;
|
||||
use crate::object_api::write_commit_context::WriteCommitContext;
|
||||
use crate::object_api::{GetObjectBodySource, get_object_body_cache_hook_suppressed};
|
||||
use crate::object_api::{
|
||||
NamespaceLockFence, ReplicationStatusWritebackCondition, ReplicationStatusWritebackMode,
|
||||
SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, WriteCompletion,
|
||||
SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY,
|
||||
};
|
||||
use crate::services::notification_sys::RemoteVersionStateFleetProofToken;
|
||||
use crate::services::tier::tier::{TierConfigMgr, TierDestinationId, TierOperationLease, tier_destination_id_from_metadata};
|
||||
@@ -2785,6 +2788,13 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
||||
)
|
||||
.await;
|
||||
if prepare_late_materialized_retry(&initial_result, &mut output, object_size) {
|
||||
#[cfg(test)]
|
||||
super::super::rename_fanout_barrier::checkpoint(
|
||||
object,
|
||||
0,
|
||||
super::super::rename_fanout_barrier::PHASE_LATE_METADATA_REFRESH,
|
||||
)
|
||||
.await;
|
||||
let (full_fi, full_parts_metadata, full_online_disks) = Self::refresh_late_metadata_fanout(
|
||||
&late_metadata_fanout_disks,
|
||||
bucket,
|
||||
@@ -3499,6 +3509,7 @@ impl SetDisks {
|
||||
opts: &ObjectOptions,
|
||||
mut publication_fence: Option<RemoteTuplePublicationFence>,
|
||||
) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
|
||||
let write_context = WriteCommitContext::from_options(opts, bucket, object, publication_fence.is_some())?;
|
||||
let protect_write = opts.shard_integrity_write_enabled();
|
||||
let source_bucket_incarnation_id = match opts.expected_bucket_incarnation_id {
|
||||
Some(incarnation_id) => Some(incarnation_id),
|
||||
@@ -3836,6 +3847,16 @@ impl SetDisks {
|
||||
.map(crate::io_support::rio::compression_index_storage_bytes);
|
||||
|
||||
let mut etag = data.stream.try_resolve_etag().unwrap_or_default();
|
||||
if data.stream.encrypted_etag {
|
||||
if opts.preserve_etag.as_ref().is_none_or(|preserved| preserved == &etag) {
|
||||
insert_str(&mut user_defined, rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG, "v1".to_owned());
|
||||
} else {
|
||||
rustfs_utils::http::metadata_compat::remove_str(
|
||||
&mut user_defined,
|
||||
rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG,
|
||||
);
|
||||
}
|
||||
}
|
||||
if let Some(ref tag) = opts.preserve_etag {
|
||||
etag = tag.clone();
|
||||
}
|
||||
@@ -4026,7 +4047,7 @@ impl SetDisks {
|
||||
self.pool_index,
|
||||
bucket,
|
||||
object,
|
||||
opts.no_lock || object_lock_guard.is_some(),
|
||||
!write_context.acquires_namespace() || object_lock_guard.is_some(),
|
||||
DecommissionCapacityAdmission::Mutation,
|
||||
)
|
||||
.await?;
|
||||
@@ -4034,7 +4055,8 @@ impl SetDisks {
|
||||
decommission_target_lock_covered = target_lock_covered;
|
||||
decommission_capacity_guard = capacity_guard;
|
||||
}
|
||||
if publication_fence.is_some() || (!opts.no_lock && object_lock_guard.is_none() && !decommission_target_lock_covered)
|
||||
if publication_fence.is_some()
|
||||
|| (write_context.acquires_namespace() && object_lock_guard.is_none() && !decommission_target_lock_covered)
|
||||
{
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pause_put_object_commit(bucket, object, PutObjectCommitPause::BeforeNamespace).await;
|
||||
@@ -4382,10 +4404,11 @@ impl SetDisks {
|
||||
// complete rename fan-out drains. Keep this path synchronous so
|
||||
// its terminal state is known before the coordinator releases
|
||||
// remote leases.
|
||||
let commit_owns_namespace_guard = commit_object_lock_guard.is_some()
|
||||
let commit_owns_namespace_guard = write_context.has_borrowed_owner()
|
||||
|| commit_object_lock_guard.is_some()
|
||||
|| commit_decommission_object_lock_guard.is_some()
|
||||
|| commit_publication_guard.is_some();
|
||||
let commit_allows_early_ack = opts.write_completion == WriteCompletion::Quorum
|
||||
let commit_allows_early_ack = write_context.allows_early_ack(opts.write_completion)
|
||||
&& !(opts.data_movement && opts.has_decommission_capacity_reservation())
|
||||
&& commit_owns_namespace_guard
|
||||
&& commit_scanner_publication_scope.is_none();
|
||||
@@ -4412,6 +4435,7 @@ impl SetDisks {
|
||||
tmp_cleanup_owned = true;
|
||||
|
||||
let commit = move |cancellation: Option<CancellationToken>| async move {
|
||||
let write_context = write_context;
|
||||
let mut _object_lock_guard = commit_object_lock_guard;
|
||||
let mut _decommission_object_lock_guard = commit_decommission_object_lock_guard;
|
||||
let mut _publication_guard = commit_publication_guard;
|
||||
@@ -4427,6 +4451,7 @@ impl SetDisks {
|
||||
pause_put_object_commit(&commit_bucket, &commit_object, PutObjectCommitPause::BeforeQuotaRename).await;
|
||||
if quota_reservation.is_lock_lost()
|
||||
|| !quota_reservation.capability_proof_matches()
|
||||
|| write_context.is_lock_lost()
|
||||
|| _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| _decommission_object_lock_guard
|
||||
.as_ref()
|
||||
@@ -4486,6 +4511,7 @@ impl SetDisks {
|
||||
}
|
||||
if quota_reservation.is_lock_lost()
|
||||
|| !quota_reservation.capability_proof_matches()
|
||||
|| write_context.is_lock_lost()
|
||||
|| _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| _decommission_object_lock_guard
|
||||
.as_ref()
|
||||
@@ -4671,6 +4697,7 @@ impl SetDisks {
|
||||
guard_release_rx,
|
||||
(
|
||||
object_lock_guard,
|
||||
write_context,
|
||||
publication_guard,
|
||||
bucket_lifecycle_guard,
|
||||
decommission_object_lock_guard,
|
||||
@@ -4691,6 +4718,7 @@ impl SetDisks {
|
||||
},
|
||||
move |(
|
||||
object_lock_guard,
|
||||
write_context,
|
||||
publication_guard,
|
||||
bucket_lifecycle_guard,
|
||||
decommission_object_lock_guard,
|
||||
@@ -4698,6 +4726,7 @@ impl SetDisks {
|
||||
),
|
||||
targets| async move {
|
||||
drop(object_lock_guard);
|
||||
drop(write_context);
|
||||
drop(publication_guard);
|
||||
drop(bucket_lifecycle_guard);
|
||||
cleanup_set
|
||||
@@ -7066,6 +7095,82 @@ async fn pause_delete_object_commit_after_publish(bucket: &str, object: &str) {
|
||||
}
|
||||
}
|
||||
|
||||
/// Pauses a delete after its per-disk commit and before the cleanup or rollback
|
||||
/// pass, the window in which an abandoned delete used to leave metadata-less
|
||||
/// residue behind.
|
||||
#[cfg(test)]
|
||||
struct DeleteCleanupBarrierState {
|
||||
bucket: String,
|
||||
arrived: tokio::sync::Notify,
|
||||
release: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) struct DeleteCleanupBarrier {
|
||||
state: Arc<DeleteCleanupBarrierState>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
static DELETE_CLEANUP_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Arc<DeleteCleanupBarrierState>>>> =
|
||||
std::sync::OnceLock::new();
|
||||
|
||||
#[cfg(test)]
|
||||
impl DeleteCleanupBarrier {
|
||||
pub(crate) fn install(bucket: &str) -> Self {
|
||||
let state = Arc::new(DeleteCleanupBarrierState {
|
||||
bucket: bucket.to_string(),
|
||||
arrived: tokio::sync::Notify::new(),
|
||||
release: tokio::sync::Notify::new(),
|
||||
});
|
||||
let mut slot = DELETE_CLEANUP_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("delete cleanup barrier mutex should not poison");
|
||||
assert!(slot.is_none(), "delete cleanup barrier must be unique");
|
||||
*slot = Some(Arc::clone(&state));
|
||||
Self { state }
|
||||
}
|
||||
|
||||
pub(crate) async fn wait_until_paused(&self) {
|
||||
tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
|
||||
.await
|
||||
.expect("delete should reach the cleanup barrier");
|
||||
}
|
||||
|
||||
pub(crate) fn release(&self) {
|
||||
self.state.release.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for DeleteCleanupBarrier {
|
||||
fn drop(&mut self) {
|
||||
self.state.release.notify_one();
|
||||
let mut slot = DELETE_CLEANUP_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("delete cleanup barrier mutex should not poison");
|
||||
if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
|
||||
*slot = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn pause_delete_before_cleanup(bucket: &str) {
|
||||
let barrier = DELETE_CLEANUP_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("delete cleanup barrier mutex should not poison")
|
||||
.as_ref()
|
||||
.filter(|barrier| barrier.bucket == bucket)
|
||||
.cloned();
|
||||
if let Some(barrier) = barrier {
|
||||
barrier.arrived.notify_one();
|
||||
barrier.release.notified().await;
|
||||
}
|
||||
}
|
||||
|
||||
fn persisted_transition_version_with_gate(
|
||||
remote_version: &str,
|
||||
remote_version_state_writer_enabled: bool,
|
||||
@@ -7646,6 +7751,8 @@ impl SetDisks {
|
||||
|
||||
let quorum_result = resolve_tiered_decommission_write_quorum_result(&errs, write_quorum, bucket, object);
|
||||
let should_rollback = quorum_result.is_err();
|
||||
#[cfg(test)]
|
||||
pause_delete_before_cleanup(bucket).await;
|
||||
let mut rollback_futures = Vec::new();
|
||||
for (index, err) in errs.iter().enumerate() {
|
||||
// backlog#1158: when rolling back, fan the idempotent undo out to every
|
||||
@@ -8655,6 +8762,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
}
|
||||
|
||||
self.record_capacity_scope_if_needed(opts.capacity_scope_token, &disks);
|
||||
#[cfg(test)]
|
||||
pause_delete_before_cleanup(bucket).await;
|
||||
|
||||
let mut rollback_futures = Vec::new();
|
||||
let committed_receipt_indices =
|
||||
@@ -10511,6 +10620,65 @@ mod object_encryption_resolver_wiring_tests {
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
#[tokio::test]
|
||||
async fn encrypted_put_attests_only_etags_produced_by_its_encryption_reader() {
|
||||
use super::hermetic_set_disks_support::hermetic_set_disks_isolated;
|
||||
use crate::io_support::rio::{WriteEncryption, WritePlan};
|
||||
use crate::storage_api_contracts::object::ObjectIO as _;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
let (_temp_dirs, disks, set_disks) = hermetic_set_disks_isolated(4).await;
|
||||
let bucket = "encrypted-etag-proof";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("create bucket volume");
|
||||
}
|
||||
let plaintext = b"Salary 2026: 85000 EUR";
|
||||
let size = plaintext.len() as i64;
|
||||
let mut control = HashReader::from_stream(Cursor::new(plaintext), size, size, None, None, false).expect("control reader");
|
||||
control.read_to_end(&mut Vec::new()).await.expect("read control plaintext");
|
||||
let plaintext_md5 = control.try_resolve_etag().expect("control MD5");
|
||||
for (object, preserved) in [("new", None), ("preserved", Some(plaintext_md5.clone()))] {
|
||||
let reader =
|
||||
HashReader::from_stream(Cursor::new(plaintext), size, size, None, None, false).expect("plaintext reader");
|
||||
let reader = WritePlan::new()
|
||||
.with_encryption(WriteEncryption::singlepart([1; 32], [2; 12]))
|
||||
.apply(reader, size)
|
||||
.expect("encrypted reader");
|
||||
let mut opts = ObjectOptions {
|
||||
preserve_etag: preserved.clone(),
|
||||
user_defined: HashMap::from([
|
||||
("x-amz-server-side-encryption-customer-algorithm".to_owned(), "AES256".to_owned()),
|
||||
(rustfs_utils::http::SSEC_ORIGINAL_SIZE_HEADER.to_owned(), size.to_string()),
|
||||
]),
|
||||
..Default::default()
|
||||
};
|
||||
// A copied producer marker cannot attest to a preserved plaintext tag.
|
||||
insert_str(&mut opts.user_defined, rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG, "v1".to_owned());
|
||||
set_disks
|
||||
.put_object(bucket, object, &mut PutObjReader::new(reader), &opts)
|
||||
.await
|
||||
.expect("store encrypted bytes");
|
||||
let stored = set_disks
|
||||
.get_object_info(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read persisted metadata");
|
||||
if let Some(preserved) = preserved {
|
||||
assert_eq!(stored.etag.as_deref(), Some(preserved.as_str()));
|
||||
assert!(!rustfs_utils::http::etag_can_be_listed(&stored.user_defined));
|
||||
} else {
|
||||
assert_ne!(stored.etag.as_ref(), Some(&plaintext_md5));
|
||||
assert_eq!(
|
||||
rustfs_utils::http::get_consistent_str(
|
||||
&stored.user_defined,
|
||||
rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG
|
||||
),
|
||||
Some("v1")
|
||||
);
|
||||
assert!(rustfs_utils::http::etag_can_be_listed(&stored.user_defined));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct CapturedLogs(Arc<Mutex<Vec<u8>>>);
|
||||
|
||||
@@ -19687,6 +19855,282 @@ mod put_object_tmp_cleanup_tests {
|
||||
.expect("three disks must publish metadata while the fourth rename remains paused");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn seeded_put_completion_pressure_drains_every_real_tail() {
|
||||
use super::hermetic_set_disks_support::hermetic_set_disks_isolated;
|
||||
use crate::disk::os;
|
||||
use rand::{Rng, SeedableRng, rngs::StdRng};
|
||||
use tokio::time::{Instant, timeout_at};
|
||||
|
||||
const SEED: u64 = 0x2248_0009;
|
||||
let mut rng = StdRng::seed_from_u64(SEED);
|
||||
for (round, (completion, cancel)) in [
|
||||
(WriteCompletion::Quorum, false),
|
||||
(WriteCompletion::Quorum, true),
|
||||
(WriteCompletion::TailDrained, false),
|
||||
(WriteCompletion::TailDrained, true),
|
||||
]
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
{
|
||||
let (dirs, disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
let bucket = "seeded-put-pressure";
|
||||
make_completion_test_bucket(&disks, bucket).await;
|
||||
let deadline = Instant::now() + Duration::from_secs(30);
|
||||
let objects: Vec<_> = (0..3).map(|slot| format!("pressure-{round}-{slot}")).collect();
|
||||
let mut bodies = Vec::with_capacity(3);
|
||||
let mut barriers = Vec::with_capacity(3);
|
||||
let mut trackers = Vec::with_capacity(3);
|
||||
let mut puts = Vec::with_capacity(3);
|
||||
let mut paused_disks = Vec::with_capacity(3);
|
||||
let quorum_barrier = (completion == WriteCompletion::Quorum && cancel)
|
||||
.then(|| PutObjectCommitBarrier::install(bucket, &objects[0], PutObjectCommitPause::AfterRenameQuorum));
|
||||
let handoff_barrier = (completion == WriteCompletion::Quorum && cancel)
|
||||
.then(|| PutObjectCommitBarrier::install(bucket, &objects[0], PutObjectCommitPause::AfterRenameHandoff));
|
||||
for (slot, object) in objects.iter().enumerate() {
|
||||
let mut body = vec![0; TEST_OBJECT_SIZE + 17 + 16 * slot];
|
||||
rng.fill_bytes(&mut body);
|
||||
let disk = usize::try_from(rng.next_u32() % 4).expect("four-disk schedule slot");
|
||||
paused_disks.push(disk);
|
||||
trackers.push(rename_fanout_barrier::observe_tasks(object));
|
||||
barriers.push(Some(rename_fanout_barrier::arm(object, disk, rename_fanout_barrier::PHASE_RENAME)));
|
||||
bodies.push(body.clone());
|
||||
let writer = Arc::clone(&set);
|
||||
let object = object.clone();
|
||||
let write_completion = if slot == 0 { completion } else { WriteCompletion::TailDrained };
|
||||
puts.push(tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(body);
|
||||
writer
|
||||
.put_object(
|
||||
bucket,
|
||||
&object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
write_completion,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
}));
|
||||
}
|
||||
for (slot, object) in objects.iter().enumerate() {
|
||||
barriers[slot]
|
||||
.as_ref()
|
||||
.expect("armed real-disk checkpoint")
|
||||
.wait_until_paused_before(deadline)
|
||||
.await;
|
||||
timeout_at(deadline, wait_for_paused_tail_metadata_quorum(&disks, bucket, object))
|
||||
.await
|
||||
.expect("target and both pressure writes must publish three real metadata records");
|
||||
assert!(trackers[slot].running() >= 1, "round {round} key {slot}: parked rename remains owned");
|
||||
}
|
||||
assert!(
|
||||
set.instance_ctx().namespace_commits_pending(),
|
||||
"parked real mutations must remain pending"
|
||||
);
|
||||
assert!(
|
||||
!puts[1].is_finished() && !puts[2].is_finished(),
|
||||
"pressure full-tail callers remain parked"
|
||||
);
|
||||
let mut target = Some(puts.remove(0));
|
||||
let mut written = None;
|
||||
if let Some(barrier) = quorum_barrier.as_ref() {
|
||||
timeout_at(deadline, barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("cancelled Quorum write reaches its actual quorum boundary");
|
||||
}
|
||||
if cancel {
|
||||
let caller = target.take().expect("target ACK waiter");
|
||||
assert!(!caller.is_finished(), "cancellation must reach an owned, still-pending ACK waiter");
|
||||
caller.abort();
|
||||
assert!(caller.await.expect_err("cancel target ACK waiter").is_cancelled());
|
||||
} else if completion == WriteCompletion::Quorum {
|
||||
written = Some(
|
||||
timeout_at(deadline, target.take().expect("Quorum ACK waiter"))
|
||||
.await
|
||||
.expect("Quorum ACK must precede the parked tail")
|
||||
.expect("Quorum caller joins")
|
||||
.expect("three real disk publications acknowledge the target"),
|
||||
);
|
||||
} else {
|
||||
assert!(
|
||||
!target.as_ref().expect("full-tail ACK waiter").is_finished(),
|
||||
"full-tail ACK waits past metadata quorum"
|
||||
);
|
||||
}
|
||||
if let Some(barrier) = quorum_barrier {
|
||||
barrier.release();
|
||||
drop(barrier);
|
||||
timeout_at(
|
||||
deadline,
|
||||
handoff_barrier
|
||||
.as_ref()
|
||||
.expect("Quorum handoff checkpoint")
|
||||
.wait_until_paused(),
|
||||
)
|
||||
.await
|
||||
.expect("cancelled Quorum continuation installs its tail owner");
|
||||
}
|
||||
let mut probe = Box::pin(set.acquire_write_lock_diag("seeded_pressure_owner_probe", bucket, &objects[0]));
|
||||
assert!(
|
||||
futures::poll!(probe.as_mut()).is_pending(),
|
||||
"round {round}: target owner survives ACK or cancellation"
|
||||
);
|
||||
assert!(trackers[0].running() >= 1, "target tail survives its caller");
|
||||
if let Some(barrier) = handoff_barrier {
|
||||
barrier.release();
|
||||
drop(barrier);
|
||||
}
|
||||
|
||||
// One pressure rename races the target; the other stays parked to
|
||||
// distinguish per-object full-tail completion from global quiescence.
|
||||
let first_pressure = 1 + usize::try_from(rng.next_u32() % 2).expect("two pressure keys");
|
||||
let held_pressure = 3 - first_pressure;
|
||||
let release_order = if rng.next_u32() & 1 == 0 {
|
||||
[first_pressure, 0]
|
||||
} else {
|
||||
[0, first_pressure]
|
||||
};
|
||||
for slot in release_order {
|
||||
let barrier = barriers[slot].take().expect("release reached checkpoint once");
|
||||
barrier.release();
|
||||
drop(barrier);
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
if let Some(caller) = target.take() {
|
||||
written = Some(
|
||||
timeout_at(deadline, caller)
|
||||
.await
|
||||
.expect("full-tail ACK follows target rename drain")
|
||||
.expect("full-tail caller joins")
|
||||
.expect("target full-tail write commits"),
|
||||
);
|
||||
assert_eq!(trackers[0].running(), 0, "full-tail ACK follows every target rename task");
|
||||
}
|
||||
drop(
|
||||
timeout_at(deadline, probe)
|
||||
.await
|
||||
.expect("target tail releases its actual namespace owner")
|
||||
.expect("next target writer acquires the namespace"),
|
||||
);
|
||||
assert_eq!(trackers[0].running(), 0, "target rename owner has drained");
|
||||
assert!(
|
||||
barriers[held_pressure]
|
||||
.as_ref()
|
||||
.expect("held pressure checkpoint")
|
||||
.is_paused()
|
||||
&& trackers[held_pressure].running() >= 1,
|
||||
"target must hand off while the other real pressure mutation remains parked"
|
||||
);
|
||||
{
|
||||
let mut read = timeout_at(
|
||||
deadline,
|
||||
set.get_object_reader(bucket, &objects[0], None, HeaderMap::new(), &ObjectOptions::default()),
|
||||
)
|
||||
.await
|
||||
.expect("target reader acquires within the round deadline")
|
||||
.expect("ACKed or cancelled-owned target remains readable");
|
||||
let mut actual = Vec::new();
|
||||
timeout_at(deadline, read.stream.read_to_end(&mut actual))
|
||||
.await
|
||||
.expect("target body drains before the round deadline")
|
||||
.expect("target body streams");
|
||||
assert_eq!(actual, bodies[0], "round {round}: exact target bytes survive the handoff");
|
||||
}
|
||||
if let Some(written) = written {
|
||||
let etag = written.etag.expect("successful target ACK must supply the CAS ETag");
|
||||
assert!(!etag.is_empty(), "same-key CAS must carry an actual If-Match condition");
|
||||
let mut successor = vec![0; TEST_OBJECT_SIZE + 65];
|
||||
rng.fill_bytes(&mut successor);
|
||||
let mut reader = PutObjReader::from_vec(successor.clone());
|
||||
timeout_at(
|
||||
deadline,
|
||||
set.put_object(
|
||||
bucket,
|
||||
&objects[0],
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_match: Some(etag),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("same-key CAS acquires the handed-off owner under pressure")
|
||||
.expect("same-key CAS accepts the exact target ETag");
|
||||
bodies[0] = successor;
|
||||
}
|
||||
let barrier = barriers[held_pressure].take().expect("final pressure checkpoint");
|
||||
barrier.release();
|
||||
drop(barrier);
|
||||
for caller in puts {
|
||||
timeout_at(deadline, caller)
|
||||
.await
|
||||
.expect("pressure writer drains within the finite round")
|
||||
.expect("pressure caller joins")
|
||||
.expect("pressure full-tail write commits");
|
||||
}
|
||||
for (slot, object) in objects.iter().enumerate() {
|
||||
let mut read = timeout_at(
|
||||
deadline,
|
||||
set.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()),
|
||||
)
|
||||
.await
|
||||
.expect("every reader acquires within the finite round")
|
||||
.expect("every pressure and target object is readable");
|
||||
let mut actual = Vec::new();
|
||||
timeout_at(deadline, read.stream.read_to_end(&mut actual))
|
||||
.await
|
||||
.expect("complete body drains within the round")
|
||||
.expect("complete body streams");
|
||||
assert_eq!(actual, bodies[slot], "round {round} key {slot}: exact production GET oracle");
|
||||
for disk in &disks {
|
||||
let fi = timeout_at(deadline, disk.read_version("", bucket, object, "", &ReadOptions::default()))
|
||||
.await
|
||||
.expect("physical metadata read remains bounded")
|
||||
.expect("every healthy disk publishes the object");
|
||||
assert!(!fi.inline_data(), "pressure must write actual external shards");
|
||||
assert_eq!(fi.parts[0].size, bodies[slot].len());
|
||||
let root = disk.path();
|
||||
let path = root.join(bucket).join(object);
|
||||
let part = path
|
||||
.join(fi.data_dir.expect("non-inline data directory").to_string())
|
||||
.join("part.1");
|
||||
assert!(
|
||||
timeout_at(deadline, tokio::fs::metadata(part))
|
||||
.await
|
||||
.expect("external shard inspection remains bounded")
|
||||
.expect("actual external shard file")
|
||||
.len()
|
||||
> 32
|
||||
);
|
||||
drop(
|
||||
timeout_at(deadline, os::acquire_rename_data_mutation_lease(&root, bucket, &path))
|
||||
.await
|
||||
.expect("physical mutation owner must release every real disk and key"),
|
||||
);
|
||||
}
|
||||
assert_eq!(trackers[slot].running(), 0, "round {round} key {slot}: no fan-out task remains");
|
||||
}
|
||||
timeout_at(deadline, async {
|
||||
while set.instance_ctx().namespace_commits_pending() || !non_trash_tmp_entries(&dirs).await.is_empty() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("all real mutation owners and staging entries must drain");
|
||||
assert!(!set.instance_ctx().namespace_commits_pending());
|
||||
eprintln!(
|
||||
"seed={SEED:#x} round={round} completion={completion:?} cancel={cancel} paused_disks={paused_disks:?} drained"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn tail_drained_put_waits_for_tail_and_allows_immediate_cas() {
|
||||
@@ -19817,6 +20261,16 @@ mod put_object_tmp_cleanup_tests {
|
||||
.await
|
||||
.expect("committed object must remain readable despite the failed tail");
|
||||
assert_eq!(info.size, TEST_OBJECT_SIZE as i64);
|
||||
let mut read = set
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("committed body must remain readable despite the failed tail");
|
||||
let mut body = Vec::new();
|
||||
read.stream
|
||||
.read_to_end(&mut body)
|
||||
.await
|
||||
.expect("committed body must stream completely after the failed tail");
|
||||
assert_eq!(body, vec![b'1'; TEST_OBJECT_SIZE]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -20002,6 +20456,93 @@ mod put_object_tmp_cleanup_tests {
|
||||
wait_for_tmp_workspace_to_drain(&dirs, "cancelled full-tail commit should release staging ownership").await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn borrowed_write_context_rejects_wrong_namespace_and_cache_flag() {
|
||||
let (_dirs, _disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "borrowed-write-target";
|
||||
let object = "owned-object";
|
||||
let guard = crate::object_api::WriteCommitGuard::acquire(
|
||||
&set.new_ns_lock(bucket, object).await.expect("namespace wrapper"),
|
||||
get_lock_acquire_timeout(),
|
||||
)
|
||||
.await
|
||||
.expect("outer write lock");
|
||||
let mut opts = ObjectOptions::default();
|
||||
opts.add_write_commit_guard(&guard);
|
||||
assert!(matches!(
|
||||
WriteCommitContext::from_options(&opts, bucket, "other-object", false),
|
||||
Err(Error::InvalidArgument(..))
|
||||
));
|
||||
assert!(matches!(
|
||||
WriteCommitContext::from_options(&opts, bucket, "other-object", true),
|
||||
Err(Error::InvalidArgument(..))
|
||||
));
|
||||
let fake = ObjectOptions {
|
||||
no_lock: true,
|
||||
metadata_cache_safe: true,
|
||||
..Default::default()
|
||||
};
|
||||
assert!(matches!(
|
||||
WriteCommitContext::from_options(&fake, bucket, object, false),
|
||||
Err(Error::InvalidArgument(..))
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn borrowed_tail_drained_put_keeps_owner_after_caller_cancel() {
|
||||
let (_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "borrowed-put-cancel";
|
||||
let object = "borrowed-put-cancel-object";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("bucket volume");
|
||||
}
|
||||
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
let tasks = rename_fanout_barrier::observe_tasks(object);
|
||||
let writer = Arc::clone(&set);
|
||||
let put = tokio::spawn(async move {
|
||||
let guard = crate::object_api::WriteCommitGuard::acquire(
|
||||
&writer.new_ns_lock(bucket, object).await.expect("namespace wrapper"),
|
||||
get_lock_acquire_timeout(),
|
||||
)
|
||||
.await
|
||||
.expect("outer write guard");
|
||||
let mut opts = ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
};
|
||||
opts.add_write_commit_guard(&guard);
|
||||
let mut reader = PutObjReader::from_vec(vec![b'b'; TEST_OBJECT_SIZE]);
|
||||
writer.put_object(bucket, object, &mut reader, &opts).await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("borrowed commit reaches rename phase");
|
||||
wait_for_paused_tail_metadata_quorum(&disks, bucket, object).await;
|
||||
assert!(!put.is_finished(), "full-tail borrowed write remains pending");
|
||||
put.abort();
|
||||
assert!(put.await.expect_err("caller was canceled").is_cancelled());
|
||||
let mut probe = Box::pin(set.acquire_write_lock_diag("borrowed_cancel_probe", bucket, object));
|
||||
assert!(
|
||||
futures::poll!(probe.as_mut()).is_pending(),
|
||||
"commit must retain the real borrowed namespace owner"
|
||||
);
|
||||
barrier.release();
|
||||
drop(
|
||||
tokio::time::timeout(Duration::from_secs(30), probe)
|
||||
.await
|
||||
.expect("borrowed owner drains")
|
||||
.expect("next writer acquires"),
|
||||
);
|
||||
assert_eq!(tasks.running(), 0, "all borrowed rename tasks are reaped before releasing the owner");
|
||||
let info = set
|
||||
.get_object_info(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("committed borrowed write remains readable");
|
||||
assert_eq!(info.size, TEST_OBJECT_SIZE as i64);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn no_lock_put_waits_for_rename_tail_under_outer_guard() {
|
||||
@@ -20013,27 +20554,23 @@ mod put_object_tmp_cleanup_tests {
|
||||
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
||||
}
|
||||
|
||||
let outer_guard = set_disks
|
||||
.acquire_write_lock_diag("outer_no_lock_put", bucket, object)
|
||||
.await
|
||||
.expect("the outer caller should hold the namespace guard");
|
||||
let outer_guard = crate::object_api::WriteCommitGuard::acquire(
|
||||
&set_disks.new_ns_lock(bucket, object).await.expect("outer namespace wrapper"),
|
||||
get_lock_acquire_timeout(),
|
||||
)
|
||||
.await
|
||||
.expect("the outer caller should hold the namespace guard");
|
||||
let rename_tasks = rename_fanout_barrier::observe_tasks(object);
|
||||
let rename_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
let mut opts = ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
};
|
||||
opts.add_write_commit_guard(&outer_guard);
|
||||
let put_store = Arc::clone(&set_disks);
|
||||
let put = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
|
||||
put_store
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
put_store.put_object(bucket, object, &mut reader, &opts).await
|
||||
});
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(30), rename_barrier.wait_until_paused())
|
||||
@@ -22427,6 +22964,119 @@ mod body_cache_hook_e2e_tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod marker_purge_receipt_tests {
|
||||
use super::hermetic_set_disks_support::hermetic_set_disks_isolated;
|
||||
use super::*;
|
||||
use crate::disk::ReadOptions;
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn quorum_marker_purge_receipt_survives_reopen_with_stale_member() {
|
||||
let (_dirs, disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
let bucket = "marker-purge-receipt";
|
||||
let object = "explicit-marker-version";
|
||||
let version = Uuid::new_v4();
|
||||
let incarnation = Uuid::new_v4();
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("fixture bucket");
|
||||
}
|
||||
let mut marker = FileInfo {
|
||||
name: object.to_string(),
|
||||
version_id: Some(version),
|
||||
deleted: true,
|
||||
mark_deleted: true,
|
||||
mod_time: Some(OffsetDateTime::now_utc()),
|
||||
..Default::default()
|
||||
};
|
||||
marker.set_delete_marker_incarnation(incarnation);
|
||||
let owner = set
|
||||
.acquire_write_lock_diag("marker_purge_receipt_test", bucket, object)
|
||||
.await
|
||||
.expect("own the deleted object namespace");
|
||||
set.delete_object_version_with_purge(bucket, object, &marker, true, None, Some(incarnation))
|
||||
.await
|
||||
.expect("seed the real marker on every disk");
|
||||
let marker = disks[0]
|
||||
.read_version("", bucket, object, &version.to_string(), &ReadOptions::default())
|
||||
.await
|
||||
.expect("read the persisted marker identity");
|
||||
let purge = delete_marker_purge_candidate(&marker, Some(incarnation))
|
||||
.expect("the canonical marker must authorize its exact purge");
|
||||
let request = FileInfo {
|
||||
name: object.to_string(),
|
||||
version_id: Some(version),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// A member that misses the explicit DELETE retains its original
|
||||
// marker. The other three disks still satisfy configured write quorum.
|
||||
set.disks.write().await[0] = None;
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(10),
|
||||
set.delete_object_version_with_purge(bucket, object, &request, false, Some(purge.clone()), Some(incarnation)),
|
||||
)
|
||||
.await
|
||||
.expect("the receipt must not reacquire the deleted object's owner")
|
||||
.expect("the explicit marker purge must reach write quorum");
|
||||
drop(owner);
|
||||
for disk in &disks[1..] {
|
||||
assert!(
|
||||
matches!(
|
||||
disk.read_version("", bucket, object, &version.to_string(), &ReadOptions::default())
|
||||
.await,
|
||||
Err(DiskError::FileNotFound | DiskError::FileVersionNotFound)
|
||||
),
|
||||
"acknowledging members must remove the exact marker"
|
||||
);
|
||||
}
|
||||
let stale = disks[0]
|
||||
.read_version("", bucket, object, &version.to_string(), &ReadOptions::default())
|
||||
.await
|
||||
.expect("the offline member must retain its marker");
|
||||
assert!(stale.deleted);
|
||||
assert_eq!(stale.version_id, Some(version));
|
||||
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
ctx.update_erasure_type(crate::layout::endpoints::SetupType::Erasure).await;
|
||||
let reopened = SetDisks::new_with_instance_ctx(
|
||||
set.locker_owner.clone(),
|
||||
Arc::new(tokio::sync::RwLock::new(disks.iter().cloned().map(Some).collect())),
|
||||
set.set_drive_count,
|
||||
set.default_parity_count,
|
||||
set.set_index,
|
||||
set.pool_index,
|
||||
set.set_endpoints.clone(),
|
||||
set.format.clone(),
|
||||
Vec::new(),
|
||||
ctx,
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
reopened.has_marker_purge_receipt(bucket, object, version, &purge).await,
|
||||
"a successful explicit DELETE must leave durable proof for stale-marker cleanup after reopen"
|
||||
);
|
||||
let receipt = crate::set_disk::marker_purge_receipt_path(bucket, object, version, &purge);
|
||||
assert_eq!(
|
||||
crate::config::com::read_config_limited_preserve_empty(reopened.clone(), &receipt, 128)
|
||||
.await
|
||||
.expect("read the physical quorum receipt"),
|
||||
b"rustfs-marker-purge-receipt-v1"
|
||||
);
|
||||
assert!(
|
||||
!reopened
|
||||
.has_marker_purge_receipt(bucket, object, Uuid::new_v4(), &purge)
|
||||
.await,
|
||||
"the receipt must not authorize another marker version"
|
||||
);
|
||||
reopened.consume_marker_purge_receipt(bucket, object, version, &purge).await;
|
||||
assert!(
|
||||
!reopened.has_marker_purge_receipt(bucket, object, version, &purge).await,
|
||||
"cleanup must be able to consume the persisted receipt"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod single_delete_namespace_owner_tests {
|
||||
use super::hermetic_set_disks_support::hermetic_set_disks_isolated;
|
||||
|
||||
@@ -113,7 +113,28 @@ use super::GetCodecStreamingObjectClass;
|
||||
use super::GetCodecStreamingRollout;
|
||||
#[cfg(test)]
|
||||
use super::classify_get_codec_streaming_object_class;
|
||||
use super::core::io_primitives::*;
|
||||
use super::core::io_primitives::{
|
||||
BitrotReaderSetup, BitrotReaderSetupAttribution, BitrotReaderSetupMode, DeferredReaderReopener, EVENT_SET_DISK_READ,
|
||||
GetCodecStreamingReaderBuildOutcome, MetadataCacheLookup, ObjectBitrotReader, ReadRepairAdmissionSubmitter,
|
||||
ReadRepairHealSubmission, SLOW_OBJECT_READ_LOG_THRESHOLD, codec_streaming_reader_setup_fallback_reason,
|
||||
create_bitrot_readers_until_quorum_with_preference, create_data_block_bitrot_readers, resolved_read_repair_version_id,
|
||||
send_read_repair_heal_request, shard_read_costs_for_disks, submit_read_repair_heal, submit_read_repair_heal_with_submitter,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use super::core::io_primitives::{
|
||||
ENV_RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_READER_SETUP, ENV_RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP,
|
||||
MultipartCodecStreamingReader, ReadRepairAdmissionFuture, ReadRepairAdmissionOutcome, collect_read_multiple_results,
|
||||
collect_read_parts_results, create_bitrot_readers_until_quorum, create_bitrot_readers_until_quorum_all_shards,
|
||||
release_read_repair_heal_reservation, reserve_read_repair_heal, resolve_read_part_from_responses, shard_read_cost_for_disk,
|
||||
shard_read_cost_for_endpoint,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use super::core::metadata_quorum::{MetadataEarlyStopDecision, MetadataQuorumAccumulator};
|
||||
#[cfg(test)]
|
||||
use super::core::metadata_read::{
|
||||
MetadataFanoutDiagnostics, MetadataFanoutObservation, metadata_early_stop_permitted, should_allow_metadata_early_stop,
|
||||
};
|
||||
use super::core::metadata_read::{late_materialization_candidate_is_safe, non_inline_data_read_early_stop_allowed};
|
||||
#[cfg(test)]
|
||||
use super::get_codec_streaming_config_cached_core;
|
||||
#[cfg(test)]
|
||||
@@ -607,11 +628,11 @@ impl SetDisks {
|
||||
};
|
||||
|
||||
// Early-stop for safe metadata reads is handled inside
|
||||
// read_all_fileinfo_observed (see read_all_fileinfo_early_stop in
|
||||
// core/io_primitives.rs); unsafe requests and callers that opt out
|
||||
// read_metadata_observed (see read_all_fileinfo_early_stop in
|
||||
// core/metadata_read.rs); unsafe requests and callers that opt out
|
||||
// (allow_early_stop=false) fall back to full-wait.
|
||||
let (mut parts_metadata, errs, metadata_fanout_diagnostics) = if allow_read_version_coalescing {
|
||||
Self::read_all_fileinfo_observed_for_get_object(
|
||||
let metadata_read = if allow_read_version_coalescing {
|
||||
Self::read_metadata_for_get_object(
|
||||
&disks,
|
||||
"",
|
||||
bucket,
|
||||
@@ -624,7 +645,7 @@ impl SetDisks {
|
||||
)
|
||||
.await?
|
||||
} else {
|
||||
Self::read_all_fileinfo_observed(
|
||||
Self::read_metadata_observed(
|
||||
&disks,
|
||||
"",
|
||||
bucket,
|
||||
@@ -638,13 +659,14 @@ impl SetDisks {
|
||||
)
|
||||
.await?
|
||||
};
|
||||
let metadata_fanout_complete = metadata_read.is_complete();
|
||||
let (mut parts_metadata, errs, metadata_fanout_diagnostics) = metadata_read.into_legacy();
|
||||
let metadata_metrics_path = if crate::bucket::utils::is_meta_bucketname(bucket) {
|
||||
GET_OBJECT_PATH_INTERNAL_META
|
||||
} else {
|
||||
GET_OBJECT_PATH_LEGACY_DUPLEX
|
||||
};
|
||||
metadata_fanout_diagnostics.record(metadata_metrics_path);
|
||||
let metadata_fanout_complete = metadata_fanout_diagnostics.total_responses() >= disks.len();
|
||||
// warn!("get_object_fileinfo parts_metadata {:?}", &parts_metadata);
|
||||
// warn!("get_object_fileinfo {}/{} errs {:?}", bucket, object, &errs);
|
||||
|
||||
@@ -2117,7 +2139,7 @@ impl SetDisks {
|
||||
expected: &LateMetadataIdentity,
|
||||
metrics_path: &'static str,
|
||||
) -> Result<(FileInfo, Vec<FileInfo>, Vec<Option<DiskStore>>)> {
|
||||
let (mut parts_metadata, errs, diagnostics) = SetDisks::read_all_fileinfo_observed(
|
||||
let metadata_read = SetDisks::read_metadata_observed(
|
||||
fallback_disks,
|
||||
"",
|
||||
bucket,
|
||||
@@ -2130,6 +2152,7 @@ impl SetDisks {
|
||||
expected.parity_blocks,
|
||||
)
|
||||
.await?;
|
||||
let (mut parts_metadata, errs, diagnostics) = metadata_read.into_legacy();
|
||||
diagnostics.record(metrics_path);
|
||||
|
||||
let (read_quorum, write_quorum) = SetDisks::object_quorum_from_meta(&parts_metadata, &errs, expected.parity_blocks)
|
||||
@@ -3534,6 +3557,245 @@ mod metadata_cache_tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn metadata_cache_real_get_and_range_preserve_body_across_read_modes() {
|
||||
use crate::object_api::{PutObjReader, WriteCompletion};
|
||||
use crate::set_disk::core::io_primitives::disk_call_counters;
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions};
|
||||
use crate::storage_api_contracts::object::ObjectIO;
|
||||
use crate::storage_api_contracts::range::HTTPRangeSpec;
|
||||
use http::HeaderMap;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
for size in [4097, 1024 * 1024 + 321] {
|
||||
for early_stop in [false, true] {
|
||||
for offline in [false, true] {
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let (_dirs, set) = crate::ecstore_validation_blackbox::make_local_set_disks_with_ctx(4, 2, ctx).await;
|
||||
let bucket = "metadata-cache-real-body";
|
||||
let object = format!("body-{size}-{early_stop}-{offline}");
|
||||
let body = (0..size)
|
||||
.map(|index| u8::try_from(index % 251).expect("bounded fixture byte"))
|
||||
.collect::<Vec<_>>();
|
||||
set.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create real bucket");
|
||||
set.put_object(
|
||||
bucket,
|
||||
&object,
|
||||
&mut PutObjReader::from_vec(body.clone()),
|
||||
&ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("commit all real shards before reading");
|
||||
if offline {
|
||||
set.disks.write().await[0] = None;
|
||||
}
|
||||
let opts = ObjectOptions {
|
||||
suppress_read_repair: true,
|
||||
..Default::default()
|
||||
};
|
||||
assert!(
|
||||
set.get_object_metadata_cache_bypass_reason(bucket, &opts, true)
|
||||
.await
|
||||
.is_none()
|
||||
);
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some(if early_stop { "true" } else { "false" })),
|
||||
(
|
||||
"RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE",
|
||||
Some(if early_stop { "true" } else { "false" }),
|
||||
),
|
||||
(
|
||||
"RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT",
|
||||
Some(if early_stop { "true" } else { "false" }),
|
||||
),
|
||||
],
|
||||
async {
|
||||
let calls = disk_call_counters::observe(&object);
|
||||
for range in [
|
||||
None,
|
||||
Some(HTTPRangeSpec {
|
||||
is_suffix_length: false,
|
||||
start: 17,
|
||||
end: 103,
|
||||
}),
|
||||
] {
|
||||
set.invalidate_get_object_metadata_cache(bucket, &object).await;
|
||||
assert!(
|
||||
set.cached_get_object_fileinfo(bucket, &object).await.is_none(),
|
||||
"miss fixture must be uncached"
|
||||
);
|
||||
let expected = if range.is_some() { &body[17..104] } else { body.as_slice() };
|
||||
let before_miss = calls.total(disk_call_counters::KIND_READ_VERSION);
|
||||
let mut miss = set
|
||||
.get_object_reader(bucket, &object, range.clone(), HeaderMap::new(), &opts)
|
||||
.await
|
||||
.expect("real miss reader");
|
||||
assert_eq!(miss.object_info.size, i64::try_from(body.len()).expect("fixture object size"));
|
||||
let mut miss_body = Vec::new();
|
||||
miss.stream.read_to_end(&mut miss_body).await.expect("stream real cache miss");
|
||||
assert_eq!(miss_body, expected);
|
||||
assert!(
|
||||
calls.total(disk_call_counters::KIND_READ_VERSION) > before_miss,
|
||||
"miss must issue disk metadata reads"
|
||||
);
|
||||
|
||||
// Partial early-stop snapshots cannot be cached. Warm through the
|
||||
// existing full-wait read, retaining its normal publication rules.
|
||||
set.get_object_fileinfo_gated(bucket, &object, &opts, true, false)
|
||||
.await
|
||||
.expect("complete metadata snapshot");
|
||||
let cached = set
|
||||
.cached_get_object_fileinfo(bucket, &object)
|
||||
.await
|
||||
.expect("real complete snapshot published");
|
||||
assert!(cached.read_quorum > 0);
|
||||
assert_eq!(
|
||||
cached.fi.inline_data(),
|
||||
size == 4097,
|
||||
"the real disk fixture must reach the intended storage shape"
|
||||
);
|
||||
assert!(cached.online_disks.iter().filter(|disk| disk.is_some()).count() >= cached.read_quorum);
|
||||
assert_eq!(cached.online_disks.len(), 4, "snapshot must retain every disk slot");
|
||||
assert_eq!(
|
||||
cached.online_disks.iter().flatten().count(),
|
||||
if offline { 3 } else { 4 },
|
||||
"snapshot must include every available disk rather than only the first quorum"
|
||||
);
|
||||
let before_hit = calls.total(disk_call_counters::KIND_READ_VERSION);
|
||||
let snapshot = set
|
||||
.get_object_fileinfo_gated(bucket, &object, &opts, true, false)
|
||||
.await
|
||||
.expect("real metadata cache hit");
|
||||
assert!(
|
||||
snapshot.shared_entry().is_some_and(|entry| Arc::ptr_eq(entry, &cached)),
|
||||
"hit must share the published disk snapshot"
|
||||
);
|
||||
let mut hit = set
|
||||
.get_object_reader(bucket, &object, range, HeaderMap::new(), &opts)
|
||||
.await
|
||||
.expect("real hit reader");
|
||||
assert_eq!(hit.object_info.size, miss.object_info.size);
|
||||
let mut hit_body = Vec::new();
|
||||
hit.stream.read_to_end(&mut hit_body).await.expect("stream real cache hit");
|
||||
assert_eq!(hit_body.len(), expected.len());
|
||||
assert_eq!(hit_body, miss_body);
|
||||
assert_eq!(
|
||||
calls.total(disk_call_counters::KIND_READ_VERSION),
|
||||
before_hit,
|
||||
"cached GET and Range must not refetch metadata"
|
||||
);
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn late_metadata_refresh_rejects_changed_generation_and_mapped_shards() {
|
||||
use crate::object_api::{PutObjReader, WriteCompletion};
|
||||
use crate::set_disk::core::io_primitives::disk_call_counters;
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions};
|
||||
use crate::storage_api_contracts::object::ObjectIO;
|
||||
|
||||
let bucket = "late-metadata-identity";
|
||||
let object = "late-metadata-identity-object";
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let (_dirs, set) = crate::ecstore_validation_blackbox::make_local_set_disks_with_ctx(4, 2, ctx).await;
|
||||
set.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create real late-read bucket");
|
||||
set.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![7; 1024 * 1024 + 321]),
|
||||
&ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("commit real metadata and all shard files before late refresh");
|
||||
let disks = set.get_disks_internal().await;
|
||||
let original = SetDisks::read_metadata_observed(&disks, "", bucket, object, "", true, false, false, false, 2)
|
||||
.await
|
||||
.expect("read original real disk metadata");
|
||||
assert!(original.is_complete());
|
||||
let (baseline, errors, _) = original.into_legacy();
|
||||
assert!(errors.iter().all(Option::is_none), "original real shards must all be readable");
|
||||
let (_, selected, _) =
|
||||
SetDisks::select_valid_fileinfo(&disks, &baseline, &errors, "", 2, 3).expect("original metadata quorum");
|
||||
assert!(!selected.inline_data(), "late fixture must retain real non-inline shard files");
|
||||
let expected = LateMetadataIdentity::from_file_info(&selected);
|
||||
let (_, _, original_online) =
|
||||
SetDisks::refresh_late_metadata_fanout(&disks, bucket, object, &expected, GET_OBJECT_PATH_LEGACY_DUPLEX)
|
||||
.await
|
||||
.expect("matching late metadata remains usable");
|
||||
assert_eq!(original_online.iter().flatten().count(), 4);
|
||||
|
||||
for changed_generation in [true, false] {
|
||||
let changed_directory = uuid::Uuid::new_v4();
|
||||
for (index, disk) in disks.iter().enumerate() {
|
||||
let mut metadata = baseline[index].clone();
|
||||
if changed_generation {
|
||||
metadata.data_dir = Some(changed_directory);
|
||||
} else {
|
||||
metadata.erasure.index = metadata.erasure.index % 4 + 1;
|
||||
}
|
||||
assert!(
|
||||
metadata.validate_for_metadata_read().is_ok(),
|
||||
"mutated late metadata must remain decodable"
|
||||
);
|
||||
disk.as_ref()
|
||||
.expect("real disk slot")
|
||||
.write_metadata(bucket, bucket, object, metadata)
|
||||
.await
|
||||
.expect("persist late identity change");
|
||||
}
|
||||
let changed = SetDisks::read_metadata_observed(&disks, "", bucket, object, "", true, false, false, false, 2)
|
||||
.await
|
||||
.expect("read changed real disk metadata");
|
||||
assert!(changed.is_complete(), "late fixture must reach complete metadata observation");
|
||||
let (metadata, errors, _) = changed.into_legacy();
|
||||
let (online, selected, _) = SetDisks::select_valid_fileinfo(&disks, &metadata, &errors, "", 2, 3)
|
||||
.expect("changed metadata still reaches selection quorum");
|
||||
assert_eq!(online.iter().flatten().count(), 4, "the late identity guard must decide this failure");
|
||||
if changed_generation {
|
||||
assert_ne!(SetDisks::file_info_quorum_hash(&selected), expected.quorum_hash);
|
||||
} else {
|
||||
assert_eq!(SetDisks::file_info_quorum_hash(&selected), expected.quorum_hash);
|
||||
assert!(
|
||||
metadata
|
||||
.iter()
|
||||
.enumerate()
|
||||
.all(|(index, item)| expected.distribution[index] != item.erasure.index)
|
||||
);
|
||||
}
|
||||
let calls = disk_call_counters::observe(object);
|
||||
let result =
|
||||
SetDisks::refresh_late_metadata_fanout(&disks, bucket, object, &expected, GET_OBJECT_PATH_LEGACY_DUPLEX).await;
|
||||
assert_eq!(
|
||||
calls.total(disk_call_counters::KIND_READ_VERSION),
|
||||
4,
|
||||
"late refresh must read all real disk slots before rejection"
|
||||
);
|
||||
assert!(
|
||||
matches!(result, Err(Error::InsufficientReadQuorum(ref failed_bucket, ref failed_object))
|
||||
if failed_bucket == bucket && failed_object == object),
|
||||
"changed late identity cannot yield usable shards (changed_generation={changed_generation})"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_object_metadata_cache_rejects_deleted_and_invalid_fileinfo() {
|
||||
let set = new_metadata_cache_test_set().await;
|
||||
|
||||
@@ -121,9 +121,7 @@ async fn table_catalog_metadata_exists(ctx: &crate::runtime::instance::InstanceC
|
||||
}
|
||||
|
||||
async fn validate_table_bucket_delete_guard(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result<()> {
|
||||
let table_bucket_enabled = metadata_sys::get_in(ctx, bucket)
|
||||
.await
|
||||
.is_ok_and(|metadata| metadata.table_bucket_enabled());
|
||||
let table_bucket_enabled = table_bucket_enabled_in(ctx, bucket).await?;
|
||||
if table_bucket_enabled {
|
||||
validate_table_bucket_delete_allowed(bucket, true, table_catalog_metadata_exists(ctx, bucket).await?)?;
|
||||
}
|
||||
@@ -131,6 +129,21 @@ async fn validate_table_bucket_delete_guard(ctx: &crate::runtime::instance::Inst
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn table_bucket_enabled_in(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result<bool> {
|
||||
match metadata_sys::get_in(ctx, bucket).await {
|
||||
Ok(metadata) => Ok(metadata.table_bucket_enabled()),
|
||||
// A live bucket may be absent from the metadata cache during lazy
|
||||
// startup. Re-read persisted metadata before deciding that it is an
|
||||
// ordinary bucket; an actual metadata backend failure must never turn
|
||||
// a protected-bucket check into a best-effort operation.
|
||||
Err(Error::ConfigNotFound) => {
|
||||
let (metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(ctx, bucket).await?;
|
||||
Ok(persisted && metadata.table_bucket_enabled())
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
fn bucket_delete_metadata_cleanup_prefixes(bucket: &str) -> [String; 2] {
|
||||
[
|
||||
table_bucket_catalog_metadata_prefix(bucket),
|
||||
@@ -191,6 +204,52 @@ where
|
||||
.await
|
||||
}
|
||||
|
||||
fn bucket_recovery_lock_unavailable(bucket: &str, object: &str) -> StorageError {
|
||||
StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "write",
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
}
|
||||
}
|
||||
|
||||
async fn await_orphan_bucket_recovery<T, F>(
|
||||
bucket: &str,
|
||||
publication_guard: &rustfs_lock::NamespaceLockGuard,
|
||||
lifecycle_guard: &rustfs_lock::NamespaceLockGuard,
|
||||
metadata_guard: &rustfs_lock::NamespaceLockGuard,
|
||||
namespace_guard: &rustfs_lock::NamespaceLockGuard,
|
||||
future: F,
|
||||
) -> Result<T>
|
||||
where
|
||||
F: Future<Output = Result<T>>,
|
||||
{
|
||||
let metadata_lock_object = crate::bucket::metadata_sys::bucket_metadata_transaction_lock_key(bucket);
|
||||
for ((lock_bucket, lock_object), guard) in [
|
||||
(
|
||||
(bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH),
|
||||
publication_guard,
|
||||
),
|
||||
((bucket, BUCKET_LIFECYCLE_LOCK_OBJECT), lifecycle_guard),
|
||||
((RUSTFS_META_BUCKET, metadata_lock_object.as_str()), metadata_guard),
|
||||
((bucket, bucket), namespace_guard),
|
||||
] {
|
||||
if guard.is_lock_lost() {
|
||||
return Err(bucket_recovery_lock_unavailable(lock_bucket, lock_object));
|
||||
}
|
||||
}
|
||||
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = publication_guard.lock_lost_notified() => Err(bucket_recovery_lock_unavailable(bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)),
|
||||
_ = lifecycle_guard.lock_lost_notified() => Err(bucket_recovery_lock_unavailable(bucket, BUCKET_LIFECYCLE_LOCK_OBJECT)),
|
||||
_ = metadata_guard.lock_lost_notified() => Err(bucket_recovery_lock_unavailable(RUSTFS_META_BUCKET, &metadata_lock_object)),
|
||||
_ = namespace_guard.lock_lost_notified() => Err(bucket_recovery_lock_unavailable(bucket, bucket)),
|
||||
result = future => result,
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_bucket_usage_cleanup<F>(guard: Option<&rustfs_lock::NamespaceLockGuard>, bucket: &str, future: F) -> Result<()>
|
||||
where
|
||||
F: Future<Output = Result<()>>,
|
||||
@@ -259,6 +318,20 @@ async fn bucket_delete_local_blocker(
|
||||
}
|
||||
|
||||
impl ECStore {
|
||||
/// Run a multi-step storage mutation on the instance task tracker so its
|
||||
/// caller may stop waiting without releasing guards before the mutation
|
||||
/// finishes.
|
||||
pub async fn run_detached_mutation<F>(&self, mutation: F) -> Result<F::Output>
|
||||
where
|
||||
F: Future + Send + 'static,
|
||||
F::Output: Send + 'static,
|
||||
{
|
||||
self.ctx
|
||||
.run_detached_mutation(mutation)
|
||||
.await
|
||||
.map_err(|error| StorageError::other_with_context("detached ECStore mutation failed", error))
|
||||
}
|
||||
|
||||
fn bucket_sets(&self) -> impl Iterator<Item = (usize, usize, Arc<crate::set_disk::SetDisks>)> + '_ {
|
||||
self.pools.iter().flat_map(|pool| {
|
||||
pool.disk_set
|
||||
@@ -267,6 +340,113 @@ impl ECStore {
|
||||
})
|
||||
}
|
||||
|
||||
async fn ensure_bucket_volume_absent_on_all_sets(&self, bucket: &str) -> Result<()> {
|
||||
let sets: Vec<_> = self.bucket_sets().map(|(_, _, set)| set).collect();
|
||||
if sets.is_empty() {
|
||||
return Err(StorageError::ErasureReadQuorum);
|
||||
}
|
||||
let results = join_all(sets.iter().map(|set| set.ensure_bucket_volume_absent_on_every_disk(bucket))).await;
|
||||
for result in results {
|
||||
result?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Return whether an old bucket generation has a durable retirement record.
|
||||
/// A live matching incarnation is never considered retired, even if a stale
|
||||
/// record exists from a previous generation.
|
||||
pub async fn is_bucket_incarnation_retired(&self, bucket: &str, expected: Uuid) -> Result<bool> {
|
||||
if expected.is_nil() {
|
||||
return Ok(false);
|
||||
}
|
||||
let retirement_store = metadata_sys::object_store_if_initialized_in(&self.ctx).await.ok_or_else(|| {
|
||||
StorageError::InsufficientReadQuorum(RUSTFS_META_BUCKET.to_string(), "bucket-retirements".to_string())
|
||||
})?;
|
||||
if crate::bucket::metadata::load_bucket_incarnation(retirement_store.clone(), bucket).await? == Some(expected) {
|
||||
return Ok(false);
|
||||
}
|
||||
crate::bucket::retirement::is_retired(retirement_store, bucket, expected).await
|
||||
}
|
||||
|
||||
/// Reconcile bucket metadata after an operator removed all physical volume
|
||||
/// directories. The expected generation, all-disk absence proof and durable
|
||||
/// retirement record make retries safe after partial metadata cleanup.
|
||||
pub fn recover_orphaned_bucket<'a>(&'a self, bucket: &'a str, expected: Uuid) -> futures::future::BoxFuture<'a, Result<()>> {
|
||||
Box::pin(async move { Box::pin(self.recover_orphaned_bucket_inner(bucket, expected)).await })
|
||||
}
|
||||
|
||||
async fn recover_orphaned_bucket_inner(&self, bucket: &str, expected: Uuid) -> Result<()> {
|
||||
if let Err(error) = check_valid_bucket_name_strict(bucket) {
|
||||
return Err(StorageError::BucketNameInvalid(error.to_string()));
|
||||
}
|
||||
if is_meta_bucketname(bucket) || expected.is_nil() {
|
||||
return Err(StorageError::InvalidArgument(
|
||||
"RecoverOrphanedBucket".to_string(),
|
||||
"bucket".to_string(),
|
||||
"a user bucket and non-nil incarnation are required".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
// Lock order matches bucket deletion: publication -> lifecycle ->
|
||||
// metadata transaction -> exact namespace.
|
||||
let publication_guard = self.acquire_bucket_publication_write_lock(bucket).await?;
|
||||
let lifecycle_guard = self.acquire_bucket_lifecycle_write_lock(bucket).await?;
|
||||
let metadata_guard = metadata_sys::acquire_bucket_metadata_transaction_lock_in(&self.ctx, bucket).await?;
|
||||
let ns_lock = self.new_ns_lock(bucket, bucket).await?;
|
||||
let ns_guard = ns_lock
|
||||
.get_write_lock(get_lock_acquire_timeout())
|
||||
.await
|
||||
.map_err(|error| match error {
|
||||
rustfs_lock::LockError::QuorumNotReached { required, achieved } => StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "write",
|
||||
bucket: bucket.to_string(),
|
||||
object: bucket.to_string(),
|
||||
required,
|
||||
achieved,
|
||||
},
|
||||
other => StorageError::Lock(other),
|
||||
})?;
|
||||
|
||||
let recover = Box::pin(async {
|
||||
self.ensure_bucket_volume_absent_on_all_sets(bucket).await?;
|
||||
let retirement_store = metadata_sys::object_store_if_initialized_in(&self.ctx).await.ok_or_else(|| {
|
||||
StorageError::InsufficientReadQuorum(RUSTFS_META_BUCKET.to_string(), "bucket-retirements".to_string())
|
||||
})?;
|
||||
let current_incarnation = crate::bucket::metadata::load_bucket_incarnation(retirement_store.clone(), bucket).await?;
|
||||
if current_incarnation.is_some_and(|current| current != expected) {
|
||||
return Err(StorageError::PreconditionFailed);
|
||||
}
|
||||
|
||||
let retired = crate::bucket::retirement::is_retired(retirement_store.clone(), bucket, expected).await?;
|
||||
if current_incarnation.is_none() && !retired {
|
||||
return Err(StorageError::PreconditionFailed);
|
||||
}
|
||||
if !retired {
|
||||
let mut record_opts = ObjectOptions {
|
||||
max_parity: true,
|
||||
..Default::default()
|
||||
};
|
||||
record_opts.add_bucket_lifecycle_lock_guard(&lifecycle_guard);
|
||||
record_opts.add_namespace_lock_guard(&ns_guard);
|
||||
let publish_retirement = Box::pin(crate::bucket::retirement::commit_retirement(
|
||||
retirement_store.clone(),
|
||||
bucket,
|
||||
expected,
|
||||
&record_opts,
|
||||
));
|
||||
publish_retirement.await?;
|
||||
}
|
||||
|
||||
crate::bucket::quota::reservation::cleanup_retired_bucket_ledger(retirement_store.clone(), bucket, expected).await?;
|
||||
crate::data_usage::prepare_bucket_usage_for_namespace_change(bucket, Some(&ns_guard)).await?;
|
||||
crate::data_usage::remove_bucket_usage_from_backend_with_guard_fenced(self, bucket, Some(&ns_guard)).await?;
|
||||
self.cleanup_deleted_bucket_metadata_strict(bucket, true).await?;
|
||||
crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
|
||||
Ok(())
|
||||
});
|
||||
await_orphan_bucket_recovery(bucket, &publication_guard, &lifecycle_guard, &metadata_guard, &ns_guard, recover).await
|
||||
}
|
||||
|
||||
pub async fn get_bucket_metadata(&self, bucket: &str) -> Result<Arc<BucketMetadata>> {
|
||||
let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?;
|
||||
sys.read().await.get(bucket).await
|
||||
@@ -298,6 +478,10 @@ impl ECStore {
|
||||
Ok(tags)
|
||||
}
|
||||
|
||||
pub async fn table_bucket_enabled(&self, bucket: &str) -> Result<bool> {
|
||||
table_bucket_enabled_in(&self.ctx, bucket).await
|
||||
}
|
||||
|
||||
pub async fn get_bucket_policy(&self, bucket: &str) -> Result<(BucketPolicy, OffsetDateTime)> {
|
||||
let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?;
|
||||
sys.read().await.get_bucket_policy(bucket).await
|
||||
@@ -318,6 +502,21 @@ impl ECStore {
|
||||
metadata_sys::update_in(&self.ctx, bucket, config_file, data).await
|
||||
}
|
||||
|
||||
/// Validate an external fence while holding the bucket metadata transaction.
|
||||
/// Callers acquire their external fence before the lifecycle and transaction locks.
|
||||
pub async fn update_bucket_metadata_config_validated<F>(
|
||||
&self,
|
||||
bucket: &str,
|
||||
config_file: &str,
|
||||
data: Vec<u8>,
|
||||
validate: F,
|
||||
) -> Result<OffsetDateTime>
|
||||
where
|
||||
F: FnOnce() -> Result<()> + Send,
|
||||
{
|
||||
metadata_sys::update_validated_in(&self.ctx, bucket, config_file, data, validate).await
|
||||
}
|
||||
|
||||
pub async fn bucket_incarnation_id(&self, bucket: &str) -> Result<Uuid> {
|
||||
metadata_sys::get_cached_bucket_incarnation_id_in(&self.ctx, bucket).await
|
||||
}
|
||||
@@ -421,6 +620,26 @@ impl ECStore {
|
||||
})
|
||||
}
|
||||
|
||||
async fn acquire_bucket_publication_write_lock(&self, bucket: &str) -> Result<rustfs_lock::NamespaceLockGuard> {
|
||||
let lock = self
|
||||
.new_ns_lock(bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
|
||||
.await?;
|
||||
lock.get_write_lock(get_lock_acquire_timeout())
|
||||
.await
|
||||
.map_err(|err| match err {
|
||||
rustfs_lock::error::LockError::QuorumNotReached { required, achieved } => {
|
||||
StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "table_bucket_publication_write",
|
||||
bucket: bucket.to_string(),
|
||||
object: rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH.to_string(),
|
||||
required,
|
||||
achieved,
|
||||
}
|
||||
}
|
||||
other => StorageError::Lock(other),
|
||||
})
|
||||
}
|
||||
|
||||
async fn mark_bucket_deleted(&self, bucket: &str) -> Result<()> {
|
||||
let marker_volume = bucket_deleted_marker_volume(bucket);
|
||||
|
||||
@@ -552,6 +771,25 @@ impl ECStore {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn cleanup_deleted_bucket_metadata_strict(&self, bucket: &str, include_deleted_marker: bool) -> Result<()> {
|
||||
let options = ObjectOptions {
|
||||
delete_prefix_object: true,
|
||||
..Default::default()
|
||||
};
|
||||
for prefix in bucket_delete_metadata_cleanup_prefixes(bucket) {
|
||||
self.delete_prefix(RUSTFS_META_BUCKET, &prefix, &options).await?;
|
||||
}
|
||||
|
||||
if include_deleted_marker {
|
||||
let marker_prefix = bucket_deleted_marker_prefix(bucket);
|
||||
self.delete_prefix(RUSTFS_META_BUCKET, &marker_prefix, &options).await?;
|
||||
}
|
||||
|
||||
metadata_sys::remove_bucket_metadata_in(&self.ctx, bucket).await?;
|
||||
runtime_sources::delete_bucket_monitor_entry(bucket);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn cleanup_bucket_usage(&self, bucket: &str, guard: Option<&rustfs_lock::NamespaceLockGuard>) -> Result<()> {
|
||||
run_bucket_usage_cleanup(guard, bucket, async {
|
||||
crate::data_usage::prepare_bucket_usage_for_namespace_change(bucket, guard).await?;
|
||||
@@ -663,7 +901,46 @@ impl ECStore {
|
||||
}
|
||||
};
|
||||
let confirmed_missing = existing_bucket_info.is_none();
|
||||
let existing_metadata = if opts.force_create && !confirmed_missing && !is_meta_bucketname(bucket) {
|
||||
let mut pending_creation_metadata = None;
|
||||
if confirmed_missing
|
||||
&& !opts.no_lock
|
||||
&& !is_meta_bucketname(bucket)
|
||||
&& let Some(metadata_store) = metadata_sys::object_store_if_initialized_in(&self.ctx).await
|
||||
&& let Some(old_incarnation) = crate::bucket::metadata::load_bucket_incarnation(metadata_store, bucket).await?
|
||||
{
|
||||
// A quorum-level absence is not proof that every disk lost the old
|
||||
// generation. Require the explicit recovery path to verify all disks
|
||||
// and retire this incarnation before a same-name create can mutate
|
||||
// any volume.
|
||||
self.ensure_bucket_volume_absent_on_all_sets(bucket)
|
||||
.await
|
||||
.map_err(|error| match error {
|
||||
StorageError::BucketExists(_) => StorageError::InvalidArgument(
|
||||
"CreateBucket".to_string(),
|
||||
"bucket".to_string(),
|
||||
"an old bucket volume remains on disk; administrator recovery refused".to_string(),
|
||||
),
|
||||
other => other,
|
||||
})?;
|
||||
let (metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&self.ctx, bucket).await?;
|
||||
let metadata_store = metadata_sys::object_store_in(&self.ctx).await?;
|
||||
let pending_intent = persisted
|
||||
&& metadata.bucket_incarnation_id == old_incarnation
|
||||
&& metadata.needs_bucket_creation_commit()
|
||||
&& !crate::bucket::retirement::is_retired(metadata_store, bucket, old_incarnation).await?;
|
||||
if pending_intent {
|
||||
pending_creation_metadata = Some(metadata);
|
||||
} else {
|
||||
return Err(StorageError::InvalidArgument(
|
||||
"CreateBucket".to_string(),
|
||||
"bucket".to_string(),
|
||||
format!("orphaned bucket generation {old_incarnation} requires administrator recovery before recreation"),
|
||||
));
|
||||
}
|
||||
}
|
||||
let existing_metadata = if let Some(metadata) = pending_creation_metadata {
|
||||
Some(metadata)
|
||||
} else if opts.force_create && !confirmed_missing && !is_meta_bucketname(bucket) {
|
||||
let (mut metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&self.ctx, bucket).await?;
|
||||
if !persisted {
|
||||
metadata = BucketMetadata::new(bucket);
|
||||
@@ -1041,7 +1318,7 @@ impl ECStore {
|
||||
&self,
|
||||
bucket: &str,
|
||||
opts: &DeleteBucketOptions,
|
||||
mut diagnostic_budget: BucketDeleteDiagnosticBudget,
|
||||
diagnostic_budget: BucketDeleteDiagnosticBudget,
|
||||
) -> Result<()> {
|
||||
if is_meta_bucketname(bucket) {
|
||||
return Err(StorageError::BucketNameInvalid(bucket.to_string()));
|
||||
@@ -1051,11 +1328,55 @@ impl ECStore {
|
||||
return Err(StorageError::BucketNameInvalid(err.to_string()));
|
||||
}
|
||||
|
||||
// Bucket deletion is a publication mutation too: a durable catalog
|
||||
// backup must not enumerate or verify table objects while their
|
||||
// containing bucket is being physically removed. Acquire this fence
|
||||
// before the lifecycle and namespace locks to match object writers.
|
||||
// `no_lock` is reserved for callers that already own the enclosing
|
||||
// mutation locks (for example, failed bucket-creation rollback).
|
||||
let publication_guard = if !opts.no_lock {
|
||||
Some(self.acquire_bucket_publication_write_lock(bucket).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
await_bucket_namespace_operation(
|
||||
publication_guard.as_ref(),
|
||||
bucket,
|
||||
"table-bucket publication fence during bucket deletion",
|
||||
self.handle_delete_bucket_under_publication_lock(bucket, opts, diagnostic_budget),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn handle_delete_bucket_under_publication_lock(
|
||||
&self,
|
||||
bucket: &str,
|
||||
opts: &DeleteBucketOptions,
|
||||
mut diagnostic_budget: BucketDeleteDiagnosticBudget,
|
||||
) -> Result<()> {
|
||||
let bucket_lifecycle_guard = if !opts.no_lock {
|
||||
Some(self.acquire_bucket_lifecycle_write_lock(bucket).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
// Match metadata migration: lifecycle -> metadata transaction ->
|
||||
// bucket namespace. Holding the transaction while waiting for the
|
||||
// namespace prevents a migration writer and force delete from
|
||||
// acquiring these two locks in opposite orders.
|
||||
let object_lock_metadata_transaction = if opts.force && !is_meta_bucketname(bucket) {
|
||||
Some(
|
||||
await_bucket_namespace_operation(
|
||||
bucket_lifecycle_guard.as_ref(),
|
||||
bucket,
|
||||
"Object Lock metadata transaction acquisition",
|
||||
crate::bucket::metadata_sys::acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket),
|
||||
)
|
||||
.await?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let ns_guard = if !opts.no_lock {
|
||||
let ns_lock = self.new_ns_lock(bucket, bucket).await?;
|
||||
Some(
|
||||
@@ -1111,6 +1432,29 @@ impl ECStore {
|
||||
}
|
||||
}
|
||||
|
||||
// Keep configuration changes fenced until data deletion finishes;
|
||||
// a force delete cannot bypass any Object Lock protection.
|
||||
let object_lock_metadata_guard = if bucket_exists && let Some(guard) = object_lock_metadata_transaction {
|
||||
let (state, incarnation, _) =
|
||||
crate::bucket::metadata_sys::get_object_lock_config_and_incarnation_from_disk_with_guard_in(
|
||||
&self.ctx,
|
||||
bucket,
|
||||
Some(ns_guard.as_ref().unwrap_or(&guard)),
|
||||
)
|
||||
.await?;
|
||||
match state {
|
||||
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent => Some((guard, incarnation)),
|
||||
crate::bucket::metadata_sys::ObjectLockConfigState::Configured { .. } => {
|
||||
return Err(StorageError::PrefixAccessDenied(bucket.to_owned(), String::new()));
|
||||
}
|
||||
crate::bucket::metadata_sys::ObjectLockConfigState::Fabricated => {
|
||||
return Err(StorageError::FileCorrupt);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
if sr_delete && !bucket_exists {
|
||||
delete_opts.force_if_empty = true;
|
||||
}
|
||||
@@ -1124,7 +1468,12 @@ impl ECStore {
|
||||
ns_guard.as_ref(),
|
||||
bucket,
|
||||
"bucket delete marker creation",
|
||||
self.mark_bucket_deleted(bucket),
|
||||
await_bucket_namespace_operation(
|
||||
object_lock_metadata_guard.as_ref().map(|(guard, _)| guard),
|
||||
bucket,
|
||||
"Object Lock configuration fence during bucket retirement",
|
||||
self.mark_bucket_deleted(bucket),
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
@@ -1133,9 +1482,13 @@ impl ECStore {
|
||||
// Legacy buckets without a stamp cannot produce retirement authority.
|
||||
let retirement = if bucket_exists && bucket_lifecycle_guard.is_some() && !is_meta_bucketname(bucket) {
|
||||
if let Some(store) = metadata_sys::object_store_if_initialized_in(&self.ctx).await {
|
||||
crate::bucket::metadata::load_bucket_incarnation(store.clone(), bucket)
|
||||
.await?
|
||||
.map(|incarnation| (store, incarnation))
|
||||
if let Some((_, incarnation)) = object_lock_metadata_guard.as_ref() {
|
||||
Some((store, *incarnation))
|
||||
} else {
|
||||
crate::bucket::metadata::load_bucket_incarnation(store.clone(), bucket)
|
||||
.await?
|
||||
.map(|incarnation| (store, incarnation))
|
||||
}
|
||||
} else {
|
||||
None
|
||||
}
|
||||
@@ -1147,13 +1500,19 @@ impl ECStore {
|
||||
bucket_lifecycle_guard.as_ref(),
|
||||
bucket,
|
||||
"physical bucket deletion",
|
||||
run_physical_bucket_deletion(ns_guard.as_ref(), bucket, async {
|
||||
self.delete_bucket_on_sets(bucket, &delete_opts)
|
||||
.await
|
||||
.map_err(|err| to_object_err(err, vec![bucket]))
|
||||
}),
|
||||
await_bucket_namespace_operation(
|
||||
object_lock_metadata_guard.as_ref().map(|(guard, _)| guard),
|
||||
bucket,
|
||||
"Object Lock configuration fence during bucket deletion",
|
||||
run_physical_bucket_deletion(ns_guard.as_ref(), bucket, async {
|
||||
self.delete_bucket_on_sets(bucket, &delete_opts)
|
||||
.await
|
||||
.map_err(|err| to_object_err(err, vec![bucket]))
|
||||
}),
|
||||
),
|
||||
)
|
||||
.await;
|
||||
drop(object_lock_metadata_guard);
|
||||
if let Err(err) = delete_result
|
||||
&& (!sr_delete || !is_err_strict_volume_not_found(&err))
|
||||
{
|
||||
@@ -1212,9 +1571,9 @@ mod tests {
|
||||
BUCKET_DELETE_DIAGNOSTIC_MAX_ELAPSED, BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES, BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES,
|
||||
BucketDeleteBlockerKind, BucketDeleteDiagnosticBudget, BucketMetadataLessResidue, SCANNER_BUCKET_LIST_SET_CONCURRENCY,
|
||||
await_bucket_namespace_operation, bucket_delete_metadata_cleanup_prefixes, bucket_deleted_marker_prefix,
|
||||
bucket_deleted_marker_volume, bucket_list_set_concurrency, record_bucket_delete_blocker, run_bucket_usage_cleanup,
|
||||
run_physical_bucket_deletion, scan_metadata_less_residue, scan_metadata_less_residue_with_budget,
|
||||
should_override_created_from_metadata, validate_table_bucket_delete_allowed,
|
||||
bucket_deleted_marker_volume, bucket_list_set_concurrency, get_lock_acquire_timeout, record_bucket_delete_blocker,
|
||||
run_bucket_usage_cleanup, run_physical_bucket_deletion, scan_metadata_less_residue,
|
||||
scan_metadata_less_residue_with_budget, should_override_created_from_metadata, validate_table_bucket_delete_allowed,
|
||||
};
|
||||
use crate::bucket::metadata::{BucketMetadata, table_bucket_catalog_metadata_prefix};
|
||||
use crate::bucket::metadata_sys;
|
||||
@@ -1531,6 +1890,65 @@ mod tests {
|
||||
setup_bucket_quorum_test_env(&[4, 4], None).await
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn orphaned_bucket_recovery_requires_every_volume_absent_and_is_retryable() {
|
||||
let (temp_dir, store) = setup_multi_pool_bucket_test_env().await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("orphan-recovery-{}", Uuid::new_v4().simple());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket before simulated operator removal");
|
||||
let incarnation = store
|
||||
.bucket_incarnation_id_from_disk(&bucket)
|
||||
.await
|
||||
.expect("persisted bucket generation");
|
||||
|
||||
let first_disk_volume = temp_dir.path().join("pool0-disk0").join(&bucket);
|
||||
tokio::fs::remove_dir_all(&first_disk_volume)
|
||||
.await
|
||||
.expect("simulate operator removal on one disk");
|
||||
let blocked = store
|
||||
.recover_orphaned_bucket(&bucket, incarnation)
|
||||
.await
|
||||
.expect_err("recovery must refuse while any configured disk still has the volume");
|
||||
assert!(matches!(blocked, StorageError::BucketExists(_)), "unexpected error: {blocked}");
|
||||
assert_eq!(
|
||||
crate::bucket::metadata::load_bucket_incarnation(store.clone(), &bucket)
|
||||
.await
|
||||
.expect("generation sidecar remains readable"),
|
||||
Some(incarnation),
|
||||
"a refused recovery must preserve the old generation identity"
|
||||
);
|
||||
|
||||
for disk_index in 1..4 {
|
||||
tokio::fs::remove_dir_all(temp_dir.path().join(format!("pool0-disk{disk_index}")).join(&bucket))
|
||||
.await
|
||||
.expect("remove remaining volume from first set");
|
||||
}
|
||||
for disk_index in 0..4 {
|
||||
tokio::fs::remove_dir_all(temp_dir.path().join(format!("pool1-disk{disk_index}")).join(&bucket))
|
||||
.await
|
||||
.expect("remove volume from second set");
|
||||
}
|
||||
|
||||
store
|
||||
.recover_orphaned_bucket(&bucket, incarnation)
|
||||
.await
|
||||
.expect("recovery should retire the exact generation and clean stale metadata");
|
||||
assert!(
|
||||
store
|
||||
.is_bucket_incarnation_retired(&bucket, incarnation)
|
||||
.await
|
||||
.expect("retirement proof remains readable")
|
||||
);
|
||||
store
|
||||
.recover_orphaned_bucket(&bucket, incarnation)
|
||||
.await
|
||||
.expect("retry after metadata cleanup should converge from durable retirement proof");
|
||||
}
|
||||
|
||||
async fn setup_bucket_quorum_test_env(
|
||||
drives_per_pool: &[usize],
|
||||
standard_parity: Option<usize>,
|
||||
@@ -3281,6 +3699,40 @@ mod tests {
|
||||
assert!(any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn bucket_delete_waits_for_table_publication_readers() {
|
||||
let (_disk_paths, ecstore) = setup_bucket_delete_test_env().await;
|
||||
let bucket = format!("bucket-delete-publication-{}", Uuid::new_v4().simple());
|
||||
ecstore
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
|
||||
let publication_lock = ecstore
|
||||
.new_ns_lock(&bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
|
||||
.await
|
||||
.expect("publication lock should be created");
|
||||
let publication_reader = publication_lock
|
||||
.get_read_lock(get_lock_acquire_timeout())
|
||||
.await
|
||||
.expect("publication reader should be acquired");
|
||||
|
||||
let delete_store = Arc::clone(&ecstore);
|
||||
let mut delete = tokio::spawn(async move { delete_store.delete_bucket(&bucket, &DeleteBucketOptions::default()).await });
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(250), &mut delete).await.is_err(),
|
||||
"bucket deletion must remain behind the table publication reader"
|
||||
);
|
||||
|
||||
drop(publication_reader);
|
||||
tokio::time::timeout(Duration::from_secs(10), &mut delete)
|
||||
.await
|
||||
.expect("bucket deletion should proceed after publication reader release")
|
||||
.expect("bucket deletion task should join")
|
||||
.expect("empty bucket deletion should succeed");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn bucket_delete_mark_delete_rejects_non_empty_bucket_without_force() {
|
||||
@@ -3305,6 +3757,116 @@ mod tests {
|
||||
assert!(!any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn force_bucket_delete_preserves_object_lock_versions() {
|
||||
let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], Some(2)).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("force-delete-worm-{}", Uuid::new_v4().simple());
|
||||
store
|
||||
.make_bucket(
|
||||
&bucket,
|
||||
&MakeBucketOptions {
|
||||
lock_enabled: true,
|
||||
versioning_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create Object Lock bucket");
|
||||
let until = (OffsetDateTime::now_utc() + time::Duration::days(365))
|
||||
.format(&time::format_description::well_known::Rfc3339)
|
||||
.expect("format future retention date");
|
||||
let written = store
|
||||
.put_object(
|
||||
&bucket,
|
||||
"retained",
|
||||
&mut PutObjReader::from_vec(b"retained bytes".to_vec()),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
user_defined: HashMap::from([
|
||||
("x-amz-object-lock-mode".to_owned(), "COMPLIANCE".to_owned()),
|
||||
("x-amz-object-lock-retain-until-date".to_owned(), until),
|
||||
]),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write retained object");
|
||||
for operation in [SRBucketDeleteOp::NoOp, SRBucketDeleteOp::MarkDelete, SRBucketDeleteOp::Purge] {
|
||||
let err = store
|
||||
.delete_bucket(
|
||||
&bucket,
|
||||
&DeleteBucketOptions {
|
||||
force: true,
|
||||
srdelete_op: operation,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("force deletion must not bypass Object Lock");
|
||||
assert!(matches!(err, StorageError::PrefixAccessDenied(_, _)), "unexpected denial: {err}");
|
||||
let info = store
|
||||
.get_object_info(
|
||||
&bucket,
|
||||
"retained",
|
||||
&ObjectOptions {
|
||||
version_id: written.version_id.map(|version| version.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("retained version must survive the rejected bucket deletion");
|
||||
assert_eq!(info.etag, written.etag);
|
||||
assert_eq!(info.size, written.size);
|
||||
assert_eq!(info.version_id, written.version_id);
|
||||
}
|
||||
metadata_sys::inject_object_lock_disk_read_error_in(&store.ctx, &bucket)
|
||||
.await
|
||||
.expect("inject authoritative metadata read failure");
|
||||
let error = store
|
||||
.delete_bucket(
|
||||
&bucket,
|
||||
&DeleteBucketOptions {
|
||||
force: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("an unreadable Object Lock configuration must fail closed");
|
||||
assert!(error.to_string().contains("injected Object Lock metadata disk read failure"));
|
||||
let info = store
|
||||
.get_object_info(&bucket, "retained", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("metadata read failure must leave the retained object intact");
|
||||
assert_eq!(info.version_id, written.version_id);
|
||||
|
||||
let unlocked = format!("force-delete-unlocked-{}", Uuid::new_v4().simple());
|
||||
store
|
||||
.make_bucket(&unlocked, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create unlocked control bucket");
|
||||
store
|
||||
.put_object(
|
||||
&unlocked,
|
||||
"ordinary",
|
||||
&mut PutObjReader::from_vec(b"ordinary bytes".to_vec()),
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("write ordinary control object");
|
||||
store
|
||||
.delete_bucket(
|
||||
&unlocked,
|
||||
&DeleteBucketOptions {
|
||||
force: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("force deletion must still work for an unlocked bucket");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn bucket_delete_mark_delete_rejects_hidden_object_paths_without_force() {
|
||||
|
||||
@@ -14521,15 +14521,19 @@ mod tests {
|
||||
let barrier = TierDeleteChunkTestBarrier::install(stage);
|
||||
let worker_store = store.clone();
|
||||
let worker_bucket = bucket.clone();
|
||||
// Simulate a crash at the barrier. Aborting a caller of the public entry
|
||||
// point no longer stops the delete, which runs on a detached task, so
|
||||
// drive the delete below that task.
|
||||
let worker = tokio::spawn(async move {
|
||||
worker_store
|
||||
.delete_object_with_tier_delete_journal(
|
||||
.handle_delete_object_with_journal(
|
||||
&worker_bucket,
|
||||
prefix,
|
||||
ObjectOptions {
|
||||
delete_prefix: true,
|
||||
..Default::default()
|
||||
},
|
||||
Some(worker_store.clone()),
|
||||
)
|
||||
.await
|
||||
});
|
||||
@@ -14538,6 +14542,11 @@ mod tests {
|
||||
.unwrap_or_else(|_| panic!("chunk delete did not reach crash boundary {stage:?}"));
|
||||
worker.abort();
|
||||
let _ = worker.await;
|
||||
assert_eq!(
|
||||
ctx.detached_mutation_count(),
|
||||
0,
|
||||
"the crash must stop the delete below the detached task, so nothing keeps running after the abort"
|
||||
);
|
||||
drop(barrier);
|
||||
let released_bucket_guard =
|
||||
tokio::time::timeout(Duration::from_secs(5), store.acquire_bucket_lifecycle_write_lock(&bucket))
|
||||
@@ -14683,15 +14692,19 @@ mod tests {
|
||||
|
||||
let barrier = TierDeleteChunkTestBarrier::install(TierDeleteChunkTestStage::DispatchAuthorized);
|
||||
let worker_store = store.clone();
|
||||
// Simulate a crash at the barrier. Aborting a caller of the public entry
|
||||
// point no longer stops the delete, which runs on a detached task, so
|
||||
// drive the delete below that task.
|
||||
let worker = tokio::spawn(async move {
|
||||
worker_store
|
||||
.delete_object_with_tier_delete_journal(
|
||||
.handle_delete_object_with_journal(
|
||||
bucket,
|
||||
prefix,
|
||||
ObjectOptions {
|
||||
delete_prefix: true,
|
||||
..Default::default()
|
||||
},
|
||||
Some(worker_store.clone()),
|
||||
)
|
||||
.await
|
||||
});
|
||||
@@ -14700,6 +14713,11 @@ mod tests {
|
||||
.expect("chunk delete should persist its authorization before corruption injection");
|
||||
worker.abort();
|
||||
let _ = worker.await;
|
||||
assert_eq!(
|
||||
ctx.detached_mutation_count(),
|
||||
0,
|
||||
"the crash must stop the delete below the detached task, so nothing keeps running after the abort"
|
||||
);
|
||||
drop(barrier);
|
||||
|
||||
let records = store
|
||||
@@ -17536,6 +17554,332 @@ mod tests {
|
||||
.expect("new-generation object must survive the old batch delete");
|
||||
}
|
||||
|
||||
async fn put_detached_delete_fixture(store: &Arc<crate::store::ECStore>, bucket: &str, object: &str) {
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create detached delete bucket");
|
||||
// Large enough for a data dir, so the delete commit leaves a marked
|
||||
// data dir and a rollback backup until its cleanup pass runs.
|
||||
let mut reader = PutObjReader::from_vec(vec![7u8; 2 * 1024 * 1024]);
|
||||
store
|
||||
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("put detached delete fixture");
|
||||
}
|
||||
|
||||
async fn assert_delete_lock_moved_to_detached_task(store: &Arc<crate::store::ECStore>, bucket: &str, object: &str) {
|
||||
let lock = store
|
||||
.handle_new_ns_lock(bucket, object)
|
||||
.await
|
||||
.expect("namespace lock handle for the deleted key");
|
||||
lock.get_write_lock_quiet(Duration::from_millis(200))
|
||||
.await
|
||||
.expect_err("the detached delete must keep its object lock after the caller is dropped");
|
||||
}
|
||||
|
||||
fn assert_no_delete_residue(temp_dir: &std::path::Path, bucket: &str, object: &str, disks: usize) {
|
||||
for disk in 0..disks {
|
||||
let object_dir = temp_dir.join(format!("pool0/set0/disk{disk}")).join(bucket).join(object);
|
||||
assert!(
|
||||
!object_dir.exists(),
|
||||
"disk {disk} must hold neither metadata, rollback backup nor data of the deleted object: {object_dir:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// #6898: a DeleteObjects request dropped between the per-disk commit and the
|
||||
// cleanup pass (client disconnect) must not leave metadata-less residue.
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn delete_objects_finishes_cleanup_after_caller_is_dropped() {
|
||||
let temp = tempfile::tempdir().expect("create temp store dir");
|
||||
let (ctx, store, _shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp.path(), "detached-batch-delete", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("detached-batch-delete-{}", uuid::Uuid::new_v4());
|
||||
let object = "residue/batch.bin";
|
||||
put_detached_delete_fixture(&store, &bucket, object).await;
|
||||
|
||||
let barrier = crate::set_disk::DeleteCleanupBarrier::install(&bucket);
|
||||
let delete = store.delete_objects_with_tier_delete_journal_and_accounting(
|
||||
&bucket,
|
||||
vec![ObjectToDelete {
|
||||
object_name: object.to_string(),
|
||||
..Default::default()
|
||||
}],
|
||||
ObjectOptions::default(),
|
||||
);
|
||||
tokio::select! {
|
||||
() = barrier.wait_until_paused() => {}
|
||||
_ = delete => panic!("batch delete must pause between its commit and its cleanup"),
|
||||
}
|
||||
assert_eq!(ctx.detached_mutation_count(), 1, "the dropped request must leave its delete running");
|
||||
assert_delete_lock_moved_to_detached_task(&store, &bucket, object).await;
|
||||
|
||||
barrier.release();
|
||||
assert!(
|
||||
ctx.wait_for_detached_mutations(Duration::from_secs(30)).await,
|
||||
"the detached batch delete should finish"
|
||||
);
|
||||
assert_no_delete_residue(temp.path(), &bucket, object, 4);
|
||||
store
|
||||
.get_object_info(&bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("the object must be deleted");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn delete_object_finishes_cleanup_after_caller_is_dropped() {
|
||||
let temp = tempfile::tempdir().expect("create temp store dir");
|
||||
let (ctx, store, _shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp.path(), "detached-single-delete", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("detached-single-delete-{}", uuid::Uuid::new_v4());
|
||||
let object = "residue/single.bin";
|
||||
put_detached_delete_fixture(&store, &bucket, object).await;
|
||||
|
||||
let barrier = crate::set_disk::DeleteCleanupBarrier::install(&bucket);
|
||||
let delete = store.delete_object_with_tier_delete_journal(&bucket, object, ObjectOptions::default());
|
||||
tokio::select! {
|
||||
() = barrier.wait_until_paused() => {}
|
||||
_ = delete => panic!("single delete must pause between its commit and its cleanup"),
|
||||
}
|
||||
assert_eq!(ctx.detached_mutation_count(), 1, "the dropped request must leave its delete running");
|
||||
assert_delete_lock_moved_to_detached_task(&store, &bucket, object).await;
|
||||
|
||||
barrier.release();
|
||||
assert!(
|
||||
ctx.wait_for_detached_mutations(Duration::from_secs(30)).await,
|
||||
"the detached single delete should finish"
|
||||
);
|
||||
assert_no_delete_residue(temp.path(), &bucket, object, 4);
|
||||
store
|
||||
.get_object_info(&bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("the object must be deleted");
|
||||
}
|
||||
|
||||
// A recursive (force) delete runs under its caller's bucket lifecycle write
|
||||
// lock and carries only that lock's fence. If the caller is dropped, the
|
||||
// detached delete must still hold the lock: otherwise a PUT under the prefix
|
||||
// succeeds and is then removed by the delete that is still running.
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn recursive_delete_keeps_bucket_lock_after_caller_is_dropped() {
|
||||
let temp = tempfile::tempdir().expect("create temp store dir");
|
||||
let (ctx, store, _shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp.path(), "detached-recursive-delete", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("detached-recursive-delete-{}", uuid::Uuid::new_v4());
|
||||
let old_object = "folder/old.bin";
|
||||
let new_object = "folder/new.bin";
|
||||
put_detached_delete_fixture(&store, &bucket, old_object).await;
|
||||
|
||||
// The S3 force-delete path: the caller owns the lock, opts carry its fence.
|
||||
let guard = store
|
||||
.lock_bucket_for_recursive_delete(&bucket)
|
||||
.await
|
||||
.expect("recursive delete should lock the bucket");
|
||||
let mut opts = ObjectOptions {
|
||||
delete_prefix: true,
|
||||
..Default::default()
|
||||
};
|
||||
opts.add_bucket_lifecycle_lock_guard(&guard);
|
||||
let barrier = crate::store::object::DeleteAfterObjectLockSnapshotBarrier::install(&bucket);
|
||||
let delete = store.delete_object_with_tier_delete_journal_and_guards(&bucket, "folder/", opts, Some(guard));
|
||||
tokio::select! {
|
||||
() = barrier.wait_until_paused() => {}
|
||||
_ = delete => panic!("recursive delete must pause inside its detached task"),
|
||||
}
|
||||
assert_eq!(ctx.detached_mutation_count(), 1, "the dropped caller must leave its delete running");
|
||||
|
||||
let put_store = Arc::clone(&store);
|
||||
let put_bucket = bucket.clone();
|
||||
let mut put = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(b"written while the delete runs".to_vec());
|
||||
put_store
|
||||
.put_object(&put_bucket, new_object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(500), &mut put).await.is_err(),
|
||||
"a PUT under the prefix must wait while the detached recursive delete runs"
|
||||
);
|
||||
|
||||
barrier.release();
|
||||
assert!(
|
||||
ctx.wait_for_detached_mutations(Duration::from_secs(30)).await,
|
||||
"the detached recursive delete should finish"
|
||||
);
|
||||
put.await
|
||||
.expect("PUT task should join")
|
||||
.expect("the PUT should succeed once the recursive delete has finished");
|
||||
store
|
||||
.get_object_info(&bucket, old_object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("the recursive delete must remove the object it covered");
|
||||
store
|
||||
.get_object_info(&bucket, new_object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("the PUT that waited for the recursive delete must survive it");
|
||||
for disk in 0..4 {
|
||||
let metadata = temp
|
||||
.path()
|
||||
.join(format!("pool0/set0/disk{disk}"))
|
||||
.join(&bucket)
|
||||
.join(new_object)
|
||||
.join(crate::disk::STORAGE_FORMAT_FILE);
|
||||
assert!(
|
||||
metadata.exists(),
|
||||
"disk {disk} must keep the object written after the delete: {metadata:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Lifecycle expiry holds the table-bucket publication read lock and passes
|
||||
// only its lock-lost signal to the delete. A cancelled expiry worker must not
|
||||
// release that lock while its detached delete is still running.
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn lifecycle_expiry_keeps_publication_lock_after_worker_is_dropped() {
|
||||
let temp = tempfile::tempdir().expect("create temp store dir");
|
||||
let (ctx, store, _shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp.path(), "detached-lifecycle-expiry", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("detached-lifecycle-expiry-{}", uuid::Uuid::new_v4());
|
||||
let object = "expired.bin";
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create lifecycle bucket");
|
||||
let mut reader = PutObjReader::from_vec(b"expired data".to_vec());
|
||||
let object_info = store
|
||||
.put_object(&bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("put expired object");
|
||||
let bucket_incarnation_id = store.bucket_incarnation_id(&bucket).await.expect("read bucket incarnation");
|
||||
|
||||
let barrier = crate::store::object::DeleteAfterObjectLockSnapshotBarrier::install(&bucket);
|
||||
let expiry_store = Arc::clone(&store);
|
||||
let expiry = tokio::spawn(async move {
|
||||
crate::bucket::lifecycle::bucket_lifecycle_ops::apply_expiry_on_non_transitioned_objects(
|
||||
expiry_store,
|
||||
&object_info,
|
||||
&crate::bucket::lifecycle::lifecycle::Event {
|
||||
action: crate::bucket::lifecycle::lifecycle::IlmAction::DeleteAction,
|
||||
..Default::default()
|
||||
},
|
||||
&crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc::Scanner,
|
||||
bucket_incarnation_id,
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("lifecycle delete should pause inside its detached task");
|
||||
// A cancelled worker drops its future while the delete keeps running.
|
||||
expiry.abort();
|
||||
assert!(
|
||||
expiry
|
||||
.await
|
||||
.expect_err("aborted expiry worker should not join")
|
||||
.is_cancelled(),
|
||||
"expiry worker should be cancelled"
|
||||
);
|
||||
assert_eq!(ctx.detached_mutation_count(), 1, "the cancelled worker must leave its delete running");
|
||||
|
||||
let publication_lock = store
|
||||
.handle_new_ns_lock(&bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
|
||||
.await
|
||||
.expect("table-bucket publication lock handle");
|
||||
publication_lock
|
||||
.get_write_lock_quiet(Duration::from_millis(300))
|
||||
.await
|
||||
.expect_err("table-bucket publication must wait for the detached lifecycle delete");
|
||||
|
||||
barrier.release();
|
||||
assert!(
|
||||
ctx.wait_for_detached_mutations(Duration::from_secs(30)).await,
|
||||
"the detached lifecycle delete should finish"
|
||||
);
|
||||
let publication_guard = publication_lock
|
||||
.get_write_lock_quiet(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await
|
||||
.expect("table-bucket publication should proceed once the lifecycle delete has finished");
|
||||
drop(publication_guard);
|
||||
store
|
||||
.get_object_info(&bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("the detached lifecycle delete should still remove the expired object");
|
||||
}
|
||||
|
||||
// #6898: residue that an older build stranded (every disk committed the
|
||||
// delete, the cleanup pass never ran) is reclaimed by deleting the key again.
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn repeated_delete_objects_finishes_committed_delete_residue() {
|
||||
use crate::disk::DiskAPI as _;
|
||||
|
||||
for drives in [1, 4] {
|
||||
let temp = tempfile::tempdir().expect("create temp store dir");
|
||||
let (_ctx, store, _shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp.path(), "residue-retry", &[drives])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("residue-retry-{drives}-{}", uuid::Uuid::new_v4());
|
||||
let object = "residue/retry.bin";
|
||||
put_detached_delete_fixture(&store, &bucket, object).await;
|
||||
|
||||
let transaction = uuid::Uuid::new_v4();
|
||||
let disks = store.pools[0].disk_set[0].disks.read().await.clone();
|
||||
for disk in disks.iter().flatten() {
|
||||
let errors = disk
|
||||
.delete_versions(
|
||||
&bucket,
|
||||
vec![crate::disk::FileInfoVersions {
|
||||
name: object.to_string(),
|
||||
versions: vec![rustfs_filemeta::FileInfo {
|
||||
name: object.to_string(),
|
||||
..Default::default()
|
||||
}],
|
||||
..Default::default()
|
||||
}],
|
||||
crate::disk::DeleteOptions {
|
||||
old_data_dir: Some(transaction),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(errors.iter().all(Option::is_none), "the delete should commit on every disk: {errors:?}");
|
||||
}
|
||||
for disk in 0..drives {
|
||||
let object_dir = temp.path().join(format!("pool0/set0/disk{disk}")).join(&bucket).join(object);
|
||||
assert!(!object_dir.join(crate::disk::STORAGE_FORMAT_FILE).exists());
|
||||
assert!(
|
||||
object_dir
|
||||
.join(transaction.to_string())
|
||||
.join(crate::disk::STORAGE_FORMAT_FILE_BACKUP)
|
||||
.exists(),
|
||||
"disk {disk} should hold the stranded rollback backup"
|
||||
);
|
||||
}
|
||||
|
||||
let (_deleted, errors) = store
|
||||
.delete_objects(
|
||||
&bucket,
|
||||
vec![ObjectToDelete {
|
||||
object_name: object.to_string(),
|
||||
..Default::default()
|
||||
}],
|
||||
ObjectOptions::default(),
|
||||
)
|
||||
.await;
|
||||
assert!(errors.iter().all(Option::is_none), "a missing key deletes successfully: {errors:?}");
|
||||
assert_no_delete_residue(temp.path(), &bucket, object, drives);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn prefix_delete_serializes_with_concurrent_object_lock_metadata_updates() {
|
||||
@@ -17974,7 +18318,7 @@ mod tests {
|
||||
let bucket = format!("lock-create-intent-retry-{}", uuid::Uuid::new_v4());
|
||||
let mut intent = crate::bucket::metadata::BucketMetadata::new(&bucket);
|
||||
intent.lock_enabled = true;
|
||||
crate::bucket::metadata_sys::set_new_bucket_metadata_in(&ctx, intent)
|
||||
crate::bucket::metadata_sys::set_new_bucket_metadata_intent_in(&ctx, intent)
|
||||
.await
|
||||
.expect("persist pre-visibility lock intent");
|
||||
assert!(
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
+171
-19
@@ -42,7 +42,7 @@ use crate::error::{
|
||||
is_err_strict_volume_not_found, is_err_version_not_found, to_object_err,
|
||||
};
|
||||
use crate::runtime::global::DISK_RESERVE_FRACTION;
|
||||
use crate::runtime::instance::InstanceContext;
|
||||
use crate::runtime::instance::{InstanceContext, ScannerPublicationLeasePurpose};
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use crate::services::rebalance::{RebalStatus, RebalanceMeta, is_rebalance_conflicting_with_decommission};
|
||||
use crate::storage_api_contracts::{
|
||||
@@ -1098,36 +1098,67 @@ impl ECStore {
|
||||
&self,
|
||||
expected_generation: u64,
|
||||
ttl: std::time::Duration,
|
||||
) -> Result<(Uuid, u64)> {
|
||||
self.acquire_scanner_publication_lease_with_purpose(expected_generation, ttl, false)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn acquire_scanner_publication_lease_with_purpose(
|
||||
&self,
|
||||
expected_generation: u64,
|
||||
ttl: std::time::Duration,
|
||||
observational_only: bool,
|
||||
) -> Result<(Uuid, u64)> {
|
||||
if ttl != crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL {
|
||||
return Err(Error::other("scanner publication lease TTL is not supported"));
|
||||
}
|
||||
|
||||
// Bind the original activity generation across the asynchronous checks;
|
||||
// a completed namespace commit must never refresh an existing proof.
|
||||
let namespace_generation = self.scanner_namespace_mutation_generation();
|
||||
let purpose = if observational_only {
|
||||
ScannerPublicationLeasePurpose::Observational
|
||||
} else {
|
||||
ScannerPublicationLeasePurpose::Authoritative
|
||||
};
|
||||
// Strict leases bind the namespace generation before their async
|
||||
// admission checks. An observation only needs the movement fence, but
|
||||
// still fails closed if generation tracking is exhausted.
|
||||
let observed_namespace_generation = self.scanner_namespace_mutation_generation();
|
||||
let operation_gate = self.ctx.data_movement_operation_gate();
|
||||
let operation_guard = operation_gate.read_owned().await;
|
||||
if self.ctx.data_movement_generation_exhausted()
|
||||
|| self.ctx.data_movement_operation_epoch_exhausted()
|
||||
|| self.ctx.data_movement_generation() != expected_generation
|
||||
|| namespace_generation == u64::MAX
|
||||
|| observed_namespace_generation == u64::MAX
|
||||
{
|
||||
return Err(Error::other("scanner publication lease generation is stale"));
|
||||
}
|
||||
if self.scanner_data_movement_snapshot_locked().await.1 || self.ctx.namespace_commits_pending() {
|
||||
if self.scanner_data_movement_snapshot_locked().await.1 {
|
||||
return Err(Error::other("scanner publication lease is blocked by data movement"));
|
||||
}
|
||||
|
||||
if self.scanner_namespace_mutation_generation() != namespace_generation {
|
||||
return Err(Error::other("scanner publication lease generation is stale"));
|
||||
}
|
||||
let namespace_generation = if observational_only {
|
||||
None
|
||||
} else {
|
||||
if self.ctx.namespace_commits_pending() {
|
||||
return Err(Error::other("scanner publication lease is blocked by pending namespace commit"));
|
||||
}
|
||||
if self.scanner_namespace_mutation_generation() != observed_namespace_generation {
|
||||
return Err(Error::other("scanner publication lease namespace changed during acquisition"));
|
||||
}
|
||||
Some(observed_namespace_generation)
|
||||
};
|
||||
|
||||
let token = Uuid::new_v4();
|
||||
let expires_at = tokio::time::Instant::now() + ttl;
|
||||
if !self
|
||||
.ctx
|
||||
.install_scanner_publication_lease(token, expires_at, expected_generation, namespace_generation, operation_guard)
|
||||
.install_scanner_publication_lease(
|
||||
token,
|
||||
expires_at,
|
||||
expected_generation,
|
||||
namespace_generation,
|
||||
purpose,
|
||||
operation_guard,
|
||||
)
|
||||
.await
|
||||
{
|
||||
return Err(Error::other("scanner publication lease capacity is exhausted"));
|
||||
@@ -1151,24 +1182,46 @@ impl ECStore {
|
||||
/// admission; a restarted context has no old token and therefore fails
|
||||
/// closed even if its generation counter has returned to zero.
|
||||
pub async fn validate_scanner_publication_lease(&self, token: Uuid, expected_generation: u64) -> Result<()> {
|
||||
let _operation_guard = self.acquire_scanner_publication_lease_guard(token).await?;
|
||||
self.validate_scanner_publication_lease_with_purpose(token, expected_generation, false)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn validate_scanner_publication_lease_with_purpose(
|
||||
&self,
|
||||
token: Uuid,
|
||||
expected_generation: u64,
|
||||
observational_only: bool,
|
||||
) -> Result<()> {
|
||||
let expected_purpose = if observational_only {
|
||||
ScannerPublicationLeasePurpose::Observational
|
||||
} else {
|
||||
ScannerPublicationLeasePurpose::Authoritative
|
||||
};
|
||||
let _operation_guard = self
|
||||
.acquire_scanner_publication_lease_guard_with_target(token, None, true, Some(expected_purpose))
|
||||
.await?;
|
||||
if self.ctx.data_movement_generation_exhausted()
|
||||
|| self.ctx.data_movement_operation_epoch_exhausted()
|
||||
|| self.ctx.data_movement_generation() != expected_generation
|
||||
{
|
||||
return Err(Error::other("scanner publication lease generation is stale"));
|
||||
}
|
||||
if self.scanner_data_movement_snapshot_locked().await.1 || self.ctx.namespace_commits_pending() {
|
||||
if self.scanner_data_movement_snapshot_locked().await.1 {
|
||||
return Err(Error::other("scanner publication lease is blocked by data movement"));
|
||||
}
|
||||
let Some((lease_generation, lease_namespace_generation)) = self.ctx.scanner_publication_lease_generations(token).await
|
||||
let Some((lease_generation, lease_namespace_generation, purpose)) =
|
||||
self.ctx.scanner_publication_lease_generations(token).await
|
||||
else {
|
||||
return Err(Error::other("scanner publication lease is unknown or expired"));
|
||||
};
|
||||
if purpose == ScannerPublicationLeasePurpose::Authoritative && self.ctx.namespace_commits_pending() {
|
||||
return Err(Error::other("scanner publication lease is blocked by pending namespace commit"));
|
||||
}
|
||||
let namespace_generation = self.scanner_namespace_mutation_generation();
|
||||
if lease_generation != self.ctx.data_movement_generation()
|
||||
|| lease_namespace_generation != namespace_generation
|
||||
|| namespace_generation == u64::MAX
|
||||
|| (purpose == ScannerPublicationLeasePurpose::Authoritative
|
||||
&& lease_namespace_generation != Some(namespace_generation))
|
||||
{
|
||||
return Err(Error::other("scanner publication lease generation is stale"));
|
||||
}
|
||||
@@ -1181,22 +1234,65 @@ impl ECStore {
|
||||
/// restart, expiry, generation changes, or blocked movement all reject it
|
||||
/// before the target disk is touched.
|
||||
pub async fn acquire_scanner_publication_lease_guard(&self, token: Uuid) -> Result<tokio::sync::OwnedRwLockReadGuard<()>> {
|
||||
self.acquire_scanner_publication_lease_guard_with_target(
|
||||
token,
|
||||
None,
|
||||
false,
|
||||
Some(ScannerPublicationLeasePurpose::Authoritative),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn acquire_scanner_publication_lease_guard_for_rename(
|
||||
&self,
|
||||
token: Uuid,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
) -> Result<tokio::sync::OwnedRwLockReadGuard<()>> {
|
||||
let target = format!(
|
||||
"{}/{}",
|
||||
crate::disk::BUCKET_META_PREFIX,
|
||||
rustfs_data_usage::DATA_USAGE_OBSERVED_OBJECT_NAME
|
||||
);
|
||||
let target = (volume == RUSTFS_META_BUCKET && path == target).then_some((volume, path));
|
||||
self.acquire_scanner_publication_lease_guard_with_target(token, target, false, None)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn acquire_scanner_publication_lease_guard_with_target(
|
||||
&self,
|
||||
token: Uuid,
|
||||
observed_target: Option<(&str, &str)>,
|
||||
validation_only: bool,
|
||||
expected_purpose: Option<ScannerPublicationLeasePurpose>,
|
||||
) -> Result<tokio::sync::OwnedRwLockReadGuard<()>> {
|
||||
let operation_gate = self.ctx.data_movement_operation_gate();
|
||||
let operation_guard = operation_gate.read_owned().await;
|
||||
if self.ctx.data_movement_generation_exhausted() || self.ctx.data_movement_operation_epoch_exhausted() {
|
||||
return Err(Error::other("scanner publication lease generation is exhausted"));
|
||||
}
|
||||
if self.scanner_data_movement_snapshot_locked().await.1 || self.ctx.namespace_commits_pending() {
|
||||
if self.scanner_data_movement_snapshot_locked().await.1 {
|
||||
return Err(Error::other("scanner publication lease is blocked by data movement"));
|
||||
}
|
||||
let Some((lease_generation, lease_namespace_generation)) = self.ctx.scanner_publication_lease_generations(token).await
|
||||
let Some((lease_generation, lease_namespace_generation, purpose)) =
|
||||
self.ctx.scanner_publication_lease_generations(token).await
|
||||
else {
|
||||
return Err(Error::other("scanner publication lease is unknown or expired"));
|
||||
};
|
||||
if expected_purpose.is_some_and(|expected| expected != purpose) {
|
||||
return Err(Error::other("scanner publication lease purpose does not match its request"));
|
||||
}
|
||||
if purpose == ScannerPublicationLeasePurpose::Observational && observed_target.is_none() && !validation_only {
|
||||
return Err(Error::other("observational lease is limited to observed usage publication"));
|
||||
}
|
||||
if purpose == ScannerPublicationLeasePurpose::Authoritative && self.ctx.namespace_commits_pending() {
|
||||
return Err(Error::other("scanner publication lease is blocked by pending namespace commit"));
|
||||
}
|
||||
let namespace_generation = self.scanner_namespace_mutation_generation();
|
||||
if lease_generation != self.ctx.data_movement_generation()
|
||||
|| lease_namespace_generation != namespace_generation
|
||||
|| namespace_generation == u64::MAX
|
||||
|| (purpose == ScannerPublicationLeasePurpose::Authoritative
|
||||
&& lease_namespace_generation != Some(namespace_generation))
|
||||
{
|
||||
return Err(Error::other("scanner publication lease generation is stale"));
|
||||
}
|
||||
@@ -2571,6 +2667,59 @@ mod tests {
|
||||
assert!(!store.release_scanner_publication_lease(expiring_token).await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn observational_lease_only_allows_observed_usage_rename_during_namespace_commits() {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
let store = build_store_with_ctx(ctx.clone());
|
||||
let namespace_commit = ctx.begin_namespace_commit();
|
||||
let strict_error = store
|
||||
.acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
|
||||
.await
|
||||
.expect_err("authoritative lease must wait for namespace commits");
|
||||
assert!(strict_error.to_string().contains("pending namespace commit"));
|
||||
|
||||
let (token, movement_generation) = store
|
||||
.acquire_scanner_publication_lease_with_purpose(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL, true)
|
||||
.await
|
||||
.expect("an observation may lease-fence movement while a namespace commit is active");
|
||||
assert_eq!(movement_generation, 0);
|
||||
store
|
||||
.acquire_scanner_publication_lease_guard_for_rename(token, RUSTFS_META_BUCKET, "buckets/.usage.observed.json")
|
||||
.await
|
||||
.expect("the lease must admit the exact observational object");
|
||||
assert!(
|
||||
store
|
||||
.acquire_scanner_publication_lease_guard_for_rename(token, RUSTFS_META_BUCKET, "buckets/.usage.v2.json")
|
||||
.await
|
||||
.is_err(),
|
||||
"an observational lease must not admit the authoritative object"
|
||||
);
|
||||
assert!(
|
||||
store.acquire_scanner_publication_lease_guard(token).await.is_err(),
|
||||
"an observational lease must not authorize an unscoped mutation such as delete or cleanup"
|
||||
);
|
||||
|
||||
drop(namespace_commit);
|
||||
assert!(store.ctx.namespace_commit_generation() > 0);
|
||||
store
|
||||
.validate_scanner_publication_lease_with_purpose(token, movement_generation, true)
|
||||
.await
|
||||
.expect("ordinary namespace generation changes must not invalidate an observation");
|
||||
let error = store
|
||||
.validate_scanner_publication_lease_with_purpose(token, movement_generation, false)
|
||||
.await
|
||||
.expect_err("the same token must not validate as authoritative");
|
||||
assert!(error.to_string().contains("purpose"));
|
||||
|
||||
let operation_gate = store.ctx.data_movement_operation_gate();
|
||||
assert!(
|
||||
operation_gate.clone().try_write_owned().is_err(),
|
||||
"the observational lease must still block data movement"
|
||||
);
|
||||
assert!(store.release_scanner_publication_lease(token).await);
|
||||
assert!(operation_gate.try_write_owned().is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_publication_lease_rejects_a_new_movement_generation() {
|
||||
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
|
||||
@@ -2613,8 +2762,11 @@ mod tests {
|
||||
let error = tokio::time::timeout(Duration::from_secs(1), acquire)
|
||||
.await
|
||||
.expect("snapshot admission must finish")
|
||||
.expect_err("acquisition must preserve its original namespace generation");
|
||||
assert_eq!(error.to_string(), "Io error: scanner publication lease generation is stale");
|
||||
.expect_err("a namespace change during admission must not install a lease");
|
||||
assert_eq!(
|
||||
error.to_string(),
|
||||
"Io error: scanner publication lease namespace changed during acquisition"
|
||||
);
|
||||
assert!(ctx.data_movement_operation_gate().try_write_owned().is_ok(), "no lease was installed");
|
||||
}
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user