mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-05 10:26:56 +00:00
Compare commits
110 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 93fc2c1440 | |||
| 262e890fb2 | |||
| c2958f4320 | |||
| 5ae075666f | |||
| ea8c4bbb79 | |||
| 57df9181d0 | |||
| 371f5e4ffc | |||
| 5505e4f7d7 | |||
| 7e045fe985 | |||
| 8a5a2c9c93 | |||
| cf730f1691 | |||
| b62c5e3654 | |||
| 34c6896768 | |||
| babfeaed00 | |||
| 9026271113 | |||
| 004d872a27 | |||
| 98b633d17a | |||
| d8e52174da | |||
| d96bd52117 | |||
| 736bf777b4 | |||
| faf636e1c7 | |||
| 2401a8d8ac | |||
| 0aa9ba496b | |||
| 7babc7d621 | |||
| d15dde7bb5 | |||
| 0c56ea85db | |||
| 4d46bd25f8 | |||
| ff5f491c5a | |||
| 3483a48fcb | |||
| 7f5fa4b8f6 | |||
| a97c65895c | |||
| 73e7afa01a | |||
| a8be876fd9 | |||
| e0f26a540d | |||
| 3b3cc018aa | |||
| fe3ada0689 | |||
| f4f5635aab | |||
| 926355ae1f | |||
| 36ad5a6b87 | |||
| 6c14128072 | |||
| addc258a24 | |||
| 73ea527f55 | |||
| 55da14ad82 | |||
| 116a5299b2 | |||
| fbc54f7212 | |||
| 3aa492f5ee | |||
| 00a81b4048 | |||
| 20ef9ebf15 | |||
| cc08e0d11b | |||
| 69294523d5 | |||
| cd4d86f28a | |||
| 2f55cf69e1 | |||
| fff6e94378 | |||
| 41b13aeb8b | |||
| 88587a8f94 | |||
| 8c8b5da63f | |||
| 00a238f838 | |||
| 965d805a4f | |||
| 9d66e04ae3 | |||
| 12c58d25b5 | |||
| 1ac1d883f4 | |||
| 45ba33291a | |||
| 636eded277 | |||
| ed4099ebe1 | |||
| 3a1fac16e3 | |||
| b00e85f1a4 | |||
| ecbddc188a | |||
| 2d63931bba | |||
| d429c6f792 | |||
| e494dc2b4b | |||
| 2b7c927ee8 | |||
| 647846956f | |||
| 04690f7da5 | |||
| 3c0829049d | |||
| 890a463218 | |||
| 25f45f6309 | |||
| e3700614a1 | |||
| d4ab7704b3 | |||
| 422e724051 | |||
| e0bf54197d | |||
| b4af43c884 | |||
| 1cb4c0ebe2 | |||
| e5563e6004 | |||
| 0a72b29a34 | |||
| 51275cb7d8 | |||
| 3bf270b49f | |||
| d8d064bd18 | |||
| e5653ca055 | |||
| 6413a11889 | |||
| 9e4ea9fdd1 | |||
| 1a841fb7c7 | |||
| b8ee0301e9 | |||
| 0e5fe8c658 | |||
| e46877dd1c | |||
| 3fc610c8a2 | |||
| 4992034cc2 | |||
| b4c9bf9abb | |||
| 7d45685d70 | |||
| 76bab81aa6 | |||
| 72ff5dbbad | |||
| 8ece50ff8c | |||
| 7f13ebe2a5 | |||
| 696dc1496e | |||
| 68ee16bbbe | |||
| bc0706b2e3 | |||
| c6152954f6 | |||
| 0d2bf69e2d | |||
| 7e536843ea | |||
| 7672867476 | |||
| c8f7f02817 |
@@ -18,6 +18,7 @@ These implementation, style, and quality rules apply to all contributions, regar
|
||||
- Do not include agent, tool, contributor, account, or organization names in branch names.
|
||||
- Push to the user-requested remote or the repository's configured push remote. Do not hard-code or infer a remote from an account name.
|
||||
- Check free space before artifact-heavy builds, tests, coverage, or downloads. Re-check before a broad gate when space is tight.
|
||||
- Before handoff, remove the Cargo target directories and temporary worktrees the task created unless they stay intentionally or hold unmerged work. Identify build output by Cargo's markers (`.rustc_info.json`, `CACHEDIR.TAG`), not by directory name: `target` is also a source directory here.
|
||||
- Remove only task-owned temporary/build artifacts. Never delete another task's worktree or uncommitted data.
|
||||
- At handoff, mention disk or cleanup details only when they affected execution or artifacts/worktrees remain intentionally.
|
||||
- For Docker verification, ensure the tested image contains the intended binary. `buildx build` without `--load` leaves the image in the build cache; use the Makefile's loaded image and rebuild without cache only when stale artifacts are evidenced.
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-linux=eb33067d90ee66c0a97e8f3658e46c152e47f897d2fbfdc6cb5ad93cfa55414b
|
||||
sha256-darwin=eb33067d90ee66c0a97e8f3658e46c152e47f897d2fbfdc6cb5ad93cfa55414b
|
||||
sha256-linux=2f97cbbeb2dbf52919b704a3d07336c162a41f95079d941d74251f9279c494e6
|
||||
sha256-darwin=2f97cbbeb2dbf52919b704a3d07336c162a41f95079d941d74251f9279c494e6
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=67251cb7d9978d38e899268cac4d7898297ff7e5b5c48c9c1546dc760b93568d
|
||||
sha256-linux=673da76c2e5710c4622fe2fa32d2b297730895769487fcf1152ee1c2366f32f4
|
||||
sha256-darwin=a3e166a140f3150218b4a857fedb8749940fad0c8b053d0e3c1b957321bf5beb
|
||||
sha256-linux=dce4a8eb634a27b4589bcf6d2ebf248ede97aa59701acc8696391d2812a3177d
|
||||
|
||||
@@ -1 +1 @@
|
||||
sha256=e9dc896774122a304e2b8b90f28c82d4d4e0af8ded36a82b147b30945ec994e3
|
||||
sha256=5bdd4c2eadbc41625ae3184fff34062e000d328108436259b2b7c4fe2531ecaf
|
||||
|
||||
+13
-7
@@ -234,7 +234,7 @@ test-group = 'embedded-test-ports'
|
||||
# processes cannot exhaust those budgets. Keep deadlines and assertions intact.
|
||||
# The short HTTP fixtures are tracked in rustfs/backlog#2714.
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))'
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|cleanup_accepts_missing_object_and_still_deletes_the_bucket|cleanup_diagnostics_identify_each_disconnected_transport_stage|cleanup_diagnostics_identify_each_failed_http_stage_without_private_context|cleanup_diagnostics_identify_timeouts_without_resetting_the_shared_deadline|oversized_response_is_rejected_without_buffering_it)$/)))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# This cancellation probe hashes the serving executable before acquiring its
|
||||
@@ -548,7 +548,7 @@ filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnost
|
||||
test-group = 'embedded-test-ports'
|
||||
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))'
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|cleanup_accepts_missing_object_and_still_deletes_the_bucket|cleanup_diagnostics_identify_each_disconnected_transport_stage|cleanup_diagnostics_identify_each_failed_http_stage_without_private_context|cleanup_diagnostics_identify_timeouts_without_resetting_the_shared_deadline|oversized_response_is_rejected_without_buffering_it)$/)))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# Preserve the cancellation probe's startup budget and assertions in CI too.
|
||||
@@ -701,7 +701,7 @@ test-group = 'ecstore-serial-flaky'
|
||||
[profile.e2e-smoke]
|
||||
default-filter = """
|
||||
package(e2e_test) & (
|
||||
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat|data_usage)_test::|^fake_s3_target::/)
|
||||
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|multipart_upload_id|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat|data_usage)_test::|^fake_s3_target::/)
|
||||
| test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/)
|
||||
| test(/^reliant::lifecycle::/)
|
||||
| test(/^conditional_delete_objects_test::/)
|
||||
@@ -827,7 +827,7 @@ slow-timeout = { period = "120s", terminate-after = 12, grace-period = "10s" }
|
||||
# ---------------------------------------------------------------------------
|
||||
# e2e-distributed profile — 4-node 4-disk Actions suite
|
||||
# ---------------------------------------------------------------------------
|
||||
# Storage-sensitive PR / nightly / dispatch lane owned by
|
||||
# Nightly and manual-dispatch lane owned by
|
||||
# .github/workflows/e2e-distributed.yml.
|
||||
# Each case starts four rustfs processes (and for site replication, two
|
||||
# clusters). Upgrade cases also require RUSTFS_UPGRADE_SOURCE_BINARY.
|
||||
@@ -970,10 +970,16 @@ threads-required = "num-test-threads"
|
||||
filter = 'package(e2e_test) & (test(/^kms::kms_vault_test::/) | test(/^kms::kms_rekey_sweep_test::/) | test(/^kms::configured_roundtrip_test::test_configured_vault_kms_admin_and_versioned_cleanup$/))'
|
||||
test-group = 'e2e-vault'
|
||||
|
||||
# Keep storage-heavy cases isolated in the full suite too. The large pagination
|
||||
# fixture alone seeds 2,000 objects and can otherwise starve other servers.
|
||||
# Keep the storage-heavy smoke cases isolated in the full suite too. They
|
||||
# start multiple server fixtures and issue many sequential S3 operations.
|
||||
[[profile.e2e-full.overrides]]
|
||||
filter = 'package(e2e_test) & (test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter|test_list_objects_v2_raw_exceeds_maxkeys_but_visible_below)$/) | test(=reliant::tiering::test_hermetic_transition_restore_failure_expiry_and_retry))'
|
||||
filter = 'package(e2e_test) & (test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter)$/) | test(=reliant::tiering::test_hermetic_transition_restore_failure_expiry_and_retry))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# The mc fixture uploads hundreds of objects concurrently before measuring listing.
|
||||
# Reserve the runner so other server fixtures do not starve its disk operations.
|
||||
[[profile.e2e-full.overrides]]
|
||||
filter = 'package(e2e_test) & test(=mc_mirror_small_bucket_test::test_mc_mirror_small_bucket_completes_without_list_timeout)'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# Fixed, small contract for every executable server change. These existing
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"plan_id": "rustfs-release-validation",
|
||||
"description": "Versioned certification plan for every public RustFS release exit (rustfs/backlog#2662, R01). preview, prerelease and stable all require this explicit plan. Scripts/release/release_gate.py fail-closes publication on any missing, duplicate, stale, substituted or non-PASS required item.",
|
||||
"release_types": ["preview", "prerelease", "stable"],
|
||||
"targets": [
|
||||
{"os": "linux", "arch": "x86_64", "abi": "musl"},
|
||||
{"os": "linux", "arch": "aarch64", "abi": "musl"},
|
||||
{"os": "linux", "arch": "x86_64", "abi": "gnu"},
|
||||
{"os": "linux", "arch": "aarch64", "abi": "gnu"},
|
||||
{"os": "macos", "arch": "aarch64", "abi": "none"},
|
||||
{"os": "windows", "arch": "x86_64", "abi": "msvc"}
|
||||
],
|
||||
"targets_basis": "Platform matrix of .github/workflows/build.yml (prepare-platform-matrix). gnu and macOS targets build with the pyroscope feature; musl and Windows targets build with no extra features.",
|
||||
"features": ["pyroscope"],
|
||||
"trusted_workflows": [
|
||||
{"repository": "rustfs/s3chaos", "workflow_id": "release-gate.yml"}
|
||||
],
|
||||
"trusted_workflows_basis": "The shared evidence envelope producer lives in rustfs/s3chaos (G01, fault/release_gate.rs) and its plan/live split workflow is .github/workflows/release-gate.yml there. Only results from these identities may enter a release certification; PR-uploaded JSON never gains release authority. R02 must extend this list when additional verifier workflows land, never widen it to untrusted sources.",
|
||||
"lanes": [
|
||||
{
|
||||
"lane": "identity",
|
||||
"harness_repository": "rustfs/s3chaos",
|
||||
"harness_sha": "9a48f55c1d6d58ce63a8fb1ea7317e024f32f11d",
|
||||
"harness_basis": "Fixed G01 evidence-envelope producer revision (rustfs/s3chaos PR #102 head recorded in rustfs/backlog#2658). R02 re-pins this when the live verifier ships.",
|
||||
"timeout_seconds": 1800,
|
||||
"environment_capabilities": ["process-exec"],
|
||||
"cases": ["artifact-identity", "startup"],
|
||||
"applies_to": {
|
||||
"release_types": ["preview", "prerelease", "stable"],
|
||||
"platforms": "all"
|
||||
},
|
||||
"applies_basis": "Artifact identity and startup checks are mandatory for every release type, preview included, on every shipped target (rustfs/backlog#2662 frozen design item 5)."
|
||||
},
|
||||
{
|
||||
"lane": "upgrade",
|
||||
"harness_repository": "rustfs/s3chaos",
|
||||
"harness_sha": "9a48f55c1d6d58ce63a8fb1ea7317e024f32f11d",
|
||||
"harness_basis": "Same fixed envelope producer as the identity lane; the workflow SHA and the product build SHA are bound separately and need not be equal.",
|
||||
"timeout_seconds": 5400,
|
||||
"environment_capabilities": ["process-exec", "previous-release-artifact"],
|
||||
"cases": [
|
||||
"direct-upgrade",
|
||||
"mixed-version-upgrade",
|
||||
"bucket-config-upgrade",
|
||||
"bucket-config-rollback",
|
||||
"multipart-layout-upgrade",
|
||||
"multipart-layout-baseline"
|
||||
],
|
||||
"cases_basis": "Case IDs of the pinned previous-release matrix in .github/workflows/e2e-upgrade.yml (source.version/asset/sha256 and the case list mapped there).",
|
||||
"applies_to": {
|
||||
"release_types": ["prerelease", "stable"],
|
||||
"platforms": [{"os": "linux", "arch": "x86_64", "abi": "gnu"}]
|
||||
},
|
||||
"applies_basis": "preview releases use the pre-declared smaller set (identity lane only). The upgrade lane is only exercised on linux-x86_64-gnu, the platform of the pinned source assets."
|
||||
}
|
||||
],
|
||||
"upgrade_sources": {
|
||||
"basis": "Exactly the two previous-release sources pinned in .github/workflows/e2e-upgrade.yml are exercised today (source.version / source.asset / source.sha256). No other N-1/N-2 compatibility is claimed, and these digests are not inferred from tags.",
|
||||
"sources": [
|
||||
{
|
||||
"version": "1.0.1",
|
||||
"role": "N-1",
|
||||
"artifact_name": "rustfs-linux-x86_64-gnu-v1.0.1.zip",
|
||||
"artifact_sha256": "9742e183cb0dfb0d48fa909797af572fe47b7f827569036baa4156e0d3958ee5"
|
||||
},
|
||||
{
|
||||
"version": "1.0.0-rc.5",
|
||||
"role": "N-2",
|
||||
"artifact_name": "rustfs-linux-x86_64-gnu-v1.0.0-rc.5.zip",
|
||||
"artifact_sha256": "3ee8df71e8edcfada533be452c4135868f697bc515460ae97b027313eade7a3d"
|
||||
}
|
||||
],
|
||||
"rollback_supported": false,
|
||||
"rollback_basis": "Only the bucket-config-rollback case (rollback_to_previous_release_reads_current_bucket_metadata) is exercised. Full data-plane rollback is not certified and is never a default claim."
|
||||
}
|
||||
}
|
||||
+3
-3
@@ -22,9 +22,9 @@ The following files in the project root are essential for Docker operations:
|
||||
| File | Description | Usage |
|
||||
| :--- | :--- | :--- |
|
||||
| **`docker-buildx.sh`** | **Multi-Arch Build Script**<br>Automates building and pushing Docker images for `amd64` and `arm64`. Supports release and dev channels. | `./docker-buildx.sh --push` |
|
||||
| **`Dockerfile`** | **Production Image (Alpine)**<br>Lightweight image using musl libc. Downloads pre-built binaries from GitHub Releases. | `docker build -t rustfs:latest .` |
|
||||
| **`Dockerfile.glibc`** | **Production Image (Ubuntu)**<br>Standard image using glibc. Useful if you need specific dynamic libraries. | `docker build -f Dockerfile.glibc .` |
|
||||
| **`Dockerfile.source`** | **Development Image**<br>Builds RustFS from source code. Includes build tools. Ideal for local development and CI. | `docker build -f Dockerfile.source .` |
|
||||
| **`Dockerfile`** | **Production Image (Alpine)**<br>Lightweight image using musl libc. Downloads verified RustFS and `rc` binaries from GitHub Releases and bundles `/etc/rustfs/bootstrap`. | `docker build -t rustfs:latest .` |
|
||||
| **`Dockerfile.glibc`** | **Production Image (Ubuntu)**<br>Standard image using glibc. Includes the bundled bootstrap configuration. | `docker build -f Dockerfile.glibc .` |
|
||||
| **`Dockerfile.source`** | **Development Image**<br>Builds RustFS from source code and bundles `rc` and the default bootstrap configuration. | `docker build -f Dockerfile.source .` |
|
||||
|
||||
### Docker Compose Configurations
|
||||
|
||||
|
||||
@@ -32,6 +32,8 @@ updates:
|
||||
- "overtrue"
|
||||
- "majinghe"
|
||||
ignore:
|
||||
- dependency-name: "pulsar"
|
||||
versions: [ "6.9.x" ]
|
||||
- dependency-name: "object_store"
|
||||
versions: [ "0.13.x" ]
|
||||
- dependency-name: "libunftp"
|
||||
|
||||
@@ -296,6 +296,32 @@
|
||||
"origin": "baseline",
|
||||
"alert_contract": "scheduled-validation"
|
||||
},
|
||||
{
|
||||
"workflow": ".github/workflows/rustfs-validation-sweep.yml",
|
||||
"name": "RustFS Validation Sweep",
|
||||
"roles": [
|
||||
"periodic",
|
||||
"manual"
|
||||
],
|
||||
"events": [
|
||||
"schedule",
|
||||
"workflow_dispatch"
|
||||
],
|
||||
"cadence": "nightly",
|
||||
"timezone": "UTC",
|
||||
"schedule": [
|
||||
"30 19 * * *"
|
||||
],
|
||||
"owner": "@overtrue",
|
||||
"policy": "active",
|
||||
"monitor": "workflow",
|
||||
"max_age_hours": 60,
|
||||
"result_owner": "rustfs-validation-sweep.yml",
|
||||
"disposition": "canonical validation cases executed in rotating batches",
|
||||
"failure_owner": "workflow",
|
||||
"origin": "baseline",
|
||||
"alert_contract": "scheduled-validation"
|
||||
},
|
||||
{
|
||||
"workflow": ".github/workflows/btf-prototype-reference.yml",
|
||||
"name": "BTF prototype reference",
|
||||
@@ -743,12 +769,10 @@
|
||||
"workflow": ".github/workflows/e2e-distributed.yml",
|
||||
"name": "e2e-distributed",
|
||||
"roles": [
|
||||
"pr",
|
||||
"periodic",
|
||||
"manual"
|
||||
],
|
||||
"events": [
|
||||
"pull_request",
|
||||
"workflow_dispatch",
|
||||
"schedule"
|
||||
],
|
||||
@@ -823,12 +847,11 @@
|
||||
"workflow": ".github/workflows/e2e-upgrade.yml",
|
||||
"name": "Upgrade Compatibility",
|
||||
"roles": [
|
||||
"pr",
|
||||
"periodic",
|
||||
"manual"
|
||||
"manual",
|
||||
"release"
|
||||
],
|
||||
"events": [
|
||||
"pull_request",
|
||||
"push",
|
||||
"schedule",
|
||||
"workflow_dispatch"
|
||||
|
||||
+311
-5
@@ -814,13 +814,18 @@ jobs:
|
||||
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
|
||||
export AWS_DEFAULT_REGION="auto"
|
||||
|
||||
# Determine upload path based on build type
|
||||
# Determine upload path based on build type. Release and prerelease
|
||||
# builds only stage their bytes here: the official
|
||||
# artifacts/rustfs/release/ path is written solely by the
|
||||
# certification-gated promote-release-artifacts job, from these same
|
||||
# staged objects. Development builds keep the uncertified dev channel
|
||||
# and never touch official or latest channels.
|
||||
if [[ "$BUILD_TYPE" == "development" ]]; then
|
||||
R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/dev/"
|
||||
echo "📤 Uploading development build to R2 dev directory"
|
||||
else
|
||||
R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/release/"
|
||||
echo "📤 Uploading release build to R2 release directory"
|
||||
R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
|
||||
echo "📤 Staging release build for certification-gated promotion"
|
||||
fi
|
||||
|
||||
# Upload all rustfs zip files to R2 using S3-compatible API
|
||||
@@ -869,6 +874,221 @@ jobs:
|
||||
echo "This run does not trigger Docker publishing."
|
||||
fi
|
||||
|
||||
# Aggregate the parent candidate batch. This job always collects results and
|
||||
# retains its evidence, including on failure, and every publication job
|
||||
# below requires its success. Missing, stale, substituted or non-PASS
|
||||
# evidence fails closed here: only the trusted verification sources
|
||||
# (rustfs/s3chaos release-gate envelopes, rustfs/backlog#2658) can satisfy
|
||||
# it, so a PR-uploaded JSON set never gains release authority.
|
||||
release-certification:
|
||||
name: Release Certification
|
||||
needs: [ build-check, build-rustfs ]
|
||||
if: ${{ !cancelled() && needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') }}
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download all build artifacts
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
path: ./artifacts
|
||||
pattern: rustfs-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Write candidate and collect trusted validation results
|
||||
env:
|
||||
# Cross-repo read of the trusted verification run. Falls back to the
|
||||
# workflow token, which cannot read rustfs/s3chaos; a missing token
|
||||
# therefore fail-closes instead of certifying.
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
PRODUCT_COMMIT: ${{ github.sha }}
|
||||
VERSION: ${{ needs.build-check.outputs.version }}
|
||||
BUILD_TYPE: ${{ needs.build-check.outputs.build_type }}
|
||||
FEATURES: ${{ needs.build-check.outputs.build_type == 'development' && '' || 'pyroscope' }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
mkdir -p results release-gate-output
|
||||
PLAN_SHA256=$(python3 -c "import hashlib,json,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('.config/release-validation-plan.json'))))")
|
||||
|
||||
python3 - "$PLAN_SHA256" > candidate.json <<'PY'
|
||||
import hashlib, json, os, sys, zipfile
|
||||
|
||||
plan_sha256 = sys.argv[1]
|
||||
profile = {"release": "stable", "prerelease": "prerelease", "preview": "preview"}.get(os.environ["BUILD_TYPE"])
|
||||
if profile is None:
|
||||
raise SystemExit("release certification only covers preview, prerelease and stable builds")
|
||||
|
||||
def sha256(path):
|
||||
digest = hashlib.sha256()
|
||||
with open(path, "rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
deliverables = []
|
||||
for name in sorted(os.listdir("artifacts")):
|
||||
if not name.startswith("rustfs-") or not name.endswith(".zip"):
|
||||
continue
|
||||
path = os.path.join("artifacts", name)
|
||||
members = zipfile.ZipFile(path).namelist()
|
||||
binary_name = "rustfs.exe" if "rustfs.exe" in members else "rustfs"
|
||||
with zipfile.ZipFile(path) as archive:
|
||||
binary_bytes = archive.read(binary_name)
|
||||
import tempfile
|
||||
with tempfile.NamedTemporaryFile() as handle:
|
||||
handle.write(binary_bytes)
|
||||
handle.flush()
|
||||
binary_sha = sha256(handle.name)
|
||||
deliverables.append({
|
||||
"kind": "zip",
|
||||
"name": name,
|
||||
"location": "https://github.com/rustfs/rustfs/releases/download/{}/{}".format(os.environ["VERSION"], name),
|
||||
"sha256": sha256(path),
|
||||
"size": os.path.getsize(path),
|
||||
"binary_sha256": binary_sha,
|
||||
})
|
||||
if not deliverables:
|
||||
raise SystemExit("no built deliverables found to certify")
|
||||
candidate = {
|
||||
"schema_version": 1,
|
||||
"product_commit": os.environ["PRODUCT_COMMIT"],
|
||||
"version": os.environ["VERSION"],
|
||||
"release_profile": profile,
|
||||
"features": [f for f in os.environ.get("FEATURES", "").split(",") if f],
|
||||
"plan_sha256": plan_sha256,
|
||||
"validation_batch": {"run_id": os.environ["GITHUB_RUN_ID"], "run_attempt": int(os.environ["GITHUB_RUN_ATTEMPT"])},
|
||||
"deliverables": deliverables,
|
||||
}
|
||||
json.dump(candidate, sys.stdout, indent=2, sort_keys=True)
|
||||
print()
|
||||
PY
|
||||
|
||||
# Trusted verification evidence only: the release-gate envelopes of
|
||||
# rustfs/s3chaos for this exact product commit. Absence blocks below.
|
||||
TRUSTED_REPO="rustfs/s3chaos"
|
||||
TRUSTED_WORKFLOW="release-gate.yml"
|
||||
TRUSTED_RUN=$(gh api "repos/${TRUSTED_REPO}/actions/workflows/${TRUSTED_WORKFLOW}/runs?head_sha=${PRODUCT_COMMIT}&status=success&per_page=1" --jq '.workflow_runs[0].id // empty' || true)
|
||||
if [[ -z "$TRUSTED_RUN" ]]; then
|
||||
echo "❌ No trusted verification run of ${TRUSTED_REPO}/${TRUSTED_WORKFLOW} for ${PRODUCT_COMMIT}; refusing to certify" >&2
|
||||
else
|
||||
gh run download "$TRUSTED_RUN" -R "$TRUSTED_REPO" -n release-validation-evidence -D results || \
|
||||
echo "❌ Trusted verification run ${TRUSTED_RUN} published no release-validation-evidence artifact" >&2
|
||||
fi
|
||||
|
||||
python3 scripts/release/release_gate.py validate \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate candidate.json \
|
||||
--results results \
|
||||
--output release-gate-output
|
||||
|
||||
- name: Upload certification evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: release-certification
|
||||
path: |
|
||||
candidate.json
|
||||
results/
|
||||
release-gate-output/
|
||||
if-no-files-found: warn
|
||||
retention-days: 30
|
||||
|
||||
# The only writer of the official R2 release path. Promotion re-checks the
|
||||
# exact bytes about to be published against the passed certification list
|
||||
# and copies the staged objects byte-for-byte: no rebuild-after-verify.
|
||||
promote-release-artifacts:
|
||||
name: Promote Release Artifacts
|
||||
needs: [ build-check, build-rustfs, release-certification ]
|
||||
# Target-channel lock for the official R2 release path.
|
||||
concurrency:
|
||||
group: release-channel-r2-release
|
||||
cancel-in-progress: false
|
||||
if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
env:
|
||||
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
|
||||
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
|
||||
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
|
||||
R2_BUCKET: ${{ secrets.R2_BUCKET }}
|
||||
AWS_EC2_METADATA_DISABLED: true
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download build and certification artifacts
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
path: ./artifacts
|
||||
pattern: rustfs-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Download certification evidence
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: release-certification
|
||||
path: ./certification
|
||||
|
||||
- name: Verify and promote staged release artifacts
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
if [[ -z "$R2_ACCESS_KEY_ID" || -z "$R2_SECRET_ACCESS_KEY" || -z "$R2_ENDPOINT" || -z "$R2_BUCKET" ]]; then
|
||||
echo "❌ R2 credentials missing; refusing to promote to the official channel" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for zip_file in artifacts/rustfs-*.zip; do
|
||||
[[ -f "$zip_file" ]] || continue
|
||||
name="$(basename "$zip_file")"
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$zip_file" \
|
||||
--artifact-name "$name"
|
||||
done
|
||||
|
||||
if ! command -v aws >/dev/null 2>&1; then
|
||||
sudo apt-get update && sudo apt-get install -y awscli
|
||||
fi
|
||||
export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
|
||||
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
|
||||
export AWS_DEFAULT_REGION="auto"
|
||||
|
||||
STAGING_PATH="s3://${R2_BUCKET}/artifacts/rustfs/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
|
||||
RELEASE_PATH="s3://${R2_BUCKET}/artifacts/rustfs/release/"
|
||||
for zip_file in artifacts/rustfs-*.zip; do
|
||||
[[ -f "$zip_file" ]] || continue
|
||||
name="$(basename "$zip_file")"
|
||||
staged="$(mktemp)"
|
||||
aws s3 cp "${STAGING_PATH}${name}" "$staged" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
staged_sha="$(sha256sum "$staged" | awk '{print $1}')"
|
||||
built_sha="$(sha256sum "$zip_file" | awk '{print $1}')"
|
||||
if [[ "$staged_sha" != "$built_sha" ]]; then
|
||||
echo "❌ Staged object ${name} does not match the verified bytes" >&2
|
||||
exit 1
|
||||
fi
|
||||
aws s3 cp "$staged" "${RELEASE_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
rm -f "$staged"
|
||||
echo "✅ Promoted ${name} (${built_sha})"
|
||||
done
|
||||
|
||||
# Create GitHub Release for every valid release tag, including previews
|
||||
create-release:
|
||||
name: Create GitHub Release
|
||||
@@ -1028,6 +1248,10 @@ jobs:
|
||||
update-latest-version:
|
||||
name: Update Latest Version
|
||||
needs: [ build-check, publish-release ]
|
||||
# Target-channel lock for the latest.json version pointer.
|
||||
concurrency:
|
||||
group: release-channel-oss-latest
|
||||
cancel-in-progress: false
|
||||
if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
@@ -1068,6 +1292,45 @@ jobs:
|
||||
rm -rf "$OSSUTIL_DIR" "$OSSUTIL_ZIP"
|
||||
chmod +x "$OSSUTIL_BIN"
|
||||
|
||||
# Candidate/pointer ordering: an older candidate must never overwrite
|
||||
# a newer pointer, even if its promotion job finishes later.
|
||||
CURRENT_LATEST="${RUNNER_TEMP}/current-latest.json"
|
||||
if "$OSSUTIL_BIN" cp oss://rustfs-version/latest.json "$CURRENT_LATEST" 2>/dev/null; then
|
||||
python3 - "$CURRENT_LATEST" "$VERSION" <<'PY'
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
|
||||
def key(version):
|
||||
match = re.match(r"^(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?$", version)
|
||||
if not match:
|
||||
raise SystemExit(3)
|
||||
core = tuple(int(part) for part in match.groups()[:3])
|
||||
prerelease = match.group(4)
|
||||
if prerelease is None:
|
||||
return core + (1,)
|
||||
return core + (0, tuple(int(p) if p.isdigit() else p for p in re.split(r"[.-]", prerelease)))
|
||||
|
||||
try:
|
||||
current = json.load(open(sys.argv[1], encoding="utf-8")).get("version", "")
|
||||
proposed = sys.argv[2]
|
||||
if current and key(current) > key(proposed):
|
||||
raise SystemExit(1)
|
||||
except SystemExit:
|
||||
raise
|
||||
except (OSError, ValueError, TypeError):
|
||||
raise SystemExit(3)
|
||||
PY
|
||||
guard_status=$?
|
||||
if [[ "$guard_status" -eq 1 ]]; then
|
||||
echo "Current latest.json points at a newer version; refusing to downgrade" >&2
|
||||
exit 0
|
||||
elif [[ "$guard_status" -ne 0 ]]; then
|
||||
echo "Existing latest.json is unparsable; refusing to overwrite it blindly" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Create latest.json
|
||||
cat > latest.json << EOF
|
||||
{
|
||||
@@ -1084,16 +1347,59 @@ jobs:
|
||||
|
||||
echo "✅ Updated latest.json for ${RELEASE_TYPE} release $VERSION"
|
||||
|
||||
# Publish release (remove draft status)
|
||||
# Publish release (remove draft status). Requires the release-certification
|
||||
# aggregate: without a complete legal success there is no public write.
|
||||
publish-release:
|
||||
name: Publish Release
|
||||
needs: [ build-check, create-release, upload-release-assets ]
|
||||
needs: [ build-check, create-release, upload-release-assets, release-certification ]
|
||||
# Target-channel lock: concurrent releases serialize on this channel so an
|
||||
# older candidate cannot interleave its publish with a newer one.
|
||||
concurrency:
|
||||
group: release-channel-github-release
|
||||
cancel-in-progress: false
|
||||
if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'preview' || needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: write
|
||||
actions: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download certification evidence
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: release-certification
|
||||
path: ./certification
|
||||
|
||||
- name: Verify release assets against certification
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
TAG="${{ needs.build-check.outputs.version }}"
|
||||
mkdir -p "$RUNNER_TEMP/promotion-assets"
|
||||
gh release download "$TAG" --dir "$RUNNER_TEMP/promotion-assets" --pattern '*.zip'
|
||||
found=0
|
||||
for zip_file in "$RUNNER_TEMP"/promotion-assets/*.zip; do
|
||||
[[ -f "$zip_file" ]] || continue
|
||||
found=1
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$zip_file" \
|
||||
--artifact-name "$(basename "$zip_file")"
|
||||
done
|
||||
if [[ "$found" -ne 1 ]]; then
|
||||
echo "No release zip assets to verify against the certification" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Publish release
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
|
||||
@@ -96,7 +96,7 @@ env:
|
||||
RUST_BACKTRACE: 1
|
||||
|
||||
jobs:
|
||||
# Readers: test-and-lint (workspace and ILM), build-rustfs-debug-binary,
|
||||
# Readers: workspace-lint-and-build, workspace-tests (workspace and ILM), build-rustfs-debug-binary,
|
||||
# e2e-tests, e2e-full.
|
||||
warm-ci-dev:
|
||||
name: Warm ci-dev
|
||||
@@ -146,7 +146,7 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
# --all-targets covers the test binaries nextest builds, including
|
||||
# e2e_test, which test-and-lint's own run excludes. The second build adds
|
||||
# e2e_test, which the workspace run excludes. The second build adds
|
||||
# the e2e-test-hooks feature resolution that build-rustfs-debug-binary uses
|
||||
# and that no lint lane enables.
|
||||
- name: Build ci-dev superset
|
||||
|
||||
+95
-34
@@ -113,7 +113,7 @@ jobs:
|
||||
printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
|
||||
echo "Base CI policy unavailable; running the full matrix."
|
||||
fi
|
||||
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^rehearse_shards=' "$GITHUB_OUTPUT"; then
|
||||
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=3$' "$GITHUB_OUTPUT"; then
|
||||
printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'rehearse_shards=false' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
|
||||
echo "Base policy does not cover this workflow contract; retaining the full matrix."
|
||||
fi
|
||||
@@ -160,7 +160,7 @@ jobs:
|
||||
- name: Run shared quick checks
|
||||
uses: ./.github/actions/quick-checks
|
||||
|
||||
test-and-lint:
|
||||
workspace-lint-and-build:
|
||||
name: Workspace Lint and Build
|
||||
if: contains(fromJSON('["full", "e2e"]'), needs.classify-changes.outputs.mode) && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
@@ -193,13 +193,13 @@ jobs:
|
||||
|
||||
- name: Prepare test evidence
|
||||
run: |
|
||||
mkdir -p artifacts/test-and-lint
|
||||
mkdir -p artifacts/workspace
|
||||
{
|
||||
echo "run_id=${GITHUB_RUN_ID}"
|
||||
echo "job=${GITHUB_JOB}"
|
||||
echo "runner=${RUNNER_NAME}"
|
||||
echo "started_at=$(date --utc --iso-8601=seconds)"
|
||||
} > artifacts/test-and-lint/run-metadata.txt
|
||||
} > artifacts/workspace/run-metadata.txt
|
||||
|
||||
# Clippy gates every event, pull requests included. When it ran only after
|
||||
# merge, a lint regression merged freely and then failed every main run
|
||||
@@ -233,11 +233,11 @@ jobs:
|
||||
|
||||
- name: Run documentation tests
|
||||
run: |
|
||||
mkdir -p artifacts/test-and-lint
|
||||
mkdir -p artifacts/workspace
|
||||
set +e
|
||||
timeout --verbose --signal=TERM --kill-after=30s 15m \
|
||||
python3 scripts/ci_test_artifact.py cargo doc --configuration workspace \
|
||||
2>&1 | tee artifacts/test-and-lint/doctest.log
|
||||
2>&1 | tee artifacts/workspace/doctest.log
|
||||
status=${PIPESTATUS[0]}
|
||||
{
|
||||
echo "command=cargo test --all --exclude e2e_test --doc"
|
||||
@@ -246,7 +246,7 @@ jobs:
|
||||
echo
|
||||
echo "Remaining test-related processes:"
|
||||
pgrep -af 'cargo|rustdoc|target/.*/deps/' || true
|
||||
} > artifacts/test-and-lint/doctest-diagnostics.txt
|
||||
} > artifacts/workspace/doctest-diagnostics.txt
|
||||
exit "${status}"
|
||||
|
||||
- name: Upload test reports and diagnostics
|
||||
@@ -255,7 +255,7 @@ jobs:
|
||||
with:
|
||||
name: junit-test-and-lint-${{ github.run_number }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
artifacts/test-and-lint
|
||||
artifacts/workspace
|
||||
retention-days: 3
|
||||
if-no-files-found: error
|
||||
|
||||
@@ -268,7 +268,7 @@ jobs:
|
||||
|
||||
workspace-tests:
|
||||
name: Workspace tests
|
||||
needs: [ test-and-lint, classify-changes ]
|
||||
needs: [ workspace-lint-and-build, classify-changes ]
|
||||
# The archive retains compile-time fixture paths and the large test set
|
||||
# requires the same runner image until the hosted-runtime pilot passes.
|
||||
runs-on: sm-standard-4
|
||||
@@ -293,14 +293,14 @@ jobs:
|
||||
|
||||
- name: Extract and list workspace tests
|
||||
run: |
|
||||
mkdir -p artifacts/test-and-lint
|
||||
mkdir -p artifacts/workspace
|
||||
cargo nextest list --profile ci --archive-file "$RUNNER_TEMP/workspace-tests.tar.zst" --workspace-remap "$GITHUB_WORKSPACE" --extract-to "$GITHUB_WORKSPACE" --extract-overwrite --message-format json \
|
||||
> artifacts/test-and-lint/core-test-listing.json
|
||||
python3 scripts/check_test_wiring.py --check-core artifacts/test-and-lint/core-test-listing.json
|
||||
> artifacts/workspace/core-test-listing.json
|
||||
python3 scripts/check_test_wiring.py --check-core artifacts/workspace/core-test-listing.json
|
||||
|
||||
- name: Run nextest tests
|
||||
run: |
|
||||
mkdir -p artifacts/test-and-lint
|
||||
mkdir -p artifacts/workspace
|
||||
rm -f target/nextest/ci/junit.xml
|
||||
./scripts/ci/resource_sampler.sh start nextest
|
||||
trap './scripts/ci/resource_sampler.sh stop' EXIT
|
||||
@@ -309,7 +309,7 @@ jobs:
|
||||
cargo nextest run --profile ci --cargo-metadata target/nextest/cargo-metadata.json --binaries-metadata target/nextest/binaries-metadata.json \
|
||||
--workspace-remap "$GITHUB_WORKSPACE" --target-dir-remap "$GITHUB_WORKSPACE/target" \
|
||||
--status-level all --final-status-level all \
|
||||
2>&1 | tee artifacts/test-and-lint/nextest.log
|
||||
2>&1 | tee artifacts/workspace/nextest.log
|
||||
status=${PIPESTATUS[0]}
|
||||
{
|
||||
echo "command=workspace archive execution"
|
||||
@@ -321,13 +321,13 @@ jobs:
|
||||
echo
|
||||
echo "Kernel OOM / kill events:"
|
||||
dmesg -T 2>/dev/null | grep -iE 'oom|out of memory|killed process' | tail -20 || true
|
||||
} > artifacts/test-and-lint/nextest-diagnostics.txt
|
||||
} > artifacts/workspace/nextest-diagnostics.txt
|
||||
exit "${status}"
|
||||
|
||||
- name: Verify workspace execution completeness
|
||||
run: |
|
||||
python3 scripts/ci_test_artifact.py evidence artifacts/test-and-lint/core-test-listing.json target/nextest/ci/junit.xml
|
||||
./scripts/check_migration_gate_count.sh evidence artifacts/test-and-lint/core-test-listing.json target/nextest/ci/junit.xml
|
||||
python3 scripts/ci_test_artifact.py evidence artifacts/workspace/core-test-listing.json target/nextest/ci/junit.xml
|
||||
./scripts/check_migration_gate_count.sh evidence artifacts/workspace/core-test-listing.json target/nextest/ci/junit.xml
|
||||
|
||||
- name: Run ignored ILM integration tests serially
|
||||
if: needs.classify-changes.outputs.mode == 'full'
|
||||
@@ -358,7 +358,7 @@ jobs:
|
||||
name: workspace-test-evidence-${{ github.run_attempt }}
|
||||
path: |
|
||||
target/nextest/ci/junit.xml
|
||||
artifacts/test-and-lint
|
||||
artifacts/workspace
|
||||
artifacts/ilm-integration
|
||||
retention-days: 3
|
||||
if-no-files-found: error
|
||||
@@ -368,7 +368,7 @@ jobs:
|
||||
if: needs.classify-changes.outputs.rehearse_shards == 'true'
|
||||
run: |
|
||||
python3 scripts/ci_test_shards.py prepare \
|
||||
--listing artifacts/test-and-lint/core-test-listing.json \
|
||||
--listing artifacts/workspace/core-test-listing.json \
|
||||
--junit target/nextest/ci/junit.xml --output artifacts/workspace-shard-plan
|
||||
- name: Upload same-run shard plan
|
||||
if: needs.classify-changes.outputs.rehearse_shards == 'true'
|
||||
@@ -449,7 +449,7 @@ jobs:
|
||||
|
||||
connect-feature-boundaries:
|
||||
name: Connect Feature Boundaries
|
||||
if: needs.classify-changes.outputs.mode == 'full' && needs.classify-changes.outputs.connect != 'false' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
if: needs.classify-changes.outputs.mode == 'full' && needs.classify-changes.outputs.connect != 'false' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
@@ -497,13 +497,72 @@ jobs:
|
||||
- name: Check offline enrollment E2E root boundary
|
||||
run: ./scripts/check_offline_enrollment_e2e.sh
|
||||
|
||||
fips-provider:
|
||||
name: FIPS provider boundaries
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
# This builds the RustFS library test target, which exceeds hosted-runner disk capacity.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
mode: [default, fips]
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
CARGO_BUILD_JOBS: "2"
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: ci-fips-provider
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
- name: Install FIPS build tools
|
||||
if: matrix.mode == 'fips'
|
||||
run: sudo apt-get install -y cmake golang clang libclang-dev
|
||||
|
||||
- name: Verify provider installation
|
||||
shell: bash
|
||||
run: |
|
||||
args=()
|
||||
if [[ '${{ matrix.mode }}' == 'fips' ]]; then
|
||||
args+=(--features fips)
|
||||
fi
|
||||
cargo nextest run --locked -p rustfs-tls-runtime --lib "${args[@]}" --no-tests fail -E 'test(provider_policy_isolated)'
|
||||
cargo nextest run --locked -p rustfs-s3-client --lib "${args[@]}" --no-tests fail -E 'test(overflowing_closed_range_is_rejected_before_network_io)'
|
||||
cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(validate_pulsar)'
|
||||
cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(webhook_provider_policy_isolated)'
|
||||
if [[ '${{ matrix.mode }}' == 'fips' ]]; then
|
||||
cargo nextest run --locked -p rustfs-protocols --lib --features ftps,webdav,fips --no-tests fail -E 'test(protocols_reject_incompatible_host_provider)'
|
||||
cargo nextest run --locked -p rustfs-targets --lib --features fips --no-tests fail -E 'test(fips_postgres_rejects_incompatible_host_provider)'
|
||||
cargo nextest run --locked -p rustfs-s3-client --lib --features fips --no-tests fail -E 'test(fips_transition_rejects_incompatible_host_provider)'
|
||||
fi
|
||||
tests=(provider_installation_isolated prepared_generation_shares_crypto_and_serves_sni_with_mtls)
|
||||
if [[ '${{ matrix.mode }}' == 'fips' ]]; then
|
||||
tests+=(fips_provider_builds_fips_tls_configs fips_provider_tls_handshake_accepts_and_rejects_algorithms)
|
||||
fi
|
||||
for test_name in "${tests[@]}"; do
|
||||
cargo nextest run --locked -p rustfs --lib "${args[@]}" --no-tests fail -E "test($test_name)"
|
||||
done
|
||||
cargo clippy --locked -p rustfs --lib "${args[@]}" -- -D warnings
|
||||
|
||||
test-and-lint-rio-v2:
|
||||
name: Test and Lint (rio-v2)
|
||||
# Alternate feature configurations run on the nightly schedule, merge
|
||||
# groups and manual dispatch. The default workspace remains the PR gate and
|
||||
# the per-push main baseline: repeating four feature builds on every merge
|
||||
# bought nothing the nightly run does not report within a day.
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["merge_group", "schedule", "workflow_dispatch"]'), github.event_name)
|
||||
# Specialized boundary, hardware and alternate-feature lanes run only on
|
||||
# the nightly schedule and manual dispatch. They do not gate a merge: the
|
||||
# nightly run reports a regression within a day, and repeating these builds
|
||||
# on every PR revision and merge group bought no earlier signal worth the
|
||||
# runner occupancy. Workspace lint and tests, the core contract, the debug
|
||||
# build, E2E smoke and the S3 lanes remain the pull-request gate.
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
@@ -540,12 +599,12 @@ jobs:
|
||||
# --profile ci so the quarantine list (and its junit flaky markers)
|
||||
# covers this leg too; the default profile is the local no-retry
|
||||
# profile and silently ignored quarantined flakes here (rustfs#6703).
|
||||
mkdir -p artifacts/test-and-lint
|
||||
mkdir -p artifacts/workspace
|
||||
rm -f target/nextest/ci/junit.xml
|
||||
./scripts/ci/resource_sampler.sh start nextest-rio-v2
|
||||
trap './scripts/ci/resource_sampler.sh stop' EXIT
|
||||
cargo nextest run --profile ci -p rustfs -p rustfs-ecstore --features rio-v2 \
|
||||
2>&1 | tee artifacts/test-and-lint/nextest.log
|
||||
2>&1 | tee artifacts/workspace/nextest.log
|
||||
cargo test -p rustfs --doc --features rio-v2
|
||||
|
||||
- name: Upload rio-v2 test reports and diagnostics
|
||||
@@ -555,14 +614,14 @@ jobs:
|
||||
name: junit-test-and-lint-rio-v2-${{ github.run_number }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
target/nextest/ci/junit.xml
|
||||
artifacts/test-and-lint
|
||||
artifacts/workspace
|
||||
retention-days: 3
|
||||
if-no-files-found: error
|
||||
|
||||
test-and-lint-protocols:
|
||||
name: "Test and Lint (${{ matrix.features.name }})"
|
||||
# Same cadence as test-and-lint-rio-v2 above.
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["merge_group", "schedule", "workflow_dispatch"]'), github.event_name)
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
@@ -631,7 +690,7 @@ jobs:
|
||||
path: |
|
||||
target/nextest/ci/junit.xml
|
||||
artifacts/protocol-tests
|
||||
artifacts/test-and-lint
|
||||
artifacts/workspace
|
||||
retention-days: 3
|
||||
if-no-files-found: warn
|
||||
|
||||
@@ -783,7 +842,7 @@ jobs:
|
||||
# job had neither, so each closed/merged PR really ran the whole io_uring
|
||||
# suite (measured 4m17s / 7m19s / 7m31s on runs 30678272341 / 30678117601 /
|
||||
# 30662728539) and kept the cancellation run in progress for minutes.
|
||||
if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
# GitHub-hosted ubuntu-latest runs a recent kernel with io_uring and, unlike
|
||||
# a container, applies no seccomp filter that would block io_uring_setup — so
|
||||
@@ -978,7 +1037,7 @@ jobs:
|
||||
path: |
|
||||
${{ runner.temp }}/rustfs-e2e-smoke-logs/
|
||||
${{ runner.temp }}/rustfs-e2e-smoke-list.json
|
||||
artifacts/test-and-lint
|
||||
artifacts/workspace
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Upload e2e smoke JUnit report
|
||||
@@ -1389,12 +1448,13 @@ jobs:
|
||||
- classify-changes
|
||||
- typos
|
||||
- quick-checks
|
||||
- test-and-lint
|
||||
- workspace-lint-and-build
|
||||
- workspace-tests
|
||||
- workspace-shards
|
||||
- workspace-shard-evidence
|
||||
- core-contract
|
||||
- connect-feature-boundaries
|
||||
- fips-provider
|
||||
- test-and-lint-rio-v2
|
||||
- test-and-lint-protocols
|
||||
- build-rustfs-debug-binary
|
||||
@@ -1422,10 +1482,11 @@ jobs:
|
||||
needs:
|
||||
- classify-changes
|
||||
- connect-feature-boundaries
|
||||
- fips-provider
|
||||
- required-checks
|
||||
- typos
|
||||
- quick-checks
|
||||
- test-and-lint
|
||||
- workspace-lint-and-build
|
||||
- workspace-tests
|
||||
- core-contract
|
||||
- test-and-lint-rio-v2
|
||||
|
||||
@@ -329,13 +329,75 @@ jobs:
|
||||
echo " - Create latest: $create_latest"
|
||||
echo " - Source ref: $source_ref"
|
||||
|
||||
# Consume the trusted release certification for the exact version this
|
||||
# workflow wraps and verify the certified input digests. A manual dispatch
|
||||
# without that certification fails here and pushes no official tag; only
|
||||
# results from the trusted verification sources carry release authority.
|
||||
release-certification:
|
||||
name: Release Certification
|
||||
needs: build-check
|
||||
if: needs.build-check.outputs.should_build == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ needs.build-check.outputs.source_ref }}
|
||||
|
||||
- name: Download and verify release certification
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
VERSION: ${{ needs.build-check.outputs.version }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification
|
||||
run_id="$TRIGGERING_RUN_ID"
|
||||
if [[ -z "$run_id" ]]; then
|
||||
run_id="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
|
||||
-f branch="$VERSION" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
fi
|
||||
if [[ -z "$run_id" ]]; then
|
||||
echo "No certified Build and Release run found for ${VERSION}; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! gh run download "$run_id" -n release-certification -D certification; then
|
||||
echo "Build run ${run_id} published no release-certification artifact; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
|
||||
--artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
|
||||
|
||||
# Input digest binding: every zip this image wraps must be the
|
||||
# certified byte-for-byte content, never a mutable latest pointer.
|
||||
while IFS=$'\t' read -r name location expected; do
|
||||
echo "Verifying certified input ${name}"
|
||||
actual="$(curl -fsSL "$location" | sha256sum | awk '{print $1}')"
|
||||
if [[ "$actual" != "$expected" ]]; then
|
||||
echo "Certified input ${name} does not match its recorded digest" >&2
|
||||
exit 1
|
||||
fi
|
||||
done < <(jq -r '.deliverables[] | select(.name | contains("linux")) | [.name, .location, .sha256] | @tsv' certification/candidate.json)
|
||||
|
||||
# Build multi-arch Docker images
|
||||
# Strategy: Build images using pre-built binaries from dl.rustfs.com
|
||||
# Supports both release and dev channel binaries based on build context
|
||||
# Only runs when should_build is true (which includes workflow success check)
|
||||
build-docker:
|
||||
name: Build Docker Images
|
||||
needs: build-check
|
||||
needs: [ build-check, release-certification ]
|
||||
if: needs.build-check.outputs.should_build == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
@@ -443,6 +505,14 @@ jobs:
|
||||
# Output tags
|
||||
echo "tags=$TAGS" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Staging identity for this run's promotion: official tags are only
|
||||
# created after the staged image verifies against the certification,
|
||||
# and they point at this same digest (no rebuild-after-verify).
|
||||
STAGING_TAG="staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${VARIANT_SUFFIX}"
|
||||
STAGING_TAGS="${{ env.REGISTRY_DOCKERHUB }}:${STAGING_TAG},${{ env.REGISTRY_GHCR }}:${STAGING_TAG},${{ env.REGISTRY_QUAY }}:${STAGING_TAG}"
|
||||
echo "staging_tags=$STAGING_TAGS" >> "$GITHUB_OUTPUT"
|
||||
echo "staging_tag=$STAGING_TAG" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Generate labels
|
||||
LABELS="org.opencontainers.image.title=RustFS"
|
||||
LABELS="$LABELS,org.opencontainers.image.description=RustFS distributed object storage system"
|
||||
@@ -467,7 +537,9 @@ jobs:
|
||||
file: ${{ matrix.file }}
|
||||
platforms: ${{ env.DOCKER_PLATFORMS }}
|
||||
push: ${{ needs.build-check.outputs.should_push == 'true' }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
# Push to staging tags only: official tags are created by the
|
||||
# certification-gated promotion step below from this same digest.
|
||||
tags: ${{ steps.meta.outputs.staging_tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
# No layer cache. This build compiles nothing — it downloads a
|
||||
# release zip and runs apk/apt — so the cache could only save the
|
||||
@@ -493,6 +565,114 @@ jobs:
|
||||
no-cache: false
|
||||
pull: true
|
||||
|
||||
# A later-built wrapper: the staged image must complete this platform's
|
||||
# own binary identity verification before any official tag is written.
|
||||
# It may not inherit an unverified package's conclusion.
|
||||
- name: Verify staged image against certification
|
||||
if: needs.build-check.outputs.should_push == 'true'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
VERSION: ${{ needs.build-check.outputs.version }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification
|
||||
run_id="$TRIGGERING_RUN_ID"
|
||||
if [[ -z "$run_id" ]]; then
|
||||
run_id="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
|
||||
-f branch="$VERSION" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
fi
|
||||
gh run download "$run_id" -n release-certification -D certification
|
||||
|
||||
STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
|
||||
case "${{ matrix.variant }}" in
|
||||
musl) libc="musl" ;;
|
||||
glibc) libc="gnu" ;;
|
||||
esac
|
||||
for platform in linux/amd64 linux/arm64; do
|
||||
case "$platform" in
|
||||
linux/amd64) arch_sub="x86_64" ;;
|
||||
linux/arm64) arch_sub="aarch64" ;;
|
||||
esac
|
||||
docker pull --platform "$platform" "$STAGING_REF"
|
||||
container="$(docker create --platform "$platform" "$STAGING_REF")"
|
||||
docker cp "$container:/usr/bin/rustfs" "$RUNNER_TEMP/staged-rustfs"
|
||||
docker rm "$container" >/dev/null
|
||||
actual="$(sha256sum "$RUNNER_TEMP/staged-rustfs" | awk '{print $1}')"
|
||||
expected="$(jq -r --arg needle "linux-${arch_sub}-${libc}" \
|
||||
'[.deliverables[] | select(.name | contains($needle)) | .binary_sha256][0] // empty' \
|
||||
certification/candidate.json)"
|
||||
if [[ -z "$expected" || "$actual" != "$expected" ]]; then
|
||||
echo "Staged image binary for ${platform} does not match the certified binary digest" >&2
|
||||
exit 1
|
||||
fi
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$RUNNER_TEMP/staged-rustfs" \
|
||||
--artifact-role binary
|
||||
done
|
||||
|
||||
- name: Promote staged image to official tags
|
||||
if: needs.build-check.outputs.should_push == 'true'
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
|
||||
IFS=',' read -r -a official_tags <<< "${{ steps.meta.outputs.tags }}"
|
||||
for tag in "${official_tags[@]}"; do
|
||||
docker buildx imagetools create -t "$tag" "$STAGING_REF"
|
||||
echo "Promoted $tag -> $STAGING_REF"
|
||||
done
|
||||
|
||||
# Downstream consumers (Helm) verify this record instead of inheriting
|
||||
# an unverified wrapper's conclusion.
|
||||
- name: Record image promotion
|
||||
if: needs.build-check.outputs.should_push == 'true'
|
||||
shell: bash
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
VERSION: ${{ needs.build-check.outputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification
|
||||
run_id="$TRIGGERING_RUN_ID"
|
||||
if [[ -z "$run_id" ]]; then
|
||||
run_id="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
|
||||
-f branch="$VERSION" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
fi
|
||||
gh run download "$run_id" -n release-certification -D certification
|
||||
STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
|
||||
digest="$(docker buildx imagetools inspect "$STAGING_REF" --format '{{json .Manifest}}' | jq -r '.digest')"
|
||||
candidate_sha="$(python3 -c "import json,hashlib,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('certification/candidate.json'))))")"
|
||||
jq -n \
|
||||
--arg version "$VERSION" \
|
||||
--arg digest "$digest" \
|
||||
--arg staging_ref "$STAGING_REF" \
|
||||
--arg candidate_sha256 "$candidate_sha" \
|
||||
--arg tags "${{ steps.meta.outputs.tags }}" \
|
||||
--slurpfile candidate certification/candidate.json \
|
||||
'{schema_version: 1, version: $version, image_digest: $digest, staging_ref: $staging_ref,
|
||||
promoted_tags: ($tags | split(",")), candidate_sha256: $candidate_sha256,
|
||||
binary_sha256s: ($candidate[0].deliverables | map(.binary_sha256) | unique)}' \
|
||||
> image-promotion-record.json
|
||||
|
||||
- name: Upload image promotion record
|
||||
if: needs.build-check.outputs.should_push == 'true'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: image-promotion-record${{ matrix.suffix }}
|
||||
path: image-promotion-record.json
|
||||
if-no-files-found: error
|
||||
retention-days: 30
|
||||
|
||||
# Note: Manifest creation is no longer needed as we only build one variant
|
||||
# Multi-arch manifests are automatically created by docker/build-push-action
|
||||
|
||||
|
||||
@@ -17,8 +17,8 @@
|
||||
# Each selected test starts a real localhost cluster via
|
||||
# `RustFSTestClusterEnvironment` (4 processes; 4 drives per node unless the
|
||||
# case is a two-site 4-node 1-drive pair or a 4-node upgrade). Membership is
|
||||
# `[profile.e2e-distributed]` in `.config/nextest.toml`. Storage-sensitive PRs,
|
||||
# nightly runs, and manual dispatches all execute the same fail-closed suite.
|
||||
# `[profile.e2e-distributed]` in `.config/nextest.toml`. The nightly schedule
|
||||
# and manual dispatches execute the same fail-closed suite.
|
||||
# Upgrade cases download the same pinned previous release as e2e-upgrade.yml.
|
||||
#
|
||||
# Isolated pool filesystems: expand/decommission/rebalance cases require
|
||||
@@ -33,32 +33,6 @@
|
||||
name: e2e-distributed
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- "Cargo.lock"
|
||||
- "Cargo.toml"
|
||||
- ".config/nextest.toml"
|
||||
- ".config/e2e-distributed-selection.txt"
|
||||
- ".github/workflows/e2e-distributed.yml"
|
||||
- "crates/audit/**"
|
||||
- "crates/common/**"
|
||||
- "crates/config/**"
|
||||
- "crates/e2e_test/**"
|
||||
- "crates/ecstore/**"
|
||||
- "crates/filemeta/**"
|
||||
- "crates/heal/**"
|
||||
- "crates/iam/**"
|
||||
- "crates/lock/**"
|
||||
- "crates/madmin/**"
|
||||
- "crates/notify/**"
|
||||
- "crates/replication/**"
|
||||
- "crates/s3-client/**"
|
||||
- "crates/s3-ops/**"
|
||||
- "crates/s3-types/**"
|
||||
- "crates/scanner/**"
|
||||
- "crates/storage-api/**"
|
||||
- "crates/utils/**"
|
||||
- "rustfs/**"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
filter:
|
||||
|
||||
@@ -12,30 +12,19 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Single-node upgrade and rollback lane.
|
||||
#
|
||||
# The weekly Monday schedule, version tags, and workflow_dispatch run this
|
||||
# suite. Distributed upgrade cases stay on the nightly e2e-distributed lane.
|
||||
|
||||
name: Upgrade Compatibility
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- ".github/workflows/e2e-upgrade.yml"
|
||||
- "crates/e2e_test/src/common.rs"
|
||||
- "crates/e2e_test/src/distributed/**"
|
||||
- "crates/e2e_test/src/fake_s3_target/**"
|
||||
- "crates/e2e_test/src/lib.rs"
|
||||
- "crates/e2e_test/src/replication_extension_test.rs"
|
||||
- "crates/e2e_test/src/rpc_fault_proxy.rs"
|
||||
- "crates/e2e_test/src/upgrade_compatibility_test.rs"
|
||||
- "crates/ecstore/**"
|
||||
- "crates/filemeta/**"
|
||||
- "crates/heal/**"
|
||||
- "crates/kms/**"
|
||||
- "crates/storage-api/**"
|
||||
- "rustfs/**"
|
||||
- "Cargo.lock"
|
||||
push:
|
||||
tags:
|
||||
- "[0-9]*.[0-9]*.[0-9]*"
|
||||
schedule:
|
||||
# 03:17 UTC every Monday.
|
||||
- cron: "17 3 * * 1"
|
||||
workflow_dispatch:
|
||||
|
||||
@@ -44,7 +33,7 @@ permissions:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
|
||||
@@ -30,9 +30,102 @@ permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
# Consume the same trusted release certification as every other publication
|
||||
# exit and verify the image promotion record of the Docker workflow. The
|
||||
# chart wraps no binary of its own, so it may only reference an image whose
|
||||
# platform verification already passed; it must not inherit an unverified
|
||||
# package's conclusion. A run without that evidence fails closed here and
|
||||
# never reaches the chart repository.
|
||||
release-certification:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
outputs:
|
||||
version: ${{ steps.version.outputs.raw_tag }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Normalize release version
|
||||
id: version
|
||||
env:
|
||||
RAW_INPUT: ${{ github.event.inputs.version }}
|
||||
RAW_BRANCH: ${{ github.event.workflow_run.head_branch }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
RAW="$RAW_INPUT"
|
||||
else
|
||||
RAW="$RAW_BRANCH"
|
||||
fi
|
||||
case "$RAW" in
|
||||
refs/tags/*) RAW_TAG="${RAW#refs/tags/}" ;;
|
||||
*) RAW_TAG="$RAW" ;;
|
||||
esac
|
||||
echo "raw_tag=$RAW_TAG" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Download and verify release certification and image record
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
RAW_TAG: ${{ steps.version.outputs.raw_tag }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification image-record
|
||||
|
||||
build_run="$TRIGGERING_RUN_ID"
|
||||
if [[ -z "$build_run" ]]; then
|
||||
build_run="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
|
||||
-f branch="$RAW_TAG" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
fi
|
||||
if [[ -z "$build_run" ]]; then
|
||||
echo "No certified Build and Release run found for ${RAW_TAG}; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! gh run download "$build_run" -n release-certification -D certification; then
|
||||
echo "Build run ${build_run} published no release-certification artifact; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
|
||||
--artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
|
||||
|
||||
docker_run="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/docker.yml/runs" \
|
||||
-f branch="$RAW_TAG" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
if [[ -z "$docker_run" ]] || ! gh run download "$docker_run" -n image-promotion-record -D image-record; then
|
||||
echo "No image promotion record for ${RAW_TAG}; a chart may not wrap an unverified image" >&2
|
||||
exit 1
|
||||
fi
|
||||
candidate_sha="$(python3 -c "import json,hashlib,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('certification/candidate.json'))))")"
|
||||
record_sha="$(jq -r '.candidate_sha256 // empty' image-record/image-promotion-record.json)"
|
||||
if [[ "$record_sha" != "$candidate_sha" ]]; then
|
||||
echo "Image promotion record does not belong to this certification" >&2
|
||||
exit 1
|
||||
fi
|
||||
overlap="$(jq -r --slurpfile candidate certification/candidate.json \
|
||||
'[.binary_sha256s[]] - ($candidate[0].deliverables | map(.binary_sha256)) | length' \
|
||||
image-record/image-promotion-record.json)"
|
||||
if [[ "$overlap" -ne 0 ]]; then
|
||||
echo "Image promotion record carries binary digests outside the certification" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
build-helm-package:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
needs: [ release-certification ]
|
||||
if: |
|
||||
(github.event_name == 'workflow_dispatch' && !contains(github.event.inputs.version, '-preview')) ||
|
||||
(
|
||||
|
||||
@@ -87,18 +87,206 @@ jobs:
|
||||
install-build-packaging-tools: 'false'
|
||||
install-test-tools: 'false'
|
||||
|
||||
- name: Download static console assets
|
||||
shell: bash
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
# Packaging tools are off on this lane, so setup does not install unzip.
|
||||
if ! command -v unzip >/dev/null 2>&1; then
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq unzip
|
||||
fi
|
||||
if ! command -v unzip >/dev/null 2>&1; then
|
||||
echo "unzip is required to extract console assets" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p ./rustfs/static
|
||||
|
||||
verify_sha256() {
|
||||
local expected="$1"
|
||||
local file="$2"
|
||||
local actual=""
|
||||
local actual_line=""
|
||||
|
||||
if command -v sha256sum >/dev/null 2>&1; then
|
||||
actual_line=$(sha256sum -- "$file") || return 1
|
||||
elif command -v shasum >/dev/null 2>&1; then
|
||||
actual_line=$(shasum -a 256 -- "$file") || return 1
|
||||
else
|
||||
echo "No SHA-256 verification tool found" >&2
|
||||
return 1
|
||||
fi
|
||||
actual="${actual_line%%[[:space:]]*}"
|
||||
|
||||
if [ "$actual" = "$expected" ]; then
|
||||
echo "SHA256 verified OK: $actual"
|
||||
return 0
|
||||
else
|
||||
echo "SHA256 mismatch: expected=$expected actual=$actual" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
download_console_assets() {
|
||||
local curl_bin="curl"
|
||||
local python_bin="python3"
|
||||
local console_api="https://api.github.com/repos/rustfs/console/releases/latest"
|
||||
local console_json="console-release.json"
|
||||
local console_url
|
||||
local console_sha256
|
||||
local curl_auth_args=()
|
||||
console_tag=""
|
||||
|
||||
chmod +w ./rustfs/static/LICENSE || true
|
||||
|
||||
if ! command -v "$python_bin" >/dev/null 2>&1; then
|
||||
python_bin="python"
|
||||
fi
|
||||
if ! command -v "$python_bin" >/dev/null 2>&1; then
|
||||
echo "No Python interpreter found for release metadata parsing" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ -n "${GITHUB_TOKEN:-}" ]]; then
|
||||
curl_auth_args=(-H "Authorization: Bearer ${GITHUB_TOKEN}" -H "X-GitHub-Api-Version: 2022-11-28")
|
||||
fi
|
||||
|
||||
"$curl_bin" "${curl_auth_args[@]}" --fail -L "$console_api" \
|
||||
-o "$console_json" --retry 3 --retry-delay 5 --max-time 300 || return 1
|
||||
|
||||
read -r console_tag console_url console_sha256 < <("$python_bin" - "$console_json" <<'PY'
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
|
||||
with open(sys.argv[1], encoding="utf-8") as handle:
|
||||
release = json.load(handle)
|
||||
|
||||
tag = release.get("tag_name", "") or "unknown"
|
||||
|
||||
for asset in release.get("assets", []):
|
||||
name = asset.get("name", "")
|
||||
digest = asset.get("digest", "")
|
||||
url = asset.get("browser_download_url", "")
|
||||
if name.startswith("rustfs-console-") and name.endswith(".zip") and digest.startswith("sha256:") and url:
|
||||
sha256 = digest.split(":", 1)[1]
|
||||
if not re.fullmatch(r"[0-9a-fA-F]{64}", sha256):
|
||||
raise SystemExit(f"console zip asset has invalid sha256 digest: {sha256}")
|
||||
sys.stdout.buffer.write(f"{tag} {url} {sha256}\n".encode("utf-8"))
|
||||
break
|
||||
else:
|
||||
raise SystemExit("no console zip asset with sha256 digest found")
|
||||
PY
|
||||
) || return 1
|
||||
|
||||
if [[ -z "$console_url" || -z "$console_sha256" ]]; then
|
||||
echo "Console release metadata is missing URL or SHA-256 digest" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "Console release: ${console_tag}"
|
||||
echo "Downloading console asset: ${console_url}"
|
||||
"$curl_bin" --fail -L "$console_url" -o console.zip --retry 3 --retry-delay 5 --max-time 300 || return 1
|
||||
verify_sha256 "$console_sha256" console.zip || return 2
|
||||
unzip -o console.zip -d ./rustfs/static || return 2
|
||||
}
|
||||
|
||||
if download_console_assets; then
|
||||
rm -f console.zip console-release.json
|
||||
else
|
||||
status=$?
|
||||
rm -f console.zip console-release.json
|
||||
if [[ "$status" -eq 2 ]]; then
|
||||
echo "Console asset integrity verification failed" >&2
|
||||
fi
|
||||
echo "Failed to download verified console assets" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -s ./rustfs/static/index.html ]]; then
|
||||
echo "Console asset archive is missing static/index.html" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
asset_count=$(find ./rustfs/static -type f | wc -l | tr -d '[:space:]')
|
||||
echo "Console assets ready: version=${console_tag:-unknown}, ${asset_count} files extracted to ./rustfs/static"
|
||||
|
||||
- name: Build RustFS
|
||||
# Pin the glibc baseline (`<target>.<glibc>` for cargo-zigbuild) so
|
||||
# nightlies run on older distros such as RHEL/Rocky 9 (glibc 2.34); #8062.
|
||||
run: cargo zigbuild --release --locked --target x86_64-unknown-linux-gnu.2.31 -p rustfs --bins
|
||||
shell: bash
|
||||
run: |
|
||||
# Force rebuild by touching build.rs so a restored cache cannot
|
||||
# reuse a rustfs artifact compiled before these console assets existed.
|
||||
touch rustfs/build.rs
|
||||
cargo zigbuild --release --locked --target x86_64-unknown-linux-gnu.2.31 -p rustfs --bin rustfs
|
||||
|
||||
- name: Verify glibc symbol baseline
|
||||
run: |
|
||||
python3 scripts/release/check_glibc.py 2.31 \
|
||||
target/x86_64-unknown-linux-gnu/release/rustfs \
|
||||
target/x86_64-unknown-linux-gnu/release/rustfs-cli
|
||||
target/x86_64-unknown-linux-gnu/release/rustfs
|
||||
target/x86_64-unknown-linux-gnu/release/rustfs --version | grep -Fx 'build target : x86_64-unknown-linux-gnu'
|
||||
|
||||
- name: Verify console
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
find_free_port() {
|
||||
python3 - <<'PY'
|
||||
import socket
|
||||
|
||||
with socket.socket() as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
print(sock.getsockname()[1])
|
||||
PY
|
||||
}
|
||||
|
||||
binary="target/x86_64-unknown-linux-gnu/release/rustfs"
|
||||
data_dir="$(mktemp -d)"
|
||||
log_file="${RUNNER_TEMP}/rustfs-console-smoke.log"
|
||||
server_pid=""
|
||||
trap '
|
||||
if [[ -n "${server_pid:-}" ]]; then
|
||||
kill "$server_pid" 2>/dev/null || true
|
||||
wait "$server_pid" 2>/dev/null || true
|
||||
fi
|
||||
rm -rf "$data_dir"
|
||||
' EXIT
|
||||
|
||||
if [[ ! -x "$binary" ]]; then
|
||||
echo "rustfs binary not found: ${binary}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
api_port="$(find_free_port)"
|
||||
console_port="$(find_free_port)"
|
||||
console_url="http://127.0.0.1:${console_port}/rustfs/console/"
|
||||
|
||||
"$binary" server \
|
||||
--address "127.0.0.1:${api_port}" \
|
||||
--console-enable \
|
||||
--console-address "127.0.0.1:${console_port}" \
|
||||
--access-key console-smoke \
|
||||
--secret-key console-smoke-secret \
|
||||
"$data_dir" >"$log_file" 2>&1 &
|
||||
server_pid=$!
|
||||
|
||||
response=""
|
||||
for _ in {1..40}; do
|
||||
response="$(curl --silent --output /dev/null --write-out '%{http_code} %{content_type}' "$console_url" || true)"
|
||||
if [[ "$response" == 200\ text/html* ]]; then
|
||||
exit 0
|
||||
fi
|
||||
sleep 0.25
|
||||
done
|
||||
|
||||
echo "Console endpoint did not return 200 text/html: $response" >&2
|
||||
cat "$log_file"
|
||||
exit 1
|
||||
|
||||
- name: Build DEB package
|
||||
id: deb
|
||||
shell: bash
|
||||
@@ -346,12 +534,12 @@ jobs:
|
||||
path: ${{ steps.publish.outputs.candidate_file }}
|
||||
if-no-files-found: error
|
||||
|
||||
# Publish the deb/rpm pair to the auto-testing repo's `assets` branch so
|
||||
# engineers can download and install the nightly directly. The branch is
|
||||
# a single-commit orphan rewritten on every build, which keeps the repo
|
||||
# small while the latest files stay reachable at stable raw URLs.
|
||||
# Publish the deb/rpm pair as assets of the rolling `nightly` release on
|
||||
# rustfs/auto-testing (see scripts/release/publish_nightly_assets.sh).
|
||||
# The dispatch below runs only after this upload and the immutable R2
|
||||
# candidate both succeed. It does not replace rustfs-functional-chain.yml,
|
||||
# which still starts from this workflow's workflow_run completion.
|
||||
# kms-vault-lane and kms-vault-ha-failover stay parallel with this job.
|
||||
- name: Publish packages to auto-testing release assets
|
||||
env:
|
||||
ASSETS_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
@@ -360,6 +548,90 @@ jobs:
|
||||
DEB_DATE: ${{ steps.deb.outputs.deb_date }}
|
||||
BUILD_REF: ${{ env.NIGHTLY_BUILD_REF }}
|
||||
run: bash scripts/release/publish_nightly_assets.sh
|
||||
|
||||
# Cross-repo repository_dispatch cannot use GITHUB_TOKEN. The secret is a
|
||||
# PAT or GitHub App token for rustfs/auto-testing: classic `repo` scope,
|
||||
# or fine-grained Contents write (the permission GitHub checks for
|
||||
# POST /repos/{owner}/{repo}/dispatches). A dispatch failure fails this
|
||||
# step, and therefore this job, so the scheduled failure alert fires.
|
||||
# GitHub allows at most 10 top-level client_payload properties, so the
|
||||
# schema 2 candidate object is nested under `candidate` and the release
|
||||
# fields sit beside it.
|
||||
- name: Dispatch nightly candidate to auto-testing
|
||||
if: ${{ steps.publish.outputs.candidate_file != '' }}
|
||||
env:
|
||||
DISPATCH_TOKEN: ${{ secrets.AUTO_TESTING_DISPATCH_TOKEN }}
|
||||
CANDIDATE_FILE: ${{ steps.publish.outputs.candidate_file }}
|
||||
DEB_DATE: ${{ steps.deb.outputs.deb_date }}
|
||||
SOURCE_REPO: ${{ github.repository }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
: "${DISPATCH_TOKEN:?AUTO_TESTING_DISPATCH_TOKEN is required to dispatch rustfs/auto-testing}"
|
||||
: "${CANDIDATE_FILE:?nightly candidate manifest is required}"
|
||||
: "${DEB_DATE:?DEB_DATE is required}"
|
||||
[[ "${DEB_DATE}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]
|
||||
[[ "${SOURCE_REPO}" == "rustfs/rustfs" ]]
|
||||
|
||||
payload="${RUNNER_TEMP}/nightly-candidate-dispatch-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
|
||||
jq -n --slurpfile candidate "${CANDIDATE_FILE}" \
|
||||
--arg event_type "nightly-candidate-ready" \
|
||||
--arg source_repo "${SOURCE_REPO}" \
|
||||
--arg latest_alias_url "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb" \
|
||||
--arg auto_testing_release "https://github.com/rustfs/auto-testing/releases/tag/nightly" \
|
||||
--arg channel "nightly" \
|
||||
--arg deb_date "${DEB_DATE}" \
|
||||
--arg nightly_workflow ".github/workflows/nightly-gnu.yml" \
|
||||
'{
|
||||
event_type: $event_type,
|
||||
source_repo: $source_repo,
|
||||
latest_alias_url: $latest_alias_url,
|
||||
auto_testing_release: $auto_testing_release,
|
||||
channel: $channel,
|
||||
deb_date: $deb_date,
|
||||
nightly_workflow: $nightly_workflow,
|
||||
candidate: $candidate[0]
|
||||
}' > "${payload}"
|
||||
if ! jq -e '
|
||||
.event_type == "nightly-candidate-ready"
|
||||
and .source_repo == "rustfs/rustfs"
|
||||
and .channel == "nightly"
|
||||
and .deb_date == env.DEB_DATE
|
||||
and .nightly_workflow == ".github/workflows/nightly-gnu.yml"
|
||||
and .latest_alias_url == "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb"
|
||||
and .auto_testing_release == "https://github.com/rustfs/auto-testing/releases/tag/nightly"
|
||||
and .candidate.schema == 2
|
||||
and ((.candidate | keys | sort) == ["build_run_attempt","build_run_id","package_sha256","package_url","schema","source_ref","source_sha","workflow_sha"])
|
||||
and ((keys | sort) == ["auto_testing_release","candidate","channel","deb_date","event_type","latest_alias_url","nightly_workflow","source_repo"])
|
||||
' "${payload}" >/dev/null; then
|
||||
echo "nightly dispatch payload does not match the nightly-candidate-ready contract" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
body="${RUNNER_TEMP}/nightly-candidate-dispatch-body-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
|
||||
jq -n --slurpfile client_payload "${payload}" \
|
||||
'{event_type: "nightly-candidate-ready", client_payload: $client_payload[0]}' > "${body}"
|
||||
response="${RUNNER_TEMP}/nightly-candidate-dispatch-response-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.txt"
|
||||
status="$(curl -sS --retry 3 --connect-timeout 15 --max-time 60 \
|
||||
-o "${response}" -w '%{http_code}' \
|
||||
-X POST \
|
||||
-H "Authorization: Bearer ${DISPATCH_TOKEN}" \
|
||||
-H "Accept: application/vnd.github+json" \
|
||||
-H "X-GitHub-Api-Version: 2022-11-28" \
|
||||
-H "Content-Type: application/json" \
|
||||
--data-binary @"${body}" \
|
||||
https://api.github.com/repos/rustfs/auto-testing/dispatches)" || {
|
||||
echo "repository_dispatch request to rustfs/auto-testing failed before an HTTP status" >&2
|
||||
exit 1
|
||||
}
|
||||
if [[ "${status}" != "204" ]]; then
|
||||
echo "repository_dispatch to rustfs/auto-testing failed: HTTP ${status}" >&2
|
||||
if [[ -s "${response}" ]]; then
|
||||
cat "${response}" >&2
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
echo "dispatched nightly-candidate-ready to rustfs/auto-testing (deb_date=${DEB_DATE}, run=${GITHUB_RUN_ID}, attempt=${GITHUB_RUN_ATTEMPT})"
|
||||
# Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
|
||||
#
|
||||
# RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and
|
||||
|
||||
+210
-14
@@ -225,10 +225,57 @@ jobs:
|
||||
echo " Build run ID: $BUILD_RUN_ID"
|
||||
echo " Build run number: $RUN_NUMBER"
|
||||
|
||||
# Consume the trusted release certification of the resolved build run and
|
||||
# verify the certified input digests before any package is wrapped. A run
|
||||
# without that certification fails closed here; packages built later must
|
||||
# complete their own verification and cannot inherit an unverified
|
||||
# conclusion.
|
||||
release-certification:
|
||||
name: Release Certification
|
||||
needs: resolve
|
||||
if: needs.resolve.result == 'success'
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download and verify release certification
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
BUILD_RUN_ID: ${{ needs.resolve.outputs.build_run_id }}
|
||||
BUILD_TYPE: ${{ needs.resolve.outputs.build_type }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification
|
||||
if [[ "$BUILD_TYPE" == "development" ]]; then
|
||||
# Development artifacts are explicitly uncertified and never touch
|
||||
# official or latest channels; only the staging/dev paths remain.
|
||||
jq -n '{schema_version: 1, certified: false, reason: "development artifacts are explicitly uncertified"}' \
|
||||
> certification/development-uncertified.json
|
||||
exit 0
|
||||
fi
|
||||
if ! gh run download "$BUILD_RUN_ID" -n release-certification -D certification; then
|
||||
echo "Build run ${BUILD_RUN_ID} published no release-certification artifact; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
|
||||
--artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
|
||||
|
||||
# Build DEB and RPM packages for each architecture and libc variant
|
||||
package:
|
||||
name: Package (${{ matrix.arch }}-${{ matrix.libc }})
|
||||
needs: resolve
|
||||
needs: [ resolve, release-certification ]
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
@@ -494,6 +541,55 @@ jobs:
|
||||
echo "rpm_file=$RPM_FILE" >> "$GITHUB_OUTPUT"
|
||||
echo "✅ RPM built: $RPM_FILE"
|
||||
|
||||
- name: Download certification evidence
|
||||
if: needs.resolve.outputs.build_type != 'development'
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: release-certification
|
||||
path: ./certification
|
||||
|
||||
# Later-built wrapper verification: the binary inside each package must
|
||||
# match the certified binary digest for this platform before anything is
|
||||
# uploaded, and the package itself stays on staging until promotion.
|
||||
- name: Verify wrapped binary against certification
|
||||
if: needs.resolve.outputs.build_type != 'development'
|
||||
shell: bash
|
||||
env:
|
||||
DEB_FILE: ${{ steps.deb.outputs.deb_file }}
|
||||
RPM_FILE: ${{ steps.rpm.outputs.rpm_file }}
|
||||
ARTIFACT_NAME: ${{ matrix.artifact_name }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
expected="$(jq -r --arg needle "$ARTIFACT_NAME" \
|
||||
'[.deliverables[] | select(.name | contains($needle)) | .binary_sha256][0] // empty' \
|
||||
certification/candidate.json)"
|
||||
if [[ -z "$expected" ]]; then
|
||||
echo "No certified binary digest recorded for ${ARTIFACT_NAME}" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! command -v rpm2cpio >/dev/null 2>&1; then
|
||||
sudo apt-get update && sudo apt-get install -y rpm2cpio
|
||||
fi
|
||||
if ! command -v cpio >/dev/null 2>&1; then
|
||||
sudo apt-get update && sudo apt-get install -y cpio
|
||||
fi
|
||||
for package_file in "$DEB_FILE" "$RPM_FILE"; do
|
||||
case "$package_file" in
|
||||
*.deb)
|
||||
dpkg-deb --fsys-tarfile "$package_file" | tar -xOf - ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
|
||||
;;
|
||||
*.rpm)
|
||||
rpm2cpio "$package_file" | cpio -i --to-stdout ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
|
||||
;;
|
||||
esac
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$RUNNER_TEMP/wrapped-rustfs" \
|
||||
--artifact-role binary
|
||||
done
|
||||
|
||||
- name: Upload packages to artifacts
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
@@ -531,10 +627,14 @@ jobs:
|
||||
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
|
||||
export AWS_DEFAULT_REGION="auto"
|
||||
|
||||
# Build-phase writes stay on staging for release types: the official
|
||||
# packages/release and packages/latest paths are written only by the
|
||||
# certification-gated promote-packages job, from these same staged
|
||||
# objects. Development packages keep the uncertified dev channel.
|
||||
if [[ "$BUILD_TYPE" == "development" ]]; then
|
||||
R2_PREFIX="artifacts/rustfs/packages/dev"
|
||||
else
|
||||
R2_PREFIX="artifacts/rustfs/packages/release"
|
||||
R2_PREFIX="artifacts/rustfs/packages/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}"
|
||||
fi
|
||||
R2_PATH="s3://${R2_BUCKET}/${R2_PREFIX}/"
|
||||
|
||||
@@ -549,18 +649,6 @@ jobs:
|
||||
|
||||
echo "✅ Upload complete"
|
||||
|
||||
# Also upload as latest for release/prerelease
|
||||
if [[ "$BUILD_TYPE" == "release" || "$BUILD_TYPE" == "prerelease" ]]; then
|
||||
LATEST_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/latest/"
|
||||
for f in "$DEB_FILE" "$RPM_FILE"; do
|
||||
if [[ -n "$f" && -f "$f" ]]; then
|
||||
echo "Uploading latest: $(basename "$f")"
|
||||
aws s3 cp "$f" "$LATEST_PATH" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
fi
|
||||
done
|
||||
echo "✅ Latest packages updated"
|
||||
fi
|
||||
|
||||
- name: Upload packages to GitHub Release
|
||||
if: needs.resolve.outputs.tag != ''
|
||||
env:
|
||||
@@ -614,6 +702,114 @@ jobs:
|
||||
|
||||
echo "✅ GitHub release assets updated"
|
||||
|
||||
# The only writer of the official packages/release and packages/latest R2
|
||||
# paths. Promotion re-checks each package's wrapped binary against the
|
||||
# passed certification list and copies the staged objects byte-for-byte:
|
||||
# no rebuild-after-verify, no mutable latest resolution downstream.
|
||||
promote-packages:
|
||||
name: Promote Packages
|
||||
needs: [ resolve, package, release-certification ]
|
||||
# Target-channel lock for the official package paths.
|
||||
concurrency:
|
||||
group: release-channel-r2-packages
|
||||
cancel-in-progress: false
|
||||
if: needs.resolve.outputs.tag != '' && (needs.resolve.outputs.build_type == 'release' || needs.resolve.outputs.build_type == 'prerelease')
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
env:
|
||||
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
|
||||
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
|
||||
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
|
||||
R2_BUCKET: ${{ secrets.R2_BUCKET }}
|
||||
AWS_EC2_METADATA_DISABLED: true
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download packages and certification
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
path: ./packages
|
||||
pattern: packages-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Download certification evidence
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: release-certification
|
||||
path: ./certification
|
||||
|
||||
- name: Verify and promote staged packages
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
if [[ -z "$R2_ACCESS_KEY_ID" || -z "$R2_SECRET_ACCESS_KEY" || -z "$R2_ENDPOINT" || -z "$R2_BUCKET" ]]; then
|
||||
echo "❌ R2 credentials missing; refusing to promote to the official channel" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
STAGING_PREFIX="s3://${R2_BUCKET}/artifacts/rustfs/packages/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
|
||||
RELEASE_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/release/"
|
||||
LATEST_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/latest/"
|
||||
|
||||
if ! command -v aws >/dev/null 2>&1; then
|
||||
sudo apt-get update && sudo apt-get install -y awscli
|
||||
fi
|
||||
export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
|
||||
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
|
||||
export AWS_DEFAULT_REGION="auto"
|
||||
|
||||
if ! command -v rpm2cpio >/dev/null 2>&1; then
|
||||
sudo apt-get install -y rpm2cpio
|
||||
fi
|
||||
if ! command -v cpio >/dev/null 2>&1; then
|
||||
sudo apt-get install -y cpio
|
||||
fi
|
||||
|
||||
promoted=0
|
||||
for package_file in packages/*.deb packages/*.rpm; do
|
||||
[[ -f "$package_file" ]] || continue
|
||||
name="$(basename "$package_file")"
|
||||
case "$package_file" in
|
||||
*.deb)
|
||||
dpkg-deb --fsys-tarfile "$package_file" | tar -xOf - ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
|
||||
;;
|
||||
*.rpm)
|
||||
rpm2cpio "$package_file" | cpio -i --to-stdout ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
|
||||
;;
|
||||
esac
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$RUNNER_TEMP/wrapped-rustfs" \
|
||||
--artifact-role binary
|
||||
|
||||
staged="$(mktemp)"
|
||||
aws s3 cp "${STAGING_PREFIX}${name}" "$staged" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
staged_sha="$(sha256sum "$staged" | awk '{print $1}')"
|
||||
built_sha="$(sha256sum "$package_file" | awk '{print $1}')"
|
||||
if [[ "$staged_sha" != "$built_sha" ]]; then
|
||||
echo "❌ Staged package ${name} does not match the verified bytes" >&2
|
||||
exit 1
|
||||
fi
|
||||
aws s3 cp "$staged" "${RELEASE_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
aws s3 cp "$staged" "${LATEST_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
rm -f "$staged"
|
||||
promoted=$((promoted + 1))
|
||||
echo "✅ Promoted ${name} (${built_sha})"
|
||||
done
|
||||
if [[ "$promoted" -eq 0 ]]; then
|
||||
echo "No staged packages found to promote" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Summary
|
||||
summary:
|
||||
name: Summary
|
||||
|
||||
@@ -0,0 +1,457 @@
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: RustFS Validation Sweep
|
||||
|
||||
# Executes the migrated canonical validation cases (rustfs/auto-testing#113)
|
||||
# on the shared functional fleet. Everything with a lane executor is swept in
|
||||
# rotating nightly batches so the whole set is covered roughly weekly; a
|
||||
# single case or one module can be dispatched manually. That is the
|
||||
# implemented catalog (66 today) plus the eight planned cases whose dedicated
|
||||
# executors already exist and have live PASS evidence - fresh nightly
|
||||
# evidence feeds their acceptance review. heal.* cases stay on the
|
||||
# interactive validation flow: they require reviewed native/tool collectors
|
||||
# built by separate validation-tools runs before fleet execution.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# After the nightly functional chain settles (chain starts ~16:48 UTC and
|
||||
# runs for about two hours).
|
||||
- cron: "30 19 * * *"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
case:
|
||||
description: 'Single canonical validation case id (overrides module and rotation), e.g. pool.decommission'
|
||||
type: string
|
||||
required: false
|
||||
default: ''
|
||||
module:
|
||||
description: 'Restrict the sweep to one catalog module, e.g. bucket-replication'
|
||||
type: string
|
||||
required: false
|
||||
default: ''
|
||||
limit:
|
||||
description: 'Maximum cases per sweep (0 = run everything selected)'
|
||||
type: number
|
||||
required: false
|
||||
default: 12
|
||||
build_run_id:
|
||||
description: 'Successful nightly package run; empty resolves the latest'
|
||||
type: string
|
||||
required: false
|
||||
default: ''
|
||||
build_run_attempt:
|
||||
description: 'Attempt of that package run'
|
||||
type: string
|
||||
required: false
|
||||
default: '1'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: rustfs-validation-sweep
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES || 'vm000 vm001 vm002 vm003' }}
|
||||
RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER || 'azureuser' }}
|
||||
RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
|
||||
RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
|
||||
RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
|
||||
PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
jobs:
|
||||
sweep:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 150
|
||||
steps:
|
||||
- name: Checkout chain tooling
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Resolve nightly candidate
|
||||
id: candidate
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
BUILD_RUN_ID: ${{ inputs.build_run_id }}
|
||||
BUILD_RUN_ATTEMPT: ${{ inputs.build_run_attempt }}
|
||||
CHAIN_OUTPUT: ${{ runner.temp }}/validation-sweep-candidate.json
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Scheduled runs carry no inputs and the resolver only derives a
|
||||
# build from a workflow_run payload (which a schedule cannot
|
||||
# provide), so resolve the newest successful nightly build
|
||||
# explicitly before invoking it.
|
||||
if [ -z "${BUILD_RUN_ID:-}" ]; then
|
||||
LATEST="$(gh api "repos/rustfs/rustfs/actions/workflows/nightly-gnu.yml/runs?branch=main&status=success&per_page=1" --jq '.workflow_runs[0]')"
|
||||
BUILD_RUN_ID="$(printf '%s' "${LATEST}" | jq -r '.id')"
|
||||
BUILD_RUN_ATTEMPT="$(printf '%s' "${LATEST}" | jq -r '.run_attempt')"
|
||||
export BUILD_RUN_ID BUILD_RUN_ATTEMPT
|
||||
echo "scheduled sweep pinned to nightly ${BUILD_RUN_ID} attempt ${BUILD_RUN_ATTEMPT}"
|
||||
fi
|
||||
python3 scripts/resolve_functional_candidate.py
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
root = Path(os.environ["RUNNER_TEMP"]) / "validation-sweep"
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
chain = json.loads((Path(os.environ["RUNNER_TEMP"]) / "validation-sweep-candidate.json").read_text())
|
||||
manifest = chain["candidate"]["manifest"]
|
||||
(root / "candidate.json").write_text(json.dumps({
|
||||
"package_url": manifest["package_url"],
|
||||
"package_sha256": manifest["package_sha256"],
|
||||
"source_sha": manifest["source_sha"],
|
||||
}, sort_keys=True) + "\n")
|
||||
with open(os.environ["GITHUB_OUTPUT"], "a") as output:
|
||||
output.write("testing_sha=" + chain["testing_sha"] + "\n")
|
||||
output.write("package_url=" + manifest["package_url"] + "\n")
|
||||
output.write("source_sha=" + manifest["source_sha"] + "\n")
|
||||
PY
|
||||
|
||||
- name: Checkout auto-testing scripts
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
repository: rustfs/auto-testing
|
||||
ref: ${{ steps.candidate.outputs.testing_sha }}
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
path: auto-testing
|
||||
persist-credentials: false
|
||||
|
||||
- name: Prepare validation Python runtime
|
||||
# The pool / ilm / replication validation runners require a managed
|
||||
# Python, exactly like the chain's validation job provides.
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install validation Python requirements
|
||||
# Case scripts call python3 and import boto3 at startup. Install the
|
||||
# functional chain's client into a per-run env so that interpreter
|
||||
# sees it without changing the runner's shared Python.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
validation_python="${RUNNER_TEMP}/rustfs-validation-python-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
test ! -e "${validation_python}"
|
||||
python3 -m venv "${validation_python}"
|
||||
"${validation_python}/bin/python" -m pip install \
|
||||
-r auto-testing/scripts/validation_ilm/requirements.txt
|
||||
printf '%s\n' "${validation_python}/bin" >> "${GITHUB_PATH}"
|
||||
|
||||
- name: Select sweep cases
|
||||
id: select
|
||||
env:
|
||||
SELECTED_CASE: ${{ inputs.case }}
|
||||
SELECTED_MODULE: ${{ inputs.module }}
|
||||
CASE_LIMIT: ${{ inputs.limit }}
|
||||
SWEEP_EVENT: ${{ github.event_name }}
|
||||
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 - <<'PY'
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
sweep_root = Path(os.environ["SWEEP_ROOT"])
|
||||
|
||||
catalog = json.loads(Path("auto-testing/validation-cases.json").read_text())
|
||||
def sweepable(case):
|
||||
status = case.get("source_status")
|
||||
if status == "implemented":
|
||||
return True
|
||||
# The remaining planned non-heal cases route to the lane
|
||||
# executors and already have live PASS evidence; sweeping them
|
||||
# nightly produces fresh evidence for their acceptance review.
|
||||
return status == "planned" and not case["id"].startswith("heal.")
|
||||
|
||||
selected = [(case["id"], module["id"]) for module in catalog["modules"]
|
||||
for case in module["cases"]
|
||||
if sweepable(case)]
|
||||
if not selected:
|
||||
raise SystemExit("catalog has no sweepable cases")
|
||||
|
||||
one_case = os.environ.get("SELECTED_CASE", "").strip()
|
||||
module_filter = os.environ.get("SELECTED_MODULE", "").strip()
|
||||
known = {case_id for case_id, _ in selected}
|
||||
modules = {module_id for _, module_id in selected}
|
||||
if one_case:
|
||||
if one_case not in known:
|
||||
raise SystemExit(f"unknown case or no lane executor for it: {one_case}")
|
||||
selected = [(one_case, dict(selected)[one_case])]
|
||||
elif module_filter:
|
||||
if module_filter not in modules:
|
||||
raise SystemExit(f"unknown module or no implemented cases: {module_filter}")
|
||||
selected = [pair for pair in selected if pair[1] == module_filter]
|
||||
elif os.environ.get("SWEEP_EVENT") == "schedule":
|
||||
# Stateless rotation: every night take one window of the catalog
|
||||
# so a full sweep needs len(selected)/limit nights and no state
|
||||
# has to be carried between runs.
|
||||
limit = max(1, int(os.environ.get("CASE_LIMIT") or 12))
|
||||
day = datetime.datetime.now(datetime.timezone.utc).timetuple().tm_yday
|
||||
start = (day * limit) % len(selected)
|
||||
selected = (selected[start:] + selected[:start])[:limit]
|
||||
limit = int(os.environ.get("CASE_LIMIT") or 0)
|
||||
if one_case or module_filter or (not os.environ.get("SWEEP_EVENT") == "schedule"):
|
||||
if limit > 0:
|
||||
selected = selected[:limit]
|
||||
|
||||
(sweep_root / "sweep-cases.txt").write_text("".join(f"{case_id}\n" for case_id, _ in selected))
|
||||
with open(os.environ["GITHUB_OUTPUT"], "a") as output:
|
||||
output.write(f"count={len(selected)}\n")
|
||||
print(f"selected {len(selected)} case(s):", ", ".join(c for c, _ in selected))
|
||||
PY
|
||||
|
||||
- name: Admit clean shared environment
|
||||
id: environment
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-validation-sweep.json
|
||||
run: python3 scripts/functional_environment.py enter --suite heal
|
||||
|
||||
- name: Run validation cases
|
||||
id: run_cases
|
||||
timeout-minutes: 120
|
||||
env:
|
||||
AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
|
||||
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
|
||||
run: |
|
||||
set -uo pipefail
|
||||
mkdir -p "${SWEEP_ROOT}/results"
|
||||
: > "${SWEEP_ROOT}/results.txt"
|
||||
while read -r CASE_ID; do
|
||||
[ -n "${CASE_ID}" ] || continue
|
||||
echo "::group::case ${CASE_ID}"
|
||||
rm -rf "${SWEEP_ROOT}/run"
|
||||
if ./auto-testing/scripts/run_validation_case.sh \
|
||||
"${CASE_ID}" "${SWEEP_ROOT}/candidate.json" \
|
||||
"${AUTO_TESTING_SHA}" "${SWEEP_ROOT}/run" \
|
||||
> "${SWEEP_ROOT}/results/${CASE_ID//\//_}.log" 2>&1; then
|
||||
printf '%s\tPASS\n' "${CASE_ID}" >> "${SWEEP_ROOT}/results.txt"
|
||||
echo "case ${CASE_ID}: PASS"
|
||||
else
|
||||
printf '%s\tFAIL\n' "${CASE_ID}" >> "${SWEEP_ROOT}/results.txt"
|
||||
echo "case ${CASE_ID}: FAIL (log retained in the sweep artifact)"
|
||||
fi
|
||||
if [ -f "${SWEEP_ROOT}/run/report.md" ]; then
|
||||
cp "${SWEEP_ROOT}/run/report.md" "${SWEEP_ROOT}/results/${CASE_ID//\//_}.report.md" 2>/dev/null || true
|
||||
fi
|
||||
echo "::endgroup::"
|
||||
# Fleet hygiene between cases: the case runners set up their own
|
||||
# topology, this purge guarantees the next case starts clean even
|
||||
# when the previous one died mid-run.
|
||||
for node in ${RUSTFS_NODES}; do
|
||||
# -n keeps ssh from consuming the case list this loop reads from stdin
|
||||
ssh -n -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
${SUDO} dpkg -P rustfs
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -f /etc/default/rustfs
|
||||
' || echo "inter-case cleanup failed on ${node} (continuing)"
|
||||
done
|
||||
done < "${SWEEP_ROOT}/sweep-cases.txt"
|
||||
echo "sweep finished: $(wc -l < "${SWEEP_ROOT}/results.txt") case(s) executed"
|
||||
|
||||
- name: Generate sweep report
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
env:
|
||||
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
|
||||
PACKAGE_URL: ${{ steps.candidate.outputs.package_url }}
|
||||
SOURCE_SHA: ${{ steps.candidate.outputs.source_sha }}
|
||||
AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
root = Path(os.environ["SWEEP_ROOT"])
|
||||
catalog = json.loads(Path("auto-testing/validation-cases.json").read_text())
|
||||
module_of = {case["id"]: module["id"] for module in catalog["modules"] for case in module["cases"]}
|
||||
rows = []
|
||||
for line in (root / "results.txt").read_text().splitlines():
|
||||
if not line:
|
||||
continue
|
||||
case_id, status = line.split("\t")
|
||||
rows.append((case_id, module_of.get(case_id, "?"), status))
|
||||
passed = sum(1 for _, _, s in rows if s == "PASS")
|
||||
failed = len(rows) - passed
|
||||
run_url = (f"{os.environ.get('GITHUB_SERVER_URL', 'https://github.com')}/"
|
||||
f"{os.environ.get('GITHUB_REPOSITORY', 'rustfs/rustfs')}/actions/runs/"
|
||||
f"{os.environ.get('GITHUB_RUN_ID', '')}")
|
||||
|
||||
report = [
|
||||
"# RustFS validation sweep report", "",
|
||||
f"- Run: {run_url}",
|
||||
f"- Attempt: {os.environ.get('GITHUB_RUN_ATTEMPT', '')}",
|
||||
f"- Trigger: {os.environ.get('GITHUB_EVENT_NAME', '')}",
|
||||
f"- Package: {os.environ.get('PACKAGE_URL', 'nightly (R2 latest)')}",
|
||||
f"- Source SHA: {os.environ.get('SOURCE_SHA', '')}",
|
||||
f"- auto-testing: {os.environ.get('AUTO_TESTING_SHA', '')}", "",
|
||||
"Canonical validation cases (rustfs/auto-testing#113) executed by the sweep, grouped by catalog module.", "",
|
||||
]
|
||||
# Group the executed cases by catalog module (the major category:
|
||||
# pool lifecycle, heal, ILM expiration, bucket/site replication, kms,
|
||||
# data services, identity, s3 compatibility). Per-module subtotals
|
||||
# deliberately use "- Subtotal:" so the dashboard overview regexes
|
||||
# (which anchor on "- Total:") keep reading the grand totals below.
|
||||
by_module = {}
|
||||
module_order = []
|
||||
for case_id, module, status in rows:
|
||||
if module not in by_module:
|
||||
by_module[module] = []
|
||||
module_order.append(module)
|
||||
by_module[module].append((case_id, status))
|
||||
report += ["## Case Summary", ""]
|
||||
for module in module_order:
|
||||
cases = by_module[module]
|
||||
mod_pass = sum(1 for _, st in cases if st == "PASS")
|
||||
mod_fail = len(cases) - mod_pass
|
||||
report += [f"### {module} ({len(cases)})", "",
|
||||
"| Case | Status |", "| --- | --- |"]
|
||||
report += [f"| {case_id} | {status} |" for case_id, status in cases]
|
||||
report += ["", f"- Subtotal: {mod_pass} passed, {mod_fail} failed", ""]
|
||||
report += [
|
||||
"## Totals", "",
|
||||
f"- Total: {len(rows)}",
|
||||
f"- PASS: {passed}",
|
||||
f"- FAIL: {failed}",
|
||||
"- UNSUPPORTED: 0", "",
|
||||
"Failing cases keep this workflow red only as a report; the sweep artifact",
|
||||
"retains every case log and per-case report for diagnosis.", "",
|
||||
]
|
||||
(root / "report.md").write_text("\n".join(report))
|
||||
(root / "cases.md").write_text("\n".join(f"| {c} | {m} | {s} |" for c, m, s in rows) + "\n")
|
||||
PY
|
||||
cat "${SWEEP_ROOT}/report.md" >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 2
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
|
||||
REPORT_FILE: ${{ runner.temp }}/validation-sweep/report.md
|
||||
SUITE: validation
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
|
||||
exit 0
|
||||
fi
|
||||
DATE="$(date -u +%Y-%m-%d)"
|
||||
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
|
||||
B64_FILE="$(mktemp)"
|
||||
python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
|
||||
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${SHA}" ]; then
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
|
||||
'{message:$msg, content:($content|rtrimstr("\n"))}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
fi
|
||||
echo "dashboard updated: ${REPORT_PATH}"
|
||||
|
||||
- name: Retain sweep evidence
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: rustfs-validation-sweep-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
${{ runner.temp }}/validation-sweep/report.md
|
||||
${{ runner.temp }}/validation-sweep/results.txt
|
||||
${{ runner.temp }}/validation-sweep/sweep-cases.txt
|
||||
${{ runner.temp }}/validation-sweep/candidate.json
|
||||
${{ runner.temp }}/validation-sweep/results/
|
||||
${{ runner.temp }}/validation-sweep/run/
|
||||
# warn (not error): an early infra failure legitimately leaves no
|
||||
# per-case evidence, and the failing step is already red.
|
||||
if-no-files-found: warn
|
||||
retention-days: 30
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
id: environment_cleanup
|
||||
timeout-minutes: 5
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
cleanup_status=0
|
||||
for node in ${RUSTFS_NODES}; do
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
|
||||
set -euo pipefail
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
${SUDO} dpkg -P rustfs
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -f /etc/default/rustfs
|
||||
' || cleanup_status=$?
|
||||
done
|
||||
exit "${cleanup_status}"
|
||||
|
||||
- name: Release environment only after successful cleanup
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-validation-sweep.json
|
||||
CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
|
||||
run: python3 scripts/functional_environment.py finish --suite heal --cleanup-outcome "$CLEANUP_OUTCOME"
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
run: echo "RustFS validation sweep failed (harness/environment breakdown); see the sweep artifact for case logs."
|
||||
|
||||
- name: Enforce sweep result
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
env:
|
||||
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if grep -q $'\tFAIL$' "${SWEEP_ROOT}/results.txt"; then
|
||||
echo "validation sweep has failing cases:"
|
||||
grep $'\tFAIL$' "${SWEEP_ROOT}/results.txt"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
# A scheduled sweep failure usually means a canonical validation case
|
||||
# regressed or the fleet environment broke; surface it as a tracking
|
||||
# issue instead of leaving it in the Actions tab.
|
||||
needs: [sweep]
|
||||
if: github.repository == 'rustfs/rustfs' && (always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure'))
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Open or update failure-tracking issue
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}
|
||||
@@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- IAM snapshot/deletion replay, target-assigned delete-marker purges, timestamp ordering, and best-effort peer broadcast now close the control-plane gaps found by the R6 review ([#7195](https://github.com/rustfs/rustfs/pull/7195)).
|
||||
- A `set-remote-target?update=true&creds=true` request on a site-replication peer's bucket target is now rejected with `400 InvalidRequest`, naming the target and why it counts as peer-owned. It used to return success while keeping the stored endpoint, credentials, and TLS settings, so an endpoint edit from the Console's bucket replication form appeared to apply but did not. Change a peer's address with a site-replication edit instead; see "Changing a site's address" in `docs/operations/site-replication-operations.md`. The `sync`, `bandwidth`, `proxy`, and `path` update groups are unchanged.
|
||||
- Site-replication edits that omit the site name, such as `mc admin replicate update --deployment-id <ID> --endpoint <URL>`, now keep the stored name on the editing site and on every receiver instead of renaming the site after the new endpoint host. No persisted format changes.
|
||||
- The persisted replication MRF journal is now replayed every 5 minutes while the server runs, not only at startup. Delete-marker purges whose targets were unreachable, and replications that overflowed the worker queues, now converge within one replay interval after the target returns instead of waiting for a restart. A target that stays down is re-attempted once per retained entry per interval across the cluster: the start time of the last pass is shared through a small `config/replication/mrf.bin.replay` object, so nodes do not each replay once per interval. Older versions ignore that object; existing persisted formats are unchanged.
|
||||
- Upgrade and rollback: upgrade every node in one site consecutively and verify reconciliation before moving to the next site; do not intentionally run a site mixed-version. Target-version ledger keys are harmless on rollback, although old code cannot use their routing. Before rolling back past [#7307](https://github.com/rustfs/rustfs/pull/7307), drain or repair every pending version purge: older code can free a retained version's data directory before its remote purge is acknowledged. See `docs/operations/site-replication-operations.md`.
|
||||
|
||||
### Security
|
||||
|
||||
Generated
+449
-234
File diff suppressed because it is too large
Load Diff
+20
-22
@@ -147,7 +147,7 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.2" }
|
||||
async-channel = "2.5.0"
|
||||
async_zip = { default-features = false, version = "0.0.19" }
|
||||
mysql_async = { default-features = false, version = "0.37.1" }
|
||||
async-compression = { version = "0.4.51" }
|
||||
async-compression = { version = "0.4.52" }
|
||||
async-trait = "0.1.92"
|
||||
async-nats = { version = "0.51.0", default-features = false }
|
||||
axum = "0.8.9"
|
||||
@@ -168,16 +168,16 @@ http-body-util = "0.1.5"
|
||||
memchr = "2.8.3"
|
||||
minlz = "1.2.3"
|
||||
reqwest = "0.13.5"
|
||||
rustfs-kafka-async = { version = "1.4.0" }
|
||||
rustfs-kafka-async = { version = "1.4.1" }
|
||||
socket2 = { version = "0.6.5" }
|
||||
tokio = { version = "1.54.1" }
|
||||
tokio-rustls = { default-features = false, version = "0.26.6" }
|
||||
tokio-stream = { version = "0.1.19" }
|
||||
tokio-test = "0.4.6"
|
||||
tokio-util = { version = "0.7.20" }
|
||||
tonic = { version = "0.14.8" }
|
||||
tonic-prost = { version = "0.14.8" }
|
||||
tonic-prost-build = { version = "0.14.8" }
|
||||
tonic = { version = "0.14.9" }
|
||||
tonic-prost = { version = "0.14.9" }
|
||||
tonic-prost-build = { version = "0.14.9" }
|
||||
tower = { version = "0.5.3" }
|
||||
tower-http = { version = "0.7.1" }
|
||||
h3 = "0.0.8"
|
||||
@@ -191,6 +191,7 @@ quinn = { version = "0.11.12", default-features = false, features = [
|
||||
# Serialization and Data Formats
|
||||
apache-avro = { version = "0.22.0", features = ["snappy", "zstandard"] }
|
||||
bytes = { version = "1.12.1" }
|
||||
csv-core = "0.1.13"
|
||||
bytesize = "2.7.0"
|
||||
byteorder = "1.5.0"
|
||||
flatbuffers = "25.12.19"
|
||||
@@ -212,12 +213,13 @@ serde_urlencoded = "0.7.1"
|
||||
# releases.
|
||||
aes-gcm = { version = "0.11.1" }
|
||||
argon2 = { version = "0.6.0" }
|
||||
aws-lc-rs = { version = "1.19.0", default-features = false }
|
||||
blake2 = "0.11.0"
|
||||
chacha20poly1305 = { version = "0.11.0" }
|
||||
crc-fast = "1.10.0"
|
||||
hmac = { version = "0.13.0" }
|
||||
jsonwebtoken = { version = "11.1.0" }
|
||||
openidconnect = { default-features = false, version = "4.0" }
|
||||
openidconnect = { default-features = false, version = "4.1.0" }
|
||||
pbkdf2 = "0.13.0"
|
||||
p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
|
||||
rsa = { version = "=0.10.0-rc.18" }
|
||||
@@ -234,7 +236,7 @@ proptest = "1"
|
||||
# Time and Date
|
||||
chrono = { version = "0.4.45" }
|
||||
humantime = "2.4.0"
|
||||
jiff = { version = "0.2.38" }
|
||||
jiff = { version = "0.2.39" }
|
||||
time = { version = "0.3.55" }
|
||||
|
||||
# Database
|
||||
@@ -253,9 +255,9 @@ atoi = "3.1.1"
|
||||
atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.13.0" }
|
||||
aws-credential-types = { version = "1.3.1" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.128.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.156.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.123.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.129.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.157.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.124.0" }
|
||||
aws-smithy-async = { version = "1.3.1" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.5.2" }
|
||||
aws-smithy-runtime-api = { version = "1.19.2" }
|
||||
@@ -319,7 +321,7 @@ rustfs-erasure-codec = { version = "9.0.0" }
|
||||
reed-solomon-simd = "3.1.0"
|
||||
regex = { version = "1.13.1" }
|
||||
rumqttc = { package = "rumqttc-next", version = "0.34.0" }
|
||||
redis = { version = "1.8.0" }
|
||||
redis = { version = "1.8.1" }
|
||||
rustify = { version = "0.7", default-features = false }
|
||||
rustix = { version = "1.1.6" }
|
||||
rust-embed = { version = "8.13.0" }
|
||||
@@ -334,10 +336,10 @@ compact_str = "0.10.0"
|
||||
snap = "1.1.2"
|
||||
starshard = { version = "2.4.0" }
|
||||
strum = { version = "0.28.0" }
|
||||
sysinfo = "0.40.0"
|
||||
sysinfo = "0.40.1"
|
||||
temp-env = "0.3.6"
|
||||
tempfile = "3.27.0"
|
||||
test-case = "3.4.0"
|
||||
test-case = "3.4.1"
|
||||
thiserror = "2.0.21"
|
||||
tracing = { version = "0.1.44" }
|
||||
tracing-appender = "0.2.5"
|
||||
@@ -356,13 +358,13 @@ winapi-util = "0.1.11"
|
||||
windows = { version = "0.62.2" }
|
||||
windows-sys = "0.61.2"
|
||||
xxhash-rust = { version = "0.8.19" }
|
||||
zip = "9.1.3"
|
||||
zip = "9.3.0"
|
||||
zstd = "0.14.3"
|
||||
|
||||
# Observability and Metrics
|
||||
metrics = "0.24.6"
|
||||
metrics-util = "0.20"
|
||||
dial9 = { version = "0.5.6", default-features = false }
|
||||
dial9 = { version = "0.5.7", default-features = false }
|
||||
opentelemetry = { version = "0.33.1" }
|
||||
opentelemetry-appender-tracing = { version = "0.33.1" }
|
||||
opentelemetry-otlp = { version = "0.33.1" }
|
||||
@@ -377,16 +379,16 @@ libunftp = { version = "0.24.0" }
|
||||
unftp-core = "0.1.0"
|
||||
suppaftp = { version = "12.2.1" }
|
||||
rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
russh = { version = "0.65.0" }
|
||||
russh = { version = "0.65.1" }
|
||||
russh-sftp = "3.0.1"
|
||||
|
||||
# WebDAV
|
||||
dav-server = "0.11.0"
|
||||
|
||||
# Performance Analysis and Memory Profiling
|
||||
rustfs-mimalloc = { version = "0.7.2" }
|
||||
rustfs-mimalloc = { version = "0.7.3" }
|
||||
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
|
||||
hotpath = { version = "0.30.1", default-features = false }
|
||||
hotpath = { version = "0.30.3", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.49" }
|
||||
|
||||
@@ -427,7 +429,3 @@ strip = "none"
|
||||
# Pin upstream fixes until compatible crates.io releases are available.
|
||||
[patch.crates-io]
|
||||
zstd = { path = "crates/zstd-compat" }
|
||||
# Preserve RustFS OIDC flows while migrating the old elliptic-curve dependency chain.
|
||||
# Upstream: openidconnect-rs#244 and oauth2-rs#352.
|
||||
openidconnect = { git = "https://github.com/ramosbugs/openidconnect-rs", rev = "2f77ee73bc4554e1cb6279adb386005429fb4e41" }
|
||||
oauth2 = { git = "https://github.com/ramosbugs/oauth2-rs", rev = "89585f4a280afd7b285eaf1c9be0068874e03d22" }
|
||||
|
||||
+11
-3
@@ -16,6 +16,7 @@ FROM alpine:3.24.2 AS build
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RELEASE=latest
|
||||
ARG RC_RELEASE=latest
|
||||
|
||||
RUN apk add --no-cache ca-certificates curl jq unzip
|
||||
WORKDIR /build
|
||||
@@ -69,6 +70,9 @@ RUN set -eux; \
|
||||
chmod +x /build/rustfs; \
|
||||
rm -rf rustfs.zip /build/.tmp || true
|
||||
|
||||
COPY scripts/download_rc.sh /usr/local/bin/download_rc.sh
|
||||
RUN sh /usr/local/bin/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /build/rc
|
||||
|
||||
|
||||
FROM alpine:3.24.2
|
||||
|
||||
@@ -95,14 +99,18 @@ RUN apk upgrade --no-cache && \
|
||||
ca-certificates \
|
||||
coreutils \
|
||||
curl \
|
||||
jq \
|
||||
tzdata \
|
||||
&& test "$(TZ=Asia/Kolkata date +%z)" = "+0530"
|
||||
|
||||
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
|
||||
COPY --from=build /build/rustfs /usr/bin/rustfs
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
COPY --from=build /build/rc /usr/bin/rc
|
||||
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
|
||||
COPY bootstrap/ /etc/rustfs/bootstrap/
|
||||
|
||||
RUN chmod +x /usr/bin/rustfs /entrypoint.sh
|
||||
RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
|
||||
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
|
||||
|
||||
RUN addgroup -g 10001 -S rustfs && \
|
||||
adduser -u 10001 -G rustfs -S rustfs -D && \
|
||||
@@ -124,6 +132,6 @@ VOLUME ["/data"]
|
||||
|
||||
USER rustfs
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
|
||||
|
||||
CMD ["rustfs"]
|
||||
|
||||
@@ -12,7 +12,10 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
FROM rust:1.97.1-trixie
|
||||
FROM rust:1.99.1-trixie
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RC_RELEASE=latest
|
||||
|
||||
RUN set -eux; \
|
||||
export DEBIAN_FRONTEND=noninteractive; \
|
||||
@@ -21,6 +24,7 @@ RUN set -eux; \
|
||||
build-essential \
|
||||
ca-certificates \
|
||||
curl \
|
||||
jq \
|
||||
git \
|
||||
pkg-config \
|
||||
libssl-dev \
|
||||
@@ -36,6 +40,7 @@ COPY . .
|
||||
RUN ./scripts/static.sh
|
||||
RUN cargo run --bin gproto
|
||||
RUN cargo build --release --locked --bin rustfs
|
||||
RUN sh /usr/src/rustfs/scripts/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /usr/local/bin/rc
|
||||
|
||||
RUN set -eux; \
|
||||
groupadd -g 10001 rustfs; \
|
||||
@@ -50,8 +55,12 @@ RUN set -eux; \
|
||||
chown -R rustfs:rustfs /data /logs /app; \
|
||||
chmod 0750 /data /logs
|
||||
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
RUN install -m 0755 /usr/src/rustfs/target/release/rustfs /usr/bin/rustfs && chmod +x /entrypoint.sh
|
||||
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
|
||||
COPY bootstrap/ /etc/rustfs/bootstrap/
|
||||
RUN install -m 0755 /usr/src/rustfs/target/release/rustfs /usr/bin/rustfs && \
|
||||
install -m 0755 /usr/local/bin/rc /usr/bin/rc && \
|
||||
chmod +x /etc/rustfs/entrypoint.sh && \
|
||||
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
|
||||
|
||||
ENV RUSTFS_VOLUMES="/data" \
|
||||
RUST_LOG="warn" \
|
||||
@@ -63,5 +72,5 @@ ENV RUSTFS_VOLUMES="/data" \
|
||||
|
||||
EXPOSE 9000 9001
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
|
||||
CMD ["/usr/bin/rustfs"]
|
||||
|
||||
+11
-3
@@ -16,6 +16,7 @@ FROM ubuntu:26.04 AS build
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RELEASE=latest
|
||||
ARG RC_RELEASE=latest
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
@@ -76,6 +77,9 @@ RUN set -eux; \
|
||||
chmod +x /build/rustfs; \
|
||||
rm -rf rustfs.zip /build/.tmp || true
|
||||
|
||||
COPY scripts/download_rc.sh /usr/local/bin/download_rc.sh
|
||||
RUN sh /usr/local/bin/download_rc.sh "$TARGETARCH" gnu "$RC_RELEASE" /build/rc
|
||||
|
||||
FROM ubuntu:26.04
|
||||
|
||||
ARG RELEASE=latest
|
||||
@@ -99,14 +103,18 @@ RUN apt-get update && apt-get upgrade -y \
|
||||
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
curl \
|
||||
jq \
|
||||
tzdata \
|
||||
&& test "$(TZ=Asia/Kolkata date +%z)" = "+0530" \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY --from=build /build/rustfs /usr/bin/rustfs
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
COPY --from=build /build/rc /usr/bin/rc
|
||||
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
|
||||
COPY bootstrap/ /etc/rustfs/bootstrap/
|
||||
|
||||
RUN chmod +x /usr/bin/rustfs /entrypoint.sh
|
||||
RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
|
||||
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
|
||||
|
||||
RUN groupadd -g 10001 rustfs && \
|
||||
useradd -u 10001 -g rustfs -m -s /sbin/nologin rustfs && \
|
||||
@@ -131,6 +139,6 @@ VOLUME ["/data"]
|
||||
|
||||
USER rustfs
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
|
||||
|
||||
CMD ["rustfs"]
|
||||
|
||||
+19
-7
@@ -28,17 +28,19 @@ ARG TARGETPLATFORM
|
||||
ARG BUILDPLATFORM
|
||||
ARG TARGETARCH
|
||||
ARG RUSTFS_BUILD_FEATURES=""
|
||||
ARG RC_RELEASE=latest
|
||||
|
||||
# -----------------------------
|
||||
# Build stage
|
||||
# -----------------------------
|
||||
FROM rust:1.97.1-trixie AS builder
|
||||
FROM rust:1.99.1-trixie AS builder
|
||||
|
||||
# Re-declare args after FROM
|
||||
ARG TARGETPLATFORM
|
||||
ARG BUILDPLATFORM
|
||||
ARG TARGETARCH
|
||||
ARG RUSTFS_BUILD_FEATURES
|
||||
ARG RC_RELEASE
|
||||
|
||||
# Debug: print platforms
|
||||
RUN echo "Build info -> BUILDPLATFORM=${BUILDPLATFORM}, TARGETPLATFORM=${TARGETPLATFORM}"
|
||||
@@ -52,6 +54,7 @@ RUN set -eux; \
|
||||
build-essential \
|
||||
ca-certificates \
|
||||
curl \
|
||||
jq \
|
||||
git \
|
||||
pkg-config \
|
||||
libssl-dev \
|
||||
@@ -166,6 +169,8 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
||||
;; \
|
||||
esac
|
||||
|
||||
RUN sh /usr/src/rustfs/scripts/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /usr/local/bin/rc
|
||||
|
||||
# -----------------------------
|
||||
# Development stage (keeps toolchain)
|
||||
# -----------------------------
|
||||
@@ -199,10 +204,13 @@ ENV RUSTFS_VOLUMES="/data" \
|
||||
# We rely on cargo run to build and run
|
||||
EXPOSE 9000 9001
|
||||
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
RUN chmod +x /entrypoint.sh
|
||||
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
|
||||
COPY bootstrap/ /etc/rustfs/bootstrap/
|
||||
RUN install -m 0755 /usr/local/bin/rc /usr/bin/rc && \
|
||||
chmod +x /etc/rustfs/entrypoint.sh && \
|
||||
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
|
||||
CMD ["cargo", "run", "--bin", "rustfs", "--"]
|
||||
|
||||
# -----------------------------
|
||||
@@ -227,6 +235,7 @@ RUN set -eux; \
|
||||
apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
curl \
|
||||
jq \
|
||||
tzdata \
|
||||
coreutils; \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
@@ -248,8 +257,11 @@ RUN set -eux; \
|
||||
|
||||
# Copy the freshly built binary and the entrypoint
|
||||
COPY --from=builder /usr/local/bin/rustfs /usr/bin/rustfs
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
RUN chmod +x /usr/bin/rustfs /entrypoint.sh
|
||||
COPY --from=builder /usr/local/bin/rc /usr/bin/rc
|
||||
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
|
||||
COPY bootstrap/ /etc/rustfs/bootstrap/
|
||||
RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
|
||||
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
|
||||
|
||||
# Default environment (override in docker run/compose as needed)
|
||||
ENV RUSTFS_VOLUMES="/data" \
|
||||
@@ -263,5 +275,5 @@ EXPOSE 9000
|
||||
VOLUME ["/data"]
|
||||
|
||||
# Keep root here; entrypoint will drop privileges using chroot --userspec
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
|
||||
CMD ["/usr/bin/rustfs"]
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
# RustFS image bootstrap
|
||||
|
||||
The official image bundles this directory at `/etc/rustfs/bootstrap`. The default `buckets.json` is an empty JSON array, so the image starts RustFS directly and does not require a bootstrap endpoint or bootstrap credentials.
|
||||
|
||||
To create buckets, replace `buckets.json` in a derived image with a JSON array of bucket names. The entrypoint creates each bucket if it does not exist and applies `public-access-block.json`, whose default blocks all four public access paths. Edit that file in the derived image only when public access is an explicit requirement.
|
||||
|
||||
Optional per-bucket files use the bucket name as the filename without an extension:
|
||||
|
||||
| Directory | File format | Applied operation |
|
||||
| --- | --- | --- |
|
||||
| `policies/` | S3 bucket policy JSON | `rc bucket anonymous set-json` |
|
||||
| `versioning/` | JSON boolean (`true` or `false`) | Enable or suspend bucket versioning |
|
||||
| `lifecycle/` | `rc bucket lifecycle rule import` JSON | Replace lifecycle rules |
|
||||
| `cors/` | JSON or XML accepted by `rc bucket cors set` | Replace CORS rules |
|
||||
| `notifications/` | JSON array of `{ "arn": "...", "events": ["..."] }` | Add or replace event rules by ARN (prefix/suffix filters are not currently accepted) |
|
||||
|
||||
Every per-bucket file must correspond to a bucket listed in `buckets.json`. Applying these files is repeatable: bucket creation ignores existing buckets, while each configuration operation replaces or reconciles that bucket's requested state.
|
||||
|
||||
When `buckets.json` is non-empty, set `RUSTFS_BOOTSTRAP_ENDPOINT` to the RustFS S3 endpoint reachable from the container, for example `http://rustfs:9000` for a Compose service or the internal service DNS name in Kubernetes. The entrypoint waits on that endpoint's `/health/ready`; it never assumes `localhost`. `RUSTFS_BOOTSTRAP_TIMEOUT` controls the readiness wait in seconds (default `120`, maximum `3600`). `RUSTFS_BOOTSTRAP_DIR` can point to a different in-image configuration directory; it defaults to `/etc/rustfs/bootstrap`. Set `RUSTFS_ACCESS_KEY` and `RUSTFS_SECRET_KEY`, or their `_FILE` variants, for bootstrap authentication. The endpoint and credentials are read at runtime and are not baked into the image.
|
||||
|
||||
RustFS built-in IAM policies remain unchanged; the default bootstrap installs no custom bucket policies, lifecycle rules, versioning, CORS rules, or notification rules. The bundled RustFS CLI is resolved from a published release and its release checksum is verified. Set the Docker build argument `RC_RELEASE` to a version to pin it. Build the RustFS image after a CLI release containing the `bucket public-access-block` commands is available. For Kubernetes, the configured service must route traffic to the starting pod before its readiness probe succeeds (for example, with `publishNotReadyAddresses: true`); otherwise the service endpoint cannot reach the pod while the entrypoint waits for readiness.
|
||||
@@ -0,0 +1 @@
|
||||
[]
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"block_public_acls": true,
|
||||
"ignore_public_acls": true,
|
||||
"block_public_policy": true,
|
||||
"restrict_public_buckets": true
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -10,6 +10,10 @@ use std::time::Duration;
|
||||
use tokio::sync::{Notify, Semaphore};
|
||||
use tokio::time::Instant;
|
||||
|
||||
pub mod permit_set;
|
||||
|
||||
const MAX_PERMIT_REFERENCES: usize = usize::MAX / 2;
|
||||
|
||||
/// A resource owner publishes a capacity; running units retain their permits
|
||||
/// after a reduction. Lease slots are reserved once, with no per-unit Arc
|
||||
/// allocation. Tokio wait queues may lazily initialize platform mutex storage
|
||||
@@ -36,6 +40,7 @@ pub struct AdaptiveAdmission {
|
||||
maximum: u32,
|
||||
slots: Box<[PermitSlots]>,
|
||||
next_chunk: AtomicUsize,
|
||||
completed: AtomicU64,
|
||||
}
|
||||
|
||||
struct PermitSlots {
|
||||
@@ -61,10 +66,12 @@ pub struct AdaptivePermit {
|
||||
|
||||
impl Clone for AdaptivePermit {
|
||||
fn clone(&self) -> Self {
|
||||
// Every slot reference owns a strong owner reference. Clone the Arc
|
||||
// first, so its overflow protection also bounds the slot refcount.
|
||||
// Subset carriers can retain slot references through a shared owner,
|
||||
// so the slot count needs its own bound in addition to Arc's guard.
|
||||
let owner = Arc::clone(&self.owner);
|
||||
owner.slots[self.slot / 64].references[self.slot % 64].fetch_add(1, Ordering::Relaxed);
|
||||
if owner.slots[self.slot / 64].references[self.slot % 64].fetch_add(1, Ordering::Relaxed) >= MAX_PERMIT_REFERENCES {
|
||||
std::process::abort();
|
||||
}
|
||||
Self { owner, slot: self.slot }
|
||||
}
|
||||
}
|
||||
@@ -114,11 +121,22 @@ impl NodeAdmission {
|
||||
|
||||
/// Publish feedback with this owner's actual completed-work counter.
|
||||
pub fn update(&self, controller: &mut FeedbackController, observation: Option<FeedbackObservation>) -> FeedbackDecision {
|
||||
self.update_with_probe(controller, observation, true)
|
||||
}
|
||||
|
||||
/// Coordinate growth with other resource controllers in the same window.
|
||||
/// Protective reductions never depend on permission to probe.
|
||||
pub fn update_with_probe(
|
||||
&self,
|
||||
controller: &mut FeedbackController,
|
||||
observation: Option<FeedbackObservation>,
|
||||
allow_probe: bool,
|
||||
) -> FeedbackDecision {
|
||||
let observation = observation.map(|mut observation| {
|
||||
observation.completed_units = self.completed();
|
||||
observation
|
||||
});
|
||||
controller.update(&self.capacity, observation)
|
||||
controller.update_with_probe(&self.capacity, observation, allow_probe)
|
||||
}
|
||||
|
||||
/// Current leased scheduling cap and held async units.
|
||||
@@ -249,16 +267,31 @@ impl FeedbackController {
|
||||
/// Consume one sample and publish a three-second lease to the same gate.
|
||||
/// If this function stops being called, admission expires independently.
|
||||
pub fn update(&mut self, gate: &AdaptiveAdmission, observation: Option<FeedbackObservation>) -> FeedbackDecision {
|
||||
self.update_with_probe(gate, observation, true)
|
||||
}
|
||||
|
||||
/// Allow at most the sampling owner's selected controller to probe upward.
|
||||
pub fn update_with_probe(
|
||||
&mut self,
|
||||
gate: &AdaptiveAdmission,
|
||||
observation: Option<FeedbackObservation>,
|
||||
allow_probe: bool,
|
||||
) -> FeedbackDecision {
|
||||
let observation = observation.map(|mut observation| {
|
||||
observation.max_concurrency = observation.max_concurrency.min(gate.maximum);
|
||||
observation
|
||||
});
|
||||
let decision = self.decide(Instant::now(), observation);
|
||||
let decision = self.decide_with_probe(Instant::now(), observation, allow_probe);
|
||||
gate.set_sampled_limit(decision.limit, Duration::from_secs(3));
|
||||
decision
|
||||
}
|
||||
|
||||
fn decide(&mut self, now: Instant, observation: Option<FeedbackObservation>) -> FeedbackDecision {
|
||||
fn decide_with_probe(
|
||||
&mut self,
|
||||
now: Instant,
|
||||
observation: Option<FeedbackObservation>,
|
||||
allow_probe: bool,
|
||||
) -> FeedbackDecision {
|
||||
let Some(observation) = observation else {
|
||||
self.previous = None;
|
||||
self.stable_since = None;
|
||||
@@ -303,7 +336,8 @@ impl FeedbackController {
|
||||
FeedbackReason::NewBaseline
|
||||
} else {
|
||||
let since = *self.stable_since.get_or_insert(now);
|
||||
if advanced
|
||||
if allow_probe
|
||||
&& advanced
|
||||
&& now.saturating_duration_since(since) >= Duration::from_secs(8)
|
||||
&& now.saturating_duration_since(self.last_change) >= Duration::from_secs(8)
|
||||
&& self.limit < observation.max_concurrency
|
||||
@@ -325,6 +359,13 @@ impl FeedbackController {
|
||||
}
|
||||
|
||||
impl AdaptiveAdmission {
|
||||
/// Successful async operations using this physical owner. Native clones
|
||||
/// retain capacity but never count another completion.
|
||||
pub fn completed(&self) -> Option<u64> {
|
||||
let value = self.completed.load(Ordering::Acquire);
|
||||
(value != u64::MAX).then_some(value)
|
||||
}
|
||||
|
||||
/// Construct a bounded owner, with zero initial capacity allowed for unknown
|
||||
/// input. The immutable maximum reserves at most 65,535 lease slots once;
|
||||
/// acquiring or cloning a permit does not allocate a per-unit Arc.
|
||||
@@ -351,6 +392,7 @@ impl AdaptiveAdmission {
|
||||
maximum: maximum_u32,
|
||||
slots,
|
||||
next_chunk: AtomicUsize::new(0),
|
||||
completed: AtomicU64::new(0),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -466,6 +508,21 @@ impl AdaptiveAdmission {
|
||||
self.slots[slot / 64].free.fetch_or(1_u64 << (slot % 64), Ordering::Release);
|
||||
}
|
||||
|
||||
fn release_reference(&self, slot: usize) {
|
||||
self.release_reference_with(slot, || {});
|
||||
}
|
||||
|
||||
fn release_reference_with(&self, slot: usize, before_release: impl FnOnce()) {
|
||||
if self.slots[slot / 64].references[slot % 64].fetch_sub(1, Ordering::AcqRel) != 1 {
|
||||
return;
|
||||
}
|
||||
before_release();
|
||||
// Publish the free slot before exposing another admission capacity.
|
||||
self.release_slot(slot);
|
||||
self.state.fetch_sub(1, Ordering::AcqRel);
|
||||
self.changed.notify_one();
|
||||
}
|
||||
|
||||
/// Wait for capacity. The future owns no permit until it returns; dropping
|
||||
/// a pending future leaves capacity untouched. Wrap in the caller's existing
|
||||
/// cancellation/deadline select before acquiring any storage commit locks.
|
||||
@@ -493,13 +550,7 @@ impl AdaptiveAdmission {
|
||||
|
||||
impl Drop for AdaptivePermit {
|
||||
fn drop(&mut self) {
|
||||
if self.owner.slots[self.slot / 64].references[self.slot % 64].fetch_sub(1, Ordering::AcqRel) != 1 {
|
||||
return;
|
||||
}
|
||||
// Publish the free slot before exposing another admission capacity.
|
||||
self.owner.release_slot(self.slot);
|
||||
self.owner.state.fetch_sub(1, Ordering::AcqRel);
|
||||
self.owner.changed.notify_one();
|
||||
self.owner.release_reference(self.slot);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -861,6 +912,24 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn shared_probe_permission_blocks_growth_but_never_protective_cuts() {
|
||||
let gate = AdaptiveAdmission::new(NonZeroU16::new(8).expect("bound"), 0);
|
||||
let mut controller = FeedbackController::new();
|
||||
controller.update_with_probe(&gate, Some(healthy(0)), false);
|
||||
for completed in 1..=20 {
|
||||
tokio::time::advance(Duration::from_secs(1)).await;
|
||||
assert_eq!(controller.update_with_probe(&gate, Some(healthy(completed)), false).limit, 1);
|
||||
}
|
||||
tokio::time::advance(Duration::from_secs(1)).await;
|
||||
assert_eq!(controller.update_with_probe(&gate, Some(healthy(21)), true).reason, FeedbackReason::Probe);
|
||||
assert_eq!(gate.snapshot().limit, 2);
|
||||
let mut busy = healthy(22);
|
||||
busy.pressure = ResourcePressure::Busy;
|
||||
assert_eq!(controller.update_with_probe(&gate, Some(busy), false).limit, 1);
|
||||
assert_eq!(controller.update_with_probe(&gate, None, false).limit, 0);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn controller_grows_one_unit_per_cooldown_and_cuts_pressure_immediately() {
|
||||
let gate = AdaptiveAdmission::new(NonZeroU16::new(8).expect("bounded test owner"), 0);
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -30,6 +30,16 @@ pub const COMMENT_KEY: &str = "comment";
|
||||
pub const ENV_AUDIT_ENABLE: &str = "RUSTFS_AUDIT_ENABLE";
|
||||
/// Global switch for enabling the notify module.
|
||||
pub const ENV_NOTIFY_ENABLE: &str = "RUSTFS_NOTIFY_ENABLE";
|
||||
/// Maximum audit entries per second for authorization denials of authenticated callers.
|
||||
/// `0` disables denial auditing for that class of caller.
|
||||
pub const ENV_AUDIT_DENIAL_RATE: &str = "RUSTFS_AUDIT_DENIAL_RATE";
|
||||
/// Maximum audit entries per second for authorization denials of anonymous callers.
|
||||
/// `0` disables denial auditing for that class of caller.
|
||||
pub const ENV_AUDIT_DENIAL_ANONYMOUS_RATE: &str = "RUSTFS_AUDIT_DENIAL_ANONYMOUS_RATE";
|
||||
/// Default denial audit rate for authenticated callers (entries per second).
|
||||
pub const DEFAULT_AUDIT_DENIAL_RATE: u64 = 50;
|
||||
/// Default denial audit rate for anonymous callers (entries per second).
|
||||
pub const DEFAULT_AUDIT_DENIAL_ANONYMOUS_RATE: u64 = 5;
|
||||
/// Default global audit switch (disabled by default).
|
||||
pub const DEFAULT_AUDIT_ENABLE: bool = false;
|
||||
/// Default global notify switch (disabled by default).
|
||||
@@ -328,5 +338,7 @@ mod tests {
|
||||
fn test_global_audit_notify_switch_constants() {
|
||||
assert_eq!(ENV_AUDIT_ENABLE, "RUSTFS_AUDIT_ENABLE");
|
||||
assert_eq!(ENV_NOTIFY_ENABLE, "RUSTFS_NOTIFY_ENABLE");
|
||||
assert_eq!(ENV_AUDIT_DENIAL_RATE, "RUSTFS_AUDIT_DENIAL_RATE");
|
||||
assert_eq!(ENV_AUDIT_DENIAL_ANONYMOUS_RATE, "RUSTFS_AUDIT_DENIAL_ANONYMOUS_RATE");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,10 +30,10 @@ pub const ENV_TRUSTED_PROXY_VALIDATION_MODE: &str = "RUSTFS_TRUSTED_PROXY_VALIDA
|
||||
/// Default validation mode is "hop_by_hop".
|
||||
pub const DEFAULT_TRUSTED_PROXY_VALIDATION_MODE: &str = "hop_by_hop";
|
||||
|
||||
/// Environment variable to enable RFC 7239 "Forwarded" header support.
|
||||
/// Select RFC 7239 "Forwarded" instead of "X-Forwarded-For" in legacy mode.
|
||||
pub const ENV_TRUSTED_PROXY_ENABLE_RFC7239: &str = "RUSTFS_TRUSTED_PROXY_ENABLE_RFC7239";
|
||||
/// RFC 7239 support is enabled by default.
|
||||
pub const DEFAULT_TRUSTED_PROXY_ENABLE_RFC7239: bool = true;
|
||||
/// Legacy mode uses X-Forwarded-For unless RFC 7239 is explicitly selected.
|
||||
pub const DEFAULT_TRUSTED_PROXY_ENABLE_RFC7239: bool = false;
|
||||
|
||||
/// Environment variable for the maximum allowed proxy hops.
|
||||
pub const ENV_TRUSTED_PROXY_MAX_HOPS: &str = "RUSTFS_TRUSTED_PROXY_MAX_HOPS";
|
||||
|
||||
@@ -146,3 +146,9 @@ pub const ENV_TARGET_STORE_COMPRESS: &str = "RUSTFS_TARGET_STORE_COMPRESS";
|
||||
|
||||
/// Queue-store compression is enabled by default to reduce disk footprint.
|
||||
pub const DEFAULT_TARGET_STORE_COMPRESS: bool = true;
|
||||
|
||||
/// Maximum on-disk bytes in each target queue, including failed entries and pending writes.
|
||||
pub const ENV_TARGET_STORE_MAX_BYTES: &str = "RUSTFS_TARGET_STORE_MAX_BYTES";
|
||||
|
||||
/// Queue writes fail at one GiB unless the operator sets another positive byte budget.
|
||||
pub const DEFAULT_TARGET_STORE_MAX_BYTES: u64 = 1024 * 1024 * 1024;
|
||||
|
||||
@@ -175,12 +175,17 @@ pub const ENV_HTTP_SOCKET_RECV_BUFFER_BYTES: &str = "RUSTFS_HTTP_SOCKET_RECV_BUF
|
||||
pub const DEFAULT_HTTP_SOCKET_RECV_BUFFER_BYTES: usize = 0;
|
||||
|
||||
/// Environment variable for the S3 request-body inter-chunk read timeout
|
||||
/// (seconds). Default: 300. Set to 0 to disable.
|
||||
/// (seconds). Default: 300. Set to 0 to disable the inactivity timeout.
|
||||
///
|
||||
/// This bounds how long a `PutObject`/upload may wait for the *next* body chunk
|
||||
/// while more bytes are still expected. It resets on every chunk, so it does not
|
||||
/// penalize slow-but-progressing uploads — it only fires when a peer sends a
|
||||
/// partial body and then goes silent *without* closing the connection (no EOF).
|
||||
/// Independently of this knob, `PutObject` and `UploadPart` require at least
|
||||
/// 64 KiB of body progress per 60 seconds of server wait (the minimum-progress
|
||||
/// floor, logged as `upload_body_progress_stalled`); it bounds trickle uploads
|
||||
/// that would otherwise pin upload admission indefinitely, and it stays in
|
||||
/// force when this timeout is set to 0.
|
||||
/// A reverse proxy or CDN that forwards a truncated body this way would
|
||||
/// otherwise make RustFS wait forever for bytes that never arrive; with this
|
||||
/// timeout the stalled read is aborted and logged with the received/expected
|
||||
|
||||
@@ -24,6 +24,14 @@ use std::{
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
|
||||
/// Copy an existing cache-model value without including surrounding work.
|
||||
#[inline]
|
||||
pub fn clone_cache_value<T: ToOwned + ?Sized>(source: &T) -> T::Owned {
|
||||
#[cfg(feature = "hotpath-alloc")]
|
||||
let _copy_scope = hotpath::MeasurementGuardSync::new("cache_model.owned_copy", false, false);
|
||||
source.to_owned()
|
||||
}
|
||||
|
||||
/// Maximum amount a persisted `last_update` may lead the local wall clock before the
|
||||
/// persisted timestamp is treated as untrustworthy.
|
||||
///
|
||||
@@ -342,7 +350,7 @@ impl UnknownTierStats {
|
||||
self.diagnostics_dropped = checked_saturating_add(1, self.diagnostics_dropped, &mut self.counter_overflowed);
|
||||
continue;
|
||||
}
|
||||
self.diagnostics.push(diagnostic.clone());
|
||||
self.diagnostics.push(clone_cache_value(diagnostic));
|
||||
}
|
||||
}
|
||||
if !self.diagnostics.is_empty() {
|
||||
@@ -367,7 +375,7 @@ impl UnknownTierStats {
|
||||
if !self.fits_add(other) {
|
||||
return None;
|
||||
}
|
||||
let mut merged = self.clone();
|
||||
let mut merged = clone_cache_value(self);
|
||||
merged.merge(other);
|
||||
Some(merged)
|
||||
}
|
||||
@@ -407,7 +415,7 @@ impl AllTierStats {
|
||||
if st.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let entry = self.tiers.entry(tier.clone()).or_default();
|
||||
let entry = self.tiers.entry(clone_cache_value(tier)).or_default();
|
||||
*entry = entry.add(st);
|
||||
}
|
||||
}
|
||||
@@ -437,7 +445,7 @@ impl AllTierStats {
|
||||
.tiers
|
||||
.keys()
|
||||
.filter(|tier| tier.as_str() != "STANDARD" && tier.as_str() != "REDUCED_REDUNDANCY" && !known.contains(tier.as_str()))
|
||||
.cloned()
|
||||
.map(clone_cache_value)
|
||||
.collect();
|
||||
for tier in retired_tiers {
|
||||
if let Some(stats) = self.tiers.remove(&tier) {
|
||||
@@ -1237,10 +1245,10 @@ impl DataUsageEntry {
|
||||
}
|
||||
|
||||
pub fn add_child(&mut self, hash: &DataUsageHash) {
|
||||
if self.children.contains(&hash.key()) {
|
||||
if self.children.contains(&clone_cache_value(&hash.0)) {
|
||||
return;
|
||||
}
|
||||
self.children.insert(hash.key());
|
||||
self.children.insert(clone_cache_value(&hash.0));
|
||||
}
|
||||
|
||||
pub fn merge(&mut self, other: &DataUsageEntry) {
|
||||
@@ -1257,7 +1265,7 @@ impl DataUsageEntry {
|
||||
s_rep.replica_size += o_rep.replica_size;
|
||||
s_rep.replica_count += o_rep.replica_count;
|
||||
for (arn, stat) in o_rep.targets.iter() {
|
||||
let st = s_rep.targets.entry(arn.clone()).or_default();
|
||||
let st = s_rep.targets.entry(clone_cache_value(arn)).or_default();
|
||||
st.pending_size += stat.pending_size;
|
||||
st.replicated_size += stat.replicated_size;
|
||||
st.failed_size += stat.failed_size;
|
||||
@@ -1451,7 +1459,7 @@ pub fn prefix_usage_in_cache(
|
||||
} else {
|
||||
format!("{bucket}/{prefix}")
|
||||
};
|
||||
let entry = cache.get(&hash_path(&root).key())?.clone();
|
||||
let entry = clone_cache_value(cache.get(&clone_cache_value(&hash_path(&root).0))?);
|
||||
|
||||
let usage = PrefixUsageSummary::from_entry(&flatten_entry(cache, &entry, 0)?);
|
||||
|
||||
@@ -1594,38 +1602,38 @@ pub struct DataUsageCache {
|
||||
impl DataUsageCache {
|
||||
pub fn replace(&mut self, path: &str, parent: &str, e: DataUsageEntry) {
|
||||
let hash = hash_path(path);
|
||||
self.cache.insert(hash.key(), e);
|
||||
self.cache.insert(clone_cache_value(&hash.0), e);
|
||||
if !parent.is_empty() {
|
||||
let phash = hash_path(parent);
|
||||
let p = {
|
||||
let p = self.cache.entry(phash.key()).or_default();
|
||||
let p = self.cache.entry(clone_cache_value(&phash.0)).or_default();
|
||||
p.add_child(&hash);
|
||||
p.clone()
|
||||
clone_cache_value(p)
|
||||
};
|
||||
self.cache.insert(phash.key(), p);
|
||||
self.cache.insert(clone_cache_value(&phash.0), p);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn replace_hashed(&mut self, hash: &DataUsageHash, parent: &Option<DataUsageHash>, e: &DataUsageEntry) {
|
||||
self.cache.insert(hash.key(), e.clone());
|
||||
self.cache.insert(clone_cache_value(&hash.0), clone_cache_value(e));
|
||||
if let Some(parent) = parent {
|
||||
self.cache.entry(parent.key()).or_default().add_child(hash);
|
||||
self.cache.entry(clone_cache_value(&parent.0)).or_default().add_child(hash);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn find(&self, path: &str) -> Option<DataUsageEntry> {
|
||||
self.cache.get(&hash_path(path).key()).cloned()
|
||||
self.cache.get(&clone_cache_value(&hash_path(path).0)).map(clone_cache_value)
|
||||
}
|
||||
|
||||
pub fn find_children_copy(&mut self, h: DataUsageHash) -> DataUsageHashMap {
|
||||
self.cache.entry(h.string()).or_default().children.clone()
|
||||
clone_cache_value(&self.cache.entry(clone_cache_value(&h.0)).or_default().children)
|
||||
}
|
||||
|
||||
pub fn flatten(&self, root: &DataUsageEntry) -> DataUsageEntry {
|
||||
let mut root = root.clone();
|
||||
for id in root.children.clone().iter() {
|
||||
let mut root = clone_cache_value(root);
|
||||
for id in clone_cache_value(&root.children).iter() {
|
||||
if let Some(e) = self.cache.get(id) {
|
||||
let mut e = e.clone();
|
||||
let mut e = clone_cache_value(e);
|
||||
if !e.children.is_empty() {
|
||||
e = self.flatten(&e);
|
||||
}
|
||||
@@ -1637,16 +1645,16 @@ impl DataUsageCache {
|
||||
}
|
||||
|
||||
pub fn copy_with_children(&mut self, src: &DataUsageCache, hash: &DataUsageHash, parent: &Option<DataUsageHash>) {
|
||||
if let Some(e) = src.cache.get(&hash.string()) {
|
||||
self.cache.insert(hash.key(), e.clone());
|
||||
if let Some(e) = src.cache.get(&clone_cache_value(&hash.0)) {
|
||||
self.cache.insert(clone_cache_value(&hash.0), clone_cache_value(e));
|
||||
for ch in e.children.iter() {
|
||||
if *ch == hash.key() {
|
||||
if *ch == clone_cache_value(&hash.0) {
|
||||
return;
|
||||
}
|
||||
self.copy_with_children(src, &DataUsageHash(ch.to_string()), &Some(hash.clone()));
|
||||
self.copy_with_children(src, &DataUsageHash(clone_cache_value(ch)), &Some(clone_cache_value(hash)));
|
||||
}
|
||||
if let Some(parent) = parent {
|
||||
let p = self.cache.entry(parent.key()).or_default();
|
||||
let p = self.cache.entry(clone_cache_value(&parent.0)).or_default();
|
||||
p.add_child(hash);
|
||||
}
|
||||
}
|
||||
@@ -1654,14 +1662,14 @@ impl DataUsageCache {
|
||||
|
||||
pub fn delete_recursive(&mut self, hash: &DataUsageHash) {
|
||||
let mut need_remove = Vec::new();
|
||||
if let Some(v) = self.cache.get(&hash.string()) {
|
||||
if let Some(v) = self.cache.get(&clone_cache_value(&hash.0)) {
|
||||
for child in v.children.iter() {
|
||||
need_remove.push(child.clone());
|
||||
need_remove.push(clone_cache_value(child));
|
||||
}
|
||||
}
|
||||
self.cache.remove(&hash.string());
|
||||
self.cache.remove(&clone_cache_value(&hash.0));
|
||||
need_remove.iter().for_each(|child| {
|
||||
self.delete_recursive(&DataUsageHash(child.to_string()));
|
||||
self.delete_recursive(&DataUsageHash(clone_cache_value(child)));
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1682,7 +1690,7 @@ impl DataUsageCache {
|
||||
}
|
||||
|
||||
pub fn search_parent(&self, hash: &DataUsageHash) -> Option<DataUsageHash> {
|
||||
let want = hash.key();
|
||||
let want = clone_cache_value(&hash.0);
|
||||
if let Some(last_index) = want.rfind('/')
|
||||
&& let Some(v) = self.find(&want[0..last_index])
|
||||
&& v.children.contains(&want)
|
||||
@@ -1693,7 +1701,7 @@ impl DataUsageCache {
|
||||
|
||||
for (k, v) in self.cache.iter() {
|
||||
if v.children.contains(&want) {
|
||||
let found = DataUsageHash(k.clone());
|
||||
let found = DataUsageHash(clone_cache_value(k));
|
||||
return Some(found);
|
||||
}
|
||||
}
|
||||
@@ -1701,7 +1709,7 @@ impl DataUsageCache {
|
||||
}
|
||||
|
||||
pub fn is_compacted(&self, hash: &DataUsageHash) -> bool {
|
||||
match self.cache.get(&hash.key()) {
|
||||
match self.cache.get(&clone_cache_value(&hash.0)) {
|
||||
Some(due) => due.compacted,
|
||||
None => false,
|
||||
}
|
||||
@@ -1726,7 +1734,7 @@ impl DataUsageCache {
|
||||
if self.cache.len() < limit {
|
||||
return;
|
||||
}
|
||||
let top = hash_path(&self.info.name).key();
|
||||
let top = clone_cache_value(&hash_path(&self.info.name).0);
|
||||
let top_e = match self.find(&top) {
|
||||
Some(e) => e,
|
||||
None => return,
|
||||
@@ -1753,7 +1761,7 @@ impl DataUsageCache {
|
||||
}
|
||||
|
||||
pub fn reduce_children_of(&mut self, path: &DataUsageHash, limit: usize, compact_self: bool) {
|
||||
let e = match self.cache.get(&path.key()) {
|
||||
let e = match self.cache.get(&clone_cache_value(&path.0)) {
|
||||
Some(e) => e,
|
||||
None => return,
|
||||
};
|
||||
@@ -1763,13 +1771,13 @@ impl DataUsageCache {
|
||||
}
|
||||
|
||||
if e.children.len() > limit && compact_self {
|
||||
let mut flat = self.size_recursive(&path.key()).unwrap_or_default();
|
||||
let mut flat = self.size_recursive(&clone_cache_value(&path.0)).unwrap_or_default();
|
||||
flat.compacted = true;
|
||||
self.delete_recursive(path);
|
||||
self.replace_hashed(path, &None, &flat);
|
||||
return;
|
||||
}
|
||||
let total = self.total_children_rec(&path.key());
|
||||
let total = self.total_children_rec(&clone_cache_value(&path.0));
|
||||
if total < limit {
|
||||
return;
|
||||
}
|
||||
@@ -1782,12 +1790,12 @@ impl DataUsageCache {
|
||||
let mut candidate_index = 0;
|
||||
while remove > 0 && candidate_index < candidates.len() {
|
||||
let e = &candidates[candidate_index];
|
||||
let candidate = e.path.clone();
|
||||
let candidate = clone_cache_value(&e.path);
|
||||
if candidate == *path && !compact_self {
|
||||
break;
|
||||
}
|
||||
let removing = self.total_children_rec(&candidate.key());
|
||||
let mut flat = match self.size_recursive(&candidate.key()) {
|
||||
let removing = self.total_children_rec(&clone_cache_value(&candidate.0));
|
||||
let mut flat = match self.size_recursive(&clone_cache_value(&candidate.0)) {
|
||||
Some(flat) => flat,
|
||||
None => {
|
||||
candidate_index += 1;
|
||||
@@ -1824,7 +1832,7 @@ impl DataUsageCache {
|
||||
if o.root().is_none() {
|
||||
return;
|
||||
}
|
||||
*self = o.clone();
|
||||
*self = clone_cache_value(o);
|
||||
return;
|
||||
};
|
||||
|
||||
@@ -1837,7 +1845,8 @@ impl DataUsageCache {
|
||||
}
|
||||
|
||||
existing_root.merge(&other_root);
|
||||
self.cache.insert(hash_path(&self.info.name).key(), existing_root);
|
||||
self.cache
|
||||
.insert(clone_cache_value(&hash_path(&self.info.name).0), existing_root);
|
||||
|
||||
let root_hash = self.root_hash();
|
||||
for key in other_root.children.iter() {
|
||||
@@ -1848,7 +1857,7 @@ impl DataUsageCache {
|
||||
if let Some(existing) = self.cache.get_mut(key) {
|
||||
existing.merge(&flat);
|
||||
} else {
|
||||
self.replace_hashed(&DataUsageHash(key.clone()), &Some(root_hash.clone()), &flat);
|
||||
self.replace_hashed(&DataUsageHash(clone_cache_value(key)), &Some(clone_cache_value(&root_hash)), &flat);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1892,7 +1901,7 @@ impl DataUsageCache {
|
||||
|
||||
for (arn, stat) in rs.targets.iter() {
|
||||
bui.replication_info.insert(
|
||||
arn.clone(),
|
||||
clone_cache_value(arn),
|
||||
BucketTargetUsageInfo {
|
||||
replication_pending_size: stat.pending_size,
|
||||
replicated_size: stat.replicated_size,
|
||||
@@ -1905,7 +1914,7 @@ impl DataUsageCache {
|
||||
);
|
||||
}
|
||||
}
|
||||
buckets_usage.insert(bucket_name.clone(), bui);
|
||||
buckets_usage.insert(clone_cache_value(bucket_name), bui);
|
||||
}
|
||||
|
||||
DataUsageInfo {
|
||||
@@ -1937,20 +1946,20 @@ struct Inner {
|
||||
}
|
||||
|
||||
fn add(data_usage_cache: &DataUsageCache, path: &DataUsageHash, candidates: &mut Vec<Inner>) -> usize {
|
||||
let e = match data_usage_cache.cache.get(&path.key()) {
|
||||
let e = match data_usage_cache.cache.get(&clone_cache_value(&path.0)) {
|
||||
Some(e) => e,
|
||||
None => return 0,
|
||||
};
|
||||
let mut objects = e.objects;
|
||||
for ch in e.children.iter() {
|
||||
objects += add(data_usage_cache, &DataUsageHash(ch.clone()), candidates);
|
||||
objects += add(data_usage_cache, &DataUsageHash(clone_cache_value(ch)), candidates);
|
||||
}
|
||||
// Collect internal nodes (with children) as compaction candidates.
|
||||
// Leaf nodes have no children to remove, so compacting them is a no-op.
|
||||
if !e.children.is_empty() {
|
||||
candidates.push(Inner {
|
||||
objects,
|
||||
path: path.clone(),
|
||||
path: clone_cache_value(path),
|
||||
});
|
||||
}
|
||||
objects
|
||||
@@ -1958,7 +1967,7 @@ fn add(data_usage_cache: &DataUsageCache, path: &DataUsageHash, candidates: &mut
|
||||
|
||||
fn mark(duc: &DataUsageCache, entry: &DataUsageEntry, found: &mut HashSet<String>) {
|
||||
for k in entry.children.iter() {
|
||||
found.insert(k.to_string());
|
||||
found.insert(clone_cache_value(k));
|
||||
if let Some(ch) = duc.cache.get(k) {
|
||||
mark(duc, ch, found);
|
||||
}
|
||||
@@ -2408,11 +2417,11 @@ impl SizeSummary {
|
||||
}
|
||||
|
||||
for entry in &other.size_reconciliation {
|
||||
self.record_size_reconciliation(entry.clone());
|
||||
self.record_size_reconciliation(clone_cache_value(entry));
|
||||
}
|
||||
self.size_reconciliation_truncated |= other.size_reconciliation_truncated;
|
||||
for scope in &other.reconciliation_scopes {
|
||||
self.record_reconciliation_scope(&scope.bucket, &scope.object);
|
||||
self.record_reconciliation_scope_with(&scope.bucket, &scope.object, || clone_cache_value(scope));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2437,6 +2446,13 @@ impl SizeSummary {
|
||||
/// Mark one object scope as refreshed. Duplicate scopes are suppressed so
|
||||
/// merging summaries remains bounded and deterministic.
|
||||
pub fn record_reconciliation_scope(&mut self, bucket: &str, object: &str) {
|
||||
self.record_reconciliation_scope_with(bucket, object, || SizeReconciliationScope {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
fn record_reconciliation_scope_with(&mut self, bucket: &str, object: &str, build: impl FnOnce() -> SizeReconciliationScope) {
|
||||
if !self
|
||||
.reconciliation_scopes
|
||||
.iter()
|
||||
@@ -2446,10 +2462,7 @@ impl SizeSummary {
|
||||
self.size_reconciliation_truncated = true;
|
||||
return;
|
||||
}
|
||||
self.reconciliation_scopes.push(SizeReconciliationScope {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
});
|
||||
self.reconciliation_scopes.push(build());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3404,6 +3417,42 @@ mod tests {
|
||||
assert_eq!(summary1.versions, 15);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reconciliation_scope_builds_only_after_admission() {
|
||||
let mut summary = SizeSummary::new();
|
||||
let builds = std::cell::Cell::new(0);
|
||||
let build = || {
|
||||
builds.set(builds.get() + 1);
|
||||
SizeReconciliationScope {
|
||||
bucket: "bucket".to_string(),
|
||||
object: "object".to_string(),
|
||||
}
|
||||
};
|
||||
|
||||
summary.record_reconciliation_scope_with("bucket", "object", build);
|
||||
assert_eq!(builds.get(), 1, "an admitted scope is copied once");
|
||||
assert_eq!(summary.reconciliation_scopes.len(), 1);
|
||||
assert_eq!(summary.reconciliation_scopes[0].bucket, "bucket");
|
||||
assert_eq!(summary.reconciliation_scopes[0].object, "object");
|
||||
|
||||
summary.record_reconciliation_scope_with("bucket", "object", build);
|
||||
assert_eq!(builds.get(), 1, "duplicate scopes must not be copied");
|
||||
assert_eq!(summary.reconciliation_scopes.len(), 1);
|
||||
assert!(!summary.size_reconciliation_truncated);
|
||||
|
||||
for index in 1..1024 {
|
||||
summary.record_reconciliation_scope("bucket", &index.to_string());
|
||||
}
|
||||
summary.record_reconciliation_scope_with("bucket", "object", build);
|
||||
assert_eq!(builds.get(), 1, "a duplicate at capacity must not be copied");
|
||||
assert!(!summary.size_reconciliation_truncated);
|
||||
|
||||
summary.record_reconciliation_scope_with("bucket", "overflow", build);
|
||||
assert_eq!(builds.get(), 1, "a rejected scope must not be copied");
|
||||
assert_eq!(summary.reconciliation_scopes.len(), 1024);
|
||||
assert!(summary.size_reconciliation_truncated);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_size_histogram_compat_rollup_sums_all_sub_buckets() {
|
||||
let mut hist = SizeHistogram::default();
|
||||
|
||||
@@ -25,6 +25,7 @@ workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
scanner-peer-mixed-version = []
|
||||
hotpath = [
|
||||
"hotpath/hotpath",
|
||||
"hotpath/tokio",
|
||||
|
||||
@@ -26,7 +26,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
|
||||
| **protocols** | [`src/protocols/`](src/protocols) | FTPS, WebDAV, SFTP compliance. Fixed ports, own guide: [`src/protocols/README.md`](src/protocols/README.md) |
|
||||
| **reliant** | [`src/reliant/`](src/reliant) | Tests that reuse an **externally started** server (SQL/select, conditional writes, lifecycle, deleted-object reads, node-interact). Run via [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh); see [`src/reliant/README.md`](src/reliant/README.md) |
|
||||
| **cluster** | `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `namespace_lock_quorum_test`, `admin_timeout_regression_test`, `object_lambda_test`, `replication_extension_test`, `tier_stats_cluster_test` | Multi-node scenarios via `RustFSTestClusterEnvironment` |
|
||||
| **distributed 4×4** | [`src/distributed/`](src/distributed) | Storage-sensitive PR and nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
|
||||
| **distributed 4×4** | [`src/distributed/`](src/distributed) | Nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
|
||||
| **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
|
||||
| **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
|
||||
|
||||
@@ -64,6 +64,25 @@ Each build/run holds an exclusive `rustfs.e2e.lock` marker beside the binary; co
|
||||
|
||||
The protocols suite has its own fixed-port and single-worker contract in [`src/protocols/README.md`](src/protocols/README.md). Use its command under [Troubleshooting](#troubleshooting).
|
||||
|
||||
### Scanner mixed-version gate
|
||||
|
||||
The Linux N/O scanner gate needs a separately qualified `33482bb` server, which supports scoped ACKs. Set `RUSTFS_SCANNER_COMPAT_OLD_BINARY` to that executable and `RUSTFS_SCANNER_COMPAT_OLD_SHA256` to its independently verified digest. The release binary used by ordinary upgrade CI is a different baseline. Missing or mismatched inputs fail the test.
|
||||
|
||||
Enable the harness-only `scanner-peer-mixed-version` feature explicitly; keep the server wrapper features identical between build and run:
|
||||
|
||||
```bash
|
||||
python3 scripts/e2e_binary.py build --features e2e-test-hooks
|
||||
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- \
|
||||
cargo nextest run --profile e2e-distributed -p e2e_test \
|
||||
--features scanner-peer-mixed-version --ignore-default-filter \
|
||||
-E 'test(=distributed::scanner_peer_bucket_test::scanner_peer_bucket_new_coordinator_certifies_old_peers_and_scopes_next_cycle)' \
|
||||
--test-threads 1 --retries 0 --no-tests=fail
|
||||
```
|
||||
|
||||
The separate `scanner_peer_bucket_new_coordinator_certifies_old_worker_peers_and_scopes_next_cycle` case uses the same feature and qualified old binary inputs. Select its exact name in the command above. It keeps the original N process enabled and starts the three official O replacements with `RUSTFS_SCANNER_ENABLED=false`: their autonomous scanner loops are disabled, while real S3 dirty tracking, authenticated snapshots, publication leases, scoped ACKs, and N-dispatched RemoteScanner workers remain active. It records each node's actual scanner status and retains the same identity, durability, root, ACK, data-conservation, and physical-walk requirements. This fixed-role case does not validate autonomous O scanning or all-enabled election behavior. The original all-enabled N/O test remains separate and has not passed acceptance; a fixed-role result cannot satisfy that gate.
|
||||
|
||||
For this fixed-role case, the `e2e-test-hooks` server also records the N client's complete bounded worker inventory. Requests are observed before opening the normal stream and paired only after authenticated source/plan validation and terminal EOF. Closed manifests and final health receipts must bind both cycles to their actual candidate/root; the next cycle needs exactly four sets of hot-only execution and at least one positive-progress worker mapped to a measured official O process. A zero-progress Current response, missing/extra request, unhealthy recorder, or missing physical coverage fails the gate. The recorder does not add O hooks or change any protocol or timeout. Final health reads check immutable closed-cycle evidence, not current N liveness.
|
||||
|
||||
### `#[ignore]` semantics
|
||||
|
||||
Ignored tests are excluded from the default `cargo nextest run` pass because
|
||||
@@ -175,9 +194,9 @@ the same profile for membership and execution with one nightly worker.
|
||||
| `s3s-e2e` black-box | `e2e-tests` + `e2e-tests-rio-v2` jobs | **Active** (external conformance tool) |
|
||||
| ILM / lifecycle (ignored) | `test-ilm-integration-serial` lane, `-j1` | **Active** (backlog#1148 ilm-1) |
|
||||
| KMS suite | `e2e-full` job, merge queue + main | **Active** |
|
||||
| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
|
||||
| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, release tags + weekly schedule + manual dispatch | **Active** |
|
||||
| Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
|
||||
| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (storage-sensitive PR / nightly / dispatch) |
|
||||
| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (nightly / dispatch) |
|
||||
| Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
|
||||
| Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
|
||||
| Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) |
|
||||
|
||||
@@ -502,10 +502,163 @@ mod tests {
|
||||
assert!(response.contains(if expected == reqwest::StatusCode::FORBIDDEN {
|
||||
"<Code>AccessDenied</Code>"
|
||||
} else {
|
||||
"<Code>BucketAlreadyExists</Code>"
|
||||
"<Code>BucketAlreadyOwnedByYou</Code>"
|
||||
}));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A deployment owns a single bucket namespace, so an IAM user that recreates
|
||||
/// an existing bucket is the owner, not another account. In the default region
|
||||
/// every such request returns the legacy idempotent `200 OK` (AWS CreateBucket
|
||||
/// semantics), including buckets created by root and by another IAM user.
|
||||
#[tokio::test]
|
||||
async fn test_create_bucket_duplicate_in_default_region_is_idempotent_for_iam_user()
|
||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec!["--region", "us-east-1"]).await?;
|
||||
let root = env.create_s3_client();
|
||||
|
||||
admin_create_user(&env, "duplicate-iam", "duplicate-iam-secret").await?;
|
||||
admin_attach_user_policy_via(
|
||||
AdminTransport::Signed,
|
||||
&env.url,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
"readwrite",
|
||||
"duplicate-iam",
|
||||
)
|
||||
.await?;
|
||||
let iam = Client::from_conf(build_test_s3_config(
|
||||
&env.url,
|
||||
"duplicate-iam",
|
||||
"duplicate-iam-secret",
|
||||
None,
|
||||
"duplicate-create",
|
||||
));
|
||||
|
||||
root.create_bucket()
|
||||
.bucket("duplicate-root-owned")
|
||||
.send()
|
||||
.await
|
||||
.expect("root creates the first bucket");
|
||||
iam.create_bucket()
|
||||
.bucket("duplicate-root-owned")
|
||||
.send()
|
||||
.await
|
||||
.expect("an IAM user recreating a root-owned bucket in us-east-1 gets 200 OK");
|
||||
|
||||
iam.create_bucket()
|
||||
.bucket("duplicate-iam-owned")
|
||||
.send()
|
||||
.await
|
||||
.expect("the IAM user creates its own bucket");
|
||||
iam.create_bucket()
|
||||
.bucket("duplicate-iam-owned")
|
||||
.send()
|
||||
.await
|
||||
.expect("the same IAM user recreating its bucket gets 200 OK");
|
||||
root.create_bucket()
|
||||
.bucket("duplicate-iam-owned")
|
||||
.send()
|
||||
.await
|
||||
.expect("root recreating an IAM-created bucket gets 200 OK");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Outside the default region the same ownership answer becomes
|
||||
/// `409 BucketAlreadyOwnedByYou` for every authorized requester. Requests that
|
||||
/// lack `s3:CreateBucket` still fail authorization before the duplicate check.
|
||||
#[tokio::test]
|
||||
async fn test_create_bucket_duplicate_outside_default_region_reports_owned_by_you() -> Result<(), Box<dyn Error + Send + Sync>>
|
||||
{
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec!["--region", "eu-west-1"]).await?;
|
||||
let region = "eu-west-1";
|
||||
let root = Client::from_conf(
|
||||
env.create_s3_client()
|
||||
.config()
|
||||
.to_builder()
|
||||
.region(Region::new(region))
|
||||
.build(),
|
||||
);
|
||||
|
||||
admin_create_user(&env, "duplicate-eu-iam", "duplicate-eu-iam-secret").await?;
|
||||
admin_attach_user_policy_via(
|
||||
AdminTransport::Signed,
|
||||
&env.url,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
"readwrite",
|
||||
"duplicate-eu-iam",
|
||||
)
|
||||
.await?;
|
||||
let iam = Client::from_conf(
|
||||
build_test_s3_config(&env.url, "duplicate-eu-iam", "duplicate-eu-iam-secret", None, "duplicate-create")
|
||||
.to_builder()
|
||||
.region(Region::new(region))
|
||||
.build(),
|
||||
);
|
||||
|
||||
admin_create_user(&env, "duplicate-eu-readonly", "duplicate-eu-readonly-secret").await?;
|
||||
admin_attach_user_policy_via(
|
||||
AdminTransport::Signed,
|
||||
&env.url,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
"readonly",
|
||||
"duplicate-eu-readonly",
|
||||
)
|
||||
.await?;
|
||||
let readonly = Client::from_conf(
|
||||
build_test_s3_config(
|
||||
&env.url,
|
||||
"duplicate-eu-readonly",
|
||||
"duplicate-eu-readonly-secret",
|
||||
None,
|
||||
"duplicate-create",
|
||||
)
|
||||
.to_builder()
|
||||
.region(Region::new(region))
|
||||
.build(),
|
||||
);
|
||||
|
||||
for (client, bucket) in [(&root, "duplicate-eu-root"), (&iam, "duplicate-eu-iam")] {
|
||||
client
|
||||
.create_bucket()
|
||||
.bucket(bucket)
|
||||
.send()
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("{bucket} must be created: {error:?}"));
|
||||
// Both the creator and the other authenticated identity own the namespace.
|
||||
for creator in [&root, &iam] {
|
||||
let error = creator
|
||||
.create_bucket()
|
||||
.bucket(bucket)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a duplicate create outside us-east-1 must report ownership");
|
||||
assert_eq!(
|
||||
error.as_service_error().and_then(ProvideErrorMetadata::code),
|
||||
Some("BucketAlreadyOwnedByYou"),
|
||||
"{bucket}"
|
||||
);
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(409), "{bucket}");
|
||||
}
|
||||
}
|
||||
|
||||
let denied = readonly
|
||||
.create_bucket()
|
||||
.bucket("duplicate-eu-root")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a requester without s3:CreateBucket must be denied before the duplicate check");
|
||||
assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,8 +16,193 @@
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::config::http::HttpResponse;
|
||||
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration};
|
||||
|
||||
fn assert_not_found<E: ProvideErrorMetadata + std::fmt::Debug>(error: &SdkError<E, HttpResponse>) {
|
||||
assert_eq!(error.raw_response().map(|r| r.status().as_u16()), Some(404), "{error:?}");
|
||||
assert_eq!(
|
||||
error.as_service_error().and_then(ProvideErrorMetadata::code),
|
||||
Some("NoSuchKey"),
|
||||
"{error:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn conditional_writes_cannot_replace_delete_marker_with_if_match() {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
|
||||
env.start_rustfs_server(vec![]).await.expect("start RustFS");
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "conditional-delete-marker-proof";
|
||||
create_versioned_bucket(&client, bucket).await;
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("source")
|
||||
.body(ByteStream::from_static(b"source"))
|
||||
.send()
|
||||
.await
|
||||
.expect("write copy source");
|
||||
|
||||
for operation in ["put", "copy", "multipart"] {
|
||||
let old = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.body(ByteStream::from_static(b"old"))
|
||||
.send()
|
||||
.await
|
||||
.expect("write historical version");
|
||||
let old_etag = old.e_tag().expect("historical ETag");
|
||||
let marker = client
|
||||
.delete_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.send()
|
||||
.await
|
||||
.expect("create delete marker");
|
||||
let marker_id = marker.version_id().expect("delete marker version");
|
||||
|
||||
let (upload_id, completed) = if operation == "multipart" {
|
||||
let upload = client
|
||||
.create_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.send()
|
||||
.await
|
||||
.expect("initiate multipart upload");
|
||||
let upload_id = upload.upload_id().expect("upload ID").to_owned();
|
||||
let part = client
|
||||
.upload_part()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.upload_id(&upload_id)
|
||||
.part_number(1)
|
||||
.body(ByteStream::from_static(b"new"))
|
||||
.send()
|
||||
.await
|
||||
.expect("upload part");
|
||||
let completed = CompletedMultipartUpload::builder()
|
||||
.parts(
|
||||
CompletedPart::builder()
|
||||
.part_number(1)
|
||||
.set_e_tag(part.e_tag().map(str::to_owned))
|
||||
.build(),
|
||||
)
|
||||
.build();
|
||||
(Some(upload_id), Some(completed))
|
||||
} else {
|
||||
(None, None)
|
||||
};
|
||||
|
||||
for etag in [old_etag, "\"00000000000000000000000000000000\""] {
|
||||
match operation {
|
||||
"put" => {
|
||||
let err = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.if_match(etag)
|
||||
.body(ByteStream::from_static(b"new"))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("If-Match must reject marker");
|
||||
assert_not_found(&err);
|
||||
}
|
||||
"copy" => {
|
||||
let err = client
|
||||
.copy_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.copy_source(format!("{bucket}/source"))
|
||||
.if_match(etag)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("If-Match copy must reject marker");
|
||||
assert_not_found(&err);
|
||||
}
|
||||
_ => {
|
||||
let err = client
|
||||
.complete_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.upload_id(upload_id.as_deref().expect("upload ID"))
|
||||
.multipart_upload(completed.clone().expect("completed parts"))
|
||||
.if_match(etag)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("If-Match completion must reject marker");
|
||||
assert_not_found(&err);
|
||||
}
|
||||
}
|
||||
let versions = client
|
||||
.list_object_versions()
|
||||
.bucket(bucket)
|
||||
.prefix(operation)
|
||||
.send()
|
||||
.await
|
||||
.expect("list versions after rejected write");
|
||||
assert_eq!(versions.versions().len(), 1, "{operation}: failed write created a version");
|
||||
assert_eq!(versions.delete_markers().len(), 1);
|
||||
assert_eq!(versions.delete_markers()[0].version_id(), Some(marker_id));
|
||||
assert_eq!(versions.delete_markers()[0].is_latest(), Some(true));
|
||||
}
|
||||
|
||||
match operation {
|
||||
"put" => {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.if_none_match("*")
|
||||
.body(ByteStream::from_static(b"new"))
|
||||
.send()
|
||||
.await
|
||||
.expect("If-None-Match PUT over marker");
|
||||
}
|
||||
"copy" => {
|
||||
client
|
||||
.copy_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.copy_source(format!("{bucket}/source"))
|
||||
.if_none_match("*")
|
||||
.send()
|
||||
.await
|
||||
.expect("If-None-Match copy over marker");
|
||||
}
|
||||
_ => {
|
||||
client
|
||||
.complete_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.upload_id(upload_id.as_deref().expect("upload ID"))
|
||||
.multipart_upload(completed.expect("completed parts"))
|
||||
.if_none_match("*")
|
||||
.send()
|
||||
.await
|
||||
.expect("If-None-Match completion over marker");
|
||||
}
|
||||
}
|
||||
let current = client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.send()
|
||||
.await
|
||||
.expect("read successful conditional write")
|
||||
.body
|
||||
.collect()
|
||||
.await
|
||||
.expect("read object body")
|
||||
.into_bytes();
|
||||
let expected: &[u8] = if operation == "copy" { b"source" } else { b"new" };
|
||||
assert_eq!(current.as_ref(), expected, "{operation}: unexpected current object");
|
||||
}
|
||||
}
|
||||
|
||||
async fn create_versioned_bucket(client: &Client, bucket: &str) {
|
||||
client
|
||||
|
||||
@@ -12,11 +12,249 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, unique_bucket};
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, payload_for, s3_probe_client, sha256_hex, unique_bucket};
|
||||
use crate::common::init_logging;
|
||||
use aws_sdk_s3::{
|
||||
Client,
|
||||
error::{ProvideErrorMetadata, SdkError},
|
||||
primitives::ByteStream,
|
||||
};
|
||||
use std::any::TypeId;
|
||||
use std::collections::BTreeSet;
|
||||
use std::fmt::Debug;
|
||||
use std::future::Future;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::Barrier;
|
||||
use tokio::time::Instant;
|
||||
|
||||
const LOCK_ADMISSION_BUDGET: Duration = Duration::from_secs(45);
|
||||
|
||||
async fn after_lock_contention<T, E, F, Fut>(operation: &str, mut call: F) -> TestResult<Result<T, SdkError<E>>>
|
||||
where
|
||||
E: Debug + ProvideErrorMetadata + 'static,
|
||||
F: FnMut() -> Fut,
|
||||
Fut: Future<Output = Result<T, SdkError<E>>>,
|
||||
{
|
||||
let started = Instant::now();
|
||||
let mut attempts = 0;
|
||||
// The workload uses unique unversioned keys. Repeat only explicitly
|
||||
// advertised lock backpressure; every step still needs its real result.
|
||||
let outcome = tokio::time::timeout(LOCK_ADMISSION_BUDGET, async {
|
||||
loop {
|
||||
attempts += 1;
|
||||
let result = call().await;
|
||||
let retry = result.as_ref().err().is_some_and(|error| {
|
||||
let Some(response) = error.raw_response() else {
|
||||
return false;
|
||||
};
|
||||
let Some(service_error) = error.as_service_error() else {
|
||||
return false;
|
||||
};
|
||||
// HEAD has no XML error body, so its SDK service error can
|
||||
// lack a code even when the server supplies retry headers.
|
||||
let namespace_code = service_error.code() == Some("ServiceUnavailable")
|
||||
|| (service_error.code().is_none()
|
||||
&& TypeId::of::<E>() == TypeId::of::<aws_sdk_s3::operation::head_object::HeadObjectError>());
|
||||
response.status().as_u16() == 503
|
||||
&& namespace_code
|
||||
&& response.headers().get("x-rustfs-error-cause") == Some("namespace_lock_timeout")
|
||||
&& response.headers().get("retry-after") == Some("1")
|
||||
});
|
||||
if !retry {
|
||||
return result;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
if attempts > 1 || !matches!(&outcome, Ok(Ok(_))) {
|
||||
eprintln!(
|
||||
"mixed workload request: operation={operation}, attempts={attempts}, elapsed={:?}",
|
||||
started.elapsed()
|
||||
);
|
||||
}
|
||||
outcome.map_err(|_| {
|
||||
format!("{operation} exceeded its {LOCK_ADMISSION_BUDGET:?} request budget after {attempts} attempts").into()
|
||||
})
|
||||
}
|
||||
|
||||
async fn assert_workload_bytes(client: &Client, bucket: &str, key: &str, expected: &[u8]) -> TestResult {
|
||||
let output = after_lock_contention(&format!("GET {key}"), || client.get_object().bucket(bucket).key(key).send()).await??;
|
||||
let actual = output.body.collect().await?.into_bytes();
|
||||
if actual.as_ref() != expected {
|
||||
return Err(format!(
|
||||
"object {key} bytes mismatch: expected {} bytes sha256={} got {} bytes sha256={}",
|
||||
expected.len(),
|
||||
sha256_hex(expected),
|
||||
actual.len(),
|
||||
sha256_hex(&actual)
|
||||
)
|
||||
.into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use aws_smithy_http_client::test_util::infallible_client_fn;
|
||||
use aws_smithy_types::body::SdkBody;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
type Reply = (u16, &'static str, Option<&'static str>, Option<&'static str>, &'static str);
|
||||
|
||||
fn client(respond: impl Fn(usize) -> Reply + Send + Sync + 'static) -> (Client, Arc<AtomicUsize>) {
|
||||
let requests = Arc::new(AtomicUsize::new(0));
|
||||
let seen = Arc::clone(&requests);
|
||||
let transport = infallible_client_fn(move |_| {
|
||||
let (status, code, cause, retry_after, body) = respond(seen.fetch_add(1, Ordering::SeqCst));
|
||||
let body = if code.is_empty() {
|
||||
body.to_string()
|
||||
} else {
|
||||
format!("<Error><Code>{code}</Code></Error>")
|
||||
};
|
||||
let mut response = http::Response::builder()
|
||||
.status(status)
|
||||
.header("content-type", "application/xml")
|
||||
.header("content-length", body.len());
|
||||
if let Some(cause) = cause {
|
||||
response = response.header("x-rustfs-error-cause", cause);
|
||||
}
|
||||
if let Some(retry_after) = retry_after {
|
||||
response = response.header("retry-after", retry_after);
|
||||
}
|
||||
response.body(SdkBody::from(body)).expect("scripted S3 response")
|
||||
});
|
||||
let config = crate::common::build_test_s3_config(
|
||||
"http://contention.example.invalid",
|
||||
"synthetic-access",
|
||||
"synthetic-secret",
|
||||
None,
|
||||
"contention-test",
|
||||
);
|
||||
let client = Client::from_conf(config.to_builder().http_client(transport).build());
|
||||
(s3_probe_client(&client), requests)
|
||||
}
|
||||
|
||||
fn contention() -> Reply {
|
||||
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("1"), "")
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn advertised_contention_recovers_after_standard_sdk_attempts_are_exhausted() -> TestResult {
|
||||
let (client, requests) = client(|attempt| {
|
||||
if attempt < 3 {
|
||||
contention()
|
||||
} else {
|
||||
(200, "", None, None, "")
|
||||
}
|
||||
});
|
||||
let standard = Client::from_conf(
|
||||
client
|
||||
.config()
|
||||
.to_builder()
|
||||
.retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(3))
|
||||
.build(),
|
||||
);
|
||||
standard
|
||||
.put_object()
|
||||
.bucket("bucket")
|
||||
.key("key")
|
||||
.body(ByteStream::from_static(b"body"))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("three advertised lock responses exhaust the standard SDK policy");
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 3);
|
||||
requests.store(0, Ordering::SeqCst);
|
||||
let started = Instant::now();
|
||||
after_lock_contention("PUT recovery", || {
|
||||
client
|
||||
.put_object()
|
||||
.bucket("bucket")
|
||||
.key("key")
|
||||
.body(ByteStream::from_static(b"body"))
|
||||
.send()
|
||||
})
|
||||
.await??;
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 4);
|
||||
assert!(started.elapsed() >= Duration::from_secs(3), "honor each Retry-After response");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn bodyless_head_contention_uses_explicit_retry_headers() -> TestResult {
|
||||
let (client, requests) = client(|attempt| {
|
||||
if attempt == 0 {
|
||||
(503, "", Some("namespace_lock_timeout"), Some("1"), "")
|
||||
} else {
|
||||
(200, "", None, None, "")
|
||||
}
|
||||
});
|
||||
after_lock_contention("HEAD recovery", || client.head_object().bucket("bucket").key("key").send()).await??;
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 2);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn unrelated_or_malformed_responses_are_not_retried() -> TestResult {
|
||||
for reply in [
|
||||
(500, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
|
||||
(403, "AccessDenied", None, None, ""),
|
||||
(404, "NoSuchKey", None, None, ""),
|
||||
(503, "ServiceUnavailable", Some("storage_write_quorum"), None, ""),
|
||||
(503, "ServiceUnavailable", Some("namespace_lock_unavailable"), Some("1"), ""),
|
||||
(503, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
|
||||
(503, "", Some("namespace_lock_timeout"), Some("1"), ""),
|
||||
(503, "ServiceUnavailable", None, Some("1"), ""),
|
||||
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), None, ""),
|
||||
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("2"), ""),
|
||||
] {
|
||||
let (client, requests) = client(move |_| reply);
|
||||
let result = after_lock_contention("PUT fatal response", || {
|
||||
client
|
||||
.put_object()
|
||||
.bucket("bucket")
|
||||
.key("key")
|
||||
.body(ByteStream::from_static(b"body"))
|
||||
.send()
|
||||
})
|
||||
.await?;
|
||||
assert!(result.is_err(), "fatal response cannot become success: {reply:?}");
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 1, "fatal response must remain visible: {reply:?}");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn persistent_lock_contention_exhausts_the_request_budget() {
|
||||
let (client, requests) = client(|_| contention());
|
||||
let started = Instant::now();
|
||||
let result = after_lock_contention("PUT unavailable", || {
|
||||
client
|
||||
.put_object()
|
||||
.bucket("bucket")
|
||||
.key("key")
|
||||
.body(ByteStream::from_static(b"body"))
|
||||
.send()
|
||||
})
|
||||
.await;
|
||||
assert!(result.is_err(), "persistent contention cannot acknowledge an operation");
|
||||
assert!(started.elapsed() >= LOCK_ADMISSION_BUDGET);
|
||||
assert!(requests.load(Ordering::SeqCst) > 1);
|
||||
assert!(requests.load(Ordering::SeqCst) <= LOCK_ADMISSION_BUDGET.as_secs() as usize + 1);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn acknowledged_wrong_bytes_fail_without_retry() {
|
||||
let (client, requests) = client(|_| (200, "", None, None, "wrong bytes"));
|
||||
let error = assert_workload_bytes(&client, "bucket", "key", b"expected bytes")
|
||||
.await
|
||||
.expect_err("a successful but incorrect read is a consistency failure");
|
||||
assert!(error.to_string().contains("bytes mismatch"));
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node() -> TestResult {
|
||||
@@ -24,7 +262,9 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
|
||||
let dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let bucket = unique_bucket("conc");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let clients = Arc::new(dist.clients()?);
|
||||
// This consistency probe owns its admission policy; unrelated responses
|
||||
// must not be hidden by the SDK's general-purpose retries.
|
||||
let clients = Arc::new(dist.clients()?.iter().map(s3_probe_client).collect::<Vec<_>>());
|
||||
let worker_count = 24;
|
||||
let rounds = 4;
|
||||
let barrier = Arc::new(Barrier::new(worker_count));
|
||||
@@ -44,31 +284,46 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
|
||||
let key = format!("source/worker-{idx:02}-round-{round}.bin");
|
||||
let copy_key = format!("retained/worker-{idx:02}-round-{round}.bin");
|
||||
let body = payload_for(&key, 64 * 1024);
|
||||
put_object(writer, &bucket, &key, body.clone()).await?;
|
||||
let context = format!("worker={idx} round={round}");
|
||||
after_lock_contention(&format!("{context} PUT source"), || {
|
||||
writer
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(&key)
|
||||
.body(ByteStream::from(body.clone()))
|
||||
.send()
|
||||
})
|
||||
.await??;
|
||||
|
||||
let head = reader.head_object().bucket(&bucket).key(&key).send().await?;
|
||||
let head = after_lock_contention(&format!("{context} HEAD source"), || {
|
||||
reader.head_object().bucket(&bucket).key(&key).send()
|
||||
})
|
||||
.await??;
|
||||
if head.content_length() != Some(body.len() as i64) {
|
||||
return Err(format!("HEAD returned the wrong size for {key}: {head:?}").into());
|
||||
}
|
||||
assert_object_bytes(reader, &bucket, &key, &body).await?;
|
||||
assert_workload_bytes(reader, &bucket, &key, &body).await?;
|
||||
|
||||
copier
|
||||
.copy_object()
|
||||
.bucket(&bucket)
|
||||
.key(©_key)
|
||||
.copy_source(format!("{bucket}/{key}"))
|
||||
.send()
|
||||
.await?;
|
||||
assert_object_bytes(writer, &bucket, ©_key, &body).await?;
|
||||
after_lock_contention(&format!("{context} COPY retained"), || {
|
||||
copier
|
||||
.copy_object()
|
||||
.bucket(&bucket)
|
||||
.key(©_key)
|
||||
.copy_source(format!("{bucket}/{key}"))
|
||||
.send()
|
||||
})
|
||||
.await??;
|
||||
assert_workload_bytes(writer, &bucket, ©_key, &body).await?;
|
||||
|
||||
writer.delete_object().bucket(&bucket).key(&key).send().await?;
|
||||
let missing = reader
|
||||
.head_object()
|
||||
.bucket(&bucket)
|
||||
.key(&key)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("deleted source key must not remain visible");
|
||||
after_lock_contention(&format!("{context} DELETE source"), || {
|
||||
writer.delete_object().bucket(&bucket).key(&key).send()
|
||||
})
|
||||
.await??;
|
||||
let missing = after_lock_contention(&format!("{context} HEAD deleted source"), || {
|
||||
reader.head_object().bucket(&bucket).key(&key).send()
|
||||
})
|
||||
.await?
|
||||
.expect_err("deleted source key must not remain visible");
|
||||
if missing.raw_response().map(|response| response.status().as_u16()) != Some(404) {
|
||||
return Err(format!("deleted source {key} returned an unexpected result: {missing:?}").into());
|
||||
}
|
||||
@@ -82,14 +337,22 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
|
||||
for handle in handles {
|
||||
inventory.extend(handle.await??);
|
||||
}
|
||||
assert_eq!(
|
||||
inventory.len(),
|
||||
worker_count * rounds,
|
||||
"every worker round must retain its acknowledged copy"
|
||||
);
|
||||
|
||||
let expected_keys: BTreeSet<_> = inventory.iter().map(|(key, _)| key.as_str()).collect();
|
||||
for (node_idx, client) in clients.iter().enumerate() {
|
||||
let listed = client.list_objects_v2().bucket(&bucket).prefix("retained/").send().await?;
|
||||
let listed = after_lock_contention(&format!("node={node_idx} LIST retained"), || {
|
||||
client.list_objects_v2().bucket(&bucket).prefix("retained/").send()
|
||||
})
|
||||
.await??;
|
||||
let listed_keys: BTreeSet<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
|
||||
assert_eq!(listed_keys, expected_keys, "node {node_idx} returned a divergent retained-key listing");
|
||||
for (key, body) in &inventory {
|
||||
assert_object_bytes(client, &bucket, key, body)
|
||||
assert_workload_bytes(client, &bucket, key, body)
|
||||
.await
|
||||
.map_err(|error| format!("node {node_idx} failed to read {key}: {error}"))?;
|
||||
}
|
||||
|
||||
@@ -39,6 +39,8 @@ mod s3_during_data_movement_test;
|
||||
mod scanner_heal_topology_fixture;
|
||||
mod scanner_heal_topology_test;
|
||||
mod scanner_heal_topology_tier_test;
|
||||
mod scanner_peer_bucket_fixture;
|
||||
mod scanner_peer_bucket_test;
|
||||
mod site_replication_test;
|
||||
mod table_catalog_test;
|
||||
mod upgrade_test;
|
||||
|
||||
@@ -18,7 +18,7 @@ use super::harness::{
|
||||
};
|
||||
use crate::common::{ClusterTopology, RustFSTestClusterEnvironment};
|
||||
use crate::rpc_fault_proxy::{RpcFaultProxy, RpcPause};
|
||||
use crate::storage_api::multipart_fence_acceptance::RUSTFS_META_MULTIPART_BUCKET;
|
||||
use crate::storage_api::multipart_fence_acceptance::{LOCK_DIAGNOSTICS_FILTER, RUSTFS_META_MULTIPART_BUCKET};
|
||||
use aws_sdk_s3::{
|
||||
Client,
|
||||
config::retry::RetryConfig,
|
||||
@@ -38,14 +38,16 @@ const DEADLINE: Duration = Duration::from_secs(45);
|
||||
async fn fixture(
|
||||
drives_per_node: usize,
|
||||
storage_class: &str,
|
||||
lock_diagnostics: bool,
|
||||
) -> TestResult<(RustFSTestClusterEnvironment, Vec<RpcFaultProxy>, Vec<Client>, String)> {
|
||||
fixture_from_binary(drives_per_node, storage_class, None).await
|
||||
fixture_from_binary(drives_per_node, storage_class, None, lock_diagnostics).await
|
||||
}
|
||||
|
||||
async fn fixture_from_binary(
|
||||
drives_per_node: usize,
|
||||
storage_class: &str,
|
||||
binary: Option<&std::path::Path>,
|
||||
lock_diagnostics: bool,
|
||||
) -> TestResult<(RustFSTestClusterEnvironment, Vec<RpcFaultProxy>, Vec<Client>, String)> {
|
||||
let mut cluster =
|
||||
RustFSTestClusterEnvironment::with_topology(ClusterTopology::single_pool_multidrive(4, drives_per_node)).await?;
|
||||
@@ -90,6 +92,10 @@ async fn fixture_from_binary(
|
||||
] {
|
||||
cluster.set_env(name, value);
|
||||
}
|
||||
if lock_diagnostics {
|
||||
cluster.set_env("RUST_LOG", LOCK_DIAGNOSTICS_FILTER);
|
||||
cluster.set_env("RUSTFS_OBJECT_LOCK_DIAG_ENABLE", "true");
|
||||
}
|
||||
let mut proxies = Vec::new();
|
||||
if binary.is_none() {
|
||||
for node in 0..4 {
|
||||
@@ -363,26 +369,120 @@ fn upload_path_scope(bucket: &str, key: &str, upload_id: &str) -> TestResult<Str
|
||||
))
|
||||
}
|
||||
|
||||
async fn with_paused_peers<T: Send + 'static>(
|
||||
async fn with_paused_peers<T, E>(
|
||||
proxies: &[RpcFaultProxy],
|
||||
methods: &[&'static str],
|
||||
limits: &[usize],
|
||||
scope: &str,
|
||||
request: impl std::future::Future<Output = T> + Send + 'static,
|
||||
) -> TestResult<T> {
|
||||
request: impl std::future::Future<Output = Result<T, SdkError<E>>> + Send + 'static,
|
||||
) -> TestResult<Result<T, SdkError<E>>>
|
||||
where
|
||||
T: Send + 'static,
|
||||
E: std::error::Error + ProvideErrorMetadata + Send + Sync + 'static,
|
||||
{
|
||||
let deadline = tokio::time::Instant::now() + DEADLINE;
|
||||
let pauses: Vec<_> = proxies[1..=limits.len()]
|
||||
.iter()
|
||||
.zip(limits)
|
||||
.map(|(proxy, limit)| proxy.pause(methods, *limit, Some(scope)))
|
||||
.collect::<Result<_, _>>()?;
|
||||
let task = tokio::spawn(request);
|
||||
wait_for_faults(&pauses, methods).await?;
|
||||
let result = tokio::time::timeout(DEADLINE, task).await??;
|
||||
let mut task = tokio::spawn(request);
|
||||
let mut joined = None;
|
||||
let mut deadline_expired = false;
|
||||
let mut barrier_result = {
|
||||
let faults = wait_for_faults(&pauses, methods);
|
||||
tokio::pin!(faults);
|
||||
match tokio::time::timeout_at(deadline, async {
|
||||
tokio::select! {
|
||||
result = &mut faults => result,
|
||||
result = &mut task => {
|
||||
joined = Some(result);
|
||||
// A ready request is not evidence that every selected fault
|
||||
// was reached. Poll the original barrier once for that proof.
|
||||
match futures::poll!(faults.as_mut()) {
|
||||
std::task::Poll::Ready(result) => result,
|
||||
std::task::Poll::Pending => Err("SDK request completed before the selected RPC fault barrier".into()),
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(result) => result,
|
||||
Err(error) => {
|
||||
deadline_expired = true;
|
||||
Err(std::io::Error::new(std::io::ErrorKind::TimedOut, error).into())
|
||||
}
|
||||
}
|
||||
};
|
||||
// Tokio's timeout polls an already-ready inner future before its timer.
|
||||
// A late observation cannot certify the original absolute deadline.
|
||||
deadline_expired |= tokio::time::Instant::now() >= deadline;
|
||||
if deadline_expired {
|
||||
barrier_result = Err(std::io::Error::new(
|
||||
std::io::ErrorKind::TimedOut,
|
||||
"selected RPC fault proof observed after the absolute MPU deadline",
|
||||
)
|
||||
.into());
|
||||
}
|
||||
if barrier_result.is_err() {
|
||||
for pause in &pauses {
|
||||
pause.discard_pending();
|
||||
}
|
||||
}
|
||||
let mut client_task_aborted = false;
|
||||
let request_result = match joined {
|
||||
Some(result) => result,
|
||||
None => match tokio::time::timeout_at(deadline, &mut task).await {
|
||||
Ok(result) => result,
|
||||
Err(error) => {
|
||||
deadline_expired = true;
|
||||
for pause in &pauses {
|
||||
pause.discard_pending();
|
||||
}
|
||||
barrier_result = Err(std::io::Error::new(std::io::ErrorKind::TimedOut, error).into());
|
||||
// This owns only the test's SDK request, not server/native work.
|
||||
// Retire the client task without granting another operation budget.
|
||||
client_task_aborted = true;
|
||||
task.abort();
|
||||
task.await
|
||||
}
|
||||
},
|
||||
};
|
||||
deadline_expired |= tokio::time::Instant::now() >= deadline;
|
||||
if deadline_expired {
|
||||
barrier_result = Err(std::io::Error::new(
|
||||
std::io::ErrorKind::TimedOut,
|
||||
"SDK request result observed after the absolute MPU deadline",
|
||||
)
|
||||
.into());
|
||||
}
|
||||
let hits: Vec<_> = pauses.iter().map(RpcPause::hits).collect();
|
||||
for pause in &pauses {
|
||||
pause.discard_pending();
|
||||
}
|
||||
drop(pauses);
|
||||
Ok(result)
|
||||
if barrier_result.is_err() {
|
||||
let sdk_error = request_result.as_ref().ok().and_then(|result| result.as_ref().err());
|
||||
let response = sdk_error.and_then(SdkError::raw_response);
|
||||
eprintln!(
|
||||
"MPU fault barrier failed: methods={methods:?}, hits={hits:?}, client_task_aborted={client_task_aborted}, deadline_expired={deadline_expired}, sdk_code={:?}, cause={:?}, request_id={:?}",
|
||||
sdk_error
|
||||
.and_then(|error| error.as_service_error())
|
||||
.and_then(|error| error.meta().code()),
|
||||
response.and_then(|response| response.headers().get("x-rustfs-error-cause")),
|
||||
response.and_then(|response| response.headers().get("x-amz-request-id")),
|
||||
);
|
||||
}
|
||||
match (barrier_result, request_result) {
|
||||
// Preserve the actual SDK source instead of replacing an early service
|
||||
// failure with an unrelated fault-wait timeout. The outer Err stays red.
|
||||
(Err(_), Ok(Err(error))) => Err(Box::new(error)),
|
||||
(Err(_), Err(error)) if !client_task_aborted => Err(Box::new(error)),
|
||||
(Err(error), _) => Err(error),
|
||||
(Ok(()), Ok(result)) => Ok(result),
|
||||
(Ok(()), Err(error)) => Err(Box::new(error)),
|
||||
}
|
||||
}
|
||||
|
||||
fn quorum_outcome<T: std::fmt::Debug, E: std::fmt::Debug + ProvideErrorMetadata>(
|
||||
@@ -1318,7 +1418,7 @@ async fn concurrent_capped_parts_respect_limit_on_distinct_nodes(clients: &[Clie
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_delay_preserves_write_quorum_and_rejects_quorum_minus_one() -> TestResult {
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2", true).await?;
|
||||
list_parts_disk_loss_after_metadata_read(&cluster, &proxies, &clients, &bucket).await?;
|
||||
concurrent_capped_parts_respect_limit_on_distinct_nodes(&clients, &bucket).await?;
|
||||
phase_quorum_matrix(&cluster, &proxies, &clients, &bucket, false, 1, 3).await
|
||||
@@ -1326,13 +1426,13 @@ async fn metadata_delay_preserves_write_quorum_and_rejects_quorum_minus_one() ->
|
||||
|
||||
#[tokio::test]
|
||||
async fn lock_delay_preserves_admission_quorum_and_reports_timeout() -> TestResult {
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
|
||||
phase_quorum_matrix(&cluster, &proxies, &clients, &bucket, true, 1, 3).await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sixteen_shard_multipart_metadata_requires_twelve_votes() -> TestResult {
|
||||
let (cluster, proxies, clients, bucket) = fixture(4, "EC:4").await?;
|
||||
let (cluster, proxies, clients, bucket) = fixture(4, "EC:4", false).await?;
|
||||
let storage = cluster_admin_ok(&cluster, http::Method::GET, "/rustfs/admin/v3/storageinfo", None).await?;
|
||||
let storage: serde_json::Value = serde_json::from_str(&storage)?;
|
||||
let storage: rustfs_madmin::StorageInfo = serde_json::from_value(storage["info"].clone())?;
|
||||
@@ -1346,7 +1446,7 @@ async fn sixteen_shard_multipart_metadata_requires_twelve_votes() -> TestResult
|
||||
#[tokio::test]
|
||||
async fn mixed_version_failed_complete_requires_current_coordinator_for_abort() -> TestResult {
|
||||
let previous = super::upgrade_test::source_binary()?;
|
||||
let (mut cluster, mut proxies, clients, bucket) = fixture_from_binary(4, "EC:4", Some(&previous)).await?;
|
||||
let (mut cluster, mut proxies, clients, bucket) = fixture_from_binary(4, "EC:4", Some(&previous), false).await?;
|
||||
assert!(
|
||||
proxies.is_empty(),
|
||||
"the genuine previous fleet boots and seeds through its normal direct transport"
|
||||
@@ -1549,7 +1649,7 @@ async fn native_certified_partial_abort_retry_reclaims_survivors_below_metadata_
|
||||
capability_sha256: Option<[u8; 32]>,
|
||||
abort: Option<AbortRef>,
|
||||
}
|
||||
let (mut cluster, proxies, clients, bucket) = fixture(4, "EC:4").await?;
|
||||
let (mut cluster, proxies, clients, bucket) = fixture(4, "EC:4", false).await?;
|
||||
let key = "partial-abort";
|
||||
let committed = payload_for("committed-before-abort", 4096);
|
||||
clients[0]
|
||||
@@ -1765,7 +1865,7 @@ async fn native_certified_partial_abort_retry_reclaims_survivors_below_metadata_
|
||||
#[tokio::test]
|
||||
async fn late_complete_packets_cannot_publish_after_abort_and_bucket_recreation() -> TestResult {
|
||||
for size in [0, 8 * 1024 * 1024] {
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
|
||||
let key = "late-complete";
|
||||
let upload = clients[0].create_multipart_upload().bucket(&bucket).key(key).send().await?;
|
||||
let upload_id = upload.upload_id().ok_or("missing upload ID")?;
|
||||
@@ -1892,7 +1992,7 @@ async fn late_complete_packets_cannot_publish_after_abort_and_bucket_recreation(
|
||||
async fn late_part_packets_preserve_retired_and_fresh_uploads() -> TestResult {
|
||||
for (method, settles) in [("PreparePartTransaction", false), ("SettlePartTransaction", true)] {
|
||||
for recreate_bucket in [false, true] {
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
|
||||
let key = "late-part-transaction";
|
||||
let previous = payload_for("committed-before-late-part", 4096);
|
||||
clients[0]
|
||||
@@ -2075,7 +2175,7 @@ async fn late_part_packets_preserve_retired_and_fresh_uploads() -> TestResult {
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelled_metadata_wait_releases_fence_for_bucket_recreation() -> TestResult {
|
||||
let (_cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
|
||||
let (_cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
|
||||
let pauses: Vec<_> = proxies[1..]
|
||||
.iter()
|
||||
.map(|proxy| proxy.pause(&["WriteMetadata"], usize::MAX, Some(&bucket)))
|
||||
@@ -2098,7 +2198,7 @@ async fn cancelled_metadata_wait_releases_fence_for_bucket_recreation() -> TestR
|
||||
async fn part_resend_owner_serializes_complete_and_abort_after_client_cancellation() -> TestResult {
|
||||
use md5::{Digest as _, Md5};
|
||||
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
|
||||
for cancel_waiter in [false, true] {
|
||||
for complete in [false, true] {
|
||||
let key = format!("resend-{cancel_waiter}-{complete}");
|
||||
@@ -2586,7 +2686,7 @@ async fn resume_at_rpc<T: Send + 'static>(
|
||||
|
||||
#[tokio::test]
|
||||
async fn each_multipart_phase_resumes_after_metadata_and_lock_delays() -> TestResult {
|
||||
let (_cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
|
||||
let (_cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
|
||||
for locks in [false, true] {
|
||||
let key = format!("phase-{locks}");
|
||||
let client = clients[0].clone();
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -283,6 +283,9 @@ mod ssec_copy_test;
|
||||
#[cfg(test)]
|
||||
mod multipart_storage_class_test;
|
||||
|
||||
#[cfg(test)]
|
||||
mod multipart_upload_id_test;
|
||||
|
||||
#[cfg(test)]
|
||||
mod storage_class_capability_test;
|
||||
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Multipart upload ID regression tests.
|
||||
//!
|
||||
//! Every operation that addresses an existing multipart upload must report an
|
||||
//! upload ID that does not name a live upload as `404 NoSuchUpload`, whether
|
||||
//! the ID is malformed or is a well-formed ID of an aborted upload.
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
||||
|
||||
type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
|
||||
|
||||
const BUCKET: &str = "multipart-upload-id";
|
||||
const KEY: &str = "target.bin";
|
||||
const COPY_SOURCE_KEY: &str = "copy-source.bin";
|
||||
// RustFS upload IDs are URL-safe base64; '.' is outside that alphabet, so
|
||||
// the object layer rejects this ID as malformed instead of looking it up.
|
||||
const MALFORMED_UPLOAD_ID: &str = "malformed.upload.id";
|
||||
const PART_NUMBER: i32 = 1;
|
||||
const PART_BODY: &[u8] = b"multipart upload id regression part";
|
||||
const PART_ETAG: &str = "\"00000000000000000000000000000000\"";
|
||||
const NO_SUCH_UPLOAD_STATUS: u16 = 404;
|
||||
const NO_SUCH_UPLOAD_CODE: &str = "NoSuchUpload";
|
||||
const OBJECT_NOT_FOUND_STATUS: u16 = 404;
|
||||
|
||||
/// Describes how `result` differs from `404 NoSuchUpload`, or returns `None` when it matches.
|
||||
fn no_such_upload_mismatch<T, E>(operation: &str, result: Result<T, SdkError<E>>) -> Option<String>
|
||||
where
|
||||
E: ProvideErrorMetadata + std::fmt::Debug,
|
||||
{
|
||||
let Err(error) = result else {
|
||||
return Some(format!("{operation}: succeeded"));
|
||||
};
|
||||
let status = error.raw_response().map(|response| response.status().as_u16());
|
||||
let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
|
||||
(status != Some(NO_SUCH_UPLOAD_STATUS) || code != Some(NO_SUCH_UPLOAD_CODE))
|
||||
.then(|| format!("{operation}: status {status:?}, code {code:?}: {error:?}"))
|
||||
}
|
||||
|
||||
async fn start_server_with_copy_source() -> Result<(RustFSTestEnvironment, Client), Box<dyn std::error::Error + Send + Sync>>
|
||||
{
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(Vec::new()).await?;
|
||||
env.create_test_bucket(BUCKET).await?;
|
||||
let client = env.create_s3_client();
|
||||
client
|
||||
.put_object()
|
||||
.bucket(BUCKET)
|
||||
.key(COPY_SOURCE_KEY)
|
||||
.body(ByteStream::from_static(PART_BODY))
|
||||
.send()
|
||||
.await?;
|
||||
Ok((env, client))
|
||||
}
|
||||
|
||||
/// Sends every operation that addresses an existing upload and asserts that all of them report `404 NoSuchUpload`.
|
||||
async fn assert_every_upload_operation_reports_no_such_upload(client: &Client, upload_id: &str) {
|
||||
let upload_part = client
|
||||
.upload_part()
|
||||
.bucket(BUCKET)
|
||||
.key(KEY)
|
||||
.upload_id(upload_id)
|
||||
.part_number(PART_NUMBER)
|
||||
.body(ByteStream::from_static(PART_BODY))
|
||||
.send()
|
||||
.await;
|
||||
let upload_part_copy = client
|
||||
.upload_part_copy()
|
||||
.bucket(BUCKET)
|
||||
.key(KEY)
|
||||
.upload_id(upload_id)
|
||||
.part_number(PART_NUMBER)
|
||||
.copy_source(format!("/{BUCKET}/{COPY_SOURCE_KEY}"))
|
||||
.send()
|
||||
.await;
|
||||
let list_parts = client.list_parts().bucket(BUCKET).key(KEY).upload_id(upload_id).send().await;
|
||||
let complete = client
|
||||
.complete_multipart_upload()
|
||||
.bucket(BUCKET)
|
||||
.key(KEY)
|
||||
.upload_id(upload_id)
|
||||
.multipart_upload(
|
||||
CompletedMultipartUpload::builder()
|
||||
.parts(CompletedPart::builder().part_number(PART_NUMBER).e_tag(PART_ETAG).build())
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await;
|
||||
let abort = client
|
||||
.abort_multipart_upload()
|
||||
.bucket(BUCKET)
|
||||
.key(KEY)
|
||||
.upload_id(upload_id)
|
||||
.send()
|
||||
.await;
|
||||
|
||||
let mismatches: Vec<String> = [
|
||||
no_such_upload_mismatch("UploadPart", upload_part),
|
||||
no_such_upload_mismatch("UploadPartCopy", upload_part_copy),
|
||||
no_such_upload_mismatch("ListParts", list_parts),
|
||||
no_such_upload_mismatch("CompleteMultipartUpload", complete),
|
||||
no_such_upload_mismatch("AbortMultipartUpload", abort),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect();
|
||||
assert!(
|
||||
mismatches.is_empty(),
|
||||
"upload ID {upload_id:?}: operations not reporting {NO_SUCH_UPLOAD_STATUS} {NO_SUCH_UPLOAD_CODE}:\n{}",
|
||||
mismatches.join("\n")
|
||||
);
|
||||
}
|
||||
|
||||
/// Asserts that the rejected operations left neither a multipart upload nor an object under `KEY`.
|
||||
async fn assert_no_upload_or_object_published(client: &Client) -> TestResult {
|
||||
let uploads = client.list_multipart_uploads().bucket(BUCKET).prefix(KEY).send().await?;
|
||||
assert!(uploads.uploads().is_empty(), "rejected operations must not leave an upload: {uploads:?}");
|
||||
let head = client.head_object().bucket(BUCKET).key(KEY).send().await;
|
||||
let status = head
|
||||
.as_ref()
|
||||
.err()
|
||||
.and_then(|error| error.raw_response())
|
||||
.map(|response| response.status().as_u16());
|
||||
assert_eq!(
|
||||
status,
|
||||
Some(OBJECT_NOT_FOUND_STATUS),
|
||||
"rejected operations must not publish {KEY}: {head:?}"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn malformed_upload_id_reports_no_such_upload_on_every_upload_operation() -> TestResult {
|
||||
let (_env, client) = start_server_with_copy_source().await?;
|
||||
|
||||
assert_every_upload_operation_reports_no_such_upload(&client, MALFORMED_UPLOAD_ID).await;
|
||||
assert_no_upload_or_object_published(&client).await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn aborted_upload_id_reports_no_such_upload_on_every_upload_operation() -> TestResult {
|
||||
let (_env, client) = start_server_with_copy_source().await?;
|
||||
let created = client.create_multipart_upload().bucket(BUCKET).key(KEY).send().await?;
|
||||
let upload_id = created.upload_id().ok_or("CreateMultipartUpload returned no upload ID")?;
|
||||
client
|
||||
.abort_multipart_upload()
|
||||
.bucket(BUCKET)
|
||||
.key(KEY)
|
||||
.upload_id(upload_id)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
assert_every_upload_operation_reports_no_such_upload(&client, upload_id).await;
|
||||
assert_no_upload_or_object_published(&client).await
|
||||
}
|
||||
}
|
||||
@@ -31,8 +31,8 @@ use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::operation::list_object_versions::ListObjectVersionsOutput;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, DeleteMarkerEntry, ObjectVersion, ServerSideEncryption,
|
||||
VersioningConfiguration,
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Delete, DeleteMarkerEntry, ObjectIdentifier, ObjectVersion,
|
||||
ServerSideEncryption, VersioningConfiguration,
|
||||
};
|
||||
use base64_simd::STANDARD as BASE64_STANDARD;
|
||||
use bytes::Bytes;
|
||||
@@ -3729,6 +3729,8 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
|
||||
init_logging();
|
||||
|
||||
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||
let source_log = format!("{}/source.log", source_env.temp_dir);
|
||||
source_env.capture_log_path = Some(source_log.clone());
|
||||
let mut source_env_vars = replication_fast_env();
|
||||
source_env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
source_env.start_rustfs_server_with_env(vec![], &source_env_vars).await?;
|
||||
@@ -3791,19 +3793,66 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
|
||||
"target did not preserve the source delete-marker version ID"
|
||||
);
|
||||
|
||||
source_client
|
||||
.delete_object()
|
||||
let wait_for_source_writeback = || async {
|
||||
let metrics_url = format!("{}/{source_bucket}?replication-metrics", source_env.url);
|
||||
timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
let response = signed_request(
|
||||
http::Method::GET,
|
||||
&metrics_url,
|
||||
&source_env.access_key,
|
||||
&source_env.secret_key,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
let metrics: serde_json::Value = response.error_for_status()?.json().await?;
|
||||
let queued = metrics["queued"]["curr"]["count"]
|
||||
.as_f64()
|
||||
.ok_or("missing replication queue count")?;
|
||||
if metrics["provider_available"] == true && queued == 0.0 {
|
||||
return Ok::<(), Box<dyn Error + Send + Sync>>(());
|
||||
}
|
||||
sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
})
|
||||
.await?
|
||||
};
|
||||
wait_for_source_writeback().await?;
|
||||
|
||||
let purged = source_client
|
||||
.delete_objects()
|
||||
.bucket(source_bucket)
|
||||
.key(object_key)
|
||||
.version_id(&purged_version_id)
|
||||
.delete(
|
||||
Delete::builder()
|
||||
.objects(
|
||||
ObjectIdentifier::builder()
|
||||
.key(object_key)
|
||||
.version_id(&purged_version_id)
|
||||
.build()?,
|
||||
)
|
||||
.objects(
|
||||
ObjectIdentifier::builder()
|
||||
.key(object_key)
|
||||
.version_id(delete_marker_version_id)
|
||||
.build()?,
|
||||
)
|
||||
.build()?,
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
assert!(purged.errors().is_empty(), "batch version purge failed: {:?}", purged.errors());
|
||||
assert_eq!(purged.deleted().len(), 2, "both the data version and marker must be purged");
|
||||
assert_replication_converged(&source_client, source_bucket, &target_client, target_bucket).await?;
|
||||
let target_state = list_replication_state(&target_client, target_bucket).await?;
|
||||
assert!(
|
||||
target_state.iter().all(|entry| entry.version_id != purged_version_id),
|
||||
"target retained the explicitly purged object version"
|
||||
);
|
||||
assert!(
|
||||
target_state.iter().all(|entry| entry.version_id != delete_marker_version_id),
|
||||
"target retained the explicitly purged delete marker"
|
||||
);
|
||||
assert!(
|
||||
target_state
|
||||
.iter()
|
||||
@@ -3819,6 +3868,19 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
|
||||
.await?;
|
||||
assert_eq!(retained.body.collect().await?.into_bytes().as_ref(), b"versioned replication payload v2");
|
||||
|
||||
// Wait for the source workers to finish their final writeback, which runs
|
||||
// after the remote purge has already become visible (rustfs#9121).
|
||||
wait_for_source_writeback().await?;
|
||||
let logs = fs::read_to_string(source_log).await?;
|
||||
let writeback_failures: Vec<_> = logs
|
||||
.lines()
|
||||
.filter(|line| line.contains("apply_replication_delete_state"))
|
||||
.collect();
|
||||
assert!(
|
||||
writeback_failures.is_empty(),
|
||||
"a completed batch purge must not report a source writeback failure: {writeback_failures:?}"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -28,7 +28,8 @@ use hyper_util::{
|
||||
use prost::Message as _;
|
||||
use rustfs_protos::proto_gen::node_service::{
|
||||
BatchGenerallyLockRequest, DeleteRequest, GenerallyLockRequest, PreparePartTransactionResponse, ReadVersionRequest,
|
||||
RecoverMultipartSourceRequest, RenameDataResponse, SettlePartTransactionRequest, SettlePartTransactionResponse,
|
||||
RecoverMultipartSourceRequest, RenameDataResponse, ScannerScopedDirtyUsageAckRequest, SettlePartTransactionRequest,
|
||||
SettlePartTransactionResponse,
|
||||
};
|
||||
use std::{
|
||||
collections::HashSet,
|
||||
@@ -49,6 +50,9 @@ use tokio::{
|
||||
type ProxyBody = BoxBody<Bytes, io::Error>;
|
||||
type ProxyClient = Client<HttpConnector, ProxyBody>;
|
||||
|
||||
mod scoped_ack;
|
||||
pub(crate) use scoped_ack::{ScopedAckFault, ScopedAckQueue, ScopedAckScope};
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
enum Disposition {
|
||||
Hold,
|
||||
@@ -121,7 +125,8 @@ impl RpcPause {
|
||||
self.state.blocked.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
/// Selected RPCs with successful HTTP/gRPC headers, excluding discarded packets.
|
||||
/// Selected RPCs with successful HTTP/gRPC response headers, excluding
|
||||
/// discarded packets. This does not prove a successful gRPC trailer or ACK apply.
|
||||
pub(crate) fn forwarded(&self) -> usize {
|
||||
self.state.forwarded.load(Ordering::SeqCst)
|
||||
}
|
||||
@@ -233,6 +238,7 @@ pub(crate) struct RpcFaultProxy {
|
||||
address: SocketAddr,
|
||||
slot: PauseSlot,
|
||||
lock_slot: PauseSlot,
|
||||
scoped_ack: scoped_ack::QueueSlot,
|
||||
task: JoinHandle<()>,
|
||||
late_requests: LateRequests,
|
||||
}
|
||||
@@ -245,6 +251,8 @@ impl RpcFaultProxy {
|
||||
let active = slot.clone();
|
||||
let lock_slot: PauseSlot = Arc::new(Mutex::new(None));
|
||||
let active_lock = lock_slot.clone();
|
||||
let scoped_ack = scoped_ack::empty_slot();
|
||||
let active_ack = scoped_ack.clone();
|
||||
let late_requests: LateRequests = Arc::new(Mutex::new(Some(Vec::new())));
|
||||
let late = late_requests.clone();
|
||||
let task = tokio::spawn(async move {
|
||||
@@ -259,6 +267,7 @@ impl RpcFaultProxy {
|
||||
let h2 = h2.clone();
|
||||
let active = active.clone();
|
||||
let active_lock = active_lock.clone();
|
||||
let active_ack = active_ack.clone();
|
||||
let late = late.clone();
|
||||
connections.spawn(async move {
|
||||
let service = service_fn(move |request: Request<Incoming>| {
|
||||
@@ -272,6 +281,7 @@ impl RpcFaultProxy {
|
||||
None
|
||||
};
|
||||
let pause = owner_pause.or_else(|| active.lock().expect("RPC pause slot").clone());
|
||||
let ack = active_ack.lock().expect("scoped ACK queue slot").clone();
|
||||
let retain = pause.as_ref().is_some_and(|pause| {
|
||||
pause.retain_after_disconnect
|
||||
&& request.uri().path().strip_prefix("/node_service.NodeService/")
|
||||
@@ -280,11 +290,11 @@ impl RpcFaultProxy {
|
||||
let late = late.clone();
|
||||
async move {
|
||||
if !retain {
|
||||
return forward_request(request, client, target, pause).await;
|
||||
return forward_request(request, client, target, pause, ack).await;
|
||||
}
|
||||
// This opt-in fault owns the packet after client cancellation.
|
||||
// Proxy teardown still aborts every retained request.
|
||||
let task = tokio::spawn(forward_request(request, client, target, pause));
|
||||
let task = tokio::spawn(forward_request(request, client, target, pause, ack));
|
||||
{
|
||||
let mut requests = late.lock().expect("late RPC ownership");
|
||||
if let Some(requests) = requests.as_mut() {
|
||||
@@ -309,6 +319,7 @@ impl RpcFaultProxy {
|
||||
address,
|
||||
slot,
|
||||
lock_slot,
|
||||
scoped_ack,
|
||||
task,
|
||||
late_requests,
|
||||
})
|
||||
@@ -318,6 +329,11 @@ impl RpcFaultProxy {
|
||||
self.address
|
||||
}
|
||||
|
||||
/// Retain only genuine scoped ACKs for one receiver/session and bucket set.
|
||||
pub(crate) fn scoped_ack_queue(&self, scope: ScopedAckScope, capacity: usize) -> io::Result<ScopedAckQueue> {
|
||||
ScopedAckQueue::arm(self.scoped_ack.clone(), scope, capacity)
|
||||
}
|
||||
|
||||
/// Pause at most limit requests to the named unary methods; other streams
|
||||
/// on the same HTTP/2 connection continue forwarding.
|
||||
pub(crate) fn pause(&self, methods: &[&'static str], limit: usize, scope: Option<&str>) -> io::Result<RpcPause> {
|
||||
@@ -469,6 +485,9 @@ fn unary_payload(body: &[u8]) -> io::Result<&[u8]> {
|
||||
}
|
||||
|
||||
fn scope_matches(method: &str, body: &[u8], scope: &[u8]) -> io::Result<bool> {
|
||||
if method == "ScannerScopedDirtyUsageAck" {
|
||||
return scoped_ack_matches(body, Some(scope));
|
||||
}
|
||||
if method == "RecoverMultipartSource" {
|
||||
let request = RecoverMultipartSourceRequest::decode(unary_payload(body)?).map_err(io::Error::other)?;
|
||||
if request.protocol_version != u32::from(MULTIPART_SOURCE_PROTOCOL_VERSION) || request.path.as_bytes() != scope {
|
||||
@@ -525,7 +544,9 @@ fn scoped_lock_owners(method: &str, body: &[u8], scope: &[u8]) -> io::Result<Vec
|
||||
let args = match method {
|
||||
"Lock" => vec![GenerallyLockRequest::decode(payload).map_err(io::Error::other)?.args],
|
||||
"LockBatch" => BatchGenerallyLockRequest::decode(payload).map_err(io::Error::other)?.args,
|
||||
_ => return Err(io::Error::new(io::ErrorKind::InvalidInput, "lock observer requires a lock method")),
|
||||
_ => {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidInput, "lock observer requires a lock method"));
|
||||
}
|
||||
};
|
||||
args.into_iter()
|
||||
.filter(|args| args.as_bytes().windows(scope.len()).any(|window| window == scope))
|
||||
@@ -540,18 +561,39 @@ fn scoped_lock_owners(method: &str, body: &[u8], scope: &[u8]) -> io::Result<Vec
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn scoped_ack_matches(body: &[u8], scope: Option<&[u8]>) -> io::Result<bool> {
|
||||
let request = ScannerScopedDirtyUsageAckRequest::decode(unary_payload(body)?).map_err(io::Error::other)?;
|
||||
Ok(!request.probe_only && scope.is_none_or(|scope| request.entries.iter().any(|entry| entry.bucket.as_bytes() == scope)))
|
||||
}
|
||||
|
||||
async fn forward_request(
|
||||
request: Request<Incoming>,
|
||||
client: ProxyClient,
|
||||
target: SocketAddr,
|
||||
pause: Option<Arc<PauseState>>,
|
||||
ack: Option<Arc<scoped_ack::QueueState>>,
|
||||
) -> io::Result<hyper::Response<ProxyBody>> {
|
||||
let (mut parts, body) = request.into_parts();
|
||||
let mut body = body.map_err(io::Error::other).boxed();
|
||||
let request = request.map(|body| body.map_err(io::Error::other).boxed());
|
||||
let request = if let Some(ack) = ack
|
||||
&& request.uri().path() == "/node_service.ScannerControlService/ScannerScopedDirtyUsageAck"
|
||||
{
|
||||
match scoped_ack::intercept(request, client.clone(), target, ack).await? {
|
||||
scoped_ack::Interception::Pass(request) => request,
|
||||
scoped_ack::Interception::Handled(response) => return Ok(response),
|
||||
}
|
||||
} else {
|
||||
request
|
||||
};
|
||||
let (parts, body) = request.into_parts();
|
||||
let mut body = body;
|
||||
let mut observed = None;
|
||||
let mut inspection = None;
|
||||
if let Some(pause) = pause {
|
||||
let method = parts.uri.path().strip_prefix("/node_service.NodeService/");
|
||||
let method = parts
|
||||
.uri
|
||||
.path()
|
||||
.strip_prefix("/node_service.NodeService/")
|
||||
.or_else(|| parts.uri.path().strip_prefix("/node_service.ScannerControlService/"));
|
||||
if let Some(method) = method.filter(|name| pause.methods.contains(name)) {
|
||||
let mut route = None;
|
||||
let mut lock_owners = Vec::new();
|
||||
@@ -599,6 +641,16 @@ async fn forward_request(
|
||||
.with_trailers(async move { trailers.map(Ok::<_, io::Error>) })
|
||||
.boxed();
|
||||
matches
|
||||
} else if method == "ScannerScopedDirtyUsageAck" {
|
||||
let collected = Limited::new(body, 1024 * 1024).collect().await.map_err(io::Error::other)?;
|
||||
let trailers = collected.trailers().cloned();
|
||||
let bytes = collected.to_bytes();
|
||||
let matches = scoped_ack_matches(&bytes, None)?;
|
||||
body = Full::new(bytes)
|
||||
.map_err(|never| match never {})
|
||||
.with_trailers(async move { trailers.map(Ok::<_, io::Error>) })
|
||||
.boxed();
|
||||
matches
|
||||
} else {
|
||||
true
|
||||
};
|
||||
@@ -642,24 +694,7 @@ async fn forward_request(
|
||||
}
|
||||
}
|
||||
}
|
||||
if !parts.headers.contains_key(http::header::HOST)
|
||||
&& let Some(authority) = parts.uri.authority()
|
||||
{
|
||||
parts.headers.insert(
|
||||
http::header::HOST,
|
||||
http::HeaderValue::from_str(authority.as_str()).map_err(io::Error::other)?,
|
||||
);
|
||||
}
|
||||
parts.uri = http::Uri::builder()
|
||||
.scheme("http")
|
||||
.authority(target.to_string())
|
||||
.path_and_query(parts.uri.path_and_query().map_or("/", |value| value.as_str()))
|
||||
.build()
|
||||
.map_err(io::Error::other)?;
|
||||
let response = client
|
||||
.request(Request::from_parts(parts, body))
|
||||
.await
|
||||
.map_err(io::Error::other)?;
|
||||
let response = forward_to_backend(Request::from_parts(parts, body), client, target).await?;
|
||||
if let Some((pause, disk)) = inspection
|
||||
&& response.status().is_success()
|
||||
&& response.headers().get("grpc-status").is_some_and(|status| status == "12")
|
||||
@@ -712,6 +747,32 @@ async fn forward_request(
|
||||
Ok(response.map(|body| body.map_err(io::Error::other).boxed()))
|
||||
}
|
||||
|
||||
async fn forward_to_backend(
|
||||
request: Request<ProxyBody>,
|
||||
client: ProxyClient,
|
||||
target: SocketAddr,
|
||||
) -> io::Result<hyper::Response<Incoming>> {
|
||||
let (mut parts, body) = request.into_parts();
|
||||
if !parts.headers.contains_key(http::header::HOST)
|
||||
&& let Some(authority) = parts.uri.authority()
|
||||
{
|
||||
parts.headers.insert(
|
||||
http::header::HOST,
|
||||
http::HeaderValue::from_str(authority.as_str()).map_err(io::Error::other)?,
|
||||
);
|
||||
}
|
||||
parts.uri = http::Uri::builder()
|
||||
.scheme("http")
|
||||
.authority(target.to_string())
|
||||
.path_and_query(parts.uri.path_and_query().map_or("/", |value| value.as_str()))
|
||||
.build()
|
||||
.map_err(io::Error::other)?;
|
||||
client
|
||||
.request(Request::from_parts(parts, body))
|
||||
.await
|
||||
.map_err(io::Error::other)
|
||||
}
|
||||
|
||||
impl Drop for RpcFaultProxy {
|
||||
fn drop(&mut self) {
|
||||
if let Some(requests) = self.late_requests.lock().expect("late RPC ownership").take() {
|
||||
@@ -731,6 +792,7 @@ mod tests {
|
||||
|
||||
struct Backend {
|
||||
received: Arc<AtomicUsize>,
|
||||
auth_headers: Arc<Mutex<Vec<(String, String)>>>,
|
||||
address: SocketAddr,
|
||||
task: JoinHandle<()>,
|
||||
}
|
||||
@@ -754,6 +816,8 @@ mod tests {
|
||||
let address = listener.local_addr()?;
|
||||
let received = Arc::new(AtomicUsize::new(0));
|
||||
let received_task = received.clone();
|
||||
let auth_headers = Arc::new(Mutex::new(Vec::new()));
|
||||
let backend_auth = auth_headers.clone();
|
||||
let task = tokio::spawn(async move {
|
||||
let mut children = JoinSet::new();
|
||||
loop {
|
||||
@@ -761,10 +825,12 @@ mod tests {
|
||||
accepted = listener.accept() => {
|
||||
let Ok((socket, _)) = accepted else { break };
|
||||
let received = received_task.clone();
|
||||
let backend_auth = backend_auth.clone();
|
||||
let reply = reply.clone();
|
||||
children.spawn(async move {
|
||||
let service = service_fn(move |request: Request<Incoming>| {
|
||||
let received = received.clone();
|
||||
let backend_auth = backend_auth.clone();
|
||||
let reply = reply.clone();
|
||||
async move {
|
||||
received.fetch_add(1, Ordering::SeqCst);
|
||||
@@ -773,6 +839,10 @@ mod tests {
|
||||
http::HeaderValue::from_str(authority.as_str()).expect("valid HTTP authority")
|
||||
})
|
||||
});
|
||||
if let (Some(nonce), Some(signature)) = (request.headers().get("x-rustfs-rpc-nonce"), request.headers().get("x-rustfs-rpc-signature-v2")) {
|
||||
backend_auth.lock().expect("backend auth header observations").push((
|
||||
nonce.to_str().expect("test nonce").to_string(), signature.to_str().expect("test signature").to_string()));
|
||||
}
|
||||
let body = request.into_body().collect().await?.to_bytes();
|
||||
let body = reply.unwrap_or(body);
|
||||
let mut trailers = http::HeaderMap::new();
|
||||
@@ -795,7 +865,12 @@ mod tests {
|
||||
}
|
||||
}
|
||||
});
|
||||
Ok(Backend { received, address, task })
|
||||
Ok(Backend {
|
||||
received,
|
||||
auth_headers,
|
||||
address,
|
||||
task,
|
||||
})
|
||||
}
|
||||
|
||||
async fn call(client: &ProxyClient, address: SocketAddr, method: &str) -> io::Result<hyper::Response<Incoming>> {
|
||||
@@ -809,9 +884,14 @@ mod tests {
|
||||
bytes: Bytes,
|
||||
) -> io::Result<hyper::Response<Incoming>> {
|
||||
let body: ProxyBody = Full::new(bytes).map_err(|never| match never {}).boxed();
|
||||
let service = if method == "ScannerScopedDirtyUsageAck" {
|
||||
"ScannerControlService"
|
||||
} else {
|
||||
"NodeService"
|
||||
};
|
||||
let request = Request::builder()
|
||||
.method("POST")
|
||||
.uri(format!("http://{address}/node_service.NodeService/{method}"))
|
||||
.uri(format!("http://{address}/node_service.{service}/{method}"))
|
||||
.header(http::header::TE, "trailers")
|
||||
.body(body)
|
||||
.map_err(io::Error::other)?;
|
||||
@@ -826,6 +906,353 @@ mod tests {
|
||||
frame.into()
|
||||
}
|
||||
|
||||
const ACK_OWNER: &str = "11111111-1111-1111-1111-111111111111";
|
||||
const ACK_INSTANCE: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
|
||||
|
||||
fn ack_scope() -> ScopedAckScope {
|
||||
ScopedAckScope {
|
||||
nonce: uuid::Uuid::new_v4().to_string(),
|
||||
owner_id: ACK_OWNER.to_string(),
|
||||
instance_id: ACK_INSTANCE.to_string(),
|
||||
buckets: std::collections::BTreeSet::from(["cold-bucket".to_string()]),
|
||||
}
|
||||
}
|
||||
|
||||
fn ack_request(bucket: &str, generation: u64) -> ScannerScopedDirtyUsageAckRequest {
|
||||
ScannerScopedDirtyUsageAckRequest {
|
||||
challenge: vec![7; 16].into(),
|
||||
protocol_version: 1,
|
||||
owner_id: ACK_OWNER.to_string(),
|
||||
instance_id: ACK_INSTANCE.to_string(),
|
||||
scope: 1,
|
||||
probe_only: false,
|
||||
entries: vec![rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageEntry {
|
||||
bucket: bucket.to_string(),
|
||||
bucket_incarnation: vec![4; 16].into(),
|
||||
generation,
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
fn ack_frame(request: &ScannerScopedDirtyUsageAckRequest) -> Bytes {
|
||||
let payload = request.encode_to_vec();
|
||||
let mut frame = vec![0];
|
||||
frame.extend_from_slice(&u32::try_from(payload.len()).expect("bounded test frame").to_be_bytes());
|
||||
frame.extend_from_slice(&payload);
|
||||
Bytes::from(frame)
|
||||
}
|
||||
|
||||
fn ack_hash(bytes: &[u8]) -> String {
|
||||
use sha2::Digest as _;
|
||||
hex_simd::encode_to_string(&sha2::Sha256::digest(bytes)[..], hex_simd::AsciiCase::Lower)
|
||||
}
|
||||
|
||||
// Transport-only fixtures: the distributed tests supply production signatures.
|
||||
async fn call_ack_frame(
|
||||
client: &ProxyClient,
|
||||
address: SocketAddr,
|
||||
frame: Bytes,
|
||||
nonce: &str,
|
||||
) -> io::Result<hyper::Response<Incoming>> {
|
||||
let request = Request::builder()
|
||||
.method("POST")
|
||||
.uri(format!("http://{address}/node_service.ScannerControlService/ScannerScopedDirtyUsageAck"))
|
||||
.header(http::header::TE, "trailers")
|
||||
.header("x-rustfs-rpc-nonce", nonce)
|
||||
.header("x-rustfs-rpc-signature-v2", format!("opaque-signature-{nonce}"))
|
||||
.body(Full::new(frame).map_err(|never| match never {}).boxed())
|
||||
.map_err(io::Error::other)?;
|
||||
client.request(request).await.map_err(io::Error::other)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scoped_ack_queue_loss_preserves_backend_boundary() {
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
for fault in [ScopedAckFault::RequestLoss, ScopedAckFault::ResponseLoss] {
|
||||
let backend = backend().await.expect("backend");
|
||||
let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
|
||||
.await
|
||||
.expect("proxy");
|
||||
let queue = proxy.scoped_ack_queue(ack_scope(), 1).expect("ACK queue");
|
||||
let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
|
||||
let frame = ack_frame(&ack_request("cold-bucket", 7));
|
||||
let address = proxy.local_addr();
|
||||
let body = frame.clone();
|
||||
let caller = tokio::spawn(async move { call_ack_frame(&client, address, body, "loss").await });
|
||||
queue.wait_for_packets(1).await.expect("captured ACK");
|
||||
assert_eq!(backend.received.load(Ordering::SeqCst), 0, "retention must precede backend dispatch");
|
||||
queue.settle(0, fault).expect("settle actual retained packet");
|
||||
assert!(
|
||||
caller.await.expect("caller task").is_err(),
|
||||
"the caller must observe the injected transport failure"
|
||||
);
|
||||
let receipt = queue.wait_for_completion(0).await.expect("complete fault receipt");
|
||||
let response_loss = fault == ScopedAckFault::ResponseLoss;
|
||||
assert_eq!(receipt["forwarded"], response_loss);
|
||||
assert_eq!(receipt["backend_complete"], response_loss);
|
||||
assert_eq!(backend.received.load(Ordering::SeqCst), usize::from(response_loss));
|
||||
if response_loss {
|
||||
assert_eq!(receipt["outcome"], "response_lost");
|
||||
assert_eq!(receipt["response"]["grpc_status"], "0", "status exists only in real backend trailers");
|
||||
assert_eq!(receipt["response"]["bytes"], frame.len());
|
||||
assert_eq!(receipt["response"]["sha256"], ack_hash(&frame));
|
||||
assert!(
|
||||
receipt["response"]["trailers_sha256"]
|
||||
.as_str()
|
||||
.is_some_and(|value| value.len() == 64)
|
||||
);
|
||||
} else {
|
||||
assert_eq!(receipt["outcome"], "request_lost");
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("bounded ACK loss checks");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scoped_ack_queue_keeps_cancelled_packets_and_releases_out_of_order() {
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
let backend = backend().await.expect("backend");
|
||||
let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
|
||||
.await
|
||||
.expect("proxy");
|
||||
let queue = proxy.scoped_ack_queue(ack_scope(), 2).expect("two-packet ACK queue");
|
||||
let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
|
||||
let first = ack_frame(&ack_request("cold-bucket", 7));
|
||||
let second = ack_frame(&ack_request("cold-bucket", 9));
|
||||
let address = proxy.local_addr();
|
||||
let old_client = client.clone();
|
||||
let old_body = first.clone();
|
||||
let old_caller = tokio::spawn(async move { call_ack_frame(&old_client, address, old_body, "first").await });
|
||||
queue.wait_for_packets(1).await.expect("first retained packet");
|
||||
old_caller.abort();
|
||||
assert!(old_caller.await.expect_err("cancel original caller").is_cancelled());
|
||||
let new_body = second.clone();
|
||||
let new_caller = tokio::spawn(async move { call_ack_frame(&client, address, new_body, "second").await });
|
||||
queue.wait_for_packets(2).await.expect("second retained packet");
|
||||
assert_eq!(backend.received.load(Ordering::SeqCst), 0);
|
||||
queue.settle(1, ScopedAckFault::Forward).expect("release newer packet first");
|
||||
let response = new_caller
|
||||
.await
|
||||
.expect("new caller task")
|
||||
.expect("new response")
|
||||
.into_body()
|
||||
.collect()
|
||||
.await
|
||||
.expect("complete body");
|
||||
assert_eq!(response.trailers().expect("response trailers")["x-proof"], "unchanged-proof");
|
||||
assert_eq!(response.to_bytes(), second);
|
||||
queue.wait_for_completion(1).await.expect("new packet complete");
|
||||
queue
|
||||
.settle(0, ScopedAckFault::Forward)
|
||||
.expect("release old packet after its caller disappeared");
|
||||
let old = queue.wait_for_completion(0).await.expect("detached old packet complete");
|
||||
assert_eq!(old["backend_complete"], true);
|
||||
assert_eq!(old["request_sha256"], ack_hash(&first));
|
||||
assert_eq!(old["response"]["sha256"], ack_hash(&first));
|
||||
assert_eq!(backend.received.load(Ordering::SeqCst), 2);
|
||||
assert_eq!(
|
||||
*backend.auth_headers.lock().expect("backend header observations"),
|
||||
vec![
|
||||
("second".to_string(), "opaque-signature-second".to_string()),
|
||||
("first".to_string(), "opaque-signature-first".to_string())
|
||||
]
|
||||
);
|
||||
assert!(
|
||||
queue.settle(0, ScopedAckFault::Forward).is_err(),
|
||||
"one original packet may only be forwarded once"
|
||||
);
|
||||
})
|
||||
.await
|
||||
.expect("bounded detached retention check");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scoped_ack_queue_bounds_capacity_and_excludes_other_scopes() {
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
let backend = backend().await.expect("backend");
|
||||
let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
|
||||
.await
|
||||
.expect("proxy");
|
||||
assert!(proxy.scoped_ack_queue(ack_scope(), 0).is_err());
|
||||
assert!(proxy.scoped_ack_queue(ack_scope(), 5).is_err());
|
||||
let mut invalid = ack_scope();
|
||||
invalid.nonce.clear();
|
||||
assert!(proxy.scoped_ack_queue(invalid, 1).is_err());
|
||||
let mut scope = ack_scope();
|
||||
scope.buckets.insert("cold-bucket-z".to_string());
|
||||
let queue = proxy.scoped_ack_queue(scope, 1).expect("bounded ACK queue");
|
||||
let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
|
||||
let mut probe = ack_request("cold-bucket", 7);
|
||||
probe.probe_only = true;
|
||||
let mut other_peer = ack_request("cold-bucket", 7);
|
||||
other_peer.owner_id = uuid::Uuid::new_v4().to_string();
|
||||
let mut other_instance = ack_request("cold-bucket", 7);
|
||||
other_instance.instance_id = "b".repeat(32);
|
||||
let mut invalid_challenge = ack_request("cold-bucket", 7);
|
||||
invalid_challenge.challenge = vec![7; 15].into();
|
||||
let mut noncanonical_entries = ack_request("cold-bucket-z", 7);
|
||||
noncanonical_entries.entries.extend(ack_request("cold-bucket", 7).entries);
|
||||
let unselected = [
|
||||
probe,
|
||||
other_peer,
|
||||
other_instance,
|
||||
ack_request("cold-bucket-other", 7),
|
||||
ack_request("cold-bucket", u64::MAX),
|
||||
invalid_challenge,
|
||||
noncanonical_entries,
|
||||
];
|
||||
let unselected_count = unselected.len();
|
||||
for (index, request) in unselected.into_iter().enumerate() {
|
||||
let body = ack_frame(&request);
|
||||
let response = call_ack_frame(&client, proxy.local_addr(), body.clone(), "unselected")
|
||||
.await
|
||||
.expect("unselected ACK progresses");
|
||||
assert_eq!(response.into_body().collect().await.expect("unselected body").to_bytes(), body);
|
||||
assert!(queue.receipts().is_empty(), "unselected or protocol-invalid ACK must not consume a slot");
|
||||
assert_eq!(backend.received.load(Ordering::SeqCst), index + 1);
|
||||
}
|
||||
assert!(queue.receipts().is_empty());
|
||||
let address = proxy.local_addr();
|
||||
let held_client = client.clone();
|
||||
let valid_frame = ack_frame(&ack_request("cold-bucket", 7));
|
||||
let held_body = valid_frame.clone();
|
||||
let held = tokio::spawn(async move { call_ack_frame(&held_client, address, held_body, "held").await });
|
||||
queue.wait_for_packets(1).await.expect("one matching packet");
|
||||
assert!(!held.is_finished(), "the subsequent valid ACK must be retained");
|
||||
assert_eq!(queue.receipts()[0]["request_sha256"], ack_hash(&valid_frame));
|
||||
assert_eq!(backend.received.load(Ordering::SeqCst), unselected_count);
|
||||
let extra = call_ack_frame(&client, address, ack_frame(&ack_request("cold-bucket", 8)), "beyond-capacity")
|
||||
.await
|
||||
.expect("capacity overflow passes without retention");
|
||||
extra.into_body().collect().await.expect("overflow response");
|
||||
assert_eq!(queue.receipts().len(), 1);
|
||||
assert_eq!(backend.received.load(Ordering::SeqCst), unselected_count + 1);
|
||||
drop(queue);
|
||||
assert!(
|
||||
held.await.expect("held caller").is_err(),
|
||||
"dropping the session must cancel its retained packet"
|
||||
);
|
||||
assert_eq!(
|
||||
backend.received.load(Ordering::SeqCst),
|
||||
unselected_count + 1,
|
||||
"session drop must never forward the held packet"
|
||||
);
|
||||
})
|
||||
.await
|
||||
.expect("bounded scope and lifecycle checks");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scoped_ack_queue_expiry_never_forwards_retained_packet() {
|
||||
let (backend, _proxy, queue, caller) = tokio::time::timeout(Duration::from_secs(10), async {
|
||||
let backend = backend().await.expect("backend");
|
||||
let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
|
||||
.await
|
||||
.expect("proxy");
|
||||
let queue = proxy.scoped_ack_queue(ack_scope(), 1).expect("ACK queue");
|
||||
let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
|
||||
let address = proxy.local_addr();
|
||||
let caller = tokio::spawn(async move {
|
||||
call_ack_frame(&client, address, ack_frame(&ack_request("cold-bucket", 7)), "expires").await
|
||||
});
|
||||
queue.wait_for_packets(1).await.expect("retained packet");
|
||||
(backend, proxy, queue, caller)
|
||||
})
|
||||
.await
|
||||
.expect("bounded queue setup");
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(Duration::from_secs(120)).await;
|
||||
tokio::time::resume();
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_secs(10), caller)
|
||||
.await
|
||||
.expect("expiry completes caller")
|
||||
.expect("caller task")
|
||||
.is_err()
|
||||
);
|
||||
let receipt = queue.receipts().into_iter().next().expect("expired receipt");
|
||||
assert_eq!(receipt["completed"], true);
|
||||
assert_eq!(receipt["forwarded"], false);
|
||||
assert_eq!(receipt["backend_complete"], false);
|
||||
assert_eq!(backend.received.load(Ordering::SeqCst), 0);
|
||||
assert_eq!(
|
||||
queue
|
||||
.settle(0, ScopedAckFault::Forward)
|
||||
.expect_err("expired packet cannot be released")
|
||||
.kind(),
|
||||
io::ErrorKind::TimedOut
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_ack_fault_matches_exact_bucket_and_excludes_probes() {
|
||||
use rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageEntry;
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
let backend = backend().await.expect("backend listener");
|
||||
let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
|
||||
.await
|
||||
.expect("proxy listener");
|
||||
let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
|
||||
let pause = proxy
|
||||
.pause(&["ScannerScopedDirtyUsageAck"], 1, Some("cold-bucket"))
|
||||
.expect("scoped ACK fault");
|
||||
let frame = |bucket: &str, probe_only: bool| {
|
||||
let payload = ScannerScopedDirtyUsageAckRequest {
|
||||
challenge: vec![3; 16].into(),
|
||||
protocol_version: 1,
|
||||
owner_id: "11111111-1111-1111-1111-111111111111".to_string(),
|
||||
instance_id: "a".repeat(32),
|
||||
scope: 1,
|
||||
probe_only,
|
||||
entries: vec![ScannerScopedDirtyUsageEntry {
|
||||
bucket: bucket.to_string(),
|
||||
bucket_incarnation: vec![4; 16].into(),
|
||||
generation: 17,
|
||||
}],
|
||||
}
|
||||
.encode_to_vec();
|
||||
let mut frame = vec![0];
|
||||
frame.extend_from_slice(&u32::try_from(payload.len()).expect("small ACK frame").to_be_bytes());
|
||||
frame.extend_from_slice(&payload);
|
||||
Bytes::from(frame)
|
||||
};
|
||||
for bytes in [frame("cold-bucket", true), frame("cold-bucket-other", false)] {
|
||||
let response = call_body(&client, proxy.local_addr(), "ScannerScopedDirtyUsageAck", bytes.clone())
|
||||
.await
|
||||
.expect("probe and other bucket must progress");
|
||||
assert_eq!(response.into_body().collect().await.expect("unselected response").to_bytes(), bytes);
|
||||
}
|
||||
assert_eq!(pause.hits(), 0);
|
||||
let signed_bytes = frame("cold-bucket", false);
|
||||
let task_bytes = signed_bytes.clone();
|
||||
let address = proxy.local_addr();
|
||||
let blocked =
|
||||
tokio::spawn(async move { call_body(&client, address, "ScannerScopedDirtyUsageAck", task_bytes).await });
|
||||
pause
|
||||
.wait_for_hits(1)
|
||||
.await
|
||||
.expect("real ScannerControlService method must reach the fault");
|
||||
assert!(!blocked.is_finished());
|
||||
assert_eq!(backend.received.load(Ordering::SeqCst), 2, "held ACK must not apply at the backend");
|
||||
pause.release();
|
||||
let collected = blocked
|
||||
.await
|
||||
.expect("ACK task")
|
||||
.expect("released ACK")
|
||||
.into_body()
|
||||
.collect()
|
||||
.await
|
||||
.expect("ACK body and trailers");
|
||||
assert_eq!(collected.trailers().expect("ACK trailers")["x-proof"], "unchanged-proof");
|
||||
assert_eq!(collected.to_bytes(), signed_bytes, "proxy must preserve the complete signed ACK frame");
|
||||
})
|
||||
.await
|
||||
.expect("scanner ACK fault deadline");
|
||||
}
|
||||
|
||||
fn native_abort_request(path: &str) -> RecoverMultipartSourceRequest {
|
||||
use crate::storage_api::multipart_source_rpc::{
|
||||
MultipartAbortCertificate, MultipartAbortProof, MultipartAbortProofKind, MultipartAbortRound, MultipartSourcePhase,
|
||||
@@ -1719,3 +2146,4 @@ mod tests {
|
||||
);
|
||||
}
|
||||
}
|
||||
// event-drill: isolated harness-only change
|
||||
|
||||
@@ -0,0 +1,404 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! Bounded retention of original scoped ACK frames, independent of caller cancellation.
|
||||
|
||||
use super::{ProxyBody, ProxyClient, forward_to_backend, unary_payload};
|
||||
use bytes::Bytes;
|
||||
use http_body_util::{BodyExt as _, Full, Limited};
|
||||
use hyper::{Request, Response};
|
||||
use prost::Message as _;
|
||||
use rustfs_protos::proto_gen::node_service::{ScannerScopedDirtyUsageAckRequest, ScannerScopedDirtyUsageAckResponse};
|
||||
use rustfs_protos::scoped_dirty_usage::validate_scoped_dirty_usage_request;
|
||||
use serde_json::{Value, json};
|
||||
use sha2::{Digest as _, Sha256};
|
||||
use std::{
|
||||
collections::BTreeSet,
|
||||
io,
|
||||
net::SocketAddr,
|
||||
sync::{
|
||||
Arc, Mutex,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
},
|
||||
time::Duration,
|
||||
};
|
||||
use tokio::{
|
||||
sync::{Notify, oneshot, watch},
|
||||
task::AbortHandle,
|
||||
time::Instant,
|
||||
};
|
||||
|
||||
const MAX_RETAINED: usize = 4;
|
||||
const MAX_BYTES: usize = 1024 * 1024;
|
||||
const RETENTION: Duration = Duration::from_secs(120);
|
||||
|
||||
pub(crate) struct ScopedAckScope {
|
||||
pub nonce: String,
|
||||
pub owner_id: String,
|
||||
pub instance_id: String,
|
||||
pub buckets: BTreeSet<String>,
|
||||
}
|
||||
|
||||
impl ScopedAckScope {
|
||||
fn validate(&self) -> io::Result<()> {
|
||||
let valid_uuid = |value: &str| uuid::Uuid::parse_str(value).is_ok_and(|value| !value.is_nil());
|
||||
if !valid_uuid(&self.nonce)
|
||||
|| !valid_uuid(&self.owner_id)
|
||||
|| self.instance_id.is_empty()
|
||||
|| self.instance_id.len() > 128
|
||||
|| self.buckets.is_empty()
|
||||
|| self.buckets.len() > 16
|
||||
|| self.buckets.iter().any(|bucket| bucket.is_empty() || bucket.len() > 63)
|
||||
{
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidInput,
|
||||
"scoped ACK queue needs a bounded session, receiver and bucket scope",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn matches(&self, request: &ScannerScopedDirtyUsageAckRequest) -> bool {
|
||||
!request.probe_only
|
||||
&& validate_scoped_dirty_usage_request(request).is_ok()
|
||||
&& request.owner_id == self.owner_id
|
||||
&& request.instance_id == self.instance_id
|
||||
&& request.entries.iter().all(|entry| self.buckets.contains(&entry.bucket))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum ScopedAckFault {
|
||||
Forward,
|
||||
RequestLoss,
|
||||
ResponseLoss,
|
||||
}
|
||||
|
||||
struct Packet {
|
||||
action: watch::Sender<Option<ScopedAckFault>>,
|
||||
receipt: Mutex<Value>,
|
||||
task: Mutex<Option<AbortHandle>>,
|
||||
}
|
||||
|
||||
pub(super) struct QueueState {
|
||||
scope: ScopedAckScope,
|
||||
capacity: usize,
|
||||
started: Instant,
|
||||
deadline: Instant,
|
||||
closed: AtomicBool,
|
||||
// The packet inventory lock is released before any packet lock or await.
|
||||
packets: Mutex<Vec<Arc<Packet>>>,
|
||||
changed: Notify,
|
||||
}
|
||||
|
||||
pub(super) type QueueSlot = Arc<Mutex<Option<Arc<QueueState>>>>;
|
||||
|
||||
pub(super) fn empty_slot() -> QueueSlot {
|
||||
Arc::new(Mutex::new(None))
|
||||
}
|
||||
|
||||
pub(crate) struct ScopedAckQueue {
|
||||
state: Arc<QueueState>,
|
||||
slot: QueueSlot,
|
||||
}
|
||||
|
||||
impl ScopedAckQueue {
|
||||
pub(super) fn arm(slot: QueueSlot, scope: ScopedAckScope, capacity: usize) -> io::Result<Self> {
|
||||
scope.validate()?;
|
||||
if !(1..=MAX_RETAINED).contains(&capacity) {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidInput, "scoped ACK capacity must be 1..=4"));
|
||||
}
|
||||
let mut active = slot.lock().expect("scoped ACK queue slot");
|
||||
if active.is_some() {
|
||||
return Err(io::Error::new(io::ErrorKind::AlreadyExists, "scoped ACK queue is already armed"));
|
||||
}
|
||||
let started = Instant::now();
|
||||
let state = Arc::new(QueueState {
|
||||
scope,
|
||||
capacity,
|
||||
started,
|
||||
deadline: started + RETENTION,
|
||||
closed: AtomicBool::new(false),
|
||||
packets: Mutex::new(Vec::new()),
|
||||
changed: Notify::new(),
|
||||
});
|
||||
*active = Some(state.clone());
|
||||
drop(active);
|
||||
Ok(Self { state, slot })
|
||||
}
|
||||
|
||||
pub(crate) fn receipts(&self) -> Vec<Value> {
|
||||
let packets = self.state.packets.lock().expect("scoped ACK packets").clone();
|
||||
packets
|
||||
.iter()
|
||||
.map(|packet| packet.receipt.lock().expect("scoped ACK receipt").clone())
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(crate) async fn wait_for_packets(&self, count: usize) -> io::Result<()> {
|
||||
self.wait_until(|| self.receipts().len() >= count).await
|
||||
}
|
||||
|
||||
pub(crate) async fn wait_for_completion(&self, id: usize) -> io::Result<Value> {
|
||||
self.wait_until(|| {
|
||||
self.receipts()
|
||||
.get(id)
|
||||
.is_some_and(|receipt| receipt["completed"].as_bool() == Some(true))
|
||||
})
|
||||
.await?;
|
||||
self.receipts()
|
||||
.get(id)
|
||||
.cloned()
|
||||
.ok_or_else(|| io::Error::other("scoped ACK receipt disappeared"))
|
||||
}
|
||||
|
||||
async fn wait_until(&self, ready: impl Fn() -> bool) -> io::Result<()> {
|
||||
tokio::time::timeout_at(self.state.deadline, async {
|
||||
loop {
|
||||
let changed = self.state.changed.notified();
|
||||
if ready() {
|
||||
return;
|
||||
}
|
||||
changed.await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "scoped ACK queue lifetime elapsed"))
|
||||
}
|
||||
|
||||
pub(crate) fn settle(&self, id: usize, fault: ScopedAckFault) -> io::Result<()> {
|
||||
if Instant::now() >= self.state.deadline {
|
||||
return Err(io::Error::new(io::ErrorKind::TimedOut, "scoped ACK retention expired"));
|
||||
}
|
||||
let packet = self
|
||||
.state
|
||||
.packets
|
||||
.lock()
|
||||
.expect("scoped ACK packets")
|
||||
.get(id)
|
||||
.cloned()
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "scoped ACK packet is not retained"))?;
|
||||
if !packet.action.send_if_modified(|current| {
|
||||
if current.is_none() {
|
||||
*current = Some(fault);
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}) {
|
||||
return Err(io::Error::new(io::ErrorKind::AlreadyExists, "scoped ACK packet was already settled"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ScopedAckQueue {
|
||||
fn drop(&mut self) {
|
||||
self.state.closed.store(true, Ordering::SeqCst);
|
||||
let mut slot = self.slot.lock().expect("scoped ACK queue slot");
|
||||
if slot.as_ref().is_some_and(|current| Arc::ptr_eq(current, &self.state)) {
|
||||
*slot = None;
|
||||
}
|
||||
drop(slot);
|
||||
let packets = self.state.packets.lock().expect("scoped ACK packets").clone();
|
||||
for packet in packets {
|
||||
if let Some(task) = packet.task.lock().expect("scoped ACK task").take() {
|
||||
task.abort();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) enum Interception {
|
||||
Pass(Request<ProxyBody>),
|
||||
Handled(Response<ProxyBody>),
|
||||
}
|
||||
|
||||
fn restored_body(bytes: Bytes, trailers: Option<http::HeaderMap>) -> ProxyBody {
|
||||
Full::new(bytes)
|
||||
.map_err(|never| match never {})
|
||||
.with_trailers(async move { trailers.map(Ok::<_, io::Error>) })
|
||||
.boxed()
|
||||
}
|
||||
|
||||
fn hash(bytes: &[u8]) -> String {
|
||||
hex(Sha256::digest(bytes).as_ref())
|
||||
}
|
||||
|
||||
fn hex(bytes: &[u8]) -> String {
|
||||
hex_simd::encode_to_string(bytes, hex_simd::AsciiCase::Lower)
|
||||
}
|
||||
|
||||
fn trailer_receipt(trailers: Option<&http::HeaderMap>) -> Value {
|
||||
trailers.map_or(Value::Null, |headers| {
|
||||
json!(
|
||||
headers
|
||||
.iter()
|
||||
.map(|(name, value)| { json!({"name": name.as_str(), "value_hex": hex(value.as_bytes())}) })
|
||||
.collect::<Vec<_>>()
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn header_hash(headers: &http::HeaderMap) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
for (name, value) in headers {
|
||||
hasher.update(name.as_str().as_bytes());
|
||||
hasher.update([0]);
|
||||
hasher.update(value.as_bytes());
|
||||
hasher.update([0]);
|
||||
}
|
||||
hex(hasher.finalize().as_ref())
|
||||
}
|
||||
|
||||
fn millis(state: &QueueState) -> u64 {
|
||||
u64::try_from(state.started.elapsed().as_millis()).unwrap_or(u64::MAX)
|
||||
}
|
||||
|
||||
pub(super) async fn intercept(
|
||||
request: Request<ProxyBody>,
|
||||
client: ProxyClient,
|
||||
target: SocketAddr,
|
||||
state: Arc<QueueState>,
|
||||
) -> io::Result<Interception> {
|
||||
if Instant::now() >= state.deadline || state.closed.load(Ordering::SeqCst) {
|
||||
return Ok(Interception::Pass(request));
|
||||
}
|
||||
let (parts, body) = request.into_parts();
|
||||
let collected = tokio::time::timeout_at(state.deadline, Limited::new(body, MAX_BYTES).collect())
|
||||
.await
|
||||
.map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "scoped ACK frame collection expired"))?
|
||||
.map_err(io::Error::other)?;
|
||||
let trailers = collected.trailers().cloned();
|
||||
let bytes = collected.to_bytes();
|
||||
let decoded = ScannerScopedDirtyUsageAckRequest::decode(unary_payload(&bytes)?).map_err(io::Error::other)?;
|
||||
let request = Request::from_parts(parts, restored_body(bytes.clone(), trailers.clone()));
|
||||
if Instant::now() >= state.deadline || !state.scope.matches(&decoded) {
|
||||
return Ok(Interception::Pass(request));
|
||||
}
|
||||
let header = |name| request.headers().get(name).and_then(|value| value.to_str().ok());
|
||||
let (action, mut selected) = watch::channel(None);
|
||||
let packet = {
|
||||
let mut packets = state.packets.lock().expect("scoped ACK packets");
|
||||
if packets.len() >= state.capacity || state.closed.load(Ordering::SeqCst) {
|
||||
return Ok(Interception::Pass(request));
|
||||
}
|
||||
let id = packets.len();
|
||||
let packet = Arc::new(Packet {
|
||||
action,
|
||||
task: Mutex::new(None),
|
||||
receipt: Mutex::new(json!({
|
||||
"id": id, "session_nonce": state.scope.nonce, "receiver": target.to_string(),
|
||||
"authority": request.uri().authority().map(|value| value.as_str()),
|
||||
"captured_at_ms": millis(&state), "completed": false, "forwarded": false,
|
||||
"backend_complete": false, "outcome": "held", "request_bytes": bytes.len(),
|
||||
"request_sha256": hash(&bytes), "request_frame_hex": hex(&bytes), "request_headers_sha256": header_hash(request.headers()),
|
||||
"request_trailers_sha256": trailers.as_ref().map(header_hash),
|
||||
"request_trailers": trailer_receipt(trailers.as_ref()),
|
||||
"auth_nonce": header("x-rustfs-rpc-nonce"), "replay_nonce": header("x-rustfs-rpc-replay-nonce"),
|
||||
"signature_sha256": header("x-rustfs-rpc-signature-v2").map(|value| hash(value.as_bytes())),
|
||||
"replay_signature_sha256": header("x-rustfs-rpc-signature-v3").map(|value| hash(value.as_bytes())),
|
||||
"request": {"owner_id": decoded.owner_id, "instance_id": decoded.instance_id,
|
||||
"probe_only": decoded.probe_only, "entries": decoded.entries.iter().map(|entry| json!({
|
||||
"bucket": entry.bucket, "bucket_incarnation": uuid::Uuid::from_slice(&entry.bucket_incarnation).ok().map(|value| value.to_string()),
|
||||
"generation": entry.generation })).collect::<Vec<_>>() }
|
||||
})),
|
||||
});
|
||||
packets.push(packet.clone());
|
||||
packet
|
||||
};
|
||||
let (reply, response) = oneshot::channel();
|
||||
let worker_packet = packet.clone();
|
||||
let worker_state = state.clone();
|
||||
// This owner survives an HTTP/2 stream reset from the original 5 s caller.
|
||||
// Every request is settled once and is discarded at the session deadline.
|
||||
let task = tokio::spawn(async move {
|
||||
let result = tokio::time::timeout_at(worker_state.deadline, async {
|
||||
let fault = loop {
|
||||
if let Some(fault) = *selected.borrow_and_update() {
|
||||
break fault;
|
||||
}
|
||||
selected
|
||||
.changed()
|
||||
.await
|
||||
.map_err(|_| io::Error::other("scoped ACK queue closed"))?;
|
||||
};
|
||||
if fault == ScopedAckFault::RequestLoss {
|
||||
worker_packet.receipt.lock().expect("scoped ACK receipt")["outcome"] = json!("request_lost");
|
||||
return Err(io::Error::new(io::ErrorKind::ConnectionAborted, "injected scoped ACK request loss"));
|
||||
}
|
||||
if worker_state.closed.load(Ordering::SeqCst) || Instant::now() >= worker_state.deadline {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::ConnectionAborted,
|
||||
"scoped ACK session ended or expired before forwarding",
|
||||
));
|
||||
}
|
||||
{
|
||||
let mut receipt = worker_packet.receipt.lock().expect("scoped ACK receipt");
|
||||
receipt["forwarded"] = json!(true);
|
||||
receipt["forwarded_at_ms"] = json!(millis(&worker_state));
|
||||
}
|
||||
let backend = forward_to_backend(request, client, target).await?;
|
||||
let (parts, body) = backend.into_parts();
|
||||
let collected = Limited::new(body, MAX_BYTES).collect().await.map_err(io::Error::other)?;
|
||||
let trailers = collected.trailers().cloned();
|
||||
let bytes = collected.to_bytes();
|
||||
let grpc_status = trailers
|
||||
.as_ref()
|
||||
.and_then(|values| values.get("grpc-status"))
|
||||
.or_else(|| parts.headers.get("grpc-status"))
|
||||
.and_then(|value| value.to_str().ok());
|
||||
let response = unary_payload(&bytes)
|
||||
.ok()
|
||||
.and_then(|payload| ScannerScopedDirtyUsageAckResponse::decode(payload).ok())
|
||||
.map(|response| {
|
||||
json!({"owner_id": response.owner_id, "instance_id": response.instance_id,
|
||||
"supported": response.supported, "cleared": response.cleared,
|
||||
"response_proof_bytes": response.response_proof.len()})
|
||||
});
|
||||
{
|
||||
let mut receipt = worker_packet.receipt.lock().expect("scoped ACK receipt");
|
||||
receipt["backend_complete"] = json!(true);
|
||||
receipt["backend_completed_at_ms"] = json!(millis(&worker_state));
|
||||
receipt["response"] = json!({"status": parts.status.as_u16(), "grpc_status": grpc_status,
|
||||
"bytes": bytes.len(), "sha256": hash(&bytes), "frame_hex": hex(&bytes), "headers_sha256": header_hash(&parts.headers),
|
||||
"trailers": trailer_receipt(trailers.as_ref()),
|
||||
"trailers_sha256": trailers.as_ref().map(header_hash), "ack": response});
|
||||
receipt["outcome"] = json!(if fault == ScopedAckFault::ResponseLoss {
|
||||
"response_lost"
|
||||
} else {
|
||||
"forwarded"
|
||||
});
|
||||
}
|
||||
if fault == ScopedAckFault::ResponseLoss {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::ConnectionAborted,
|
||||
"injected scoped ACK response loss after backend trailers",
|
||||
));
|
||||
}
|
||||
Ok(Response::from_parts(parts, restored_body(bytes, trailers)))
|
||||
})
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(io::Error::new(io::ErrorKind::TimedOut, "scoped ACK retention expired")));
|
||||
{
|
||||
let mut receipt = worker_packet.receipt.lock().expect("scoped ACK receipt");
|
||||
if result.is_err() && receipt["outcome"] == "held" {
|
||||
receipt["outcome"] = json!("transport_failed_or_expired");
|
||||
}
|
||||
receipt["completed"] = json!(true);
|
||||
receipt["completed_at_ms"] = json!(millis(&worker_state));
|
||||
}
|
||||
let _ = reply.send(result);
|
||||
worker_state.changed.notify_one();
|
||||
});
|
||||
*packet.task.lock().expect("scoped ACK task") = Some(task.abort_handle());
|
||||
if state.closed.load(Ordering::SeqCst) {
|
||||
task.abort();
|
||||
}
|
||||
drop(task);
|
||||
state.changed.notify_one();
|
||||
response
|
||||
.await
|
||||
.map_err(|_| io::Error::new(io::ErrorKind::ConnectionAborted, "scoped ACK queue ended"))?
|
||||
.map(Interception::Handled)
|
||||
}
|
||||
@@ -70,6 +70,9 @@ pub(crate) mod multipart_rpc_delay {
|
||||
#[cfg(test)]
|
||||
pub(crate) mod multipart_fence_acceptance {
|
||||
pub(crate) use super::RUSTFS_META_MULTIPART_BUCKET;
|
||||
|
||||
/// Targeted lock diagnostics for the metadata-delay quorum case.
|
||||
pub(crate) const LOCK_DIAGNOSTICS_FILTER: &str = "warn,rustfs_lock::distributed_lock=debug,rustfs_ecstore::object_lock_diag=trace,rustfs_ecstore::cluster::rpc::remote_locker=info";
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -80,3 +83,23 @@ pub(crate) mod multipart_source_rpc {
|
||||
PublicationIdentity, SourceGeneration, SourceGeometry, SourceLineage,
|
||||
};
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod scanner_peer_bucket {
|
||||
use rustfs_ecstore::api::disk as ecstore_disk;
|
||||
use rustfs_ecstore::api::erasure as ecstore_erasure;
|
||||
use rustfs_ecstore::api::rpc as ecstore_rpc;
|
||||
|
||||
pub(crate) use ecstore_disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
|
||||
pub(crate) use ecstore_erasure::{ReedSolomonEncoder, calc_shard_size};
|
||||
pub(crate) use ecstore_rpc::PeerRestClient;
|
||||
}
|
||||
|
||||
/// Storage handles used to seed a pre-validation bucket record while the fixture server is stopped.
|
||||
#[cfg(test)]
|
||||
pub(crate) mod bucket_tag_fixture {
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata::load_bucket_metadata;
|
||||
pub(crate) use rustfs_ecstore::api::layout::{DisksLayout, EndpointServerPools};
|
||||
pub(crate) use rustfs_ecstore::api::runtime::InstanceContext;
|
||||
pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks_with_instance_ctx};
|
||||
}
|
||||
|
||||
@@ -360,12 +360,26 @@ impl OpaMock {
|
||||
|
||||
pub(crate) async fn expect_bucket_tags(&mut self, action: &str, bucket: &str, department: Option<&str>) -> TestResult {
|
||||
let payload = self.next_request().await?;
|
||||
Self::assert_bucket_tags(&payload, action, bucket, department);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn expect_versioned_tag_permission(&mut self, bucket: &str, object: &str, version: &str) -> TestResult {
|
||||
let payload = self.next_request().await?;
|
||||
Self::assert_bucket_tags(&payload, "s3:GetObjectVersionTagging", bucket, Some("finance"));
|
||||
assert_eq!(payload["input"]["resource"]["object"], object);
|
||||
assert_eq!(payload["input"]["context"]["conditions"]["versionid"], serde_json::json!([version]));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn assert_bucket_tags(payload: &Value, action: &str, bucket: &str, department: Option<&str>) {
|
||||
let input = &payload["input"];
|
||||
assert_eq!(input["identity"]["account"], "opabuckettags");
|
||||
assert_eq!(input["identity"]["is_owner"], false);
|
||||
assert_eq!(input["action"], action);
|
||||
assert_eq!(input["resource"]["bucket"], bucket);
|
||||
let expected = department.map(|value| serde_json::json!([value]));
|
||||
assert_eq!(input["context"]["conditions"].get("ExistingBucketTag/department"), expected.as_ref());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn wait_for_validation(&mut self) -> TestResult {
|
||||
@@ -678,22 +692,61 @@ async fn test_list_buckets_opa_contract() -> TestResult {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Stop and reap the directory's owning server before opening the fixture store.
|
||||
async fn seed_legacy_duplicate_bucket_tags(env: &mut RustFSTestEnvironment, bucket: &str) -> TestResult {
|
||||
use crate::storage_api::bucket_tag_fixture::{
|
||||
DisksLayout, ECStore, EndpointServerPools, InstanceContext, init_local_disks_with_instance_ctx, load_bucket_metadata,
|
||||
};
|
||||
|
||||
let server = env
|
||||
.process
|
||||
.as_mut()
|
||||
.ok_or("missing server process before offline fixture write")?;
|
||||
server.kill()?;
|
||||
server.wait()?;
|
||||
env.process = None;
|
||||
let layout = DisksLayout::from_volumes(std::slice::from_ref(&env.temp_dir))?;
|
||||
let (pools, _) = EndpointServerPools::create_server_endpoints(&env.address, &layout).await?;
|
||||
let context = std::sync::Arc::new(InstanceContext::new());
|
||||
init_local_disks_with_instance_ctx(&context, pools.clone()).await?;
|
||||
let store = ECStore::new_with_instance_ctx(env.address.parse()?, pools, Default::default(), context.clone()).await?;
|
||||
let _cancel = context
|
||||
.background_cancel_token()
|
||||
.expect("store binds its shutdown token")
|
||||
.drop_guard();
|
||||
let mut metadata = load_bucket_metadata(store.clone(), bucket).await?;
|
||||
let incarnation = metadata.bucket_incarnation_id;
|
||||
let policy = metadata.policy_config_json.clone();
|
||||
assert!(!incarnation.is_nil());
|
||||
assert!(!policy.is_empty(), "the bucket-policy Allow must survive legacy seeding");
|
||||
// Encode the duplicate tag set with the same bucket record and erasure writer used by the store.
|
||||
let xml = b"<Tagging><TagSet><Tag><Key>department</Key><Value>finance</Value></Tag><Tag><Key>department</Key><Value>engineering</Value></Tag></TagSet></Tagging>";
|
||||
metadata.tagging_config_xml = xml.to_vec();
|
||||
metadata.save_with_store(store.clone()).await?;
|
||||
let persisted = load_bucket_metadata(store, bucket).await?;
|
||||
assert_eq!(persisted.bucket_incarnation_id, incarnation);
|
||||
assert_eq!(persisted.policy_config_json, policy);
|
||||
assert_eq!(persisted.tagging_config_xml.as_slice(), xml);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Tag, Tagging, VersioningConfiguration,
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Tag, Tagging, TaggingDirective, VersioningConfiguration,
|
||||
};
|
||||
|
||||
init_logging();
|
||||
let mut opa = OpaMock::start().await?;
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_without_cleanup_with_env(&[
|
||||
("RUSTFS_POLICY_PLUGIN_URL", opa.url.as_str()),
|
||||
let opa_url = opa.url.clone();
|
||||
let server_env = [
|
||||
("RUSTFS_POLICY_PLUGIN_URL", opa_url.as_str()),
|
||||
("RUSTFS_POLICY_PLUGIN_AUTH_TOKEN", OPA_AUTH_TOKEN),
|
||||
("NO_PROXY", "127.0.0.1,localhost"),
|
||||
])
|
||||
.await?;
|
||||
];
|
||||
env.start_rustfs_server_without_cleanup_with_env(&server_env).await?;
|
||||
let secret = uuid::Uuid::new_v4().to_string();
|
||||
create_user(&env, "opabuckettags", &secret).await?;
|
||||
let client = aws_sdk_s3::Client::from_conf(build_test_s3_config(&env.url, "opabuckettags", &secret, None, "e2e-bucket-tags"));
|
||||
@@ -733,7 +786,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
.await?;
|
||||
assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
|
||||
client
|
||||
let head = client
|
||||
.head_object()
|
||||
.bucket(source)
|
||||
.key("report")
|
||||
@@ -741,6 +794,8 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
.send()
|
||||
.await?;
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
|
||||
assert_eq!(head.tag_count(), Some(1));
|
||||
opa.expect_versioned_tag_permission(source, "report", version).await?;
|
||||
client.list_objects_v2().bucket(source).send().await?;
|
||||
opa.expect_bucket_tags("s3:ListBucket", source, Some("finance")).await?;
|
||||
client.list_object_versions().bucket(source).send().await?;
|
||||
@@ -750,28 +805,65 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
admin.create_bucket().bucket(destination).send().await?;
|
||||
set_department(&admin, destination, "engineering").await?;
|
||||
let copy_source = format!("{source}/report?versionId={version}");
|
||||
let error = client
|
||||
.copy_object()
|
||||
.bucket(destination)
|
||||
.key("copied")
|
||||
.copy_source(©_source)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("source tags must not authorize a different destination");
|
||||
assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
|
||||
opa.expect_bucket_tags("s3:PutObject", destination, Some("engineering"))
|
||||
.await?;
|
||||
for directive in [TaggingDirective::Copy, TaggingDirective::Replace] {
|
||||
let error = client
|
||||
.copy_object()
|
||||
.bucket(destination)
|
||||
.key("copied")
|
||||
.copy_source(©_source)
|
||||
.set_tagging_directive((directive == TaggingDirective::Replace).then_some(TaggingDirective::Replace))
|
||||
.set_tagging((directive == TaggingDirective::Replace).then(|| "department=legal".to_owned()))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("source tags must not authorize a different destination");
|
||||
assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
|
||||
if directive == TaggingDirective::Copy {
|
||||
opa.expect_versioned_tag_permission(source, "report", version).await?;
|
||||
}
|
||||
opa.expect_bucket_tags("s3:PutObject", destination, Some("engineering"))
|
||||
.await?;
|
||||
let missing = admin
|
||||
.head_object()
|
||||
.bucket(destination)
|
||||
.key("copied")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("denied COPY and REPLACE must leave no destination object");
|
||||
assert_eq!(missing.raw_response().map(|response| response.status().as_u16()), Some(404));
|
||||
}
|
||||
set_department(&admin, destination, "finance").await?;
|
||||
client
|
||||
.copy_object()
|
||||
.bucket(destination)
|
||||
.key("copied")
|
||||
.copy_source(©_source)
|
||||
.send()
|
||||
.await?;
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
|
||||
opa.expect_bucket_tags("s3:PutObject", destination, Some("finance")).await?;
|
||||
for directive in [TaggingDirective::Copy, TaggingDirective::Replace] {
|
||||
client
|
||||
.copy_object()
|
||||
.bucket(destination)
|
||||
.key("copied")
|
||||
.copy_source(©_source)
|
||||
.set_tagging_directive((directive == TaggingDirective::Replace).then_some(TaggingDirective::Replace))
|
||||
.set_tagging((directive == TaggingDirective::Replace).then(|| "department=legal".to_owned()))
|
||||
.send()
|
||||
.await?;
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
|
||||
// The access hook checks preliminary tags, then the copy use case checks
|
||||
// the exact source reader's tags before publishing the destination.
|
||||
for _ in 0..2 {
|
||||
if directive == TaggingDirective::Copy {
|
||||
opa.expect_versioned_tag_permission(source, "report", version).await?;
|
||||
}
|
||||
opa.expect_bucket_tags("s3:PutObject", destination, Some("finance")).await?;
|
||||
}
|
||||
let tags = admin.get_object_tagging().bucket(destination).key("copied").send().await?;
|
||||
assert_eq!(tags.tag_set().len(), 1);
|
||||
assert_eq!(tags.tag_set()[0].key(), "department");
|
||||
assert_eq!(
|
||||
tags.tag_set()[0].value(),
|
||||
if directive == TaggingDirective::Copy {
|
||||
"engineering"
|
||||
} else {
|
||||
"legal"
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
let upload = client
|
||||
.create_multipart_upload()
|
||||
@@ -847,6 +939,14 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("engineering"))
|
||||
.await?;
|
||||
let missing = admin
|
||||
.head_object()
|
||||
.bucket(destination)
|
||||
.key("denied-source")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a denied source must leave no destination object");
|
||||
assert_eq!(missing.raw_response().map(|response| response.status().as_u16()), Some(404));
|
||||
set_department(&client, source, "finance")
|
||||
.await
|
||||
.expect_err("requested tags must not authorize their own mutation");
|
||||
@@ -885,7 +985,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
|
||||
opa.expect_bucket_tags("s3:GetObject", destination, Some("engineering"))
|
||||
.await?;
|
||||
admin
|
||||
let invalid_write = admin
|
||||
.put_bucket_tagging()
|
||||
.bucket(destination)
|
||||
.tagging(
|
||||
@@ -895,7 +995,25 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
.build()?,
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("new duplicate tags must be rejected before publication");
|
||||
assert_eq!(invalid_write.as_service_error().and_then(ProvideErrorMetadata::code), Some("InvalidTag"));
|
||||
assert_eq!(invalid_write.raw_response().map(|response| response.status().as_u16()), Some(400));
|
||||
|
||||
set_department(&admin, source, "finance").await?;
|
||||
seed_legacy_duplicate_bucket_tags(&mut env, destination).await?;
|
||||
env.start_rustfs_server_without_cleanup_with_env(&server_env).await?;
|
||||
// Prove this process can authorize the same caller and read a healthy bucket's persisted tags.
|
||||
let read = client
|
||||
.get_object()
|
||||
.bucket(source)
|
||||
.key("report")
|
||||
.version_id(version)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
|
||||
|
||||
let error = client
|
||||
.get_object()
|
||||
.bucket(destination)
|
||||
|
||||
@@ -1112,8 +1112,9 @@ async fn wait_for_upgrade_writes(
|
||||
match response {
|
||||
Ok(Ok(_)) => break,
|
||||
Ok(Err(error)) => {
|
||||
let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
|
||||
if error.raw_response().map(|response| response.status().as_u16()) != Some(503)
|
||||
|| error.as_service_error().and_then(ProvideErrorMetadata::code) != Some("ServiceUnavailable")
|
||||
|| !matches!(code, Some("ServiceUnavailable" | "SlowDown" | "SlowDownRead"))
|
||||
{
|
||||
return Err(format!("{phase}: node {node} {probe} failed: {error:?}").into());
|
||||
}
|
||||
|
||||
@@ -31,6 +31,7 @@ workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
fips = ["rustfs-s3-client/fips", "rustfs-utils/fips"]
|
||||
diagnostic-read-version-caller = ["rustfs-io-metrics/diagnostic-read-version-caller"]
|
||||
# Sampled wall/CPU diagnostics only; does not change the read backend.
|
||||
diagnostic-local-batch = ["rustix/time"]
|
||||
@@ -225,9 +226,11 @@ metrics = { workspace = true }
|
||||
# The guard scripts/check_no_tokio_io_uring.sh allows an explicit
|
||||
# io-uring integration; only the tokio "io-uring" runtime feature is banned.
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
# SharedReadBudget and the bounded read APIs are available in the 0.3.0
|
||||
# registry release; keep the application on the published contract.
|
||||
rustfs-uring = "0.3.0"
|
||||
# Native completion owners keep physical I/O leases alive after cancellation.
|
||||
rustfs-uring = { git = "https://github.com/rustfs/uring.git", rev = "eb8fc160e71d804427de1e905f5c226d580f71bf" }
|
||||
|
||||
[target.'cfg(target_os = "linux")'.dev-dependencies]
|
||||
rustfs-uring = { git = "https://github.com/rustfs/uring.git", rev = "eb8fc160e71d804427de1e905f5c226d580f71bf", features = ["fault-injection"] }
|
||||
|
||||
[target.'cfg(windows)'.dependencies]
|
||||
winapi-util.workspace = true
|
||||
|
||||
@@ -192,6 +192,7 @@ pub mod bucket {
|
||||
table_catalog_path_hash, unreadable_config_refusal,
|
||||
};
|
||||
pub use crate::bucket::metadata::{BUCKET_DURABILITY_CONFIG, BUCKET_ON_DEMAND_MIGRATION_CONFIG};
|
||||
pub use crate::bucket::metadata::{MAX_BUCKET_TAGS, validate_tag_set_entries};
|
||||
}
|
||||
|
||||
pub mod config_parse_mode {
|
||||
@@ -331,8 +332,8 @@ pub mod bucket {
|
||||
pub mod utils {
|
||||
pub use crate::bucket::utils::{
|
||||
check_bucket_and_object_names, check_list_objs_args, check_object_name_for_length_and_slash,
|
||||
check_valid_bucket_name_strict, deserialize, has_bad_path_component, is_meta_bucketname, is_valid_object_prefix,
|
||||
serialize,
|
||||
check_valid_bucket_name_strict, deserialize, has_bad_path_component, has_xml_invalid_control_characters,
|
||||
is_meta_bucketname, is_valid_object_prefix, serialize,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -447,10 +448,19 @@ pub mod data_usage {
|
||||
}
|
||||
|
||||
pub mod disk {
|
||||
pub use crate::cluster::rpc::background_pipe::{BackgroundPipeReader, BackgroundPipeWriter, background_pipe};
|
||||
pub use crate::cluster::rpc::background_reader_stream::BackgroundReaderStream;
|
||||
pub use crate::disk::background_memory::BackgroundEncodeBuffer;
|
||||
pub use crate::disk::disk_store::get_object_disk_read_timeout;
|
||||
#[cfg(feature = "test-util")]
|
||||
pub use crate::disk::disk_store::wait_for_multipart_index_bootstrap;
|
||||
pub use crate::disk::local::{ReplacementExecutionLease, ScanGuard};
|
||||
#[cfg(feature = "test-util")]
|
||||
pub use crate::disk::local::{local_durability_modes, set_local_rename_data_fail_after_metadata_commit};
|
||||
pub use crate::disk::multipart_index::{
|
||||
MULTIPART_INDEX_RPC_MAX_BYTES, MULTIPART_INDEX_RPC_MAX_REQUEST_BYTES, MultipartIndexPage, MultipartIndexRpcReply,
|
||||
MultipartIndexRpcRequest,
|
||||
};
|
||||
pub use crate::disk::multipart_source::{
|
||||
MULTIPART_SOURCE_PROTOCOL_VERSION, MultipartAbortCertificate, MultipartAbortProof, MultipartAbortProofKind,
|
||||
MultipartAbortRound, MultipartSourceOperation, MultipartSourcePhase, MultipartSourceRequest, MultipartSourceResponse,
|
||||
@@ -463,7 +473,9 @@ pub mod disk {
|
||||
pub use crate::disk::os::{LocalPublicationPause, LocalPublicationStage};
|
||||
pub use crate::disk::{
|
||||
BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp,
|
||||
CheckPartsResp, ConditionalFileUpdate, CurrentDeleteGeneration, CurrentDeleteTransition,
|
||||
BoundedDeleteRecoveryDiscovery, BoundedDeleteRecoveryInspection, BoundedDeleteRecoverySettlement,
|
||||
BoundedDeleteRecoverySettlementReply, CheckPartsResp, ConditionalFileUpdate, CurrentDeleteGeneration,
|
||||
CurrentDeleteTransition, DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES, DELETE_RECOVERY_MAX_INSPECTION_ENTRIES,
|
||||
DELETE_RECOVERY_PROTOCOL_VERSION, DeleteOptions, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest,
|
||||
DeleteRecoveryInspection, DeleteRecoveryPage, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, DeleteRecoveryState,
|
||||
Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore, FileInfoVersions, FileReader, FileWriter,
|
||||
@@ -475,6 +487,10 @@ pub mod disk {
|
||||
UpdateMetadataOpts, VolumeInfo, WalkDirOptions, is_quota_mutation_fence_path, new_disk,
|
||||
validate_batch_read_version_item_count,
|
||||
};
|
||||
pub use crate::disk::{
|
||||
BackgroundMemoryBudget, PhysicalDiskBinding, PhysicalDiskPlan, PhysicalDiskSample, PhysicalDiskSampler,
|
||||
PhysicalResourcePlan, RemoteBackgroundIo, with_background_resource_unit, with_remote_background_io,
|
||||
};
|
||||
pub use bytes::Bytes;
|
||||
pub use endpoint::Endpoint;
|
||||
pub use error::DiskError;
|
||||
@@ -610,11 +626,12 @@ pub mod rebalance {
|
||||
pub mod rio {
|
||||
pub use crate::io_support::rio::{
|
||||
DecryptReader, DynReader, EncryptReader, HardLimitReader, HashReader, ReadStream, Reader, WriteEncryption, WritePlan,
|
||||
boxed_reader, compression_metadata_value, wrap_reader,
|
||||
boxed_reader, compression_metadata_value, validate_compression_scheme, wrap_reader,
|
||||
};
|
||||
}
|
||||
|
||||
pub mod rpc {
|
||||
pub use crate::cluster::rpc::http_auth::with_rpc_background_request;
|
||||
pub use crate::cluster::rpc::{
|
||||
AuthenticatedChannel, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, MAX_NETWORK_PROBE_BYTES, MAX_NETWORK_PROBE_DURATION,
|
||||
NetworkPeerProbeClient, NetworkPeerProbeError, NetworkPeerProbeMeasurement, NetworkPeerTarget, PEER_RESTDRY_RUN,
|
||||
@@ -624,8 +641,8 @@ pub mod rpc {
|
||||
TONIC_RPC_PREFIX, TonicInterceptor, build_put_file_auth_trailer, check_and_record_signed_rpc_nonce,
|
||||
decode_heal_bucket_rpc_options, encode_heal_bucket_rpc_options, gen_signature_headers, gen_tonic_replay_scope_headers,
|
||||
gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client,
|
||||
node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
|
||||
sign_ns_scanner_capability, sign_ns_scanner_capability_with_checkpoint_coverage,
|
||||
node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, rpc_background_admission,
|
||||
set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_ns_scanner_capability_with_checkpoint_coverage,
|
||||
sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, sign_tonic_rpc_response_proof,
|
||||
tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
|
||||
verify_ns_scanner_capability, verify_ns_scanner_capability_with_checkpoint_coverage,
|
||||
@@ -646,7 +663,9 @@ pub mod set_disk {
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
pub mod test_util {
|
||||
pub use crate::bucket::quota::reservation::fail_next_quota_ledger_save_for_test;
|
||||
pub use crate::bucket::quota::reservation::{
|
||||
fail_next_quota_ledger_save_for_test, initialize_quota_ledger_for_test, retry_quota_reconciliation_for_test,
|
||||
};
|
||||
pub use crate::set_disk::ConditionalReadBenchmarkMetadataGuard;
|
||||
pub use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, PutObjectCommitPause};
|
||||
|
||||
@@ -675,8 +694,9 @@ pub mod storage {
|
||||
pub use crate::store::HealWalkVersion;
|
||||
pub use crate::store::{
|
||||
BootstrapLocalTarget, ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk,
|
||||
all_local_disk_path, find_local_disk_by_ref, init_local_disks, init_local_disks_with_instance_ctx, init_lock_clients,
|
||||
prewarm_local_disk_id_map, prewarm_local_disk_id_map_with_instance_ctx,
|
||||
all_local_disk_path, find_local_disk_by_ref, find_local_disk_by_ref_for_rpc, init_local_disks,
|
||||
init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map,
|
||||
prewarm_local_disk_id_map_with_instance_ctx,
|
||||
};
|
||||
pub use crate::store::{HealObjectAbsenceProof, HealObjectStorageResult};
|
||||
}
|
||||
|
||||
@@ -16,7 +16,8 @@ use crate::bucket::metadata::BucketMetadata;
|
||||
use crate::bucket::metadata_sys::get_bucket_targets_config;
|
||||
use crate::bucket::metadata_sys::get_replication_config;
|
||||
use crate::bucket::remote_s3_client::{
|
||||
PathStyle, REPLICATION_TARGET_RETRY_POLICY, RemoteCredentials, RemoteS3EndpointSpec, build_remote_s3_client,
|
||||
PathStyle, REPLICATION_TARGET_RETRY_POLICY, RemoteCredentials, RemoteS3EndpointSpec, RemoteS3RetryPolicy,
|
||||
UploadResponseDeadline, build_remote_s3_client,
|
||||
};
|
||||
use crate::bucket::replication::{ObjectLockIntegrity, object_lock_put_integrity, replication_etags_match};
|
||||
use crate::bucket::replication::{ReplicationStatusType, ReplicationTargetConfigBridge};
|
||||
@@ -119,6 +120,7 @@ impl From<&BucketTarget> for RemoteS3EndpointSpec {
|
||||
ca_cert_pem: (!target.ca_cert_pem.trim().is_empty()).then(|| target.ca_cert_pem.clone()),
|
||||
connect_timeout: None,
|
||||
read_timeout: None,
|
||||
operation_timeout: Some(TargetRequestTimeouts::from_env().request),
|
||||
// Replication has no retry budget of its own on the request path,
|
||||
// so it keeps the SDK's standard three attempts; stating it here
|
||||
// pins the behaviour to this line instead of an SDK default.
|
||||
@@ -1191,6 +1193,7 @@ impl BucketTargetSys {
|
||||
health_check_duration: target.health_check_duration,
|
||||
replicate_sync: target.replication_sync,
|
||||
client: Arc::new(client),
|
||||
timeouts: TargetRequestTimeouts::from_env(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1922,6 +1925,83 @@ impl std::fmt::Display for S3ClientError {
|
||||
}
|
||||
}
|
||||
|
||||
/// Replication request budgets, snapshotted when a target client is built.
|
||||
/// Streamed GET bodies are not subject to the upload-response budget.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct TargetRequestTimeouts {
|
||||
request: Duration,
|
||||
complete: Duration,
|
||||
minimum_transfer_rate: u64,
|
||||
}
|
||||
|
||||
impl Default for TargetRequestTimeouts {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
request: Duration::from_secs(60),
|
||||
complete: Duration::from_secs(15 * 60),
|
||||
minimum_transfer_rate: 1024 * 1024,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TargetRequestTimeouts {
|
||||
fn from_env() -> Self {
|
||||
let defaults = Self::default();
|
||||
let seconds = |name: &str, fallback: Duration| {
|
||||
let value = rustfs_utils::get_env_u64(name, fallback.as_secs());
|
||||
Duration::from_secs(if (1..=86400).contains(&value) {
|
||||
value
|
||||
} else {
|
||||
fallback.as_secs()
|
||||
})
|
||||
};
|
||||
let minimum_transfer_rate =
|
||||
rustfs_utils::get_env_u64("RUSTFS_REPLICATION_MIN_TRANSFER_RATE_BYTES_PER_SEC", defaults.minimum_transfer_rate);
|
||||
Self {
|
||||
request: seconds("RUSTFS_REPLICATION_REQUEST_TIMEOUT_SECS", defaults.request),
|
||||
complete: seconds("RUSTFS_REPLICATION_COMPLETE_TIMEOUT_SECS", defaults.complete),
|
||||
minimum_transfer_rate: if minimum_transfer_rate == 0 {
|
||||
defaults.minimum_transfer_rate
|
||||
} else {
|
||||
minimum_transfer_rate
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn upload_timeout(self, size: i64) -> Result<Duration, S3ClientError> {
|
||||
let size = u64::try_from(size).map_err(|_| S3ClientError::new("replication upload size must be non-negative"))?;
|
||||
// The SDK can make three attempts. Leave transmission headroom for
|
||||
// all three; the separate response deadline still bounds a trickling
|
||||
// target after each upload finishes.
|
||||
let attempts = match REPLICATION_TARGET_RETRY_POLICY {
|
||||
RemoteS3RetryPolicy::Disabled => 1,
|
||||
RemoteS3RetryPolicy::Standard { max_attempts } => u64::from(max_attempts.max(1)),
|
||||
};
|
||||
let seconds = size
|
||||
.div_ceil(self.minimum_transfer_rate)
|
||||
.checked_mul(attempts)
|
||||
.ok_or_else(|| S3ClientError::new("replication upload timeout exceeds supported duration"))?;
|
||||
let timeout = self
|
||||
.request
|
||||
.checked_add(Duration::from_secs(seconds))
|
||||
.ok_or_else(|| S3ClientError::new("replication upload timeout exceeds supported duration"))?;
|
||||
if Instant::now().checked_add(timeout).is_none() {
|
||||
return Err(S3ClientError::new("replication upload timeout exceeds supported duration"));
|
||||
}
|
||||
Ok(timeout)
|
||||
}
|
||||
|
||||
fn upload_override(self, size: i64) -> Result<aws_sdk_s3::config::Builder, S3ClientError> {
|
||||
Ok(aws_sdk_s3::config::Builder::new()
|
||||
.timeout_config(
|
||||
aws_smithy_types::timeout::TimeoutConfig::builder()
|
||||
.operation_timeout(self.upload_timeout(size)?)
|
||||
.build(),
|
||||
)
|
||||
.interceptor(UploadResponseDeadline(self.request)))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct TargetClient {
|
||||
pub endpoint: String,
|
||||
@@ -1935,6 +2015,7 @@ pub struct TargetClient {
|
||||
pub health_check_duration: Duration,
|
||||
pub replicate_sync: bool,
|
||||
pub client: Arc<S3Client>,
|
||||
pub timeouts: TargetRequestTimeouts,
|
||||
}
|
||||
|
||||
impl TargetClient {
|
||||
@@ -2360,12 +2441,11 @@ impl TargetClient {
|
||||
|
||||
Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
|
||||
});
|
||||
let mut config = self.timeouts.upload_override(size)?;
|
||||
if forwards_source_checksum {
|
||||
operation = operation.config_override(
|
||||
aws_sdk_s3::config::Builder::new()
|
||||
.request_checksum_calculation(aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired),
|
||||
);
|
||||
config = config.request_checksum_calculation(aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired);
|
||||
}
|
||||
operation = operation.config_override(config);
|
||||
match operation.send().await {
|
||||
Ok(output) => {
|
||||
// Under SSE-KMS/DSSE or SSE-C the target's ETag is not the MD5
|
||||
@@ -2501,6 +2581,7 @@ impl TargetClient {
|
||||
}
|
||||
Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
|
||||
})
|
||||
.config_override(self.timeouts.upload_override(size)?)
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
@@ -2538,6 +2619,13 @@ impl TargetClient {
|
||||
}
|
||||
Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
|
||||
})
|
||||
.config_override(
|
||||
aws_sdk_s3::config::Builder::new().timeout_config(
|
||||
aws_smithy_types::timeout::TimeoutConfig::builder()
|
||||
.operation_timeout(self.timeouts.complete)
|
||||
.build(),
|
||||
),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
@@ -2966,11 +3054,241 @@ mod tests {
|
||||
health_check_duration: Duration::from_secs(5),
|
||||
replicate_sync: false,
|
||||
client: Arc::new(client),
|
||||
timeouts: TargetRequestTimeouts::default(),
|
||||
},
|
||||
request_headers,
|
||||
)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct TimedResponseConnector {
|
||||
upload_delay: Duration,
|
||||
response_delay: Duration,
|
||||
prefix_bytes: usize,
|
||||
response: bytes::Bytes,
|
||||
frames: Arc<std::sync::atomic::AtomicUsize>,
|
||||
dropped: Arc<std::sync::atomic::AtomicUsize>,
|
||||
}
|
||||
|
||||
struct ResponseDrop(Arc<std::sync::atomic::AtomicUsize>);
|
||||
impl Drop for ResponseDrop {
|
||||
fn drop(&mut self) {
|
||||
self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
impl SmithyHttpConnector for TimedResponseConnector {
|
||||
fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
|
||||
let settings = self.clone();
|
||||
HttpConnectorFuture::new(async move {
|
||||
http_body_util::BodyExt::collect(request.into_body())
|
||||
.await
|
||||
.map_err(aws_smithy_runtime_api::client::result::ConnectorError::io)?;
|
||||
tokio::time::sleep(settings.upload_delay).await;
|
||||
let guard = ResponseDrop(Arc::clone(&settings.dropped));
|
||||
let stream = futures::stream::unfold(
|
||||
(settings.prefix_bytes, Some(settings.response), true, guard),
|
||||
move |(remaining, mut body, first, guard)| {
|
||||
let frames = Arc::clone(&settings.frames);
|
||||
async move {
|
||||
if remaining == 0 && body.is_none() {
|
||||
return None;
|
||||
}
|
||||
if !first {
|
||||
tokio::time::sleep(settings.response_delay).await;
|
||||
}
|
||||
let bytes = if remaining > 0 {
|
||||
bytes::Bytes::from_static(b" ")
|
||||
} else {
|
||||
body.take()?
|
||||
};
|
||||
frames.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
Some((
|
||||
Ok::<_, std::io::Error>(http_body::Frame::data(bytes)),
|
||||
(remaining.saturating_sub(1), body, false, guard),
|
||||
))
|
||||
}
|
||||
},
|
||||
)
|
||||
.fuse();
|
||||
let body = SdkBody::from_body_1_x(http_body_util::StreamBody::new(stream));
|
||||
let mut response =
|
||||
HttpResponse::new(aws_smithy_runtime_api::http::StatusCode::try_from(200u16).expect("status"), body);
|
||||
response.headers_mut().insert("etag", "\"test-etag\"");
|
||||
Ok(response)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn timed_response_target(
|
||||
upload_delay: Duration,
|
||||
response_delay: Duration,
|
||||
prefix_bytes: usize,
|
||||
response: &'static [u8],
|
||||
) -> (TargetClient, Arc<std::sync::atomic::AtomicUsize>, Arc<std::sync::atomic::AtomicUsize>) {
|
||||
let frames = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let dropped = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let connector = SharedHttpConnector::new(TimedResponseConnector {
|
||||
upload_delay,
|
||||
response_delay,
|
||||
prefix_bytes,
|
||||
response: bytes::Bytes::from_static(response),
|
||||
frames: Arc::clone(&frames),
|
||||
dropped: Arc::clone(&dropped),
|
||||
});
|
||||
let http_client = http_client_fn(move |_settings, _components| connector.clone());
|
||||
let (mut target, _) = header_recording_target_client(Vec::new());
|
||||
target.timeouts = TargetRequestTimeouts {
|
||||
request: Duration::from_secs(2),
|
||||
complete: Duration::from_secs(10),
|
||||
minimum_transfer_rate: 1024 * 1024,
|
||||
};
|
||||
target.client = Arc::new(S3Client::from_conf(
|
||||
target
|
||||
.client
|
||||
.config()
|
||||
.to_builder()
|
||||
.http_client(http_client)
|
||||
.retry_config(aws_sdk_s3::config::retry::RetryConfig::disabled())
|
||||
.timeout_config(
|
||||
aws_smithy_types::timeout::TimeoutConfig::builder()
|
||||
.operation_timeout(target.timeouts.request)
|
||||
.build(),
|
||||
)
|
||||
.build(),
|
||||
));
|
||||
(target, frames, dropped)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_upload_budgets_preserve_large_transfers_and_reject_invalid_sizes() {
|
||||
let policy = TargetRequestTimeouts::default();
|
||||
assert_eq!(policy.upload_timeout(0).expect("empty upload"), Duration::from_secs(60));
|
||||
assert_eq!(
|
||||
policy.upload_timeout(5 * 1024 * 1024 * 1024).expect("maximum single PUT"),
|
||||
Duration::from_secs(15420)
|
||||
);
|
||||
assert!(policy.upload_timeout(-1).is_err());
|
||||
let slow = TargetRequestTimeouts {
|
||||
minimum_transfer_rate: 1,
|
||||
..policy
|
||||
};
|
||||
assert!(slow.upload_timeout(i64::MAX).is_err());
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn replication_upload_response_deadline_is_independent_of_size_and_checksum_override() {
|
||||
for operation in ["put", "forwarded-checksum", "part"] {
|
||||
let (target, frames, dropped) = timed_response_target(Duration::from_secs(4), Duration::from_millis(500), 100, b"");
|
||||
let body = vec![b'x'; 2 * 1024 * 1024];
|
||||
let size = i64::try_from(body.len()).expect("size");
|
||||
let started = tokio::time::Instant::now();
|
||||
let error = if operation == "part" {
|
||||
target
|
||||
.put_object_part(
|
||||
"bucket",
|
||||
"key",
|
||||
"upload",
|
||||
1,
|
||||
size,
|
||||
ByteStream::from(body),
|
||||
&PutObjectPartOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect_err("response deadline")
|
||||
} else {
|
||||
let mut opts = PutObjectOptions::default();
|
||||
if operation == "forwarded-checksum" {
|
||||
opts.user_metadata.insert(
|
||||
"x-amz-checksum-sha256".to_owned(),
|
||||
"OoJ3yNhRwv3wwtZoGqEIPrPX9xwTnfLl+ka0wStN1g0=".to_owned(),
|
||||
);
|
||||
}
|
||||
target
|
||||
.put_object("bucket", "key", size, ByteStream::from(body), &opts)
|
||||
.await
|
||||
.expect_err("response deadline")
|
||||
};
|
||||
assert!(
|
||||
started.elapsed() >= Duration::from_secs(6),
|
||||
"upload gets its full independent window: {error}"
|
||||
);
|
||||
assert!(
|
||||
started.elapsed() < target.timeouts.upload_timeout(size).expect("budget"),
|
||||
"response must end before the larger total deadline"
|
||||
);
|
||||
assert!(
|
||||
frames.load(std::sync::atomic::Ordering::SeqCst) >= 2,
|
||||
"trickle delivered progress before failure"
|
||||
);
|
||||
assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1, "transport body released");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn replication_slow_upload_preserves_transfer_window_and_streaming_get_body() {
|
||||
let (target, _, _) = timed_response_target(Duration::from_secs(4), Duration::from_millis(100), 5, b"");
|
||||
let body = vec![b'x'; 2 * 1024 * 1024];
|
||||
let size = i64::try_from(body.len()).expect("size");
|
||||
target
|
||||
.put_object("bucket", "key", size, ByteStream::from(body), &PutObjectOptions::default())
|
||||
.await
|
||||
.expect("slow upload, prompt response");
|
||||
let (target, _, dropped) = timed_response_target(Duration::ZERO, Duration::from_secs(1), 5, b"payload");
|
||||
let response = target
|
||||
.get_object("bucket", "key", None, None, None, HeaderMap::new())
|
||||
.await
|
||||
.expect("GET headers");
|
||||
let started = tokio::time::Instant::now();
|
||||
let body = response
|
||||
.body
|
||||
.collect()
|
||||
.await
|
||||
.expect("streamed GET remains outside upload deadline")
|
||||
.into_bytes();
|
||||
assert!(started.elapsed() > target.timeouts.request);
|
||||
assert_eq!(&body[..], b" payload");
|
||||
assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn replication_complete_processing_whitespace_has_a_separate_bounded_window() {
|
||||
const RESULT: &[u8] = b"<CompleteMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><ETag>\"etag\"</ETag></CompleteMultipartUploadResult>";
|
||||
for prefix in [5, 100] {
|
||||
let (target, frames, dropped) = timed_response_target(Duration::ZERO, Duration::from_secs(1), prefix, RESULT);
|
||||
let started = tokio::time::Instant::now();
|
||||
let result = target
|
||||
.complete_multipart_upload(
|
||||
"bucket",
|
||||
"key",
|
||||
"upload",
|
||||
vec![CompletedPart::builder().part_number(1).e_tag("etag").build()],
|
||||
&PutObjectOptions::default(),
|
||||
)
|
||||
.await;
|
||||
if prefix == 5 {
|
||||
assert!(result.is_ok(), "legal processing whitespace: {result:?}");
|
||||
} else {
|
||||
assert!(result.is_err());
|
||||
assert!(started.elapsed() <= Duration::from_secs(11));
|
||||
}
|
||||
assert!(started.elapsed() > target.timeouts.request);
|
||||
assert!(frames.load(std::sync::atomic::Ordering::SeqCst) >= 2);
|
||||
assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn replication_control_operation_deadline_covers_response_trickle() {
|
||||
let (target, frames, dropped) =
|
||||
timed_response_target(Duration::ZERO, Duration::from_millis(500), 100, b"<VersioningConfiguration/>");
|
||||
let started = tokio::time::Instant::now();
|
||||
assert!(target.get_bucket_versioning("bucket").await.is_err());
|
||||
assert!(started.elapsed() <= Duration::from_secs(3));
|
||||
assert!(frames.load(std::sync::atomic::Ordering::SeqCst) >= 2);
|
||||
assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
fn streaming_test_body(payload: &'static [u8]) -> ByteStream {
|
||||
let stream = tokio_util::io::ReaderStream::new(std::io::Cursor::new(payload));
|
||||
let body = http_body_util::StreamBody::new(futures::StreamExt::map(stream, |r| r.map(http_body::Frame::data)));
|
||||
@@ -3279,6 +3597,7 @@ mod tests {
|
||||
health_check_duration: Duration::from_secs(5),
|
||||
replicate_sync: false,
|
||||
client: Arc::new(client),
|
||||
timeouts: TargetRequestTimeouts::default(),
|
||||
},
|
||||
request_uris,
|
||||
)
|
||||
@@ -3591,6 +3910,7 @@ mod tests {
|
||||
health_check_duration: Duration::from_secs(5),
|
||||
replicate_sync: false,
|
||||
client: Arc::new(client),
|
||||
timeouts: TargetRequestTimeouts::default(),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -5487,6 +5487,14 @@ pub async fn put_restore_opts(
|
||||
if !oi.user_tags.is_empty() {
|
||||
meta.insert(AMZ_OBJECT_TAGGING.to_string(), (*oi.user_tags).clone());
|
||||
}
|
||||
if let Some(expires) = oi.expires {
|
||||
meta.insert(
|
||||
"expires".to_string(),
|
||||
expires
|
||||
.format(&time::format_description::well_known::Rfc3339)
|
||||
.map_err(std::io::Error::other)?,
|
||||
);
|
||||
}
|
||||
let restore_expiry = lifecycle::expected_expiry_time(OffsetDateTime::now_utc(), rreq.days.unwrap_or(1));
|
||||
meta.insert(
|
||||
metadata_keys::RESTORE.to_string(),
|
||||
@@ -5506,7 +5514,6 @@ pub async fn put_restore_opts(
|
||||
// computed MD5 is not the object's public plaintext ETag.
|
||||
preserve_etag: oi.etag.clone(),
|
||||
shard_integrity_write_mode: Some(oi.shard_integrity_write_mode()),
|
||||
//expires: oi.expires,
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
@@ -7144,6 +7151,25 @@ mod tests {
|
||||
0,
|
||||
"tier generation lease should release after EOF"
|
||||
);
|
||||
let unread = get_transitioned_object_reader_with_tier_manager(
|
||||
&object_info.bucket,
|
||||
&object_info.name,
|
||||
&None,
|
||||
&HeaderMap::new(),
|
||||
&object_info,
|
||||
&ObjectOptions::default(),
|
||||
&manager,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("open an unread transitioned response");
|
||||
assert_eq!(TierConfigMgr::active_operation_lease_count(&manager, &tier).await, 1);
|
||||
drop(unread);
|
||||
assert_eq!(
|
||||
TierConfigMgr::active_operation_lease_count(&manager, &tier).await,
|
||||
0,
|
||||
"canceling an unread response must release the tier lease"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
|
||||
@@ -256,6 +256,38 @@ pub const BUCKET_POLICY_CONFIG: &str = "policy.json";
|
||||
pub const BUCKET_NOTIFICATION_CONFIG: &str = "notification.xml";
|
||||
pub const BUCKET_LIFECYCLE_CONFIG: &str = "lifecycle.xml";
|
||||
pub const BUCKET_SSECONFIG: &str = "bucket-encryption.xml";
|
||||
/// Shared tag-entry validation for object requests and persisted bucket tags.
|
||||
pub fn validate_tag_set_entries(tag_set: &[s3s::dto::Tag]) -> std::result::Result<(), &'static str> {
|
||||
fn is_valid_tag_text(value: &str) -> bool {
|
||||
static VALID_TAG_TEXT: std::sync::OnceLock<std::result::Result<regex::Regex, regex::Error>> = std::sync::OnceLock::new();
|
||||
VALID_TAG_TEXT
|
||||
.get_or_init(|| regex::Regex::new(r"^[\p{L}\p{N}\p{Z}+\-=._:/@]*$"))
|
||||
.as_ref()
|
||||
.is_ok_and(|pattern| pattern.is_match(value))
|
||||
}
|
||||
let mut keys = std::collections::HashSet::with_capacity(tag_set.len());
|
||||
for tag in tag_set {
|
||||
let key = tag
|
||||
.key
|
||||
.as_deref()
|
||||
.filter(|key| !key.is_empty())
|
||||
.ok_or("Tag key cannot be empty")?;
|
||||
if key.encode_utf16().count() > 128 || !is_valid_tag_text(key) {
|
||||
return Err("The TagKey you have provided is invalid");
|
||||
}
|
||||
let value = tag.value.as_deref().ok_or("Tag value cannot be null")?;
|
||||
if value.encode_utf16().count() > 256 || !is_valid_tag_text(value) {
|
||||
return Err("The TagValue you have provided is invalid");
|
||||
}
|
||||
if !keys.insert(key) {
|
||||
return Err("Cannot provide multiple Tags with the same key");
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub const MAX_BUCKET_TAGS: usize = 50;
|
||||
|
||||
pub const BUCKET_TAGGING_CONFIG: &str = "tagging.xml";
|
||||
pub const BUCKET_QUOTA_CONFIG_FILE: &str = "quota.json";
|
||||
pub const OBJECT_LOCK_CONFIG: &str = "object-lock.xml";
|
||||
@@ -990,6 +1022,20 @@ impl BucketMetadata {
|
||||
self.encryption_config_updated_at = updated;
|
||||
}
|
||||
BUCKET_TAGGING_CONFIG => {
|
||||
if !data.is_empty() {
|
||||
let tagging = deserialize::<Tagging>(&data).map_err(|_| {
|
||||
Error::InvalidArgument(self.name.clone(), String::new(), "Invalid bucket tagging XML".to_owned())
|
||||
})?;
|
||||
if tagging.tag_set.len() > MAX_BUCKET_TAGS {
|
||||
return Err(Error::InvalidArgument(
|
||||
self.name.clone(),
|
||||
String::new(),
|
||||
"Cannot have more than 50 tags per bucket".to_owned(),
|
||||
));
|
||||
}
|
||||
validate_tag_set_entries(&tagging.tag_set)
|
||||
.map_err(|message| Error::InvalidArgument(self.name.clone(), String::new(), message.to_owned()))?;
|
||||
}
|
||||
self.tagging_config_xml = data;
|
||||
// Drop the parsed form (like lifecycle above) so clearing the
|
||||
// payload can't leave stale parsed tags to be cached.
|
||||
@@ -1503,7 +1549,11 @@ pub(crate) async fn load_bucket_metadata_parse_with_presence(
|
||||
bucket: &str,
|
||||
parse: bool,
|
||||
) -> Result<(BucketMetadata, bool)> {
|
||||
let (mut bm, persisted) = match read_bucket_metadata(api.clone(), bucket).await {
|
||||
let (metadata_res, incarnation_res) = tokio::join!(
|
||||
Box::pin(read_bucket_metadata(api.clone(), bucket)),
|
||||
Box::pin(load_bucket_incarnation(api.clone(), bucket)),
|
||||
);
|
||||
let (mut bm, persisted) = match metadata_res {
|
||||
Ok(res) => (res, true),
|
||||
Err(err) => {
|
||||
if err != Error::ConfigNotFound {
|
||||
@@ -1514,7 +1564,7 @@ pub(crate) async fn load_bucket_metadata_parse_with_presence(
|
||||
}
|
||||
};
|
||||
|
||||
let incarnation = load_bucket_incarnation(api.clone(), bucket).await?;
|
||||
let incarnation = incarnation_res?;
|
||||
if persisted {
|
||||
if bm.name.is_empty() {
|
||||
bm.name = bucket.to_string();
|
||||
@@ -1780,6 +1830,31 @@ mod test {
|
||||
/// backlog#2292: a replicated config is stamped with the source
|
||||
/// `updated_at` it was given, not the local clock, while the plain
|
||||
/// `update_config` entry keeps stamping the local clock.
|
||||
#[test]
|
||||
fn bucket_tag_updates_reject_unreadable_shapes_without_replacing_metadata() {
|
||||
let mut metadata = BucketMetadata::new("validated-tags");
|
||||
let valid = br#"<Tagging><TagSet><Tag><Key>team</Key><Value></Value></Tag></TagSet></Tagging>"#.to_vec();
|
||||
metadata
|
||||
.update_config(BUCKET_TAGGING_CONFIG, valid.clone())
|
||||
.expect("empty value is valid");
|
||||
let updated = metadata.tagging_config_updated_at;
|
||||
for invalid in [
|
||||
"<Tagging><TagSet><Tag><Key>team</Key></Tag></TagSet></Tagging>".to_owned(),
|
||||
"<Tagging><TagSet><Tag><Key></Key><Value>x</Value></Tag></TagSet></Tagging>".to_owned(),
|
||||
"<Tagging><TagSet><Tag><Key>team</Key><Value>x</Value></Tag><Tag><Key>team</Key><Value>y</Value></Tag></TagSet></Tagging>".to_owned(),
|
||||
format!("<Tagging><TagSet>{}</TagSet></Tagging>", (0..51).map(|i| format!("<Tag><Key>k{i}</Key><Value>v</Value></Tag>")).collect::<String>()),
|
||||
] {
|
||||
assert!(metadata.update_config_at(BUCKET_TAGGING_CONFIG, invalid.into_bytes(), updated + time::Duration::hours(1)).is_err());
|
||||
assert_eq!(metadata.tagging_config_xml, valid);
|
||||
assert_eq!(metadata.tagging_config_updated_at, updated);
|
||||
}
|
||||
metadata.tagging_config_xml = b"legacy corrupt tags".to_vec();
|
||||
metadata
|
||||
.update_config(BUCKET_TAGGING_CONFIG, Vec::new())
|
||||
.expect("deletion repairs legacy corruption");
|
||||
assert!(metadata.tagging_config_xml.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn update_config_at_stamps_the_given_time_and_update_config_stamps_now() {
|
||||
let source_time = OffsetDateTime::now_utc() - time::Duration::hours(3);
|
||||
|
||||
@@ -683,6 +683,7 @@ pub(crate) async fn update_validated_in<F>(
|
||||
bucket: &str,
|
||||
config_file: &str,
|
||||
data: Vec<u8>,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
validate: F,
|
||||
) -> Result<OffsetDateTime>
|
||||
where
|
||||
@@ -690,7 +691,7 @@ where
|
||||
{
|
||||
Box::pin(async {
|
||||
let sys = require_bucket_metadata_sys_in(ctx)?;
|
||||
let guard = acquire_config_write_guard(sys.clone(), bucket).await?;
|
||||
let guard = acquire_config_write_guard_for_incarnation(sys.clone(), bucket, expected_incarnation_id).await?;
|
||||
validate()?;
|
||||
update_under_config_write_guard(sys, &guard, config_file, data, None).await
|
||||
})
|
||||
@@ -1435,6 +1436,7 @@ pub(crate) async fn get_object_lock_config_state_in(
|
||||
|
||||
/// Re-read the Object Lock state and bucket incarnation from the same
|
||||
/// authoritative metadata blob while the caller holds the transaction lock.
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) async fn get_object_lock_config_and_incarnation_from_disk_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
@@ -1453,6 +1455,27 @@ pub(crate) async fn get_object_lock_config_and_incarnation_from_disk_with_guard_
|
||||
get_delete_config_from_disk_in(ctx, bucket, guard, None).await
|
||||
}
|
||||
|
||||
#[allow(clippy::type_complexity)]
|
||||
pub(crate) async fn get_object_lock_and_quota_config_from_disk_with_guard_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
guard: Option<&rustfs_lock::NamespaceLockGuard>,
|
||||
) -> Result<(
|
||||
ObjectLockConfigState,
|
||||
Uuid,
|
||||
OffsetDateTime,
|
||||
std::result::Result<(Option<BucketQuota>, Uuid, OffsetDateTime), String>,
|
||||
)> {
|
||||
let (state, incarnation_id, updated_at, quota_res) =
|
||||
get_delete_and_quota_config_from_disk_in(ctx, bucket, guard, None, true).await?;
|
||||
Ok((
|
||||
state,
|
||||
incarnation_id,
|
||||
updated_at,
|
||||
quota_res.unwrap_or(Ok((None, incarnation_id, updated_at))),
|
||||
))
|
||||
}
|
||||
|
||||
/// Validate queued expiration against the same fresh metadata used for Object
|
||||
/// Lock admission. The caller holds the transaction read lock through commit,
|
||||
/// so a successful lifecycle update cannot be followed by an old-policy delete.
|
||||
@@ -1462,6 +1485,24 @@ pub(crate) async fn get_delete_config_from_disk_in(
|
||||
guard: Option<&rustfs_lock::NamespaceLockGuard>,
|
||||
expiry_policy: Option<&rustfs_lifecycle::ExpiryPolicySnapshot>,
|
||||
) -> Result<(ObjectLockConfigState, Uuid, OffsetDateTime)> {
|
||||
let (state, incarnation_id, updated_at, _) =
|
||||
get_delete_and_quota_config_from_disk_in(ctx, bucket, guard, expiry_policy, false).await?;
|
||||
Ok((state, incarnation_id, updated_at))
|
||||
}
|
||||
|
||||
#[allow(clippy::type_complexity)]
|
||||
async fn get_delete_and_quota_config_from_disk_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
guard: Option<&rustfs_lock::NamespaceLockGuard>,
|
||||
expiry_policy: Option<&rustfs_lifecycle::ExpiryPolicySnapshot>,
|
||||
include_quota: bool,
|
||||
) -> Result<(
|
||||
ObjectLockConfigState,
|
||||
Uuid,
|
||||
OffsetDateTime,
|
||||
Option<std::result::Result<(Option<BucketQuota>, Uuid, OffsetDateTime), String>>,
|
||||
)> {
|
||||
let bucket_meta_sys_lock = bucket_metadata_sys_of(ctx)?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await.clone();
|
||||
|
||||
@@ -1477,7 +1518,7 @@ pub(crate) async fn get_delete_config_from_disk_in(
|
||||
.await?
|
||||
}
|
||||
};
|
||||
match authority {
|
||||
match &authority {
|
||||
BucketMetadataAuthority::Authoritative(metadata)
|
||||
if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() =>
|
||||
{
|
||||
@@ -1489,11 +1530,11 @@ pub(crate) async fn get_delete_config_from_disk_in(
|
||||
}
|
||||
}
|
||||
let object_lock_config_updated_at = metadata.object_lock_config_updated_at;
|
||||
Ok((
|
||||
object_lock_config_state_from_authoritative_metadata(&metadata)?,
|
||||
metadata.bucket_incarnation_id,
|
||||
object_lock_config_updated_at,
|
||||
))
|
||||
let state = object_lock_config_state_from_authoritative_metadata(metadata)?;
|
||||
let incarnation_id = metadata.bucket_incarnation_id;
|
||||
let quota_res = include_quota
|
||||
.then(|| quota_config_and_incarnation_from_authority(bucket, authority).map_err(|err| err.to_string()));
|
||||
Ok((state, incarnation_id, object_lock_config_updated_at, quota_res))
|
||||
}
|
||||
BucketMetadataAuthority::Authoritative(_) => {
|
||||
Err(Error::other(format!("bucket incarnation metadata is not authoritative: {bucket}")))
|
||||
@@ -1564,6 +1605,23 @@ pub(crate) async fn get_quota_config_and_incarnation_from_disk_for_options_in(
|
||||
return Err(Error::other("quota snapshot metadata transaction fence was released"));
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
let has_injected_error = sys.object_lock_disk_read_errors.read().await.contains(bucket);
|
||||
#[cfg(not(test))]
|
||||
let has_injected_error = false;
|
||||
if !ctx.lock_manager().is_disabled()
|
||||
&& !has_injected_error
|
||||
&& let Some(fresh_quota) = snapshot.take_fresh_put_quota()
|
||||
{
|
||||
if snapshot_guard.is_lock_lost()
|
||||
|| snapshot_guard.is_released()
|
||||
|| !snapshot.is_valid_for_destructive_put(store.id, bucket, opts.expected_bucket_incarnation_id.unwrap_or_default())
|
||||
{
|
||||
return Err(Error::other("quota snapshot fence or bucket incarnation changed during the disk read"));
|
||||
}
|
||||
return fresh_quota.map_err(Error::other);
|
||||
}
|
||||
|
||||
// The snapshot carries the caller's lifecycle fence, excluding deletion.
|
||||
// Its transaction guard excludes config writes, so a same-name object
|
||||
// owner need not recursively acquire the bucket namespace here.
|
||||
@@ -1750,9 +1808,22 @@ struct MetadataPublishGuard {
|
||||
_guard: tokio::sync::OwnedMutexGuard<MetadataPublishLockState>,
|
||||
}
|
||||
|
||||
static LOCAL_META_MUTATION_GEN: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);
|
||||
|
||||
pub(crate) fn bump_local_meta_mutation_gen() {
|
||||
LOCAL_META_MUTATION_GEN.fetch_add(1, std::sync::atomic::Ordering::Release);
|
||||
}
|
||||
|
||||
pub(crate) fn current_local_meta_mutation_gen() -> u64 {
|
||||
LOCAL_META_MUTATION_GEN.load(std::sync::atomic::Ordering::Acquire)
|
||||
}
|
||||
|
||||
type LocalAuthorityCache = Arc<StdMutex<HashMap<String, (u64, Arc<BucketMetadata>)>>>;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BucketMetadataSys {
|
||||
metadata_map: Arc<RwLock<HashMap<String, Arc<BucketMetadata>>>>,
|
||||
local_authority_cache: LocalAuthorityCache,
|
||||
/// Serializes metadata-map commits and their derived cache updates for one
|
||||
/// bucket. Namespace locks, when present, are acquired before this lock.
|
||||
metadata_publish_locks: Arc<MetadataPublishLockRegistry>,
|
||||
@@ -1796,6 +1867,7 @@ impl BucketMetadataSys {
|
||||
pub fn new(api: Arc<ECStore>) -> Self {
|
||||
Self {
|
||||
metadata_map: Arc::new(RwLock::new(HashMap::new())),
|
||||
local_authority_cache: Arc::new(StdMutex::new(HashMap::new())),
|
||||
metadata_publish_locks: Arc::new(MetadataPublishLockRegistry {
|
||||
locks: StdMutex::new(HashMap::new()),
|
||||
}),
|
||||
@@ -1830,6 +1902,15 @@ impl BucketMetadataSys {
|
||||
self.api.upgrade()
|
||||
}
|
||||
|
||||
/// Whether the backing `ECStore` has been dropped.
|
||||
///
|
||||
/// Used by `InstanceContext::clear_bucket_metadata_sys_if_stale` during
|
||||
/// `ECStore::drop` so a shared bootstrap context does not retain a
|
||||
/// `BucketMetadataSys` whose weak handle is dangling.
|
||||
pub(crate) fn is_store_stale(&self) -> bool {
|
||||
self.api.upgrade().is_none()
|
||||
}
|
||||
|
||||
fn metadata_publish_lock(&self, bucket: &str) -> Arc<Mutex<MetadataPublishLockState>> {
|
||||
Self::bucket_lock_in(&self.metadata_publish_locks, bucket)
|
||||
}
|
||||
@@ -2160,6 +2241,7 @@ impl BucketMetadataSys {
|
||||
}
|
||||
|
||||
pub async fn set(&self, bucket: String, bm: Arc<BucketMetadata>) {
|
||||
bump_local_meta_mutation_gen();
|
||||
if !is_meta_bucketname(&bucket) {
|
||||
let publish_lock = self.metadata_publish_lock(&bucket);
|
||||
let _publish_guard = publish_lock.lock().await;
|
||||
@@ -2178,6 +2260,7 @@ impl BucketMetadataSys {
|
||||
///
|
||||
/// Returns `true` if an entry was present. Reserved meta buckets are ignored.
|
||||
pub async fn remove(&self, bucket: &str) -> bool {
|
||||
bump_local_meta_mutation_gen();
|
||||
if is_meta_bucketname(bucket) {
|
||||
return false;
|
||||
}
|
||||
@@ -2198,6 +2281,7 @@ impl BucketMetadataSys {
|
||||
}
|
||||
|
||||
async fn _reset(&mut self) {
|
||||
bump_local_meta_mutation_gen();
|
||||
let mut map = self.metadata_map.write().await;
|
||||
map.clear();
|
||||
drop(map);
|
||||
@@ -3001,30 +3085,77 @@ impl BucketMetadataSys {
|
||||
return Err(Error::other(format!("injected Object Lock metadata disk read failure: {bucket}")));
|
||||
}
|
||||
|
||||
match await_bucket_namespace_operation(Some(guard), bucket, "bucket metadata snapshot existence check", async {
|
||||
self.object_store()
|
||||
.get_bucket_info_from_sets_at_read_quorum(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
|
||||
.await
|
||||
})
|
||||
.await
|
||||
{
|
||||
let store = self.object_store();
|
||||
let can_use_local_authority_cache = !store.ctx.lock_manager().is_disabled() && store.all_disks_local_and_online();
|
||||
let gen_before = current_local_meta_mutation_gen();
|
||||
let cached_authority = can_use_local_authority_cache
|
||||
.then(|| {
|
||||
self.local_authority_cache
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.get(bucket)
|
||||
.filter(|(cached_gen, _)| *cached_gen == gen_before)
|
||||
.map(|(_, bm)| Arc::clone(bm))
|
||||
})
|
||||
.flatten();
|
||||
if let Some(cached) = cached_authority {
|
||||
await_bucket_namespace_operation(
|
||||
Some(guard),
|
||||
bucket,
|
||||
"bucket metadata authoritative snapshot",
|
||||
std::future::ready(Ok::<_, Error>(())),
|
||||
)
|
||||
.await?;
|
||||
if current_local_meta_mutation_gen() == gen_before && store.all_disks_local_and_online() {
|
||||
return Ok(BucketMetadataAuthority::Authoritative(cached));
|
||||
}
|
||||
}
|
||||
|
||||
let (bucket_info_res, metadata_res) = tokio::join!(
|
||||
Box::pin(await_bucket_namespace_operation(
|
||||
Some(guard),
|
||||
bucket,
|
||||
"bucket metadata snapshot existence check",
|
||||
async {
|
||||
store
|
||||
.get_bucket_info_from_sets_at_read_quorum(
|
||||
bucket,
|
||||
&crate::storage_api_contracts::bucket::BucketOptions::default(),
|
||||
)
|
||||
.await
|
||||
},
|
||||
)),
|
||||
Box::pin(await_bucket_namespace_operation(
|
||||
Some(guard),
|
||||
bucket,
|
||||
"bucket metadata authoritative snapshot",
|
||||
load_bucket_metadata_parse_with_presence(Arc::clone(&store), bucket, true),
|
||||
)),
|
||||
);
|
||||
match bucket_info_res {
|
||||
Ok(_) => {}
|
||||
Err(Error::VolumeNotFound) => return Ok(BucketMetadataAuthority::MissingBucket),
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
|
||||
let (metadata, persisted) = await_bucket_namespace_operation(
|
||||
Some(guard),
|
||||
bucket,
|
||||
"bucket metadata authoritative snapshot",
|
||||
load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true),
|
||||
)
|
||||
.await?;
|
||||
let (metadata, persisted) = metadata_res?;
|
||||
if persisted && metadata.lock_enabled && !metadata.bucket_creation_committed {
|
||||
return Err(Error::ErasureWriteQuorum);
|
||||
}
|
||||
if persisted {
|
||||
Ok(BucketMetadataAuthority::Authoritative(Arc::new(metadata)))
|
||||
let metadata = Arc::new(metadata);
|
||||
if can_use_local_authority_cache
|
||||
&& metadata.bucket_incarnation_sidecar
|
||||
&& !metadata.bucket_incarnation_id.is_nil()
|
||||
&& current_local_meta_mutation_gen() == gen_before
|
||||
&& store.all_disks_local_and_online()
|
||||
{
|
||||
self.local_authority_cache
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.insert(bucket.to_string(), (gen_before, Arc::clone(&metadata)));
|
||||
}
|
||||
Ok(BucketMetadataAuthority::Authoritative(metadata))
|
||||
} else {
|
||||
Ok(BucketMetadataAuthority::Fabricated)
|
||||
}
|
||||
@@ -3297,7 +3428,7 @@ mod tests {
|
||||
),
|
||||
(
|
||||
BUCKET_TAGGING_CONFIG,
|
||||
r#"<Tagging><TagSet><Tag><Key>environment</Key><Value>测试-🦀</Value></Tag></TagSet></Tagging>"#.as_bytes(),
|
||||
r#"<Tagging><TagSet><Tag><Key>environment</Key><Value>测试-rust</Value></Tag></TagSet></Tagging>"#.as_bytes(),
|
||||
),
|
||||
(BUCKET_REPLICATION_CONFIG, NEW_WRITER_REPLICATION_XML),
|
||||
(
|
||||
@@ -4425,8 +4556,13 @@ mod tests {
|
||||
std::fs::create_dir_all(dir.path().join(bucket)).expect("bucket volume should be created");
|
||||
}
|
||||
let mut bm = BucketMetadata::new(bucket);
|
||||
bm.update_config(config_file, raw.to_vec())
|
||||
.expect("raw config should be stored");
|
||||
if config_file == crate::bucket::metadata::BUCKET_TAGGING_CONFIG {
|
||||
// Model legacy persisted corruption without passing the write-time validator.
|
||||
bm.tagging_config_xml = raw.to_vec();
|
||||
} else {
|
||||
bm.update_config(config_file, raw.to_vec())
|
||||
.expect("raw config should be stored");
|
||||
}
|
||||
sys.persist_new_and_set(bm).await.expect("initial metadata should persist");
|
||||
}
|
||||
|
||||
@@ -5079,6 +5215,61 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn validated_config_update_rejects_a_different_bucket_incarnation() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = "catalog-cleanup-incarnation";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let incarnation = store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("read bucket incarnation");
|
||||
let called = std::sync::atomic::AtomicBool::new(false);
|
||||
let result = store
|
||||
.update_bucket_metadata_config_for_incarnation_validated(
|
||||
bucket,
|
||||
"quota.json",
|
||||
br#"{"quota":1}"#.to_vec(),
|
||||
Uuid::new_v4(),
|
||||
|| {
|
||||
called.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(result, Err(Error::BucketNotFound(name)) if name == bucket));
|
||||
assert!(
|
||||
!called.load(std::sync::atomic::Ordering::SeqCst),
|
||||
"incarnation must be checked before the external fence"
|
||||
);
|
||||
let (metadata, _) = get_config_from_disk_with_presence_in(&store.ctx, bucket)
|
||||
.await
|
||||
.expect("read unchanged bucket metadata");
|
||||
assert!(
|
||||
metadata.quota_config_json.is_empty(),
|
||||
"a stale cleanup must not modify the replacement bucket"
|
||||
);
|
||||
store
|
||||
.update_bucket_metadata_config_for_incarnation_validated(
|
||||
bucket,
|
||||
"quota.json",
|
||||
br#"{"quota":1}"#.to_vec(),
|
||||
incarnation,
|
||||
|| Ok(()),
|
||||
)
|
||||
.await
|
||||
.expect("the current incarnation permits the validated update");
|
||||
let (metadata, _) = get_config_from_disk_with_presence_in(&store.ctx, bucket)
|
||||
.await
|
||||
.expect("read updated bucket metadata");
|
||||
assert_eq!(metadata.quota_config_json, br#"{"quota":1}"#);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_admission_rejects_stale_cached_no_quota() {
|
||||
@@ -5552,7 +5743,8 @@ mod tests {
|
||||
// update only needs one interleaving to show up.
|
||||
const ROUNDS: usize = 8;
|
||||
for round in 0..ROUNDS {
|
||||
let tagging = format!("<Tagging><Round>{round}</Round></Tagging>").into_bytes();
|
||||
let tagging =
|
||||
format!("<Tagging><TagSet><Tag><Key>round</Key><Value>{round}</Value></Tag></TagSet></Tagging>").into_bytes();
|
||||
let policy = format!(r#"{{"Version":"2012-10-17","Round":{round}}}"#).into_bytes();
|
||||
|
||||
let start = Arc::new(tokio::sync::Barrier::new(2));
|
||||
@@ -5625,7 +5817,7 @@ mod tests {
|
||||
|
||||
let blocked = tokio::spawn({
|
||||
let node_b = node_b.clone();
|
||||
async move { update_with_sys(node_b, bucket, BUCKET_TAGGING_CONFIG, b"<Tagging/>".to_vec()).await }
|
||||
async move { update_with_sys(node_b, bucket, BUCKET_TAGGING_CONFIG, b"<Tagging><TagSet/></Tagging>".to_vec()).await }
|
||||
});
|
||||
|
||||
// Long enough for the write to have finished had it not waited: the
|
||||
|
||||
@@ -119,6 +119,21 @@ async fn durable_quota_reporting_retires_degraded_floors_without_changing_ledger
|
||||
// Exercise the real storage entry point before testing the new
|
||||
// reporting semantics: the same ledger must admit and refuse
|
||||
// these writes even when a scanner floor claims a million bytes.
|
||||
if protocol == 1 {
|
||||
let mut bootstrap = PutObjReader::from_vec(vec![0x5a; 512]);
|
||||
assert!(
|
||||
matches!(
|
||||
store
|
||||
.put_object(&bucket, "bootstrap", &mut bootstrap, &ObjectOptions::default())
|
||||
.await,
|
||||
Err(StorageError::SlowDown)
|
||||
),
|
||||
"a missing authoritative ledger must not admit a write from scanner reporting"
|
||||
);
|
||||
crate::bucket::quota::reservation::retry_quota_reconciliation_for_test(store.clone(), &bucket)
|
||||
.await
|
||||
.expect("bootstrap authoritative usage through the production worker");
|
||||
}
|
||||
let mut too_large = PutObjReader::from_vec(vec![0x5a; 8192]);
|
||||
assert!(
|
||||
matches!(
|
||||
|
||||
@@ -19,7 +19,7 @@ use crate::data_usage::compute_bucket_usage;
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::disk::{DiskAPI, error::DiskError};
|
||||
use crate::error::{Result, StorageError, is_err_object_not_found, is_err_version_not_found};
|
||||
use crate::object_api::{ObjectInfo, ObjectOptions, QuotaAdmission};
|
||||
use crate::object_api::{ObjectInfo, ObjectLockConfigSnapshot, ObjectOptions, QuotaAdmission};
|
||||
use crate::set_disk::{SetDisks, get_lock_acquire_timeout};
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking;
|
||||
use crate::storage_api_contracts::{list::ListOperations as _, object::ObjectOperations};
|
||||
@@ -28,7 +28,7 @@ use futures::{StreamExt, stream};
|
||||
use rustfs_lock::NamespaceLockGuard;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::BTreeMap;
|
||||
use std::collections::{BTreeMap, HashSet};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use time::OffsetDateTime;
|
||||
@@ -36,6 +36,8 @@ use tracing::warn;
|
||||
use uuid::Uuid;
|
||||
|
||||
const QUOTA_LEDGER_FORMAT_VERSION: u8 = 1;
|
||||
const RECONCILE_RETRY_SECONDS: i64 = 30;
|
||||
const EVENT_QUOTA_RECONCILIATION: &str = "quota_reconciliation";
|
||||
const MAX_ORPHANS_REAPED_PER_WRITE: usize = 64;
|
||||
const MAX_ORPHAN_PROBES_PER_WRITE: usize = 128;
|
||||
const ORPHAN_PROBE_CONCURRENCY: usize = 32;
|
||||
@@ -96,6 +98,10 @@ struct QuotaLedger {
|
||||
version: u8,
|
||||
bucket_incarnation: Uuid,
|
||||
quota_revision_unix_nanos: i128,
|
||||
/// Changes on every durable write, including zero-growth reservations.
|
||||
/// Missing on an older writer's output, which invalidates a concurrent scan.
|
||||
#[serde(default)]
|
||||
writer_revision: Uuid,
|
||||
accounted_usage: u64,
|
||||
reservations: BTreeMap<Uuid, PersistedReservation>,
|
||||
#[serde(default)]
|
||||
@@ -110,6 +116,7 @@ impl QuotaLedger {
|
||||
version: QUOTA_LEDGER_FORMAT_VERSION,
|
||||
bucket_incarnation,
|
||||
quota_revision_unix_nanos: quota_revision.unix_timestamp_nanos(),
|
||||
writer_revision: Uuid::nil(),
|
||||
accounted_usage,
|
||||
reservations: BTreeMap::new(),
|
||||
reconcile_required: false,
|
||||
@@ -485,22 +492,23 @@ impl QuotaContext {
|
||||
let ledger_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &ledger_data.ledger_object).await?;
|
||||
let ledger_guard = Arc::new(ledger_lock.get_write_lock(get_lock_acquire_timeout()).await?);
|
||||
fence_namespace_mutations(&store, RUSTFS_META_BUCKET, &ledger_data.ledger_object, None).await?;
|
||||
let mut ledger = load_current_ledger_locked(
|
||||
Arc::clone(&store),
|
||||
&ledger_data.bucket,
|
||||
&ledger_data.ledger_object,
|
||||
bucket_incarnation,
|
||||
quota_revision,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let mut ledger =
|
||||
load_current_ledger_locked(Arc::clone(&store), &ledger_data.ledger_object, bucket_incarnation, quota_revision)
|
||||
.await?;
|
||||
let job = ReconcileJob::new(Arc::clone(&store), ledger_data.bucket.clone(), bucket_incarnation, quota_revision);
|
||||
let Some(mut ledger) = ledger.take() else {
|
||||
// Bootstrap has no authoritative usage yet. Release every
|
||||
// request-owned fence by returning; the worker owns no request.
|
||||
job.schedule();
|
||||
return Err(StorageError::SlowDown);
|
||||
};
|
||||
let growth = reservation.growth();
|
||||
let mut current_usage = ledger.admitted_usage()?;
|
||||
let mut expected_usage = current_usage.checked_add(growth).ok_or(StorageError::PartMissingOrCorrupt)?;
|
||||
if growth > 0 && expected_usage > quota_limit && growth <= quota_limit && ledger.should_reconcile_after_denial() {
|
||||
reconcile_exact(&store, &ledger_data.bucket, &mut ledger).await?;
|
||||
current_usage = ledger.admitted_usage()?;
|
||||
expected_usage = current_usage.checked_add(growth).ok_or(StorageError::PartMissingOrCorrupt)?;
|
||||
let current_usage = ledger.admitted_usage()?;
|
||||
let expected_usage = current_usage.checked_add(growth).ok_or(StorageError::PartMissingOrCorrupt)?;
|
||||
if ledger.reconcile_required
|
||||
|| (growth > 0 && expected_usage > quota_limit && growth <= quota_limit && ledger.should_reconcile_after_denial())
|
||||
{
|
||||
job.schedule();
|
||||
}
|
||||
if growth > 0 && expected_usage > quota_limit {
|
||||
return Err(StorageError::QuotaExceeded {
|
||||
@@ -518,7 +526,7 @@ impl QuotaContext {
|
||||
});
|
||||
}
|
||||
ledger.reserve(operation_id, reservation)?;
|
||||
save_ledger_locked(Arc::clone(&store), &ledger_data.ledger_object, &ledger, &ledger_guard).await?;
|
||||
save_ledger_locked(Arc::clone(&store), &ledger_data.ledger_object, &mut ledger, &ledger_guard).await?;
|
||||
|
||||
Ok(QuotaReservation {
|
||||
ledger: Some(ReservationLedgerData::Single(ledger_data)),
|
||||
@@ -1082,7 +1090,7 @@ async fn mark_commit_started(data: &ReservationLedgerData) -> Result<()> {
|
||||
fence_namespace_mutations(&data.store, RUSTFS_META_BUCKET, &data.ledger_object, None).await?;
|
||||
let mut ledger = load_ledger_locked(Arc::clone(&data.store), &data.ledger_object).await?;
|
||||
ledger.mark_commit_started(data.operation_id, &data.reservation)?;
|
||||
save_ledger_locked(Arc::clone(&data.store), &data.ledger_object, &ledger, &ledger_guard).await
|
||||
save_ledger_locked(Arc::clone(&data.store), &data.ledger_object, &mut ledger, &ledger_guard).await
|
||||
})
|
||||
.await
|
||||
.map_err(|err| StorageError::other_with_context("quota commit marker task failed", err))?
|
||||
@@ -1113,7 +1121,7 @@ async fn settle(data: &ReservationLedgerData, committed: bool) -> Result<()> {
|
||||
} else {
|
||||
ledger.abort(operation_id, &reservation)?;
|
||||
}
|
||||
save_ledger_locked(Arc::clone(&store), &ledger_object, &ledger, &ledger_guard).await
|
||||
save_ledger_locked(Arc::clone(&store), &ledger_object, &mut ledger, &ledger_guard).await
|
||||
})
|
||||
.await
|
||||
.map_err(|err| StorageError::other_with_context("quota ledger settlement task failed", err))?
|
||||
@@ -1121,22 +1129,15 @@ async fn settle(data: &ReservationLedgerData, committed: bool) -> Result<()> {
|
||||
|
||||
async fn load_current_ledger_locked(
|
||||
store: Arc<ECStore>,
|
||||
bucket: &str,
|
||||
ledger_object: &str,
|
||||
bucket_incarnation: Uuid,
|
||||
quota_revision: OffsetDateTime,
|
||||
) -> Result<QuotaLedger> {
|
||||
match load_ledger_locked(Arc::clone(&store), ledger_object).await {
|
||||
Ok(ledger) if ledger.matches(bucket_incarnation, quota_revision) => Ok(ledger),
|
||||
Ok(ledger) if ledger.reservations.is_empty() && !ledger.reconcile_required => {
|
||||
let usage = exact_bucket_usage(&store, bucket).await?;
|
||||
Ok(QuotaLedger::new(bucket_incarnation, quota_revision, usage))
|
||||
}
|
||||
) -> Result<Option<QuotaLedger>> {
|
||||
match load_ledger_locked(store, ledger_object).await {
|
||||
Ok(ledger) if ledger.matches(bucket_incarnation, quota_revision) => Ok(Some(ledger)),
|
||||
Ok(ledger) if ledger.reservations.is_empty() => Ok(None),
|
||||
Ok(_) => Err(StorageError::PartMissingOrCorrupt),
|
||||
Err(StorageError::ConfigNotFound) => {
|
||||
let usage = exact_bucket_usage(&store, bucket).await?;
|
||||
Ok(QuotaLedger::new(bucket_incarnation, quota_revision, usage))
|
||||
}
|
||||
Err(StorageError::ConfigNotFound) => Ok(None),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
@@ -1245,15 +1246,11 @@ async fn reap_stale_reservations(store: Arc<ECStore>, bucket: &str, ledger_objec
|
||||
for (object, target) in orphan_commit_targets {
|
||||
fence_namespace_mutations(&store, bucket, &object, target).await?;
|
||||
}
|
||||
let mut removed = remove_orphan_reservations(&mut ledger, &orphan_ids)?;
|
||||
if ledger.reservations.is_empty() && ledger.reconcile_required {
|
||||
reconcile_exact(&store, bucket, &mut ledger).await?;
|
||||
removed = true;
|
||||
}
|
||||
let removed = remove_orphan_reservations(&mut ledger, &orphan_ids)?;
|
||||
if !removed && !cursor_changed {
|
||||
return Ok(());
|
||||
}
|
||||
save_ledger_locked(store, ledger_object, &ledger, &ledger_guard).await
|
||||
save_ledger_locked(store, ledger_object, &mut ledger, &ledger_guard).await
|
||||
}
|
||||
|
||||
fn remove_orphan_reservations(ledger: &mut QuotaLedger, operation_ids: &[Uuid]) -> Result<bool> {
|
||||
@@ -1273,13 +1270,217 @@ fn remove_orphan_reservations(ledger: &mut QuotaLedger, operation_ids: &[Uuid])
|
||||
Ok(removed)
|
||||
}
|
||||
|
||||
async fn reconcile_exact(store: &Arc<ECStore>, bucket: &str, ledger: &mut QuotaLedger) -> Result<()> {
|
||||
if !ledger.reservations.is_empty() {
|
||||
return Err(StorageError::PartMissingOrCorrupt);
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
struct ReconcileControl {
|
||||
bucket_incarnation: Uuid,
|
||||
quota_revision_unix_nanos: i128,
|
||||
next_attempt_at: i64,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct ReconcileJob {
|
||||
store: Arc<ECStore>,
|
||||
bucket: String,
|
||||
bucket_incarnation: Uuid,
|
||||
quota_revision: OffsetDateTime,
|
||||
}
|
||||
|
||||
type ReconcileKey = (Uuid, String, Uuid, i128);
|
||||
|
||||
fn active_reconciliations() -> &'static std::sync::Mutex<HashSet<ReconcileKey>> {
|
||||
static ACTIVE: std::sync::OnceLock<std::sync::Mutex<HashSet<ReconcileKey>>> = std::sync::OnceLock::new();
|
||||
ACTIVE.get_or_init(Default::default)
|
||||
}
|
||||
|
||||
struct ActiveReconciliation(ReconcileKey);
|
||||
|
||||
impl Drop for ActiveReconciliation {
|
||||
fn drop(&mut self) {
|
||||
active_reconciliations()
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.remove(&self.0);
|
||||
}
|
||||
}
|
||||
|
||||
impl ReconcileJob {
|
||||
fn new(store: Arc<ECStore>, bucket: String, bucket_incarnation: Uuid, quota_revision: OffsetDateTime) -> Self {
|
||||
Self {
|
||||
store,
|
||||
bucket,
|
||||
bucket_incarnation,
|
||||
quota_revision,
|
||||
}
|
||||
}
|
||||
|
||||
fn schedule(self) {
|
||||
let key = (
|
||||
self.store.id,
|
||||
self.bucket.clone(),
|
||||
self.bucket_incarnation,
|
||||
self.quota_revision.unix_timestamp_nanos(),
|
||||
);
|
||||
if !active_reconciliations()
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.insert(key.clone())
|
||||
{
|
||||
return;
|
||||
}
|
||||
let owner = ActiveReconciliation(key);
|
||||
// The owner carries no object, metadata, lifecycle or reservation guard.
|
||||
// Cancellation of the rejected request cannot cancel its reconciliation.
|
||||
tokio::spawn(async move {
|
||||
let _owner = owner;
|
||||
if let Err(error) = self.run().await {
|
||||
warn!(event = EVENT_QUOTA_RECONCILIATION, component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_QUOTA, state = "deferred", bucket = %self.bucket,
|
||||
error = %error, "Quota reconciliation deferred");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
fn control_object(&self) -> String {
|
||||
format!(
|
||||
"{}.reconcile/{}/{}",
|
||||
ledger_object(&self.bucket),
|
||||
self.bucket_incarnation,
|
||||
self.quota_revision.unix_timestamp_nanos()
|
||||
)
|
||||
}
|
||||
|
||||
async fn save_control(&self, object: &str, control: &ReconcileControl, guard: &Arc<NamespaceLockGuard>) -> Result<()> {
|
||||
if guard.is_lock_lost() {
|
||||
return Err(StorageError::SlowDown);
|
||||
}
|
||||
let mut opts = ObjectOptions {
|
||||
max_parity: true,
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
let _ = opts.set_quota_admission(0, u64::MAX);
|
||||
opts.add_owned_write_lock(Arc::clone(guard), RUSTFS_META_BUCKET, object);
|
||||
opts.write_completion = crate::object_api::WriteCompletion::TailDrained;
|
||||
save_config_with_opts(Arc::clone(&self.store), object, serde_json::to_vec(control)?, &opts).await
|
||||
}
|
||||
|
||||
async fn run(&self) -> Result<()> {
|
||||
let control_object = self.control_object();
|
||||
let lock = self.store.new_ns_lock(RUSTFS_META_BUCKET, &control_object).await?;
|
||||
let Ok(guard) = lock.get_write_lock_quiet(Duration::from_millis(50)).await else {
|
||||
return Ok(());
|
||||
};
|
||||
let guard = Arc::new(guard);
|
||||
let now = now_unix();
|
||||
match read_config_no_lock(Arc::clone(&self.store), &control_object).await {
|
||||
Ok(bytes) => {
|
||||
let control: ReconcileControl = serde_json::from_slice(&bytes)?;
|
||||
if control.bucket_incarnation != self.bucket_incarnation
|
||||
|| control.quota_revision_unix_nanos != self.quota_revision.unix_timestamp_nanos()
|
||||
{
|
||||
return Err(StorageError::PartMissingOrCorrupt);
|
||||
}
|
||||
if control.next_attempt_at > now {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
Err(StorageError::ConfigNotFound) => {}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
fence_namespace_mutations(&self.store, RUSTFS_META_BUCKET, &control_object, None).await?;
|
||||
let mut control = ReconcileControl {
|
||||
bucket_incarnation: self.bucket_incarnation,
|
||||
quota_revision_unix_nanos: self.quota_revision.unix_timestamp_nanos(),
|
||||
next_attempt_at: now.saturating_add(RECONCILE_RETRY_SECONDS),
|
||||
};
|
||||
// Persist the throttle before work, including bootstrap and failed scans.
|
||||
self.save_control(&control_object, &control, &guard).await?;
|
||||
let result = self.scan_and_publish(&guard).await;
|
||||
control.next_attempt_at = now_unix().saturating_add(RECONCILE_RETRY_SECONDS);
|
||||
self.save_control(&control_object, &control, &guard).await?;
|
||||
result
|
||||
}
|
||||
|
||||
/// Take a short authoritative configuration snapshot without recursively
|
||||
/// acquiring a reader behind a queued metadata/lifecycle writer.
|
||||
async fn configuration_snapshot(&self) -> Result<Arc<ObjectLockConfigSnapshot>> {
|
||||
let snapshot = self.store.object_lock_config_snapshot_for_single_put(&self.bucket).await?;
|
||||
let guard = snapshot
|
||||
.metadata_transaction_guard_for(self.store.id, &self.bucket, Some(self.bucket_incarnation))
|
||||
.ok_or(StorageError::PreconditionFailed)?;
|
||||
let opts = ObjectOptions {
|
||||
expected_bucket_incarnation_id: Some(self.bucket_incarnation),
|
||||
object_lock_config_snapshot: Some(Arc::clone(&snapshot)),
|
||||
..Default::default()
|
||||
};
|
||||
let (quota, incarnation, revision) =
|
||||
metadata_sys::get_quota_config_and_incarnation_from_disk_for_options_in(&self.store.ctx, &self.bucket, &opts, &guard)
|
||||
.await?;
|
||||
if incarnation != self.bucket_incarnation
|
||||
|| revision != self.quota_revision
|
||||
|| !quota.as_ref().is_some_and(|quota| {
|
||||
quota.uses_durable_reservations() && quota.reservation_protocol != Some(QUOTA_RESERVATION_PROTOCOL_V2)
|
||||
})
|
||||
{
|
||||
return Err(StorageError::PreconditionFailed);
|
||||
}
|
||||
Ok(snapshot)
|
||||
}
|
||||
|
||||
async fn scan_and_publish(&self, owner: &NamespaceLockGuard) -> Result<()> {
|
||||
let ledger_object = ledger_object(&self.bucket);
|
||||
let expected_revision = {
|
||||
// Lock order: reconcile owner, lifecycle, metadata, ledger.
|
||||
// Request paths never wait for the reconcile owner.
|
||||
let _configuration = self.configuration_snapshot().await?;
|
||||
let lock = self.store.new_ns_lock(RUSTFS_META_BUCKET, &ledger_object).await?;
|
||||
let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await?);
|
||||
match load_ledger_locked(Arc::clone(&self.store), &ledger_object).await {
|
||||
Ok(mut ledger) => {
|
||||
if !ledger.reservations.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
// Seed a non-nil token even for a legacy ledger. An old
|
||||
// node drops this field on write and cannot pass the CAS.
|
||||
fence_namespace_mutations(&self.store, RUSTFS_META_BUCKET, &ledger_object, None).await?;
|
||||
save_ledger_locked(Arc::clone(&self.store), &ledger_object, &mut ledger, &guard).await?;
|
||||
Some(ledger.writer_revision)
|
||||
}
|
||||
Err(StorageError::ConfigNotFound) => None,
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
};
|
||||
#[cfg(test)]
|
||||
tests::pause_reconciliation(&self.bucket, tests::ReconcilePauseStage::BeforeScan).await?;
|
||||
// Only the independent reconcile owner is held during this whole-bucket
|
||||
// walk. Concurrent reserve/settle/reap writes invalidate its result.
|
||||
let usage = exact_bucket_usage(&self.store, &self.bucket).await?;
|
||||
#[cfg(test)]
|
||||
tests::pause_reconciliation(&self.bucket, tests::ReconcilePauseStage::BeforePublish).await?;
|
||||
let configuration = self.configuration_snapshot().await?;
|
||||
let lock = self.store.new_ns_lock(RUSTFS_META_BUCKET, &ledger_object).await?;
|
||||
let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await?);
|
||||
let unchanged = match load_ledger_locked(Arc::clone(&self.store), &ledger_object).await {
|
||||
Ok(ledger) => {
|
||||
expected_revision == Some(ledger.writer_revision)
|
||||
&& !ledger.writer_revision.is_nil()
|
||||
&& ledger.reservations.is_empty()
|
||||
}
|
||||
Err(StorageError::ConfigNotFound) => expected_revision.is_none(),
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
if !unchanged {
|
||||
return Ok(());
|
||||
}
|
||||
if owner.is_lock_lost()
|
||||
|| !configuration.is_valid_for_destructive_put(self.store.id, &self.bucket, self.bucket_incarnation)
|
||||
{
|
||||
return Err(StorageError::SlowDown);
|
||||
}
|
||||
fence_namespace_mutations(&self.store, RUSTFS_META_BUCKET, &ledger_object, None).await?;
|
||||
let mut ledger = QuotaLedger::new(self.bucket_incarnation, self.quota_revision, usage);
|
||||
save_ledger_locked(Arc::clone(&self.store), &ledger_object, &mut ledger, &guard).await
|
||||
}
|
||||
ledger.accounted_usage = exact_bucket_usage(store, bucket).await?;
|
||||
ledger.reconcile_required = false;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn exact_bucket_usage(store: &Arc<ECStore>, bucket: &str) -> Result<u64> {
|
||||
@@ -1935,7 +2136,7 @@ async fn load_ledger_locked(store: Arc<ECStore>, ledger_object: &str) -> Result<
|
||||
async fn save_ledger_locked(
|
||||
store: Arc<ECStore>,
|
||||
ledger_object: &str,
|
||||
ledger: &QuotaLedger,
|
||||
ledger: &mut QuotaLedger,
|
||||
ledger_guard: &Arc<NamespaceLockGuard>,
|
||||
) -> Result<()> {
|
||||
if ledger_guard.is_lock_lost() {
|
||||
@@ -1959,9 +2160,71 @@ async fn save_ledger_locked(
|
||||
let _ = opts.set_quota_admission(0, u64::MAX);
|
||||
opts.add_owned_write_lock(Arc::clone(ledger_guard), RUSTFS_META_BUCKET, ledger_object);
|
||||
opts.write_completion = crate::object_api::WriteCompletion::TailDrained;
|
||||
ledger.writer_revision = Uuid::new_v4();
|
||||
save_config_with_opts(store, ledger_object, serde_json::to_vec(ledger)?, &opts).await
|
||||
}
|
||||
|
||||
/// Advance the persisted reconciliation retry deadline for deterministic fixtures.
|
||||
/// The production reaper, ownership fence, scan and compare-and-swap still run.
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub async fn retry_quota_reconciliation_for_test(store: Arc<ECStore>, bucket: &str) -> Result<()> {
|
||||
let (_, incarnation, revision) = store.bucket_quota_and_incarnation_from_disk(bucket).await?;
|
||||
let job = ReconcileJob::new(Arc::clone(&store), bucket.to_owned(), incarnation, revision);
|
||||
let key = (store.id, bucket.to_owned(), incarnation, revision.unix_timestamp_nanos());
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
loop {
|
||||
let active = active_reconciliations()
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.contains(&key);
|
||||
if !active {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| StorageError::SlowDown)?;
|
||||
reap_stale_reservations(Arc::clone(&store), bucket, &ledger_object(bucket)).await?;
|
||||
let object = job.control_object();
|
||||
{
|
||||
let lock = store.new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
|
||||
let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await?);
|
||||
match read_config_no_lock(Arc::clone(&store), &object).await {
|
||||
Ok(bytes) => {
|
||||
let mut control: ReconcileControl = serde_json::from_slice(&bytes)?;
|
||||
control.next_attempt_at = 0;
|
||||
fence_namespace_mutations(&store, RUSTFS_META_BUCKET, &object, None).await?;
|
||||
job.save_control(&object, &control, &guard).await?;
|
||||
}
|
||||
Err(StorageError::ConfigNotFound) => {}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
job.run().await
|
||||
}
|
||||
|
||||
/// Bootstrap a test bucket through the real background reconciliation path.
|
||||
/// Fixtures call this after enabling V1 quota, before issuing writes whose
|
||||
/// contract is independent of the initial retryable admission response.
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub async fn initialize_quota_ledger_for_test(store: Arc<ECStore>, bucket: &str) -> Result<()> {
|
||||
let (quota, incarnation, revision) = store.bucket_quota_and_incarnation_from_disk(bucket).await?;
|
||||
if !quota
|
||||
.as_ref()
|
||||
.is_some_and(|quota| quota.uses_durable_reservations() && !quota.uses_sharded_reservations())
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
ReconcileJob::new(Arc::clone(&store), bucket.to_owned(), incarnation, revision)
|
||||
.run()
|
||||
.await?;
|
||||
match load_current_ledger_locked(store, &ledger_object(bucket), incarnation, revision).await? {
|
||||
Some(_) => Ok(()),
|
||||
None => Err(StorageError::SlowDown),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub fn fail_next_quota_ledger_save_for_test() {
|
||||
@@ -2003,6 +2266,434 @@ mod tests {
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use serial_test::serial;
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) enum ReconcilePauseStage {
|
||||
BeforeScan,
|
||||
BeforePublish,
|
||||
}
|
||||
|
||||
struct ReconcilePause {
|
||||
bucket: String,
|
||||
stage: ReconcilePauseStage,
|
||||
armed: std::sync::atomic::AtomicBool,
|
||||
scans: std::sync::atomic::AtomicUsize,
|
||||
fail_scan: std::sync::atomic::AtomicBool,
|
||||
reached: tokio::sync::Notify,
|
||||
release: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
static RECONCILE_PAUSE: std::sync::Mutex<Option<Arc<ReconcilePause>>> = std::sync::Mutex::new(None);
|
||||
|
||||
struct ReconcilePauseGuard(Arc<ReconcilePause>);
|
||||
|
||||
impl ReconcilePauseGuard {
|
||||
fn install(bucket: &str, stage: ReconcilePauseStage) -> Self {
|
||||
let pause = Arc::new(ReconcilePause {
|
||||
bucket: bucket.to_owned(),
|
||||
stage,
|
||||
armed: true.into(),
|
||||
scans: 0.into(),
|
||||
fail_scan: false.into(),
|
||||
reached: tokio::sync::Notify::new(),
|
||||
release: tokio::sync::Notify::new(),
|
||||
});
|
||||
*RECONCILE_PAUSE.lock().expect("reconcile hooks") = Some(Arc::clone(&pause));
|
||||
Self(pause)
|
||||
}
|
||||
|
||||
async fn reached(&self) {
|
||||
tokio::time::timeout(Duration::from_secs(10), self.0.reached.notified())
|
||||
.await
|
||||
.expect("reconciliation reached barrier");
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ReconcilePauseGuard {
|
||||
fn drop(&mut self) {
|
||||
self.0.release.notify_one();
|
||||
let mut hook = RECONCILE_PAUSE.lock().expect("reconcile hooks");
|
||||
if hook.as_ref().is_some_and(|hook| Arc::ptr_eq(hook, &self.0)) {
|
||||
*hook = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn pause_reconciliation(bucket: &str, stage: ReconcilePauseStage) -> Result<()> {
|
||||
use std::sync::atomic::Ordering;
|
||||
let hook = RECONCILE_PAUSE
|
||||
.lock()
|
||||
.expect("reconcile hooks")
|
||||
.as_ref()
|
||||
.filter(|hook| hook.bucket == bucket)
|
||||
.cloned();
|
||||
if let Some(hook) = hook {
|
||||
if stage == ReconcilePauseStage::BeforeScan {
|
||||
hook.scans.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
if stage == hook.stage && hook.armed.swap(false, Ordering::SeqCst) {
|
||||
hook.reached.notify_one();
|
||||
hook.release.notified().await;
|
||||
}
|
||||
if stage == ReconcilePauseStage::BeforeScan && hook.fail_scan.load(Ordering::SeqCst) {
|
||||
return Err(StorageError::other("injected quota usage failure"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn reconciliation_fixture(limit: u64) -> (Vec<tempfile::TempDir>, ReconcileJob) {
|
||||
use crate::object_api::PutObjReader;
|
||||
use crate::storage_api_contracts::object::ObjectIO;
|
||||
let (directories, store) = metadata_sys::test_support::isolated_store_over_temp_disks().await;
|
||||
metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
crate::services::notification_sys::install_cross_pool_fence_fleet_proof_for_test();
|
||||
let bucket = format!("quota-reconcile-{}", Uuid::new_v4().simple());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket");
|
||||
let mut seed = PutObjReader::from_vec(vec![1; 10]);
|
||||
store
|
||||
.put_object(&bucket, "seed", &mut seed, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("existing usage");
|
||||
store
|
||||
.update_bucket_metadata_config(
|
||||
&bucket,
|
||||
"quota.json",
|
||||
serde_json::to_vec(&super::super::BucketQuota::new(Some(limit))).expect("quota JSON"),
|
||||
)
|
||||
.await
|
||||
.expect("enable quota");
|
||||
let (_, incarnation, revision) = store
|
||||
.bucket_quota_and_incarnation_from_disk(&bucket)
|
||||
.await
|
||||
.expect("quota identity");
|
||||
(directories, ReconcileJob::new(store, bucket, incarnation, revision))
|
||||
}
|
||||
|
||||
async fn quota_reservation(job: &ReconcileJob, old: u64, new: u64) -> Result<QuotaReservation> {
|
||||
begin(&job.store.ctx, &job.bucket, "candidate", &ObjectOptions::default(), 0, 0)
|
||||
.await?
|
||||
.reserve(old, new)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn read_test_ledger(job: &ReconcileJob) -> QuotaLedger {
|
||||
load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket))
|
||||
.await
|
||||
.expect("ledger")
|
||||
}
|
||||
|
||||
async fn drain_reconciliation_scheduler(job: &ReconcileJob) {
|
||||
let key = (
|
||||
job.store.id,
|
||||
job.bucket.clone(),
|
||||
job.bucket_incarnation,
|
||||
job.quota_revision.unix_timestamp_nanos(),
|
||||
);
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
while active_reconciliations().lock().expect("active workers").contains(&key) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("owned reconciliation finished");
|
||||
}
|
||||
|
||||
async fn expire_reconciliation_backoff(job: &ReconcileJob) {
|
||||
drain_reconciliation_scheduler(job).await;
|
||||
let object = job.control_object();
|
||||
let lock = job.store.new_ns_lock(RUSTFS_META_BUCKET, &object).await.expect("owner lock");
|
||||
let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await.expect("owner guard"));
|
||||
let mut control: ReconcileControl =
|
||||
serde_json::from_slice(&read_config_no_lock(Arc::clone(&job.store), &object).await.expect("control"))
|
||||
.expect("control JSON");
|
||||
control.next_attempt_at = 0;
|
||||
fence_namespace_mutations(&job.store, RUSTFS_META_BUCKET, &object, None)
|
||||
.await
|
||||
.expect("control fence");
|
||||
job.save_control(&object, &control, &guard).await.expect("advance test clock");
|
||||
}
|
||||
|
||||
async fn wait_for_reconcile_owner(job: &ReconcileJob) {
|
||||
let object = job.control_object();
|
||||
let lock = job.store.new_ns_lock(RUSTFS_META_BUCKET, &object).await.expect("owner lock");
|
||||
let _guard = tokio::time::timeout(Duration::from_secs(10), lock.get_write_lock(get_lock_acquire_timeout()))
|
||||
.await
|
||||
.expect("worker finished")
|
||||
.expect("owner guard");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_reconciliation_bootstrap_is_single_flight_without_request_guards() {
|
||||
use std::sync::atomic::Ordering;
|
||||
let (_directories, job) = reconciliation_fixture(10).await;
|
||||
let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforeScan);
|
||||
assert!(
|
||||
matches!(quota_reservation(&job, 0, 1).await, Err(StorageError::SlowDown)),
|
||||
"unknown usage cannot admit a reservation"
|
||||
);
|
||||
pause.reached().await;
|
||||
assert!(
|
||||
matches!(
|
||||
load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket)).await,
|
||||
Err(StorageError::ConfigNotFound)
|
||||
),
|
||||
"bootstrap must not publish zero usage"
|
||||
);
|
||||
assert!(matches!(quota_reservation(&job, 0, 1).await, Err(StorageError::SlowDown)));
|
||||
// Bypass the local scheduler, exercising the cross-node ownership boundary.
|
||||
tokio::time::timeout(Duration::from_secs(2), job.run())
|
||||
.await
|
||||
.expect("contender must not wait for a scan")
|
||||
.expect("another owner is active");
|
||||
let lifecycle = tokio::time::timeout(Duration::from_secs(2), job.store.acquire_bucket_lifecycle_write_lock(&job.bucket))
|
||||
.await
|
||||
.expect("scan does not retain lifecycle readers")
|
||||
.expect("lifecycle writer");
|
||||
let metadata = tokio::time::timeout(
|
||||
Duration::from_secs(2),
|
||||
metadata_sys::acquire_bucket_metadata_transaction_lock_in(&job.store.ctx, &job.bucket),
|
||||
)
|
||||
.await
|
||||
.expect("scan does not retain metadata readers")
|
||||
.expect("metadata writer");
|
||||
drop(metadata);
|
||||
drop(lifecycle);
|
||||
pause.0.release.notify_one();
|
||||
wait_for_reconcile_owner(&job).await;
|
||||
let ledger = read_test_ledger(&job).await;
|
||||
assert_eq!(ledger.accounted_usage, 10);
|
||||
assert!(ledger.reservations.is_empty());
|
||||
assert!(!ledger.writer_revision.is_nil());
|
||||
for _ in 0..3 {
|
||||
assert!(matches!(quota_reservation(&job, 0, 1).await, Err(StorageError::QuotaExceeded { .. })));
|
||||
job.run().await.expect("persisted cooldown");
|
||||
}
|
||||
drain_reconciliation_scheduler(&job).await;
|
||||
assert_eq!(
|
||||
pause.0.scans.load(Ordering::SeqCst),
|
||||
1,
|
||||
"denied requests cannot repeatedly scan a full bucket"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_reconciliation_rejects_concurrent_zero_growth_settle_reap_and_legacy_writes() {
|
||||
let (_directories, job) = reconciliation_fixture(100).await;
|
||||
job.run().await.expect("bootstrap");
|
||||
for mutation in ["abort", "commit", "reap", "legacy"] {
|
||||
expire_reconciliation_backoff(&job).await;
|
||||
let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforePublish);
|
||||
let worker = tokio::spawn({
|
||||
let job = job.clone();
|
||||
async move { job.run().await }
|
||||
});
|
||||
pause.reached().await;
|
||||
let initial = read_test_ledger(&job).await.writer_revision;
|
||||
if mutation == "legacy" {
|
||||
let object = ledger_object(&job.bucket);
|
||||
let lock = job.store.new_ns_lock(RUSTFS_META_BUCKET, &object).await.expect("ledger lock");
|
||||
let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await.expect("ledger writer"));
|
||||
let mut document = serde_json::to_value(read_test_ledger(&job).await).expect("ledger JSON");
|
||||
document.as_object_mut().expect("ledger map").remove("writer_revision");
|
||||
fence_namespace_mutations(&job.store, RUSTFS_META_BUCKET, &object, None)
|
||||
.await
|
||||
.expect("ledger fence");
|
||||
let mut opts = ObjectOptions {
|
||||
max_parity: true,
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
let _ = opts.set_quota_admission(0, u64::MAX);
|
||||
opts.add_owned_write_lock(Arc::clone(&guard), RUSTFS_META_BUCKET, &object);
|
||||
opts.write_completion = crate::object_api::WriteCompletion::TailDrained;
|
||||
save_config_with_opts(
|
||||
Arc::clone(&job.store),
|
||||
&object,
|
||||
serde_json::to_vec(&document).expect("old node JSON"),
|
||||
&opts,
|
||||
)
|
||||
.await
|
||||
.expect("old node drops new field");
|
||||
} else {
|
||||
let mut reservation = quota_reservation(&job, 10, 10)
|
||||
.await
|
||||
.expect("zero-growth admission while scanning");
|
||||
let reserved = read_test_ledger(&job).await.writer_revision;
|
||||
assert_ne!(reserved, initial, "even zero-growth reserve must invalidate a scan");
|
||||
reservation.mark_commit_started().await.expect("commit marker");
|
||||
assert_ne!(read_test_ledger(&job).await.writer_revision, reserved);
|
||||
match mutation {
|
||||
"abort" => reservation.abort().await,
|
||||
"commit" => reservation.commit().await,
|
||||
"reap" => {
|
||||
reservation.defer_after_fence();
|
||||
reap_stale_reservations(Arc::clone(&job.store), &job.bucket, &ledger_object(&job.bucket))
|
||||
.await
|
||||
.expect("reap orphan");
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
let written = read_test_ledger(&job).await;
|
||||
assert!(written.reservations.is_empty());
|
||||
assert_ne!(written.writer_revision, initial);
|
||||
pause.0.release.notify_one();
|
||||
worker.await.expect("worker join").expect("stale scan discarded");
|
||||
let current = read_test_ledger(&job).await;
|
||||
assert_eq!(
|
||||
current.writer_revision, written.writer_revision,
|
||||
"{mutation}: stale usage must not replace a concurrent writer"
|
||||
);
|
||||
assert_eq!(current.accounted_usage, written.accounted_usage);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_reconciliation_failure_is_throttled_and_deletions_eventually_reclaim_usage() {
|
||||
use std::sync::atomic::Ordering;
|
||||
let (_directories, job) = reconciliation_fixture(10).await;
|
||||
let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforeScan);
|
||||
pause.0.fail_scan.store(true, Ordering::SeqCst);
|
||||
pause.0.release.notify_one();
|
||||
assert!(job.run().await.is_err());
|
||||
assert!(matches!(
|
||||
load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket)).await,
|
||||
Err(StorageError::ConfigNotFound)
|
||||
));
|
||||
job.run().await.expect("failure cooldown");
|
||||
assert_eq!(pause.0.scans.load(Ordering::SeqCst), 1);
|
||||
pause.0.fail_scan.store(false, Ordering::SeqCst);
|
||||
expire_reconciliation_backoff(&job).await;
|
||||
job.run().await.expect("recover after failure");
|
||||
assert_eq!(read_test_ledger(&job).await.accounted_usage, 10);
|
||||
job.store
|
||||
.delete_object(&job.bucket, "seed", ObjectOptions::default())
|
||||
.await
|
||||
.expect("delete stored usage");
|
||||
assert!(
|
||||
matches!(quota_reservation(&job, 0, 1).await, Err(StorageError::QuotaExceeded { .. })),
|
||||
"stale usage remains conservative during cooldown"
|
||||
);
|
||||
expire_reconciliation_backoff(&job).await;
|
||||
job.run().await.expect("refresh deleted usage");
|
||||
assert_eq!(read_test_ledger(&job).await.accounted_usage, 0);
|
||||
quota_reservation(&job, 0, 1)
|
||||
.await
|
||||
.expect("delete must not leave bucket permanently full")
|
||||
.abort()
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_reconciliation_survives_waiter_cancellation_and_rechecks_configuration() {
|
||||
let (_directories, job) = reconciliation_fixture(100).await;
|
||||
let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforePublish);
|
||||
let caller = tokio::spawn({
|
||||
let job = job.clone();
|
||||
async move {
|
||||
job.schedule();
|
||||
std::future::pending::<()>().await
|
||||
}
|
||||
});
|
||||
pause.reached().await;
|
||||
caller.abort();
|
||||
assert!(caller.await.expect_err("cancel caller").is_cancelled());
|
||||
job.store
|
||||
.update_bucket_metadata_config(
|
||||
&job.bucket,
|
||||
"quota.json",
|
||||
serde_json::to_vec(&super::super::BucketQuota::new(Some(200))).expect("updated quota"),
|
||||
)
|
||||
.await
|
||||
.expect("quota changes while scanning");
|
||||
pause.0.release.notify_one();
|
||||
wait_for_reconcile_owner(&job).await;
|
||||
assert!(
|
||||
matches!(
|
||||
load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket)).await,
|
||||
Err(StorageError::ConfigNotFound)
|
||||
),
|
||||
"old config cannot publish bootstrap usage"
|
||||
);
|
||||
drop(pause);
|
||||
let (_, incarnation, revision) = job
|
||||
.store
|
||||
.bucket_quota_and_incarnation_from_disk(&job.bucket)
|
||||
.await
|
||||
.expect("current configuration");
|
||||
let current = ReconcileJob::new(Arc::clone(&job.store), job.bucket.clone(), incarnation, revision);
|
||||
let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforeScan);
|
||||
let caller = tokio::spawn({
|
||||
let job = current.clone();
|
||||
async move {
|
||||
job.schedule();
|
||||
std::future::pending::<()>().await
|
||||
}
|
||||
});
|
||||
pause.reached().await;
|
||||
caller.abort();
|
||||
assert!(caller.await.expect_err("cancel new caller").is_cancelled());
|
||||
pause.0.release.notify_one();
|
||||
wait_for_reconcile_owner(¤t).await;
|
||||
assert!(
|
||||
read_test_ledger(¤t).await.matches(incarnation, revision),
|
||||
"owned scan publishes after its waiter is canceled"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_reconciliation_cannot_publish_into_recreated_bucket() {
|
||||
use crate::storage_api_contracts::bucket::DeleteBucketOptions;
|
||||
let (_directories, job) = reconciliation_fixture(100).await;
|
||||
let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforePublish);
|
||||
let worker = tokio::spawn({
|
||||
let job = job.clone();
|
||||
async move { job.run().await }
|
||||
});
|
||||
pause.reached().await;
|
||||
job.store
|
||||
.delete_object(&job.bucket, "seed", ObjectOptions::default())
|
||||
.await
|
||||
.expect("empty old bucket");
|
||||
job.store
|
||||
.delete_bucket(&job.bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("retire bucket while scan is paused");
|
||||
job.store
|
||||
.make_bucket(&job.bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("replacement bucket");
|
||||
job.store
|
||||
.update_bucket_metadata_config(
|
||||
&job.bucket,
|
||||
"quota.json",
|
||||
serde_json::to_vec(&super::super::BucketQuota::new(Some(100))).expect("new quota"),
|
||||
)
|
||||
.await
|
||||
.expect("replacement quota");
|
||||
pause.0.release.notify_one();
|
||||
assert!(worker.await.expect("old worker join").is_err(), "old incarnation must not publish");
|
||||
assert!(matches!(
|
||||
load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket)).await,
|
||||
Err(StorageError::ConfigNotFound)
|
||||
));
|
||||
drop(pause);
|
||||
initialize_quota_ledger_for_test(Arc::clone(&job.store), &job.bucket)
|
||||
.await
|
||||
.expect("replacement can bootstrap independently");
|
||||
let replacement = read_test_ledger(&job).await;
|
||||
assert_ne!(replacement.bucket_incarnation, job.bucket_incarnation);
|
||||
assert_eq!(replacement.accounted_usage, 0);
|
||||
}
|
||||
|
||||
fn ledger(accounted_usage: u64) -> QuotaLedger {
|
||||
QuotaLedger::new(Uuid::new_v4(), OffsetDateTime::now_utc(), accounted_usage)
|
||||
}
|
||||
@@ -2090,8 +2781,7 @@ mod tests {
|
||||
.expect("persist crash-left reservation fixture");
|
||||
|
||||
assert!(matches!(
|
||||
load_current_ledger_locked(Arc::clone(&store), &bucket, &ledger_path, new_incarnation, OffsetDateTime::now_utc())
|
||||
.await,
|
||||
load_current_ledger_locked(Arc::clone(&store), &ledger_path, new_incarnation, OffsetDateTime::now_utc()).await,
|
||||
Err(StorageError::PartMissingOrCorrupt)
|
||||
));
|
||||
assert!(matches!(
|
||||
@@ -2109,10 +2799,10 @@ mod tests {
|
||||
Err(StorageError::ConfigNotFound)
|
||||
));
|
||||
let replacement =
|
||||
load_current_ledger_locked(Arc::clone(&store), &bucket, &ledger_path, new_incarnation, OffsetDateTime::now_utc())
|
||||
load_current_ledger_locked(Arc::clone(&store), &ledger_path, new_incarnation, OffsetDateTime::now_utc())
|
||||
.await
|
||||
.expect("new quota admission must not inherit a fresh stale reservation");
|
||||
assert!(replacement.reservations.is_empty());
|
||||
assert!(replacement.is_none(), "replacement must await authoritative bootstrap before admission");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -40,7 +40,9 @@ use aws_smithy_runtime_api::client::http::{
|
||||
HttpConnector as SmithyHttpConnector, HttpConnectorFuture, SharedHttpConnector, http_client_fn,
|
||||
};
|
||||
use aws_smithy_runtime_api::client::interceptors::Intercept;
|
||||
use aws_smithy_runtime_api::client::interceptors::context::BeforeTransmitInterceptorContextMut;
|
||||
use aws_smithy_runtime_api::client::interceptors::context::{
|
||||
BeforeDeserializationInterceptorContextMut, BeforeTransmitInterceptorContextMut,
|
||||
};
|
||||
use aws_smithy_runtime_api::client::orchestrator::{HttpRequest, HttpResponse};
|
||||
use aws_smithy_runtime_api::client::result::ConnectorError;
|
||||
use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents;
|
||||
@@ -155,6 +157,8 @@ pub struct RemoteS3EndpointSpec {
|
||||
pub ca_cert_pem: Option<String>,
|
||||
pub connect_timeout: Option<Duration>,
|
||||
pub read_timeout: Option<Duration>,
|
||||
/// Entire SDK call, including non-streaming response bodies and retries.
|
||||
pub operation_timeout: Option<Duration>,
|
||||
/// How many wire requests one logical call may cost. Every consumer
|
||||
/// declares it; see [`RemoteS3RetryPolicy`].
|
||||
pub retry: RemoteS3RetryPolicy,
|
||||
@@ -252,6 +256,78 @@ pub(crate) fn remote_sdk_credentials(credentials: &RemoteCredentials, now: Syste
|
||||
Ok(builder.build())
|
||||
}
|
||||
|
||||
/// Only installed on non-streaming upload operations. The budget begins at
|
||||
/// response headers, independently of the potentially long request upload.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct UploadResponseDeadline(pub Duration);
|
||||
|
||||
impl Intercept for UploadResponseDeadline {
|
||||
fn name(&self) -> &'static str {
|
||||
"RustfsUploadResponseDeadline"
|
||||
}
|
||||
|
||||
fn modify_before_deserialization(
|
||||
&self,
|
||||
context: &mut BeforeDeserializationInterceptorContextMut<'_>,
|
||||
_runtime_components: &RuntimeComponents,
|
||||
_cfg: &mut ConfigBag,
|
||||
) -> Result<(), BoxError> {
|
||||
let response = context.response_mut();
|
||||
let inner = std::mem::replace(response.body_mut(), SdkBody::taken());
|
||||
*response.body_mut() = SdkBody::from_body_1_x(UploadResponseBody {
|
||||
inner: Some(inner),
|
||||
deadline: Box::pin(tokio::time::sleep(self.0)),
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
struct UploadResponseBody {
|
||||
inner: Option<SdkBody>,
|
||||
deadline: std::pin::Pin<Box<tokio::time::Sleep>>,
|
||||
}
|
||||
|
||||
impl http_body::Body for UploadResponseBody {
|
||||
type Data = bytes::Bytes;
|
||||
type Error = BoxError;
|
||||
|
||||
fn poll_frame(
|
||||
mut self: std::pin::Pin<&mut Self>,
|
||||
cx: &mut std::task::Context<'_>,
|
||||
) -> std::task::Poll<Option<Result<http_body::Frame<Self::Data>, Self::Error>>> {
|
||||
use std::future::Future as _;
|
||||
use std::task::Poll;
|
||||
if self.inner.is_none() {
|
||||
return Poll::Ready(None);
|
||||
}
|
||||
// The absolute response budget never renews on data, empty frames,
|
||||
// or trailers. A retry gets its own bounded response window.
|
||||
if self.deadline.as_mut().poll(cx).is_ready() {
|
||||
self.inner.take();
|
||||
return Poll::Ready(Some(Err(std::io::Error::new(
|
||||
std::io::ErrorKind::TimedOut,
|
||||
"replication response body deadline exceeded",
|
||||
)
|
||||
.into())));
|
||||
}
|
||||
let Some(inner) = self.inner.as_mut() else {
|
||||
return Poll::Ready(None);
|
||||
};
|
||||
let result = std::pin::Pin::new(inner).poll_frame(cx);
|
||||
if matches!(result, Poll::Ready(None) | Poll::Ready(Some(Err(_)))) {
|
||||
self.inner.take();
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn is_end_stream(&self) -> bool {
|
||||
self.inner.as_ref().is_none_or(http_body::Body::is_end_stream)
|
||||
}
|
||||
fn size_hint(&self) -> http_body::SizeHint {
|
||||
self.inner.as_ref().map(http_body::Body::size_hint).unwrap_or_default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Appends a caller-identifying token to the SDK `User-Agent`. Runs after
|
||||
/// signing: SigV4 excludes `user-agent` from the canonical request, so the
|
||||
/// signature stays valid.
|
||||
@@ -309,7 +385,7 @@ pub async fn build_remote_s3_config(spec: &RemoteS3EndpointSpec) -> Result<aws_s
|
||||
config_builder = config_builder.http_client(http_client);
|
||||
}
|
||||
|
||||
if spec.connect_timeout.is_some() || spec.read_timeout.is_some() {
|
||||
if spec.connect_timeout.is_some() || spec.read_timeout.is_some() || spec.operation_timeout.is_some() {
|
||||
let mut timeouts = TimeoutConfig::builder();
|
||||
if let Some(connect_timeout) = spec.connect_timeout {
|
||||
timeouts = timeouts.connect_timeout(connect_timeout);
|
||||
@@ -317,6 +393,9 @@ pub async fn build_remote_s3_config(spec: &RemoteS3EndpointSpec) -> Result<aws_s
|
||||
if let Some(read_timeout) = spec.read_timeout {
|
||||
timeouts = timeouts.read_timeout(read_timeout);
|
||||
}
|
||||
if let Some(operation_timeout) = spec.operation_timeout {
|
||||
timeouts = timeouts.operation_timeout(operation_timeout);
|
||||
}
|
||||
config_builder = config_builder.timeout_config(timeouts.build());
|
||||
}
|
||||
|
||||
@@ -674,6 +753,7 @@ mod tests {
|
||||
ca_cert_pem: None,
|
||||
connect_timeout: None,
|
||||
read_timeout: None,
|
||||
operation_timeout: None,
|
||||
retry: RemoteS3RetryPolicy::Disabled,
|
||||
user_agent_suffix: "",
|
||||
}
|
||||
@@ -1027,11 +1107,16 @@ mod tests {
|
||||
spec.user_agent_suffix = "RustFS-Test/0.0";
|
||||
spec.connect_timeout = Some(Duration::from_secs(5));
|
||||
spec.read_timeout = Some(Duration::from_secs(5));
|
||||
spec.operation_timeout = Some(Duration::from_secs(12));
|
||||
let config = build_remote_s3_config(&spec)
|
||||
.await
|
||||
.expect("spec should build")
|
||||
.http_client(http_client)
|
||||
.build();
|
||||
assert_eq!(
|
||||
config.timeout_config().and_then(TimeoutConfig::operation_timeout),
|
||||
Some(Duration::from_secs(12))
|
||||
);
|
||||
S3Client::from_conf(config)
|
||||
.head_bucket()
|
||||
.bucket("bucket")
|
||||
|
||||
@@ -33,6 +33,7 @@ pub(crate) struct ReplicationMetadataStore;
|
||||
impl ReplicationMetadataStore {
|
||||
pub(crate) const MRF_REPLICATION_FILE: &'static str = "config/replication/mrf.bin";
|
||||
pub(crate) const MRF_REPLICATION_RECOVERY_LOCK: &'static str = "config/replication/mrf.bin.recovery";
|
||||
pub(crate) const MRF_REPLICATION_REPLAY_STAMP: &'static str = "config/replication/mrf.bin.replay";
|
||||
pub(crate) const FORCE_DELETE_REPLICATION_FILE: &'static str = "config/replication/force-delete.bin";
|
||||
pub(crate) const FORCE_DELETE_REPLICATION_TRANSACTION_LOCK: &'static str = "config/replication/force-delete.bin.transaction";
|
||||
|
||||
@@ -116,6 +117,10 @@ mod tests {
|
||||
ReplicationMetadataStore::MRF_REPLICATION_RECOVERY_LOCK,
|
||||
"config/replication/mrf.bin.recovery"
|
||||
);
|
||||
assert_eq!(
|
||||
ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
|
||||
"config/replication/mrf.bin.replay"
|
||||
);
|
||||
assert_eq!(
|
||||
ReplicationMetadataStore::FORCE_DELETE_REPLICATION_FILE,
|
||||
"config/replication/force-delete.bin"
|
||||
|
||||
@@ -16,8 +16,8 @@ use super::replication_config_store::ReplicationConfigStore;
|
||||
use super::replication_error_boundary::{Error as EcstoreError, is_err_object_not_found, is_err_version_not_found};
|
||||
use super::replication_filemeta_boundary::{
|
||||
MrfOpKind, MrfReplicateEntry, REPLICATE_HEAL_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicatedTargetInfo,
|
||||
ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision, replicate_decision_for_admitted_targets,
|
||||
replication_statuses_map, version_purge_statuses_map,
|
||||
ReplicationGenerationSnapshot, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision,
|
||||
replicate_decision_for_admitted_targets, replication_statuses_map, version_purge_statuses_map,
|
||||
};
|
||||
use super::replication_lock_boundary::ReplicationLockTiming;
|
||||
use super::replication_logging::{EVENT_REPLICATION_CONFIG_LOOKUP_SKIPPED, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REPLICATION};
|
||||
@@ -86,6 +86,10 @@ const MRF_CORRUPT_FILE_PREFIX: &str = "config/replication/mrf.corrupt";
|
||||
const MRF_PENDING_CAP: usize = 200_000;
|
||||
const MRF_RETRY_INITIAL_DELAY: Duration = Duration::from_millis(100);
|
||||
const MRF_RETRY_MAX_DELAY: Duration = Duration::from_secs(5);
|
||||
/// Interval between runtime replays of the MRF journal, cluster-wide. A pass
|
||||
/// re-attempts every retained entry, so a target that stays down costs one
|
||||
/// attempt per entry per interval.
|
||||
const MRF_REPLAY_INTERVAL: Duration = Duration::from_secs(300);
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct DurableMrfBacklog {
|
||||
@@ -615,7 +619,7 @@ async fn read_mrf_entries<S: ReplicationObjectIO>(storage: Arc<S>) -> Result<Vec
|
||||
}
|
||||
|
||||
/// Acknowledge only the generation read by the recovery leader. Entries appended
|
||||
/// after that generation are retained as a suffix and replayed on the next startup.
|
||||
/// after that generation are retained as a suffix and replayed on the next pass.
|
||||
/// Lock order: recovery leader lock -> MRF journal object lock.
|
||||
async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
|
||||
storage: Arc<S>,
|
||||
@@ -667,7 +671,7 @@ async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
|
||||
Err(EcstoreError::PreconditionFailed)
|
||||
}
|
||||
|
||||
/// Acquires the MRF recovery leader lock for the startup replay.
|
||||
/// Acquires the MRF recovery leader lock for one journal replay pass.
|
||||
/// Returns `None` (after logging) when the lock cannot be created or another
|
||||
/// node is already processing the backlog.
|
||||
async fn acquire_mrf_recovery_guard<S: ReplicationStorage>(storage: &Arc<S>) -> Option<rustfs_lock::NamespaceLockGuard> {
|
||||
@@ -1018,7 +1022,7 @@ fn admitted_mrf_replicate_object(oi: ObjectInfo, entry: &MrfReplicateEntry, op_t
|
||||
}
|
||||
|
||||
/// Acknowledges the replayed MRF prefix and returns the retained backlog.
|
||||
/// On acknowledgement failure the backlog is preserved for the next startup and
|
||||
/// On acknowledgement failure the backlog is preserved for the next replay pass and
|
||||
/// re-read (falling back to the replayed snapshot) so the published summary stays accurate.
|
||||
async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
|
||||
storage: &Arc<S>,
|
||||
@@ -1033,7 +1037,7 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %error,
|
||||
"Failed to acknowledge the MRF recovery prefix; preserving it for the next startup"
|
||||
"Failed to acknowledge the MRF recovery prefix; preserving it for the next replay pass"
|
||||
);
|
||||
match read_mrf_entries(storage.clone()).await {
|
||||
Ok(current) => current,
|
||||
@@ -1051,6 +1055,120 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
|
||||
}
|
||||
}
|
||||
|
||||
fn unix_millis_now() -> i64 {
|
||||
i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000_000).unwrap_or(i64::MAX)
|
||||
}
|
||||
|
||||
/// Whether a periodic pass may run, given the start time of the last pass on
|
||||
/// any node. A stamp more than one interval ahead of this node's clock is skew
|
||||
/// or a bad value, so it does not suppress replay.
|
||||
fn mrf_replay_due(last_started_ms: Option<i64>, now_ms: i64, interval: Duration) -> bool {
|
||||
let Some(last_started_ms) = last_started_ms else {
|
||||
return true;
|
||||
};
|
||||
let interval_ms = i64::try_from(interval.as_millis()).unwrap_or(i64::MAX);
|
||||
now_ms.saturating_sub(last_started_ms) >= interval_ms || last_started_ms.saturating_sub(now_ms) >= interval_ms
|
||||
}
|
||||
|
||||
async fn read_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>) -> Option<i64> {
|
||||
let data = ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP)
|
||||
.await
|
||||
.ok()?;
|
||||
std::str::from_utf8(&data).ok()?.trim().parse().ok()
|
||||
}
|
||||
|
||||
async fn write_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>, started_ms: i64) {
|
||||
if let Err(error) = ReplicationConfigStore::save(
|
||||
storage.clone(),
|
||||
ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
|
||||
started_ms.to_string().into_bytes(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %error,
|
||||
"Failed to record the MRF replay start time"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// One MRF journal replay pass. The recovery leader reads the journal,
|
||||
/// re-drives every entry, and acknowledges the replayed prefix; entries that
|
||||
/// could not be admitted, and entries appended meanwhile, stay for the next pass.
|
||||
///
|
||||
/// Every node runs the periodic loop, and the leader lock only serializes
|
||||
/// passes. With `min_interval`, a pass is skipped when a pass on any node
|
||||
/// started within that interval, read from the shared replay stamp, so a
|
||||
/// cluster replays once per interval rather than once per node. The startup
|
||||
/// pass passes `None` and always runs. A pass that finds entries records its
|
||||
/// start time. Returns whether the journal was read.
|
||||
/// Lock order: recovery leader lock -> replay stamp object lock.
|
||||
async fn replay_mrf_journal<S: ReplicationStorage>(storage: &Arc<S>, min_interval: Option<Duration>) -> bool {
|
||||
let Some(recovery_guard) = acquire_mrf_recovery_guard(storage).await else {
|
||||
return false;
|
||||
};
|
||||
let started_ms = unix_millis_now();
|
||||
if let Some(interval) = min_interval
|
||||
&& !mrf_replay_due(read_mrf_replay_stamp(storage).await, started_ms, interval)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
let Some(entries) = load_mrf_recovery_entries(storage).await else {
|
||||
return true;
|
||||
};
|
||||
if !entries.is_empty() {
|
||||
write_mrf_replay_stamp(storage, started_ms).await;
|
||||
}
|
||||
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
|
||||
|
||||
let total = entries.len();
|
||||
let mut queued_count = 0usize;
|
||||
let mut retry_entries = Vec::new();
|
||||
|
||||
for entry in entries.iter() {
|
||||
let Some(admission) = replay_mrf_entry(entry, storage, &mut retry_entries).await else {
|
||||
continue;
|
||||
};
|
||||
|
||||
if admission == ReplicationQueueAdmission::Missed {
|
||||
retry_entries.push(entry.clone());
|
||||
} else if admission == ReplicationQueueAdmission::Queued {
|
||||
queued_count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
let retained = resolve_retained_mrf_entries(storage, &recovery_guard, &entries, &retry_entries).await;
|
||||
let retained_count = retained.len();
|
||||
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
|
||||
|
||||
if queued_count > 0 {
|
||||
info!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
recovered = queued_count,
|
||||
total,
|
||||
retained = retained_count,
|
||||
"Replayed MRF entries admitted for retry"
|
||||
);
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// Re-runs [`replay_mrf_journal`] every `period`. The recovery leader lock keeps
|
||||
/// one pass at a time across the cluster, including the startup pass, and the
|
||||
/// shared replay stamp spaces passes from different nodes by `period`.
|
||||
fn spawn_mrf_replay_loop<S: ReplicationStorage>(storage: Arc<S>, period: Duration) -> JoinHandle<()> {
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
tokio::time::sleep(period).await;
|
||||
replay_mrf_journal(&storage, Some(period)).await;
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("replication resync {active_resync_id} is already active for {bucket}/{arn}")]
|
||||
struct ResyncActiveConflictError {
|
||||
@@ -1171,6 +1289,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
// Start background tasks
|
||||
pool.start_mrf_persister().await;
|
||||
pool.start_mrf_processor().await;
|
||||
pool.start_mrf_replayer().await;
|
||||
pool.start_force_delete_processor().await;
|
||||
|
||||
pool
|
||||
@@ -1462,7 +1581,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
|
||||
/// Queues a replica task
|
||||
pub async fn queue_replica_task(&self, ri: ReplicateObjectInfo) -> ReplicationQueueAdmission {
|
||||
// A version that is already queued or being uploaded is not driven a
|
||||
// A source generation already queued or being uploaded is not driven a
|
||||
// second time: the scanner heal pass sees it as PENDING until the
|
||||
// first upload lands and would otherwise re-queue it every cycle
|
||||
// (backlog#2362). The key is released when the worker finishes, or
|
||||
@@ -1639,55 +1758,26 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
|
||||
/// Starts the MRF processor — one-shot at startup.
|
||||
///
|
||||
/// Reads the on-disk MRF file, re-injects admitted entries as Heal operations, and
|
||||
/// rewrites any entries that could not be admitted for a later startup retry.
|
||||
/// Replays the on-disk MRF journal once; [`Self::start_mrf_replayer`] repeats
|
||||
/// the pass while the server runs.
|
||||
async fn start_mrf_processor(&self) {
|
||||
let storage = self.storage.clone();
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
let Some(recovery_guard) = acquire_mrf_recovery_guard(&storage).await else {
|
||||
return;
|
||||
};
|
||||
|
||||
let Some(entries) = load_mrf_recovery_entries(&storage).await else {
|
||||
return;
|
||||
};
|
||||
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
|
||||
|
||||
let total = entries.len();
|
||||
let mut queued_count = 0usize;
|
||||
let mut retry_entries = Vec::new();
|
||||
|
||||
for entry in entries.iter() {
|
||||
let Some(admission) = replay_mrf_entry(entry, &storage, &mut retry_entries).await else {
|
||||
continue;
|
||||
};
|
||||
|
||||
if admission == ReplicationQueueAdmission::Missed {
|
||||
retry_entries.push(entry.clone());
|
||||
} else if admission == ReplicationQueueAdmission::Queued {
|
||||
queued_count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
let retained = resolve_retained_mrf_entries(&storage, &recovery_guard, &entries, &retry_entries).await;
|
||||
let retained_count = retained.len();
|
||||
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
|
||||
|
||||
if queued_count > 0 {
|
||||
info!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
recovered = queued_count,
|
||||
total,
|
||||
retained = retained_count,
|
||||
"Replayed MRF entries admitted for retry"
|
||||
);
|
||||
}
|
||||
replay_mrf_journal(&storage, None).await;
|
||||
});
|
||||
self.task_handles.lock().await.push(handle);
|
||||
}
|
||||
|
||||
/// Starts the runtime MRF replayer — ongoing background task.
|
||||
///
|
||||
/// Entries reach the journal while the server runs (worker-queue overflow,
|
||||
/// delete-marker purge intents whose watch window drained with targets still
|
||||
/// dirty). Replaying only at startup left them waiting for a restart.
|
||||
async fn start_mrf_replayer(&self) {
|
||||
let handle = spawn_mrf_replay_loop(self.storage.clone(), MRF_REPLAY_INTERVAL);
|
||||
self.task_handles.lock().await.push(handle);
|
||||
}
|
||||
|
||||
async fn start_force_delete_processor(&self) {
|
||||
let storage = self.storage.clone();
|
||||
let handle =
|
||||
@@ -1755,7 +1845,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
/// writes them to the on-disk MRF file every flush interval (default 10s,
|
||||
/// overridable via `RUSTFS_REPL_MRF_FLUSH_INTERVAL_MS`) or when 1 000 new
|
||||
/// entries accumulate. Runtime writers append under the journal lock; only the
|
||||
/// startup recovery leader may remove an acknowledged prefix.
|
||||
/// recovery leader of a replay pass may remove an acknowledged prefix.
|
||||
async fn start_mrf_persister(&self) {
|
||||
let Some(mut rx) = self.mrf_save_rx.lock().await.take() else {
|
||||
return;
|
||||
@@ -2314,28 +2404,47 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
}
|
||||
|
||||
/// Object versions currently queued or being uploaded, keyed by bucket,
|
||||
/// object name and version. `queue_replica_task` admits a version only once
|
||||
/// object name, version and source generation. `queue_replica_task` admits a generation only once
|
||||
/// while it is in flight; the scanner heal pass and MRF replays that arrive
|
||||
/// in the meantime are `Skipped` instead of driving a second complete upload
|
||||
/// (backlog#2362). Entries are removed when the worker finishes the task or
|
||||
/// when no worker accepted it.
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct ReplicationInFlight {
|
||||
keys: std::sync::Mutex<std::collections::HashSet<(String, String, Option<uuid::Uuid>)>>,
|
||||
keys: std::sync::Mutex<std::collections::HashSet<ReplicationInFlightKey>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
||||
struct ReplicationInFlightKey {
|
||||
bucket: String,
|
||||
object: String,
|
||||
version_id: Option<uuid::Uuid>,
|
||||
generation: ReplicationGenerationSnapshot,
|
||||
}
|
||||
|
||||
impl From<&ReplicateObjectInfo> for ReplicationInFlightKey {
|
||||
fn from(info: &ReplicateObjectInfo) -> Self {
|
||||
Self {
|
||||
bucket: info.bucket.clone(),
|
||||
object: info.name.clone(),
|
||||
version_id: info.version_id,
|
||||
generation: info.replication_generation.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ReplicationInFlight {
|
||||
fn lock(&self) -> std::sync::MutexGuard<'_, std::collections::HashSet<(String, String, Option<uuid::Uuid>)>> {
|
||||
fn lock(&self) -> std::sync::MutexGuard<'_, std::collections::HashSet<ReplicationInFlightKey>> {
|
||||
self.keys.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
}
|
||||
|
||||
/// Claim `ri`; `false` when the same version is already in flight.
|
||||
/// Claim `ri`; a metadata mutation is independent of its earlier generation.
|
||||
fn try_begin(&self, ri: &ReplicateObjectInfo) -> bool {
|
||||
self.lock().insert((ri.bucket.clone(), ri.name.clone(), ri.version_id))
|
||||
self.lock().insert(ReplicationInFlightKey::from(ri))
|
||||
}
|
||||
|
||||
fn finish(&self, ri: &ReplicateObjectInfo) {
|
||||
self.lock().remove(&(ri.bucket.clone(), ri.name.clone(), ri.version_id));
|
||||
self.lock().remove(&ReplicationInFlightKey::from(ri));
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -2348,26 +2457,21 @@ impl ReplicationInFlight {
|
||||
/// including when replication panics.
|
||||
struct ReplicationInFlightGuard {
|
||||
in_flight: Arc<ReplicationInFlight>,
|
||||
key: ReplicateObjectInfo,
|
||||
key: ReplicationInFlightKey,
|
||||
}
|
||||
|
||||
impl ReplicationInFlightGuard {
|
||||
fn new(in_flight: Arc<ReplicationInFlight>, ri: &ReplicateObjectInfo) -> Self {
|
||||
Self {
|
||||
in_flight,
|
||||
key: ReplicateObjectInfo {
|
||||
bucket: ri.bucket.clone(),
|
||||
name: ri.name.clone(),
|
||||
version_id: ri.version_id,
|
||||
..Default::default()
|
||||
},
|
||||
key: ReplicationInFlightKey::from(ri),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ReplicationInFlightGuard {
|
||||
fn drop(&mut self) {
|
||||
self.in_flight.finish(&self.key);
|
||||
self.in_flight.lock().remove(&self.key);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3421,6 +3525,8 @@ mod tests {
|
||||
allow_first_read: Notify,
|
||||
read_count: AtomicUsize,
|
||||
delete_storage_calls: AtomicUsize,
|
||||
delete_error: StdMutex<EcstoreError>,
|
||||
replay_stamp: StdMutex<Option<Vec<u8>>>,
|
||||
reported_size: StdMutex<Option<i64>>,
|
||||
stream_read_bytes: Arc<AtomicUsize>,
|
||||
write_count: AtomicUsize,
|
||||
@@ -3488,6 +3594,24 @@ mod tests {
|
||||
_h: Self::HeaderMap,
|
||||
_opts: &Self::ObjectOptions,
|
||||
) -> Result<Self::GetObjectReader, Self::Error> {
|
||||
if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
|
||||
let stamp = self
|
||||
.shared
|
||||
.replay_stamp
|
||||
.lock()
|
||||
.expect("test replay stamp lock should not be poisoned")
|
||||
.clone()
|
||||
.ok_or(EcstoreError::FileNotFound)?;
|
||||
return Ok(Self::GetObjectReader {
|
||||
object_info: ObjectInfo {
|
||||
size: i64::try_from(stamp.len()).expect("test stamp length should fit i64"),
|
||||
..Default::default()
|
||||
},
|
||||
stream: Box::new(Cursor::new(stamp)),
|
||||
buffered_body: None,
|
||||
body_source: Default::default(),
|
||||
});
|
||||
}
|
||||
if !object.ends_with("/.replication/resync.bin")
|
||||
&& !object.ends_with("config/replication/mrf.bin")
|
||||
&& !object.ends_with("config/replication/force-delete.bin")
|
||||
@@ -3551,6 +3675,16 @@ mod tests {
|
||||
let lock = self.new_ns_lock(bucket, object).await?;
|
||||
Some(lock.get_write_lock(Duration::from_secs(10)).await?)
|
||||
};
|
||||
if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
|
||||
let mut stamp = Vec::new();
|
||||
data.stream.read_to_end(&mut stamp).await.map_err(EcstoreError::from)?;
|
||||
*self
|
||||
.shared
|
||||
.replay_stamp
|
||||
.lock()
|
||||
.expect("test replay stamp lock should not be poisoned") = Some(stamp);
|
||||
return Ok(ObjectInfo::default());
|
||||
}
|
||||
if opts.http_preconditions.is_some()
|
||||
&& let Some(replacement) = self
|
||||
.shared
|
||||
@@ -3672,7 +3806,12 @@ mod tests {
|
||||
_opts: Self::ObjectOptions,
|
||||
) -> Result<Self::ObjectInfo, Self::Error> {
|
||||
self.shared.delete_storage_calls.fetch_add(1, Ordering::SeqCst);
|
||||
Err(EcstoreError::NotImplemented)
|
||||
Err(self
|
||||
.shared
|
||||
.delete_error
|
||||
.lock()
|
||||
.expect("delete error lock should not be poisoned")
|
||||
.clone())
|
||||
}
|
||||
|
||||
async fn delete_objects(
|
||||
@@ -3929,6 +4068,48 @@ mod tests {
|
||||
assert_eq!(current_queue(&pool, "in-flight-bucket").await, (3, 3 * 4096));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn queue_replica_task_admits_metadata_changes_during_an_initial_upload() {
|
||||
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
|
||||
let (tx, mut rx) = mpsc::channel(4);
|
||||
pool.workers.write().await.push(tx);
|
||||
let initial = ReplicateObjectInfo {
|
||||
bucket: "pending-tags-bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
version_id: Some(uuid::Uuid::new_v4()),
|
||||
size: 4096,
|
||||
replication_generation: ReplicationGenerationSnapshot {
|
||||
mutation_id: Some(uuid::Uuid::new_v4().to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
let metadata = ReplicateObjectInfo {
|
||||
op_type: ReplicationType::Metadata,
|
||||
user_tags: String::new(),
|
||||
replication_generation: ReplicationGenerationSnapshot {
|
||||
mutation_id: Some(uuid::Uuid::new_v4().to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
..initial.clone()
|
||||
};
|
||||
assert_eq!(pool.queue_replica_task(initial.clone()).await, ReplicationQueueAdmission::Queued);
|
||||
assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Queued);
|
||||
assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Skipped);
|
||||
assert_eq!(pool.in_flight.len(), 2);
|
||||
drop(ReplicationInFlightGuard::new(pool.in_flight.clone(), &initial));
|
||||
assert_eq!(pool.in_flight.len(), 1);
|
||||
assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Skipped);
|
||||
for expected in [&initial, &metadata] {
|
||||
let ReplicationOperation::Object(queued) = rx.recv().await.expect("queued operation") else {
|
||||
panic!("expected object replication");
|
||||
};
|
||||
assert_eq!(queued.replication_generation, expected.replication_generation);
|
||||
}
|
||||
drop(ReplicationInFlightGuard::new(pool.in_flight.clone(), &metadata));
|
||||
assert_eq!(pool.in_flight.len(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn queue_replica_task_releases_the_version_when_no_worker_accepts_it() {
|
||||
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
|
||||
@@ -4171,6 +4352,8 @@ mod tests {
|
||||
allow_first_read: Notify::new(),
|
||||
read_count: AtomicUsize::new(0),
|
||||
delete_storage_calls: AtomicUsize::new(0),
|
||||
delete_error: StdMutex::new(EcstoreError::NotImplemented),
|
||||
replay_stamp: StdMutex::new(None),
|
||||
reported_size: StdMutex::new(None),
|
||||
stream_read_bytes: Arc::new(AtomicUsize::new(0)),
|
||||
write_count: AtomicUsize::new(0),
|
||||
@@ -4221,6 +4404,103 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_worker_already_purged_source_completes_without_retry() {
|
||||
let arn = "arn:test";
|
||||
for delete_marker_version in [false, true] {
|
||||
for version_id in [Uuid::new_v4(), Uuid::nil()] {
|
||||
for error in [
|
||||
EcstoreError::VersionNotFound("source".to_string(), "purged".to_string(), version_id.to_string()),
|
||||
EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
|
||||
EcstoreError::FileVersionNotFound,
|
||||
EcstoreError::FileNotFound,
|
||||
] {
|
||||
let shared = empty_resync_shared_state();
|
||||
*shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
|
||||
let storage = Arc::new(LoadResyncNodeStore::new("purged-source", shared.clone()));
|
||||
let delete = DeletedObjectReplicationInfo {
|
||||
bucket: "source".to_string(),
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: "purged".to_string(),
|
||||
version_id: (!delete_marker_version).then_some(version_id),
|
||||
delete_marker_version_id: delete_marker_version.then_some(version_id),
|
||||
replication_state: Some(ReplicationState {
|
||||
replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
|
||||
purge_targets: HashMap::from([(arn.to_string(), VersionPurgeStatusType::Complete)]),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
op_type: ReplicationType::Delete,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(
|
||||
replicate_delete_with_outcome(delete, storage).await,
|
||||
"a confirmed target purge must complete when the source is already absent (marker={delete_marker_version}, version={version_id})"
|
||||
);
|
||||
assert_eq!(
|
||||
shared.delete_storage_calls.load(Ordering::SeqCst),
|
||||
2 + usize::from(!delete_marker_version && !version_id.is_nil()),
|
||||
"the worker must lock and attempt writeback, reading a non-null data version first"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_worker_missing_source_does_not_hide_unfinished_replication() {
|
||||
let arn = "arn:test";
|
||||
for (version_purge, target_status, error) in [
|
||||
(
|
||||
true,
|
||||
VersionPurgeStatusType::Pending,
|
||||
EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
|
||||
),
|
||||
(true, VersionPurgeStatusType::Complete, EcstoreError::ErasureWriteQuorum),
|
||||
(
|
||||
false,
|
||||
VersionPurgeStatusType::Empty,
|
||||
EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
|
||||
),
|
||||
] {
|
||||
let shared = empty_resync_shared_state();
|
||||
*shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
|
||||
let storage = Arc::new(LoadResyncNodeStore::new("unfinished-delete", shared));
|
||||
let delete = DeletedObjectReplicationInfo {
|
||||
bucket: "source".to_string(),
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: "purged".to_string(),
|
||||
version_id: version_purge.then(Uuid::new_v4),
|
||||
delete_marker: !version_purge,
|
||||
replication_state: Some(ReplicationState {
|
||||
replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
|
||||
targets: if version_purge {
|
||||
HashMap::new()
|
||||
} else {
|
||||
HashMap::from([(arn.to_string(), ReplicationStatusType::Completed)])
|
||||
},
|
||||
purge_targets: if version_purge {
|
||||
HashMap::from([(arn.to_string(), target_status)])
|
||||
} else {
|
||||
HashMap::new()
|
||||
},
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
op_type: ReplicationType::Delete,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(
|
||||
!replicate_delete_with_outcome(delete, storage).await,
|
||||
"unfinished target work, source I/O failures, and marker creation must remain retryable"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_worker_preserves_persisted_admission_without_live_config() {
|
||||
let arn = "arn:test";
|
||||
@@ -5304,6 +5584,8 @@ mod tests {
|
||||
allow_first_read: Notify::new(),
|
||||
read_count: AtomicUsize::new(0),
|
||||
delete_storage_calls: AtomicUsize::new(0),
|
||||
delete_error: StdMutex::new(EcstoreError::NotImplemented),
|
||||
replay_stamp: StdMutex::new(None),
|
||||
reported_size: StdMutex::new(None),
|
||||
stream_read_bytes: Arc::new(AtomicUsize::new(0)),
|
||||
write_count: AtomicUsize::new(0),
|
||||
@@ -6268,6 +6550,108 @@ mod tests {
|
||||
assert_eq!(retained.target_arns, entry.target_arns);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mrf_replay_loop_drains_entries_persisted_after_startup() {
|
||||
let shared = empty_resync_shared_state();
|
||||
let storage = Arc::new(LoadResyncNodeStore::new("mrf-runtime-replay", shared.clone()));
|
||||
// A force-delete intent from a superseded generation replays as
|
||||
// Skipped, so a pass that reaches it acknowledges it off the journal.
|
||||
let stale_intent = |object: &str| MrfReplicateEntry {
|
||||
bucket: "mrf-runtime-replay".to_string(),
|
||||
object: object.to_string(),
|
||||
op: MrfOpKind::Delete,
|
||||
force_delete_id: Some(uuid::Uuid::new_v4()),
|
||||
force_delete_local_commit: false,
|
||||
target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
|
||||
..Default::default()
|
||||
};
|
||||
let journal_drained = || async {
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while !shared.data.lock().expect("test data lock should not be poisoned").is_empty() {
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.is_ok()
|
||||
};
|
||||
|
||||
let handle = spawn_mrf_replay_loop(storage, Duration::from_millis(20));
|
||||
for object in ["first", "second"] {
|
||||
*shared.data.lock().expect("test data lock should not be poisoned") =
|
||||
encode_mrf_file(&[stale_intent(object)]).expect("MRF entry should encode");
|
||||
assert!(
|
||||
journal_drained().await,
|
||||
"an MRF entry persisted while the server runs must be replayed without a restart ({object})"
|
||||
);
|
||||
}
|
||||
handle.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mrf_periodic_replay_runs_once_per_interval_across_nodes() {
|
||||
let shared = empty_resync_shared_state();
|
||||
let node_a = Arc::new(LoadResyncNodeStore::new("mrf-node-a", shared.clone()));
|
||||
let node_b = Arc::new(LoadResyncNodeStore::new("mrf-node-b", shared.clone()));
|
||||
let interval = Duration::from_secs(1);
|
||||
let persist_backlog = |object: &str| {
|
||||
let entry = MrfReplicateEntry {
|
||||
bucket: "mrf-node-replay".to_string(),
|
||||
object: object.to_string(),
|
||||
op: MrfOpKind::Delete,
|
||||
force_delete_id: Some(uuid::Uuid::new_v4()),
|
||||
force_delete_local_commit: false,
|
||||
target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
|
||||
..Default::default()
|
||||
};
|
||||
*shared.data.lock().expect("test data lock should not be poisoned") =
|
||||
encode_mrf_file(&[entry]).expect("MRF entry should encode");
|
||||
};
|
||||
let journal_is_empty = || shared.data.lock().expect("test data lock should not be poisoned").is_empty();
|
||||
|
||||
persist_backlog("first");
|
||||
assert!(replay_mrf_journal(&node_a, Some(interval)).await, "node A's periodic pass should replay");
|
||||
assert!(journal_is_empty(), "node A's pass should acknowledge the replayed entry");
|
||||
|
||||
// Node B's timer fires within the same interval with a backlog present.
|
||||
persist_backlog("second");
|
||||
assert!(
|
||||
!replay_mrf_journal(&node_b, Some(interval)).await,
|
||||
"a node must not replay within one interval of another node's pass"
|
||||
);
|
||||
assert!(!journal_is_empty(), "the backlog should wait for the next interval");
|
||||
|
||||
assert!(
|
||||
replay_mrf_journal(&node_b, None).await,
|
||||
"the startup pass should replay regardless of the shared stamp"
|
||||
);
|
||||
assert!(journal_is_empty(), "the startup pass should acknowledge the replayed entry");
|
||||
|
||||
persist_backlog("third");
|
||||
tokio::time::sleep(interval).await;
|
||||
assert!(
|
||||
replay_mrf_journal(&node_a, Some(interval)).await,
|
||||
"a periodic pass should replay once the interval has elapsed"
|
||||
);
|
||||
assert!(journal_is_empty(), "the due pass should acknowledge the replayed entry");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mrf_replay_due_tolerates_missing_and_skewed_stamps() {
|
||||
let interval = Duration::from_secs(300);
|
||||
let now = 1_000_000_000;
|
||||
assert!(mrf_replay_due(None, now, interval));
|
||||
assert!(!mrf_replay_due(Some(now - 299_999), now, interval));
|
||||
assert!(mrf_replay_due(Some(now - 300_000), now, interval));
|
||||
assert!(
|
||||
!mrf_replay_due(Some(now + 1_000), now, interval),
|
||||
"small clock skew should still space passes"
|
||||
);
|
||||
assert!(
|
||||
mrf_replay_due(Some(now + 300_000), now, interval),
|
||||
"a stamp an interval ahead must not suppress replay"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mrf_entry_delete_marker_roundtrip() {
|
||||
let dm_vid = Uuid::new_v4();
|
||||
|
||||
@@ -2809,17 +2809,22 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
true
|
||||
}
|
||||
Err(e) => {
|
||||
error!(
|
||||
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
arn = %dobj.target_arn,
|
||||
object = %dobj.delete_object.object_name,
|
||||
operation = "apply_replication_delete_state",
|
||||
error = %e,
|
||||
"Replication target operation failed"
|
||||
);
|
||||
// A batch purge or another worker may have removed this version
|
||||
// already. Remote failures still control the final outcome below.
|
||||
let source_already_purged = is_version_purge && (is_err_object_not_found(&e) || is_err_version_not_found(&e));
|
||||
if !source_already_purged {
|
||||
error!(
|
||||
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
arn = %dobj.target_arn,
|
||||
object = %dobj.delete_object.object_name,
|
||||
operation = "apply_replication_delete_state",
|
||||
error = %e,
|
||||
"Replication target operation failed"
|
||||
);
|
||||
}
|
||||
send_event(EventArgs {
|
||||
event_name,
|
||||
bucket_name: bucket.clone(),
|
||||
@@ -2832,7 +2837,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
},
|
||||
..Default::default()
|
||||
});
|
||||
false
|
||||
source_already_purged
|
||||
}
|
||||
};
|
||||
|
||||
@@ -3010,7 +3015,7 @@ const DELETE_MARKER_PURGE_WATCH_INTERVAL: TokioDuration = TokioDuration::from_se
|
||||
/// If the marker disappears (deleted before or while the replica landed),
|
||||
/// purge the replicated marker from the targets, retrying failed targets on
|
||||
/// later rounds. When the window drains with targets still dirty, persist the
|
||||
/// purge intent as a durable MRF entry so the next startup replays it through
|
||||
/// purge intent as a durable MRF entry so the next MRF replay pass re-drives it through
|
||||
/// `purge_stale_delete_marker_targets`.
|
||||
async fn watch_and_purge_source_delete_marker<S: ReplicationStorage>(
|
||||
bucket: String,
|
||||
@@ -3101,7 +3106,7 @@ async fn enqueue_delete_marker_purge_mrf(dobj: &DeletedObjectReplicationInfo, fa
|
||||
/// Heal callers retry for the full watch window because the startup MRF
|
||||
/// processor runs before bucket metadata (and thus target clients) finishes
|
||||
/// initializing — the first pass can see `target_client_missing` and a later
|
||||
/// round resolves the client; the replay loop is serial and startup-only, so
|
||||
/// round resolves the client; replay passes are serial background work, so
|
||||
/// blocking it for up to the window per dirty entry is acceptable. Live
|
||||
/// callers run on replication workers where a down target would pin a worker
|
||||
/// for the whole window, so they attempt once and lean on the durable intent
|
||||
@@ -3282,10 +3287,18 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
|
||||
|
||||
let mut join_set = JoinSet::new();
|
||||
let mut all_succeeded = true;
|
||||
let mut target_infos = Vec::with_capacity(tgt_arns.len());
|
||||
|
||||
for arn in tgt_arns {
|
||||
let Some(tgt_client) = ReplicationTargetStore::remote_target_client(bucket, &arn).await else {
|
||||
all_succeeded = false;
|
||||
target_infos.push(force_delete_target_info(
|
||||
dobj.op_type,
|
||||
&arn,
|
||||
"",
|
||||
false,
|
||||
Err(TARGET_CLIENT_UNAVAILABLE_ERROR.to_string()),
|
||||
));
|
||||
debug!(
|
||||
event = EVENT_REPLICATION_FORCE_DELETE_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -3311,6 +3324,7 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
|
||||
|
||||
let bucket = bucket.clone();
|
||||
let object_name = object_name.clone();
|
||||
let op_type = dobj.op_type;
|
||||
|
||||
join_set.spawn(async move {
|
||||
if ReplicationTargetStore::target_is_offline(&tgt_client).await {
|
||||
@@ -3335,9 +3349,16 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
return false;
|
||||
return force_delete_target_info(
|
||||
op_type,
|
||||
&tgt_client.arn,
|
||||
&tgt_client.endpoint,
|
||||
tgt_client.secure,
|
||||
Err(FORCE_DELETE_TARGET_OFFLINE_ERROR.to_string()),
|
||||
);
|
||||
}
|
||||
|
||||
let started = Instant::now();
|
||||
if let Err(e) = tgt_client
|
||||
.remove_object(&tgt_client.bucket, &object_name, None, replication_force_delete_remove_options())
|
||||
.await
|
||||
@@ -3364,16 +3385,25 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
return false;
|
||||
return force_delete_target_info(
|
||||
op_type,
|
||||
&tgt_client.arn,
|
||||
&tgt_client.endpoint,
|
||||
tgt_client.secure,
|
||||
Err(e.to_string()),
|
||||
);
|
||||
}
|
||||
|
||||
true
|
||||
force_delete_target_info(op_type, &tgt_client.arn, &tgt_client.endpoint, tgt_client.secure, Ok(started.elapsed()))
|
||||
});
|
||||
}
|
||||
|
||||
while let Some(result) = join_set.join_next().await {
|
||||
match result {
|
||||
Ok(success) => all_succeeded &= success,
|
||||
Ok(info) => {
|
||||
all_succeeded &= info.replication_status == ReplicationStatusType::Completed;
|
||||
target_infos.push(info);
|
||||
}
|
||||
Err(error) => {
|
||||
all_succeeded = false;
|
||||
error!(
|
||||
@@ -3390,6 +3420,8 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
|
||||
}
|
||||
}
|
||||
|
||||
record_force_delete_target_stats(bucket, &target_infos).await;
|
||||
|
||||
if all_succeeded
|
||||
&& let Some(operation_id) = dobj.delete_object.force_delete_id
|
||||
&& let Err(error) = super::replication_pool::complete_force_delete_intent(storage, operation_id).await
|
||||
@@ -3409,6 +3441,53 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
|
||||
all_succeeded
|
||||
}
|
||||
|
||||
const FORCE_DELETE_TARGET_OFFLINE_ERROR: &str = "replication target is offline";
|
||||
|
||||
/// One target's outcome of a replicated force-delete in the shape the
|
||||
/// replication statistics consume. A force-delete carries no per-version
|
||||
/// replication state, so the previous status is always `Pending`: that is
|
||||
/// the transition `ReplicationStats::update` counts as a failure, whereas an
|
||||
/// `Empty` previous status would drop the outcome and leave `replication
|
||||
/// status` reporting zero failures while the peer still holds the objects.
|
||||
fn force_delete_target_info(
|
||||
op_type: ReplicationType,
|
||||
arn: &str,
|
||||
endpoint: &str,
|
||||
secure: bool,
|
||||
outcome: std::result::Result<std::time::Duration, String>,
|
||||
) -> ReplicatedTargetInfo {
|
||||
let mut rinfo = ReplicatedTargetInfo {
|
||||
arn: arn.to_string(),
|
||||
op_type,
|
||||
endpoint: endpoint.to_string(),
|
||||
secure,
|
||||
prev_replication_status: ReplicationStatusType::Pending,
|
||||
..Default::default()
|
||||
};
|
||||
match outcome {
|
||||
Ok(duration) => {
|
||||
rinfo.replication_status = ReplicationStatusType::Completed;
|
||||
rinfo.duration = duration;
|
||||
}
|
||||
Err(error) => {
|
||||
rinfo.replication_status = ReplicationStatusType::Failed;
|
||||
rinfo.error = Some(error);
|
||||
}
|
||||
}
|
||||
rinfo
|
||||
}
|
||||
|
||||
async fn record_force_delete_target_stats(bucket: &str, targets: &[ReplicatedTargetInfo]) {
|
||||
let Some(stats) = runtime_sources::replication_stats() else {
|
||||
return;
|
||||
};
|
||||
for target in targets {
|
||||
stats
|
||||
.update(bucket, target, target.replication_status.clone(), target.prev_replication_status.clone())
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
fn unavailable_delete_target_info(dobj: &DeletedObjectReplicationInfo, arn: &str) -> ReplicatedTargetInfo {
|
||||
let mut rinfo = dobj
|
||||
.delete_object
|
||||
@@ -6530,6 +6609,7 @@ mod tests {
|
||||
health_check_duration: std::time::Duration::from_secs(5),
|
||||
replicate_sync: false,
|
||||
client: Arc::new(aws_sdk_s3::Client::from_conf(config)),
|
||||
timeouts: Default::default(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -7444,6 +7524,51 @@ mod tests {
|
||||
server.join().expect("test HTTP server should finish");
|
||||
}
|
||||
|
||||
/// A force-delete the peer rejects must reach the replication failure
|
||||
/// counters. A replicated force-delete used to only log the rejection, so
|
||||
/// `replication status` reported zero failures while the sites diverged.
|
||||
#[tokio::test]
|
||||
async fn force_delete_target_outcomes_reach_replication_stats() {
|
||||
let stats = super::super::replication_state::ReplicationStats::new();
|
||||
let denied = force_delete_target_info(
|
||||
ReplicationType::Delete,
|
||||
"arn:rustfs:replication::denied:bucket",
|
||||
"http://peer-a:9000",
|
||||
false,
|
||||
Err("remove_object failed: AccessDenied".to_string()),
|
||||
);
|
||||
assert_eq!(denied.replication_status, ReplicationStatusType::Failed);
|
||||
assert_eq!(denied.prev_replication_status, ReplicationStatusType::Pending);
|
||||
assert_eq!(denied.error.as_deref(), Some("remove_object failed: AccessDenied"));
|
||||
|
||||
let accepted = force_delete_target_info(
|
||||
ReplicationType::Heal,
|
||||
"arn:rustfs:replication::accepted:bucket",
|
||||
"https://peer-b:9000",
|
||||
true,
|
||||
Ok(std::time::Duration::from_millis(3)),
|
||||
);
|
||||
assert_eq!(accepted.replication_status, ReplicationStatusType::Completed);
|
||||
assert!(accepted.error.is_none());
|
||||
|
||||
for target in [&denied, &accepted] {
|
||||
stats
|
||||
.update(
|
||||
"bucket",
|
||||
target,
|
||||
target.replication_status.clone(),
|
||||
target.prev_replication_status.clone(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
let bucket_stats = stats.get("bucket").await;
|
||||
let denied_stat = bucket_stats.stats.get(&denied.arn).expect("denied target: stats entry");
|
||||
assert_eq!((denied_stat.failed.count, denied_stat.replicated_count), (1, 0));
|
||||
let accepted_stat = bucket_stats.stats.get(&accepted.arn).expect("accepted target: stats entry");
|
||||
assert_eq!((accepted_stat.failed.count, accepted_stat.replicated_count), (0, 1));
|
||||
}
|
||||
|
||||
/// AWS answers `HEAD ?versionId=<source marker id>` with a bodiless 400.
|
||||
/// The marker is verified through the latest version instead: a
|
||||
/// versionless 404 with `x-amz-delete-marker: true` proves it propagated.
|
||||
|
||||
@@ -259,6 +259,10 @@ pub fn check_object_name_for_length_and_slash(bucket: &str, object: &str) -> Res
|
||||
pub fn check_new_object_args(bucket: &str, object: &str, data_movement: bool) -> Result<()> {
|
||||
check_put_object_args(bucket, object)?;
|
||||
|
||||
if !data_movement && !is_meta_bucketname(bucket) && has_xml_invalid_control_characters(object) {
|
||||
return Err(StorageError::ObjectNameInvalid(bucket.to_owned(), object.to_owned()));
|
||||
}
|
||||
|
||||
if cfg!(target_os = "windows") && !data_movement && !is_meta_bucketname(bucket) && object.contains('\\') {
|
||||
return Err(StorageError::InvalidArgument(
|
||||
bucket.to_owned(),
|
||||
@@ -270,6 +274,14 @@ pub fn check_new_object_args(bucket: &str, object: &str, data_movement: bool) ->
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// New public keys must remain representable in an unencoded XML listing.
|
||||
/// Existing keys still use the read/delete validators so they can be removed.
|
||||
pub fn has_xml_invalid_control_characters(object: &str) -> bool {
|
||||
object
|
||||
.chars()
|
||||
.any(|character| character < ' ' && !matches!(character, '\t' | '\n' | '\r'))
|
||||
}
|
||||
|
||||
pub fn check_copy_obj_args(bucket: &str, object: &str) -> Result<()> {
|
||||
check_bucket_and_object_names(bucket, object)
|
||||
}
|
||||
|
||||
@@ -34,7 +34,7 @@ use tokio::sync::Mutex as TokioMutex;
|
||||
use tokio::task::JoinSet;
|
||||
use tokio::time::timeout;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, error, warn};
|
||||
use tracing::{Instrument, debug, error, warn};
|
||||
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_METACACHE: &str = "metacache";
|
||||
@@ -224,6 +224,8 @@ pub struct ListPathRawOptions {
|
||||
pub recursive: bool,
|
||||
pub incl_deleted: bool,
|
||||
pub skip_hidden_prefix_check: bool,
|
||||
pub hidden_entry_budget: usize,
|
||||
pub pending_prefix: Option<String>,
|
||||
pub filter_prefix: Option<String>,
|
||||
pub forward_to: Option<String>,
|
||||
pub min_disks: usize,
|
||||
@@ -260,6 +262,8 @@ impl Clone for ListPathRawOptions {
|
||||
recursive: self.recursive,
|
||||
incl_deleted: self.incl_deleted,
|
||||
skip_hidden_prefix_check: self.skip_hidden_prefix_check,
|
||||
hidden_entry_budget: self.hidden_entry_budget,
|
||||
pending_prefix: self.pending_prefix.clone(),
|
||||
filter_prefix: self.filter_prefix.clone(),
|
||||
forward_to: self.forward_to.clone(),
|
||||
min_disks: self.min_disks,
|
||||
@@ -289,6 +293,8 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
|
||||
recursive: opts.recursive,
|
||||
incl_deleted: opts.incl_deleted,
|
||||
skip_hidden_prefix_check: opts.skip_hidden_prefix_check,
|
||||
hidden_entry_budget: opts.hidden_entry_budget,
|
||||
pending_prefix: opts.pending_prefix.clone(),
|
||||
report_notfound: opts.report_not_found,
|
||||
filter_prefix: opts.filter_prefix.clone(),
|
||||
forward_to: opts.forward_to.clone(),
|
||||
@@ -301,13 +307,15 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
|
||||
}
|
||||
|
||||
/// How a merged raw listing stream ended.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct ListPathRawOutcome {
|
||||
/// At least one producer stopped at its per-disk limit instead of at the
|
||||
/// end of its namespace, so the closed stream is not proof of exhaustion.
|
||||
/// Carried in-band by the metacache stream trailer, so remote producers
|
||||
/// report it as well.
|
||||
pub limit_reached: bool,
|
||||
/// Inclusive sorted boundary proved by every reader before a bounded stop.
|
||||
pub scan_cut: Option<String>,
|
||||
}
|
||||
|
||||
pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> disk::error::Result<()> {
|
||||
@@ -363,10 +371,15 @@ async fn list_path_raw_inner(
|
||||
let producer_errs: Arc<[OnceLock<DiskError>]> = (0..opts.disks.len()).map(|_| OnceLock::new()).collect::<Vec<_>>().into();
|
||||
|
||||
let cancel_rx = CancellationToken::new();
|
||||
let walk_options = Arc::new(walk_dir_options(&opts));
|
||||
|
||||
for (disk_idx, disk) in opts.disks.iter().enumerate() {
|
||||
let opdisk = disk.clone();
|
||||
let opts_clone = opts.clone();
|
||||
let walk_options = walk_options.clone();
|
||||
#[cfg(test)]
|
||||
let test_reader_behavior = opts.test_reader_behaviors.get(disk_idx).cloned();
|
||||
#[cfg(test)]
|
||||
let test_disk_io_context_observed = opts.test_disk_io_context_observed.clone();
|
||||
let fallback_claim_tracker = fallback_claim_tracker.clone();
|
||||
let fds_clone = fds.clone();
|
||||
#[cfg(test)]
|
||||
@@ -376,17 +389,17 @@ async fn list_path_raw_inner(
|
||||
let disk_io_context_clone = disk_io_context.clone();
|
||||
let (rd, wr) = tokio::io::duplex(64 * 1024);
|
||||
readers.push(MetacacheReader::new(rd));
|
||||
jobs.spawn(with_optional_disk_io_context(disk_io_context_clone, async move {
|
||||
let producer = with_optional_disk_io_context(disk_io_context_clone, async move {
|
||||
let mut wr = PublishedBytesWriter::new(wr);
|
||||
#[cfg(test)]
|
||||
if let Some(observer) = &opts_clone.test_disk_io_context_observed {
|
||||
if let Some(observer) = &test_disk_io_context_observed {
|
||||
observer.store(
|
||||
crate::disk::current_disk_io_context().is_some_and(|context| context.class == DiskIoClass::Background),
|
||||
std::sync::atomic::Ordering::Release,
|
||||
);
|
||||
}
|
||||
#[cfg(test)]
|
||||
let test_primary_error = if let Some(behavior) = opts_clone.test_reader_behaviors.get(disk_idx).cloned() {
|
||||
let test_primary_error = if let Some(behavior) = test_reader_behavior {
|
||||
match behavior {
|
||||
TestReaderBehavior::Eof => return Ok(()),
|
||||
TestReaderBehavior::Entries(entries) => {
|
||||
@@ -443,8 +456,6 @@ async fn list_path_raw_inner(
|
||||
None
|
||||
};
|
||||
|
||||
let wakl_opts = walk_dir_options(&opts_clone);
|
||||
|
||||
let mut need_fallback = false;
|
||||
let mut last_err = None;
|
||||
#[cfg(test)]
|
||||
@@ -459,7 +470,7 @@ async fn list_path_raw_inner(
|
||||
|
||||
if !need_fallback && let Some(disk) = opdisk {
|
||||
let primary_walk_started = std::time::Instant::now();
|
||||
match disk.walk_dir(wakl_opts, &mut wr).await {
|
||||
match disk.walk_dir((*walk_options).clone(), &mut wr).await {
|
||||
Ok(_res) => {
|
||||
rustfs_io_metrics::record_stage_duration(
|
||||
"metacache_walk_dir_primary",
|
||||
@@ -483,8 +494,8 @@ async fn list_path_raw_inner(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
bucket = %opts_clone.bucket,
|
||||
path = %opts_clone.path,
|
||||
bucket = %walk_options.bucket,
|
||||
path = %walk_options.base_dir,
|
||||
disk_index = disk_idx,
|
||||
state = "walk_dir_missing_path",
|
||||
error = ?err,
|
||||
@@ -495,8 +506,8 @@ async fn list_path_raw_inner(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
bucket = %opts_clone.bucket,
|
||||
path = %opts_clone.path,
|
||||
bucket = %walk_options.bucket,
|
||||
path = %walk_options.base_dir,
|
||||
disk_index = disk_idx,
|
||||
state = "walk_dir_failed",
|
||||
error = ?err,
|
||||
@@ -598,8 +609,8 @@ async fn list_path_raw_inner(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
bucket = %opts_clone.bucket,
|
||||
path = %opts_clone.path,
|
||||
bucket = %walk_options.bucket,
|
||||
path = %walk_options.base_dir,
|
||||
disk_index = disk_idx,
|
||||
state = "fallback_disk_missing_for_path",
|
||||
error = ?err,
|
||||
@@ -610,8 +621,8 @@ async fn list_path_raw_inner(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
bucket = %opts_clone.bucket,
|
||||
path = %opts_clone.path,
|
||||
bucket = %walk_options.bucket,
|
||||
path = %walk_options.base_dir,
|
||||
disk_index = disk_idx,
|
||||
state = "fallback_disk_missing",
|
||||
error = ?err,
|
||||
@@ -626,7 +637,7 @@ async fn list_path_raw_inner(
|
||||
}
|
||||
|
||||
let fallback_walk_started = std::time::Instant::now();
|
||||
match disk.as_ref().walk_dir(walk_dir_options(&opts_clone), &mut wr).await {
|
||||
match disk.as_ref().walk_dir((*walk_options).clone(), &mut wr).await {
|
||||
Ok(_r) => {
|
||||
rustfs_io_metrics::record_stage_duration(
|
||||
"metacache_walk_dir_fallback",
|
||||
@@ -652,8 +663,8 @@ async fn list_path_raw_inner(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
bucket = %opts_clone.bucket,
|
||||
path = %opts_clone.path,
|
||||
bucket = %walk_options.bucket,
|
||||
path = %walk_options.base_dir,
|
||||
disk_index = disk_idx,
|
||||
state = "fallback_walk_dir_missing_path",
|
||||
error = ?err,
|
||||
@@ -664,8 +675,8 @@ async fn list_path_raw_inner(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
bucket = %opts_clone.bucket,
|
||||
path = %opts_clone.path,
|
||||
bucket = %walk_options.bucket,
|
||||
path = %walk_options.base_dir,
|
||||
disk_index = disk_idx,
|
||||
state = "fallback_walk_dir_failed",
|
||||
error = ?err,
|
||||
@@ -687,12 +698,13 @@ async fn list_path_raw_inner(
|
||||
|
||||
// warn!("list_path_raw: while need_fallback done");
|
||||
Ok(())
|
||||
}));
|
||||
});
|
||||
jobs.spawn(producer.in_current_span());
|
||||
}
|
||||
|
||||
let revjob_rx = rx.clone();
|
||||
let mut revjobs = JoinSet::new();
|
||||
revjobs.spawn(async move {
|
||||
let merge = async move {
|
||||
// Consumer-side peek timeout, and a caveat worth understanding
|
||||
// (rustfs/backlog#1217).
|
||||
//
|
||||
@@ -722,6 +734,8 @@ async fn list_path_raw_inner(
|
||||
// Sticky across rounds: once any reader's trailer says its producer
|
||||
// stopped at the per-disk limit, the merged stream may hide more keys.
|
||||
let mut limit_reached = false;
|
||||
let mut scan_cut = None;
|
||||
let mut last_scanned_key: Option<String> = None;
|
||||
|
||||
loop {
|
||||
let mut current = MetaCacheEntry::default();
|
||||
@@ -733,7 +747,7 @@ async fn list_path_raw_inner(
|
||||
// );
|
||||
|
||||
if revjob_rx.is_cancelled() {
|
||||
return Ok(ListPathRawOutcome { limit_reached });
|
||||
return Ok(ListPathRawOutcome { limit_reached, scan_cut });
|
||||
}
|
||||
|
||||
let mut top_entries: Vec<Option<MetaCacheEntry>> = vec![None; readers.len()];
|
||||
@@ -747,6 +761,16 @@ async fn list_path_raw_inner(
|
||||
// Start every missing head read in the same round so one stalled
|
||||
// disk cannot multiply the wait budget by the erasure-set width.
|
||||
// Outcomes are still consumed below in stable disk-index order.
|
||||
if !revjob_rx.is_cancelled() {
|
||||
for (i, reader) in readers.iter_mut().enumerate() {
|
||||
if errs[i].is_none()
|
||||
&& pending_entries[i].is_none()
|
||||
&& let Some(Ok(Some(entry))) = reader.try_peek_buffered()
|
||||
{
|
||||
pending_entries[i] = Some(entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
let concurrent_peeks = readers.iter_mut().enumerate().filter_map(|(i, reader)| {
|
||||
if errs[i].is_some() || pending_entries[i].is_some() {
|
||||
return None;
|
||||
@@ -783,7 +807,12 @@ async fn list_path_raw_inner(
|
||||
continue;
|
||||
}
|
||||
// eof
|
||||
limit_reached |= r.limit_reached() == Some(true);
|
||||
if r.limit_reached() == Some(true) {
|
||||
limit_reached = true;
|
||||
scan_cut = Some(last_scanned_key.clone().ok_or_else(|| {
|
||||
DiskError::other("bounded listing reader ended without a scan boundary")
|
||||
})?);
|
||||
}
|
||||
at_eof += 1;
|
||||
// warn!("list_path_raw: peek eof, disk: {}", i);
|
||||
continue;
|
||||
@@ -857,7 +886,7 @@ async fn list_path_raw_inner(
|
||||
*r = MetacacheReader::new(detached_rd);
|
||||
continue;
|
||||
}
|
||||
PeekOutcome::Cancelled => return Ok(ListPathRawOutcome { limit_reached }),
|
||||
PeekOutcome::Cancelled => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
|
||||
}
|
||||
};
|
||||
|
||||
@@ -878,7 +907,7 @@ async fn list_path_raw_inner(
|
||||
}
|
||||
if entry.name == current.name {
|
||||
// If exact match, we agree.
|
||||
if current.matches_strict(&entry) {
|
||||
if current.matches_strict_borrowed(&entry) {
|
||||
top_entries[i] = Some(entry);
|
||||
agree += 1;
|
||||
} else {
|
||||
@@ -916,7 +945,7 @@ async fn list_path_raw_inner(
|
||||
if let Some(finished_fn) = opts.finished.as_ref() {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
|
||||
_ = finished_fn(&errs) => {}
|
||||
}
|
||||
}
|
||||
@@ -966,6 +995,14 @@ async fn list_path_raw_inner(
|
||||
return Err(classify_listing_quorum_failure(&failures));
|
||||
}
|
||||
|
||||
// The next head from a bounded reader is unknown. The previous
|
||||
// globally consumed group is a safe cut; consuming another reader's
|
||||
// later head would turn this absence of evidence into a skipped key.
|
||||
// Check quorum failures above before accepting the bounded result.
|
||||
if limit_reached {
|
||||
break;
|
||||
}
|
||||
|
||||
// Break if all at EOF or error.
|
||||
if at_eof + has_err == readers.len() {
|
||||
if has_err > 0
|
||||
@@ -973,7 +1010,7 @@ async fn list_path_raw_inner(
|
||||
{
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
|
||||
_ = finished_fn(&errs) => {}
|
||||
}
|
||||
}
|
||||
@@ -984,16 +1021,19 @@ async fn list_path_raw_inner(
|
||||
break;
|
||||
}
|
||||
|
||||
last_scanned_key = Some(current.name.clone());
|
||||
if agree == readers.len() && !opts.preserve_replica_metadata {
|
||||
for r in readers.iter_mut() {
|
||||
let _ = r.skip(1).await;
|
||||
if !r.skip_current() {
|
||||
let _ = r.skip(1).await;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(agreed_fn) = opts.agreed.as_ref() {
|
||||
// warn!("list_path_raw: agreed_fn start, current: {:?}", ¤t.name);
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
|
||||
result = agreed_fn(current) => result?
|
||||
}
|
||||
// warn!("list_path_raw: agreed_fn done");
|
||||
@@ -1009,7 +1049,7 @@ async fn list_path_raw_inner(
|
||||
}
|
||||
|
||||
for (i, r) in readers.iter_mut().enumerate() {
|
||||
if top_entries[i].is_some() {
|
||||
if top_entries[i].is_some() && !r.skip_current() {
|
||||
let _ = r.skip(1).await;
|
||||
}
|
||||
}
|
||||
@@ -1017,13 +1057,14 @@ async fn list_path_raw_inner(
|
||||
if let Some(partial_fn) = opts.partial.as_ref() {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
|
||||
_ = partial_fn(MetaCacheEntries(top_entries), &errs) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(ListPathRawOutcome { limit_reached })
|
||||
});
|
||||
Ok(ListPathRawOutcome { limit_reached, scan_cut })
|
||||
};
|
||||
revjobs.spawn(merge.in_current_span());
|
||||
|
||||
let merge_started = std::time::Instant::now();
|
||||
let Some(revjob) = revjobs.join_next().await else {
|
||||
@@ -1187,6 +1228,24 @@ mod tests {
|
||||
assert!(!options.skip_total_timeout);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cloned_reader_options_preserve_hidden_scan_budget_and_pending_prefix() {
|
||||
let request = ListPathRawOptions {
|
||||
bucket: "versioned".to_owned(),
|
||||
hidden_entry_budget: 100,
|
||||
pending_prefix: Some("pending/".to_owned()),
|
||||
forward_to: Some("pending/0099".to_owned()),
|
||||
..Default::default()
|
||||
};
|
||||
// Primary and fallback readers both construct their walk from the
|
||||
// per-job clone, rather than directly from the request options.
|
||||
let job = request.clone();
|
||||
let walk = walk_dir_options(&job);
|
||||
assert_eq!(walk.hidden_entry_budget, 100);
|
||||
assert_eq!(walk.pending_prefix.as_deref(), request.pending_prefix.as_deref());
|
||||
assert_eq!(walk.forward_to.as_deref(), Some("pending/0099"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_path_raw_empty_disks_returns_read_quorum() {
|
||||
let err = list_path_raw(CancellationToken::new(), ListPathRawOptions::default())
|
||||
@@ -1237,6 +1296,163 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_path_raw_fanout_retains_only_linear_disk_references() {
|
||||
let root = tempfile::tempdir().expect("create disk root");
|
||||
let endpoint = crate::disk::endpoint::Endpoint::try_from(root.path().to_str().expect("UTF-8 disk path"))
|
||||
.expect("parse disk endpoint");
|
||||
let local = Arc::new(
|
||||
crate::disk::local::LocalDisk::new(&endpoint, false)
|
||||
.await
|
||||
.expect("open local disk"),
|
||||
);
|
||||
let wrapper = crate::disk::disk_store::LocalDiskWrapper::new(local, false);
|
||||
let disk = Arc::new(crate::disk::Disk::Local(Box::new(wrapper)));
|
||||
let baseline = Arc::strong_count(&disk);
|
||||
|
||||
for primary_count in [1, 4, 16] {
|
||||
for fallback_count in [0, 2] {
|
||||
let entered: Vec<_> = (0..primary_count).map(|_| Arc::new(tokio::sync::Notify::new())).collect();
|
||||
let stopped: Vec<_> = (0..primary_count).map(|_| Arc::new(tokio::sync::Notify::new())).collect();
|
||||
let behaviors = entered
|
||||
.iter()
|
||||
.zip(&stopped)
|
||||
.map(|(entered, stopped)| TestReaderBehavior::StallWithSignals {
|
||||
entered: entered.clone(),
|
||||
stopped: stopped.clone(),
|
||||
})
|
||||
.collect();
|
||||
let cancel = CancellationToken::new();
|
||||
let listing = tokio::spawn(list_path_raw(
|
||||
cancel.clone(),
|
||||
ListPathRawOptions {
|
||||
disks: vec![Some(disk.clone()); primary_count],
|
||||
fallback_disks: vec![Some(disk.clone()); fallback_count],
|
||||
min_disks: primary_count,
|
||||
test_reader_behaviors: behaviors,
|
||||
peek_timeout: Some(Duration::from_secs(60)),
|
||||
..Default::default()
|
||||
},
|
||||
));
|
||||
for signal in &entered {
|
||||
timeout(Duration::from_secs(2), signal.notified())
|
||||
.await
|
||||
.expect("every producer must enter the real fanout before counting references");
|
||||
}
|
||||
|
||||
// The merge owns the configured primary/fallback slots, each producer
|
||||
// owns its primary, and the shared fallback queue owns each fallback.
|
||||
let expected = baseline + 2 * (primary_count + fallback_count);
|
||||
let retained = Arc::strong_count(&disk);
|
||||
cancel.cancel();
|
||||
timeout(Duration::from_secs(2), listing)
|
||||
.await
|
||||
.expect("cancellation must stop the listing")
|
||||
.expect("listing task must join")
|
||||
.expect("external cancellation must finish cleanly");
|
||||
for signal in &stopped {
|
||||
timeout(Duration::from_secs(2), signal.notified())
|
||||
.await
|
||||
.expect("cancellation must drop every producer");
|
||||
}
|
||||
assert_eq!(Arc::strong_count(&disk), baseline, "cancellation must release every fanout reference");
|
||||
assert_eq!(
|
||||
retained, expected,
|
||||
"fanout must retain linear references for {primary_count} primary and {fallback_count} fallback slots"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn list_path_raw_producer_and_merge_preserve_request_span() {
|
||||
use tracing_subscriber::{Layer, layer::SubscriberExt};
|
||||
|
||||
struct ProducerSpan(Arc<AtomicBool>);
|
||||
#[derive(Default)]
|
||||
struct StateField(bool);
|
||||
impl tracing::field::Visit for StateField {
|
||||
fn record_debug(&mut self, _: &tracing::field::Field, _: &dyn std::fmt::Debug) {}
|
||||
|
||||
fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
|
||||
self.0 |= field.name() == "state" && value == "walk_dir_missing_path";
|
||||
}
|
||||
}
|
||||
impl<S> Layer<S> for ProducerSpan
|
||||
where
|
||||
S: tracing::Subscriber + for<'a> tracing_subscriber::registry::LookupSpan<'a>,
|
||||
{
|
||||
fn on_event(&self, event: &tracing::Event<'_>, context: tracing_subscriber::layer::Context<'_, S>) {
|
||||
let mut state = StateField::default();
|
||||
event.record(&mut state);
|
||||
if state.0
|
||||
&& context
|
||||
.lookup_current()
|
||||
.is_some_and(|span| span.name() == "flink_list_request")
|
||||
{
|
||||
self.0.store(true, std::sync::atomic::Ordering::Release);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The subscriber is thread-local; keep the spawned tasks on this runtime thread.
|
||||
let producer_observed = Arc::new(AtomicBool::new(false));
|
||||
let subscriber = tracing_subscriber::registry().with(ProducerSpan(producer_observed.clone()));
|
||||
let _subscriber = tracing::subscriber::set_default(subscriber);
|
||||
let request = tracing::info_span!("flink_list_request", request_id = "list-span-regression");
|
||||
let root = tempfile::tempdir().expect("create disk root");
|
||||
let endpoint = crate::disk::endpoint::Endpoint::try_from(root.path().to_str().expect("UTF-8 disk path"))
|
||||
.expect("parse disk endpoint");
|
||||
let local = Arc::new(
|
||||
crate::disk::local::LocalDisk::new(&endpoint, false)
|
||||
.await
|
||||
.expect("open local disk"),
|
||||
);
|
||||
let wrapper = crate::disk::disk_store::LocalDiskWrapper::new(local, false);
|
||||
let disk = Arc::new(crate::disk::Disk::Local(Box::new(wrapper)));
|
||||
let error = list_path_raw(
|
||||
CancellationToken::new(),
|
||||
ListPathRawOptions {
|
||||
disks: vec![Some(disk)],
|
||||
bucket: "missing-bucket".to_owned(),
|
||||
min_disks: 1,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.instrument(request.clone())
|
||||
.await
|
||||
.expect_err("missing bucket must fail the real disk producer");
|
||||
assert_eq!(error, DiskError::VolumeNotFound);
|
||||
assert!(
|
||||
producer_observed.load(std::sync::atomic::Ordering::Acquire),
|
||||
"disk error must retain the request span"
|
||||
);
|
||||
|
||||
let merge_observed = Arc::new(AtomicBool::new(false));
|
||||
let merge_sink = merge_observed.clone();
|
||||
let request_id = request.id();
|
||||
list_path_raw(
|
||||
CancellationToken::new(),
|
||||
ListPathRawOptions {
|
||||
disks: vec![None],
|
||||
min_disks: 1,
|
||||
test_reader_behaviors: vec![TestReaderBehavior::Entries(vec![fallback_test_entry()])],
|
||||
agreed: Some(Box::new(move |_| {
|
||||
merge_sink.store(tracing::Span::current().id() == request_id, std::sync::atomic::Ordering::Release);
|
||||
Box::pin(async { Ok(()) })
|
||||
})),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.instrument(request)
|
||||
.await
|
||||
.expect("merge should deliver the test entry");
|
||||
assert!(
|
||||
merge_observed.load(std::sync::atomic::Ordering::Acquire),
|
||||
"merge callback must retain the request span"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_joinset_walk_uses_the_local_disk_background_gate() {
|
||||
let root = tempfile::tempdir().expect("local disk root should be created");
|
||||
@@ -1304,6 +1520,7 @@ mod tests {
|
||||
outcome.limit_reached,
|
||||
"a reader whose trailer reports a limit stop must mark the merged stream as possibly incomplete"
|
||||
);
|
||||
assert_eq!(outcome.scan_cut.as_deref(), Some("bucket/object"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -0,0 +1,682 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use crate::disk::background_memory::BackgroundBuffer;
|
||||
use std::io::{self, IoSlice};
|
||||
use std::pin::Pin;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::task::{Context, Poll, Waker, ready};
|
||||
use tokio::io::{AsyncRead, AsyncWrite, DuplexStream, ReadBuf};
|
||||
|
||||
pub struct BackgroundPipeReader {
|
||||
inner: PipeEnd,
|
||||
}
|
||||
|
||||
pub struct BackgroundPipeWriter {
|
||||
inner: PipeEnd,
|
||||
}
|
||||
|
||||
impl BackgroundPipeWriter {
|
||||
/// Capture only the payload owner, without retaining node or physical units.
|
||||
pub fn memory_budget(&self) -> Option<Arc<crate::disk::BackgroundMemoryBudget>> {
|
||||
match &self.inner {
|
||||
PipeEnd::Foreground(_) => None,
|
||||
PipeEnd::Background(state) => state
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.buffer
|
||||
.memory_budget(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
enum PipeEnd {
|
||||
Foreground(DuplexStream),
|
||||
Background(Arc<Mutex<PipeState>>),
|
||||
}
|
||||
|
||||
// The pipe lock may reserve memory. The memory budget never locks a pipe.
|
||||
struct PipeState {
|
||||
buffer: BackgroundBuffer,
|
||||
read_offset: usize,
|
||||
readable: usize,
|
||||
capacity: usize,
|
||||
reader_closed: bool,
|
||||
writer_closed: bool,
|
||||
admission_failure: Option<io::ErrorKind>,
|
||||
read_waker: Option<Waker>,
|
||||
write_waker: Option<Waker>,
|
||||
}
|
||||
|
||||
/// Capture only memory ownership while the request is admitted. Payload storage
|
||||
/// is reserved and initialized once, before the first nonempty background write.
|
||||
pub fn background_pipe(capacity: usize) -> (BackgroundPipeReader, BackgroundPipeWriter) {
|
||||
let capacity = capacity.max(1);
|
||||
let memory = crate::disk::current_resource_unit().and_then(|resources| resources.memory_budget());
|
||||
match memory {
|
||||
Some(memory) => {
|
||||
let state = Arc::new(Mutex::new(PipeState {
|
||||
buffer: BackgroundBuffer::new(Some(memory)),
|
||||
read_offset: 0,
|
||||
readable: 0,
|
||||
capacity,
|
||||
reader_closed: false,
|
||||
writer_closed: false,
|
||||
admission_failure: None,
|
||||
read_waker: None,
|
||||
write_waker: None,
|
||||
}));
|
||||
(
|
||||
BackgroundPipeReader {
|
||||
inner: PipeEnd::Background(state.clone()),
|
||||
},
|
||||
BackgroundPipeWriter {
|
||||
inner: PipeEnd::Background(state),
|
||||
},
|
||||
)
|
||||
}
|
||||
None => {
|
||||
let (reader, writer) = tokio::io::duplex(capacity);
|
||||
(
|
||||
BackgroundPipeReader {
|
||||
inner: PipeEnd::Foreground(reader),
|
||||
},
|
||||
BackgroundPipeWriter {
|
||||
inner: PipeEnd::Foreground(writer),
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn advance_ring(offset: usize, count: usize, capacity: usize) -> usize {
|
||||
let to_end = capacity - offset;
|
||||
if count < to_end { offset + count } else { count - to_end }
|
||||
}
|
||||
|
||||
fn poll_background_read(state: &Mutex<PipeState>, cx: &mut Context<'_>, output: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
|
||||
if output.remaining() == 0 {
|
||||
return Poll::Ready(Ok(()));
|
||||
}
|
||||
let mut state = state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if state.readable == 0 {
|
||||
if let Some(kind) = state.admission_failure.take() {
|
||||
drop(state);
|
||||
return Poll::Ready(Err(io::Error::new(kind, "background pipe storage admission failed")));
|
||||
}
|
||||
if state.writer_closed {
|
||||
return Poll::Ready(Ok(()));
|
||||
}
|
||||
let replaced = state.read_waker.replace(cx.waker().clone());
|
||||
drop(state);
|
||||
drop(replaced);
|
||||
return Poll::Pending;
|
||||
}
|
||||
let length = state.readable.min(output.remaining());
|
||||
let first = length.min(state.capacity - state.read_offset);
|
||||
output.put_slice(&state.buffer.as_slice()[state.read_offset..state.read_offset + first]);
|
||||
output.put_slice(&state.buffer.as_slice()[..length - first]);
|
||||
state.read_offset = advance_ring(state.read_offset, length, state.capacity);
|
||||
state.readable -= length;
|
||||
let wake = state.write_waker.take();
|
||||
drop(state);
|
||||
if let Some(wake) = wake {
|
||||
wake.wake();
|
||||
}
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
|
||||
fn poll_background_write(state: &Mutex<PipeState>, cx: &mut Context<'_>, input: &[IoSlice<'_>]) -> Poll<io::Result<usize>> {
|
||||
let mut state = state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if state.reader_closed || state.writer_closed {
|
||||
return Poll::Ready(Err(io::ErrorKind::BrokenPipe.into()));
|
||||
}
|
||||
if input.iter().all(|part| part.is_empty()) {
|
||||
return Poll::Ready(Ok(0));
|
||||
}
|
||||
let available = state.capacity - state.readable;
|
||||
if available == 0 {
|
||||
let replaced = state.write_waker.replace(cx.waker().clone());
|
||||
drop(state);
|
||||
drop(replaced);
|
||||
return Poll::Pending;
|
||||
}
|
||||
let capacity = state.capacity;
|
||||
if let Err(error) = state.buffer.ensure_len(capacity) {
|
||||
// Legacy walk completion may suppress ordinary producer errors. A new
|
||||
// pipe admission failure must also reach the reader instead of EOF.
|
||||
state.admission_failure = Some(error.kind());
|
||||
state.writer_closed = true;
|
||||
let wake = state.read_waker.take();
|
||||
drop(state);
|
||||
if let Some(wake) = wake {
|
||||
wake.wake();
|
||||
}
|
||||
return Poll::Ready(Err(error));
|
||||
}
|
||||
let mut offset = advance_ring(state.read_offset, state.readable, capacity);
|
||||
let mut written = 0;
|
||||
for part in input {
|
||||
let length = part.len().min(available - written);
|
||||
let first = length.min(capacity - offset);
|
||||
state.buffer.as_mut_slice()[offset..offset + first].copy_from_slice(&part[..first]);
|
||||
state.buffer.as_mut_slice()[..length - first].copy_from_slice(&part[first..length]);
|
||||
offset = advance_ring(offset, length, capacity);
|
||||
written += length;
|
||||
if written == available {
|
||||
break;
|
||||
}
|
||||
}
|
||||
state.readable += written;
|
||||
let wake = state.read_waker.take();
|
||||
drop(state);
|
||||
if let Some(wake) = wake {
|
||||
wake.wake();
|
||||
}
|
||||
Poll::Ready(Ok(written))
|
||||
}
|
||||
|
||||
impl AsyncRead for BackgroundPipeReader {
|
||||
fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, output: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
|
||||
match &mut self.get_mut().inner {
|
||||
PipeEnd::Foreground(reader) => Pin::new(reader).poll_read(cx, output),
|
||||
PipeEnd::Background(state) => {
|
||||
let coop = ready!(tokio::task::coop::poll_proceed(cx));
|
||||
let result = poll_background_read(state, cx, output);
|
||||
if result.is_ready() {
|
||||
coop.made_progress();
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncWrite for BackgroundPipeWriter {
|
||||
fn poll_write(self: Pin<&mut Self>, cx: &mut Context<'_>, input: &[u8]) -> Poll<io::Result<usize>> {
|
||||
match &mut self.get_mut().inner {
|
||||
PipeEnd::Foreground(writer) => Pin::new(writer).poll_write(cx, input),
|
||||
PipeEnd::Background(state) => {
|
||||
let coop = ready!(tokio::task::coop::poll_proceed(cx));
|
||||
let result = poll_background_write(state, cx, &[IoSlice::new(input)]);
|
||||
if result.is_ready() {
|
||||
coop.made_progress();
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_write_vectored(self: Pin<&mut Self>, cx: &mut Context<'_>, input: &[IoSlice<'_>]) -> Poll<io::Result<usize>> {
|
||||
match &mut self.get_mut().inner {
|
||||
PipeEnd::Foreground(writer) => Pin::new(writer).poll_write_vectored(cx, input),
|
||||
PipeEnd::Background(state) => {
|
||||
let coop = ready!(tokio::task::coop::poll_proceed(cx));
|
||||
let result = poll_background_write(state, cx, input);
|
||||
if result.is_ready() {
|
||||
coop.made_progress();
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn is_write_vectored(&self) -> bool {
|
||||
match &self.inner {
|
||||
PipeEnd::Foreground(writer) => writer.is_write_vectored(),
|
||||
PipeEnd::Background(_) => true,
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
match &mut self.get_mut().inner {
|
||||
PipeEnd::Foreground(writer) => Pin::new(writer).poll_flush(cx),
|
||||
PipeEnd::Background(_) => Poll::Ready(Ok(())),
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
match &mut self.get_mut().inner {
|
||||
PipeEnd::Foreground(writer) => Pin::new(writer).poll_shutdown(cx),
|
||||
PipeEnd::Background(state) => {
|
||||
let mut state = state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
state.writer_closed = true;
|
||||
let wake = state.read_waker.take();
|
||||
drop(state);
|
||||
if let Some(wake) = wake {
|
||||
wake.wake();
|
||||
}
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for BackgroundPipeReader {
|
||||
fn drop(&mut self) {
|
||||
if let PipeEnd::Background(state) = &self.inner {
|
||||
let mut state = state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
state.reader_closed = true;
|
||||
let wake = state.write_waker.take();
|
||||
let cancelled = state.read_waker.take();
|
||||
drop(state);
|
||||
drop(cancelled);
|
||||
if let Some(wake) = wake {
|
||||
wake.wake();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for BackgroundPipeWriter {
|
||||
fn drop(&mut self) {
|
||||
if let PipeEnd::Background(state) = &self.inner {
|
||||
let mut state = state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
state.writer_closed = true;
|
||||
let wake = state.read_waker.take();
|
||||
let cancelled = state.write_waker.take();
|
||||
drop(state);
|
||||
drop(cancelled);
|
||||
if let Some(wake) = wake {
|
||||
wake.wake();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::disk::BackgroundMemoryBudget;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::Duration;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
|
||||
pub(super) fn budget(capacity: usize) -> Arc<BackgroundMemoryBudget> {
|
||||
let budget = BackgroundMemoryBudget::new();
|
||||
budget.set_sampled_limit(u64::try_from(capacity).expect("test capacity fits u64"), Duration::from_secs(30));
|
||||
budget
|
||||
}
|
||||
|
||||
pub(super) fn local_pipe(
|
||||
memory: &Arc<BackgroundMemoryBudget>,
|
||||
capacity: usize,
|
||||
) -> (BackgroundPipeReader, BackgroundPipeWriter) {
|
||||
let state = Arc::new(Mutex::new(PipeState {
|
||||
buffer: BackgroundBuffer::new(Some(memory.clone())),
|
||||
read_offset: 0,
|
||||
readable: 0,
|
||||
capacity,
|
||||
reader_closed: false,
|
||||
writer_closed: false,
|
||||
admission_failure: None,
|
||||
read_waker: None,
|
||||
write_waker: None,
|
||||
}));
|
||||
(
|
||||
BackgroundPipeReader {
|
||||
inner: PipeEnd::Background(state.clone()),
|
||||
},
|
||||
BackgroundPipeWriter {
|
||||
inner: PipeEnd::Background(state),
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn write_now(writer: &mut (impl AsyncWrite + Unpin), bytes: &[u8]) -> Poll<io::Result<usize>> {
|
||||
Pin::new(writer).poll_write(&mut Context::from_waker(Waker::noop()), bytes)
|
||||
}
|
||||
|
||||
pub(super) fn read_now(reader: &mut (impl AsyncRead + Unpin), bytes: &mut [u8]) -> Poll<io::Result<usize>> {
|
||||
let mut output = ReadBuf::new(bytes);
|
||||
match Pin::new(reader).poll_read(&mut Context::from_waker(Waker::noop()), &mut output) {
|
||||
Poll::Ready(Ok(())) => Poll::Ready(Ok(output.filled().len())),
|
||||
Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
|
||||
Poll::Pending => Poll::Pending,
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn fixed_ring_allocates_lazily_once_and_keeps_last_owner_charged() {
|
||||
for capacity in [1, 7, 64 * 1024, 1024 * 1024] {
|
||||
let memory = budget(capacity);
|
||||
let (mut reader, mut writer) = local_pipe(&memory, capacity);
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
assert!(matches!(write_now(&mut writer, &[]), Poll::Ready(Ok(0))));
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
assert!(matches!(write_now(&mut writer, b"x"), Poll::Ready(Ok(1))));
|
||||
assert_eq!(memory.snapshot().used, capacity as u64);
|
||||
memory.set_sampled_limit(0, Duration::ZERO);
|
||||
let mut output = [0];
|
||||
assert!(matches!(read_now(&mut reader, &mut output), Poll::Ready(Ok(1))));
|
||||
assert_eq!(output, *b"x");
|
||||
assert!(matches!(write_now(&mut writer, b"y"), Poll::Ready(Ok(1))));
|
||||
drop(writer);
|
||||
assert_eq!(memory.snapshot().used, capacity as u64, "reader still owns the real ring");
|
||||
assert!(matches!(read_now(&mut reader, &mut output), Poll::Ready(Ok(1))));
|
||||
assert_eq!(output, *b"y");
|
||||
assert!(matches!(read_now(&mut reader, &mut output), Poll::Ready(Ok(0))));
|
||||
assert_eq!(memory.snapshot().used, capacity as u64, "draining does not destroy allocation ownership");
|
||||
drop(reader);
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn admission_error_reaches_writer_and_reader_once_without_payload() {
|
||||
for sample in [None, Some((6, Duration::from_secs(30))), Some((7, Duration::ZERO))] {
|
||||
let memory = BackgroundMemoryBudget::new();
|
||||
if let Some((limit, validity)) = sample {
|
||||
memory.set_sampled_limit(limit, validity);
|
||||
}
|
||||
let (mut reader, mut writer) = local_pipe(&memory, 7);
|
||||
let error = match write_now(&mut writer, b"x") {
|
||||
Poll::Ready(Err(e)) => e,
|
||||
_ => panic!("reject before copying first payload"),
|
||||
};
|
||||
assert!(
|
||||
crate::disk::background_memory::is_memory_pressure(&error),
|
||||
"writer retains the original budget marker"
|
||||
);
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
let error = match read_now(&mut reader, &mut [0]) {
|
||||
Poll::Ready(Err(e)) => e,
|
||||
_ => panic!("reader must see a pipe admission failure"),
|
||||
};
|
||||
assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
|
||||
assert!(matches!(read_now(&mut reader, &mut [0]), Poll::Ready(Ok(0))), "failure is delivered once");
|
||||
assert!(matches!(write_now(&mut writer,b"x"),Poll::Ready(Err(e)) if e.kind()==io::ErrorKind::BrokenPipe));
|
||||
drop((reader, writer));
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn allocation_overflow_is_a_reader_failure_without_a_leaked_reservation() {
|
||||
let memory = BackgroundMemoryBudget::new();
|
||||
memory.set_sampled_limit(u64::MAX, Duration::from_secs(30));
|
||||
let (mut reader, mut writer) = local_pipe(&memory, usize::MAX);
|
||||
assert!(matches!(write_now(&mut writer,b"x"),Poll::Ready(Err(e)) if e.kind()==io::ErrorKind::OutOfMemory));
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
assert!(matches!(read_now(&mut reader,&mut[0]),Poll::Ready(Err(e)) if e.kind()==io::ErrorKind::OutOfMemory));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scalar_and_vectored_wrap_match_a_reference_queue() {
|
||||
use std::collections::VecDeque;
|
||||
for capacity in [1, 3, 7, 31] {
|
||||
let memory = budget(capacity);
|
||||
let (mut reader, mut writer) = local_pipe(&memory, capacity);
|
||||
let mut expected = VecDeque::new();
|
||||
let mut next = 0u8;
|
||||
for turn in 0..300 {
|
||||
let input: Vec<_> = (0..(turn % 11 + 1))
|
||||
.map(|_| {
|
||||
let byte = next;
|
||||
next = next.wrapping_add(1);
|
||||
byte
|
||||
})
|
||||
.collect();
|
||||
let available = capacity - expected.len();
|
||||
let written = if turn % 2 == 0 {
|
||||
write_now(&mut writer, &input)
|
||||
} else {
|
||||
let split = input.len() / 2;
|
||||
Pin::new(&mut writer).poll_write_vectored(
|
||||
&mut Context::from_waker(Waker::noop()),
|
||||
&[
|
||||
io::IoSlice::new(&[]),
|
||||
io::IoSlice::new(&input[..split]),
|
||||
io::IoSlice::new(&input[split..]),
|
||||
],
|
||||
)
|
||||
};
|
||||
if available == 0 {
|
||||
assert!(written.is_pending());
|
||||
} else {
|
||||
let accepted = input.len().min(available);
|
||||
assert!(matches!(written,Poll::Ready(Ok(n)) if n==accepted));
|
||||
expected.extend(&input[..accepted]);
|
||||
}
|
||||
let mut output = vec![0; turn % 5 + 1];
|
||||
let count = output.len().min(expected.len());
|
||||
assert!(matches!(read_now(&mut reader,&mut output),Poll::Ready(Ok(n)) if n==count));
|
||||
for byte in &output[..count] {
|
||||
assert_eq!(Some(*byte), expected.pop_front());
|
||||
}
|
||||
assert_eq!(memory.snapshot().used, capacity as u64);
|
||||
}
|
||||
drop(writer);
|
||||
let mut output = vec![0; capacity];
|
||||
let count = expected.len();
|
||||
assert!(matches!(read_now(&mut reader,&mut output),Poll::Ready(Ok(n)) if n==count));
|
||||
assert_eq!(&output[..count], expected.into_iter().collect::<Vec<_>>().as_slice());
|
||||
assert!(matches!(read_now(&mut reader, &mut output), Poll::Ready(Ok(0))));
|
||||
drop(reader);
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reader_drop_wakes_blocked_writer_and_writer_drop_wakes_empty_reader() {
|
||||
let memory = budget(7);
|
||||
let (reader, mut writer) = local_pipe(&memory, 7);
|
||||
writer.write_all(b"1234567").await.expect("fill the ring");
|
||||
let task = tokio::spawn(async move { writer.write_all(b"blocked").await });
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!task.is_finished());
|
||||
drop(reader);
|
||||
let error = tokio::time::timeout(Duration::from_secs(1), task)
|
||||
.await
|
||||
.expect("reader drop wakes writer")
|
||||
.expect("writer task joins")
|
||||
.expect_err("closed reader is BrokenPipe");
|
||||
assert_eq!(error.kind(), io::ErrorKind::BrokenPipe);
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
let (mut reader, writer) = local_pipe(&memory, 7);
|
||||
let task = tokio::spawn(async move { reader.read(&mut [0]).await });
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!task.is_finished());
|
||||
drop(writer);
|
||||
assert_eq!(
|
||||
tokio::time::timeout(Duration::from_secs(1), task)
|
||||
.await
|
||||
.expect("writer drop wakes reader")
|
||||
.expect("reader task joins")
|
||||
.expect("clean EOF"),
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_wakes_a_full_writer_without_changing_byte_limit() {
|
||||
let memory = budget(7);
|
||||
let (mut reader, mut writer) = local_pipe(&memory, 7);
|
||||
writer.write_all(b"1234567").await.expect("fill initial byte limit");
|
||||
let task = tokio::spawn(async move {
|
||||
writer.write_all(b"x").await.expect("resume after one byte is read");
|
||||
writer
|
||||
});
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!task.is_finished());
|
||||
let mut first = [0];
|
||||
reader.read_exact(&mut first).await.expect("read one byte");
|
||||
let writer = tokio::time::timeout(Duration::from_secs(1), task)
|
||||
.await
|
||||
.expect("read wakes writer")
|
||||
.expect("writer joins");
|
||||
assert_eq!(&first, b"1");
|
||||
drop(writer);
|
||||
let mut all = Vec::new();
|
||||
reader.read_to_end(&mut all).await.expect("drain wrapped bytes");
|
||||
assert_eq!(all, b"234567x");
|
||||
drop(reader);
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shutdown_drains_then_eof_and_cancelled_writes_accept_nothing() {
|
||||
let memory = budget(7);
|
||||
let (mut reader, mut writer) = local_pipe(&memory, 7);
|
||||
writer
|
||||
.write_all(b"1234567")
|
||||
.await
|
||||
.expect("fill before cancelling a blocked write");
|
||||
{
|
||||
let blocked = writer.write(b"lost");
|
||||
tokio::pin!(blocked);
|
||||
assert!(futures::poll!(&mut blocked).is_pending());
|
||||
}
|
||||
writer.shutdown().await.expect("shutdown does not discard buffered bytes");
|
||||
assert!(matches!(write_now(&mut writer,b"x"),Poll::Ready(Err(e)) if e.kind()==io::ErrorKind::BrokenPipe));
|
||||
let mut out = Vec::new();
|
||||
reader.read_to_end(&mut out).await.expect("drain after shutdown");
|
||||
assert_eq!(out, b"1234567");
|
||||
drop(reader);
|
||||
assert_eq!(memory.snapshot().used, 7, "writer still holds the allocation");
|
||||
drop(writer);
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn foreground_keeps_native_duplex_semantics() {
|
||||
let (reader, mut writer) = background_pipe(7);
|
||||
assert!(matches!(write_now(&mut writer, b"12345678"), Poll::Ready(Ok(7))));
|
||||
assert!(write_now(&mut writer, b"x").is_pending());
|
||||
drop(reader);
|
||||
assert!(matches!(write_now(&mut writer,b"x"),Poll::Ready(Err(e)) if e.kind()==io::ErrorKind::BrokenPipe));
|
||||
let (mut reader, writer) = background_pipe(7);
|
||||
drop(writer);
|
||||
assert!(matches!(read_now(&mut reader, &mut [0]), Poll::Ready(Ok(0))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancelled_reader_leaves_the_real_ring_charged_until_writer_drop() {
|
||||
let memory = budget(7);
|
||||
let (reader, mut writer) = local_pipe(&memory, 7);
|
||||
assert!(matches!(write_now(&mut writer, b"x"), Poll::Ready(Ok(1))));
|
||||
drop(reader);
|
||||
assert_eq!(memory.snapshot().used, 7, "the surviving writer still owns the ring allocation");
|
||||
assert!(matches!(write_now(&mut writer,b"y"),Poll::Ready(Err(e)) if e.kind()==io::ErrorKind::BrokenPipe));
|
||||
drop(writer);
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_shutdown_releases_queued_writer_but_not_the_surviving_reader() {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("build isolated runtime");
|
||||
let memory = budget(7);
|
||||
let (reader, mut writer) = local_pipe(&memory, 7);
|
||||
runtime.block_on(async {
|
||||
writer.write_all(b"x").await.expect("allocate ring before runtime shutdown");
|
||||
tokio::spawn(async move {
|
||||
let _writer = writer;
|
||||
std::future::pending::<()>().await;
|
||||
});
|
||||
tokio::task::yield_now().await;
|
||||
});
|
||||
drop(runtime);
|
||||
assert_eq!(memory.snapshot().used, 7, "reader holds the actual allocation after runtime cancellation");
|
||||
drop(reader);
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ready_operations_keep_tokio_cooperative_yields() {
|
||||
let memory = budget(7);
|
||||
let (_reader, mut writer) = local_pipe(&memory, 7);
|
||||
let observed = Arc::new(AtomicUsize::new(0));
|
||||
let flag = observed.clone();
|
||||
let task = tokio::spawn(async move {
|
||||
flag.store(1, Ordering::SeqCst);
|
||||
});
|
||||
for _ in 0..1000 {
|
||||
assert_eq!(writer.write(&[]).await.expect("empty operation remains successful"), 0);
|
||||
}
|
||||
assert_eq!(
|
||||
observed.load(Ordering::SeqCst),
|
||||
1,
|
||||
"ready pipe operations must not starve other ready tasks"
|
||||
);
|
||||
task.await.expect("observer task joins");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod waker_tests {
|
||||
use super::*;
|
||||
use std::sync::{
|
||||
Weak,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
};
|
||||
use std::task::Wake;
|
||||
|
||||
struct LockCheck {
|
||||
state: Weak<Mutex<PipeState>>,
|
||||
wakes: Arc<AtomicUsize>,
|
||||
drops: Arc<AtomicUsize>,
|
||||
}
|
||||
impl LockCheck {
|
||||
fn unlocked(&self) {
|
||||
if let Some(state) = self.state.upgrade() {
|
||||
assert!(state.try_lock().is_ok(), "waker code must run outside the pipe lock");
|
||||
}
|
||||
}
|
||||
}
|
||||
impl Wake for LockCheck {
|
||||
fn wake(self: Arc<Self>) {
|
||||
self.unlocked();
|
||||
self.wakes.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
impl Drop for LockCheck {
|
||||
fn drop(&mut self) {
|
||||
self.unlocked();
|
||||
self.drops.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replaced_and_woken_reader_and_writer_wakers_run_outside_the_lock() {
|
||||
for waiting_reader in [true, false] {
|
||||
let memory = super::tests::budget(7);
|
||||
let (mut reader, mut writer) = super::tests::local_pipe(&memory, 7);
|
||||
let PipeEnd::Background(state) = &reader.inner else { panic!("background fixture") };
|
||||
let weak = Arc::downgrade(state);
|
||||
let wakes = Arc::new(AtomicUsize::new(0));
|
||||
let drops = Arc::new(AtomicUsize::new(0));
|
||||
if !waiting_reader {
|
||||
assert!(matches!(super::tests::write_now(&mut writer, b"1234567"), Poll::Ready(Ok(7))));
|
||||
}
|
||||
for _ in 0..2 {
|
||||
let waker = Waker::from(Arc::new(LockCheck {
|
||||
state: weak.clone(),
|
||||
wakes: wakes.clone(),
|
||||
drops: drops.clone(),
|
||||
}));
|
||||
let mut cx = Context::from_waker(&waker);
|
||||
if waiting_reader {
|
||||
let mut bytes = [0];
|
||||
assert!(
|
||||
Pin::new(&mut reader)
|
||||
.poll_read(&mut cx, &mut ReadBuf::new(&mut bytes))
|
||||
.is_pending()
|
||||
);
|
||||
} else {
|
||||
assert!(Pin::new(&mut writer).poll_write(&mut cx, b"x").is_pending());
|
||||
}
|
||||
drop(waker);
|
||||
}
|
||||
assert_eq!(drops.load(Ordering::SeqCst), 1, "the replaced waker has been dropped");
|
||||
if waiting_reader {
|
||||
assert!(matches!(super::tests::write_now(&mut writer, b"x"), Poll::Ready(Ok(1))));
|
||||
} else {
|
||||
assert!(matches!(super::tests::read_now(&mut reader, &mut [0]), Poll::Ready(Ok(1))));
|
||||
}
|
||||
assert_eq!(wakes.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(drops.load(Ordering::SeqCst), 2);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,586 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use crate::disk::background_memory::{BackgroundMemoryBudget, Reservation};
|
||||
use bytes::{Bytes, BytesMut};
|
||||
use futures::Stream;
|
||||
use std::io;
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use std::task::{Context, Poll};
|
||||
use tokio::io::{AsyncRead, ReadBuf};
|
||||
use tokio_util::io::ReaderStream;
|
||||
|
||||
struct ReservedReadBytes {
|
||||
// Free each payload view before releasing its shared allocation receipt.
|
||||
bytes: Bytes,
|
||||
_reservation: Arc<Reservation>,
|
||||
}
|
||||
|
||||
impl AsRef<[u8]> for ReservedReadBytes {
|
||||
fn as_ref(&self) -> &[u8] {
|
||||
&self.bytes
|
||||
}
|
||||
}
|
||||
|
||||
struct ReaderBuffer {
|
||||
// These initialized bytes are spare storage, not unconsumed source data.
|
||||
bytes: BytesMut,
|
||||
reservation: Option<Arc<Reservation>>,
|
||||
budget: Arc<BackgroundMemoryBudget>,
|
||||
capacity: usize,
|
||||
}
|
||||
|
||||
impl ReaderBuffer {
|
||||
fn new(budget: Arc<BackgroundMemoryBudget>, capacity: usize) -> Self {
|
||||
Self {
|
||||
bytes: BytesMut::new(),
|
||||
reservation: None,
|
||||
budget,
|
||||
capacity: capacity.max(1),
|
||||
}
|
||||
}
|
||||
|
||||
fn clear(&mut self) {
|
||||
self.bytes = BytesMut::new();
|
||||
self.reservation = None;
|
||||
}
|
||||
|
||||
fn ensure_readable(&mut self) -> io::Result<()> {
|
||||
if !self.bytes.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
if self.reservation.is_some() {
|
||||
if self.bytes.try_reclaim(self.capacity) {
|
||||
self.bytes.resize(self.capacity, 0);
|
||||
return Ok(());
|
||||
}
|
||||
// A live output owns this allocation. Drop our empty view before
|
||||
// its receipt, then reserve independent replacement storage.
|
||||
self.clear();
|
||||
}
|
||||
let reservation = self.budget.reserve(self.capacity)?;
|
||||
let mut bytes = Vec::new();
|
||||
bytes
|
||||
.try_reserve_exact(self.capacity)
|
||||
.map_err(|_| io::ErrorKind::OutOfMemory)?;
|
||||
bytes.resize(self.capacity, 0);
|
||||
// Fresh ordinary Bytes is unique and spans the complete allocation.
|
||||
// Unlike from_owner output, this conversion cannot require a copy.
|
||||
let bytes = Bytes::from(bytes.into_boxed_slice())
|
||||
.try_into_mut()
|
||||
.map_err(|_| io::Error::other("fresh reader storage was not unique"))?;
|
||||
self.bytes = bytes;
|
||||
self.reservation = Some(Arc::new(reservation));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn take_bytes(&mut self, length: usize) -> io::Result<Bytes> {
|
||||
let reservation = self
|
||||
.reservation
|
||||
.as_ref()
|
||||
.cloned()
|
||||
.ok_or_else(|| io::Error::other("reader storage has no memory reservation"))?;
|
||||
Ok(Bytes::from_owner(ReservedReadBytes {
|
||||
bytes: self.bytes.split_to(length).freeze(),
|
||||
_reservation: reservation,
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
/// RPC response chunks retain their background memory reservation until the
|
||||
/// last clone or slice is dropped. Foreground readers use the existing stream.
|
||||
pub struct BackgroundReaderStream<R> {
|
||||
inner: ReaderState<R>,
|
||||
}
|
||||
|
||||
enum ReaderState<R> {
|
||||
Foreground(ReaderStream<R>),
|
||||
Background { reader: Option<R>, buffer: ReaderBuffer },
|
||||
}
|
||||
|
||||
impl<R: AsyncRead> BackgroundReaderStream<R> {
|
||||
/// Capture memory ownership inside the admitted scope, before moving the
|
||||
/// response body out of it. This stream retains no node or device capacity.
|
||||
pub fn with_capacity(reader: R, capacity: usize) -> Self {
|
||||
let memory = crate::disk::current_resource_unit().and_then(|resources| resources.memory_budget());
|
||||
let inner = match memory {
|
||||
Some(memory) => ReaderState::Background {
|
||||
reader: Some(reader),
|
||||
buffer: ReaderBuffer::new(memory, capacity),
|
||||
},
|
||||
None => ReaderState::Foreground(ReaderStream::with_capacity(reader, capacity)),
|
||||
};
|
||||
Self { inner }
|
||||
}
|
||||
}
|
||||
|
||||
impl<R: AsyncRead + Unpin> Stream for BackgroundReaderStream<R> {
|
||||
type Item = io::Result<Bytes>;
|
||||
|
||||
fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
||||
let this = self.get_mut();
|
||||
match &mut this.inner {
|
||||
ReaderState::Foreground(reader) => Pin::new(reader).poll_next(cx),
|
||||
ReaderState::Background { reader, buffer } => {
|
||||
let Some(inner) = reader.as_mut() else {
|
||||
return Poll::Ready(None);
|
||||
};
|
||||
if let Err(error) = buffer.ensure_readable() {
|
||||
*reader = None;
|
||||
return Poll::Ready(Some(Err(error)));
|
||||
}
|
||||
let mut output = ReadBuf::new(&mut buffer.bytes);
|
||||
match std::task::ready!(Pin::new(inner).poll_read(cx, &mut output)) {
|
||||
Ok(()) => {
|
||||
let length = output.filled().len();
|
||||
if length == 0 {
|
||||
buffer.clear();
|
||||
*reader = None;
|
||||
Poll::Ready(None)
|
||||
} else {
|
||||
let bytes = buffer.take_bytes(length);
|
||||
if bytes.is_err() {
|
||||
buffer.clear();
|
||||
*reader = None;
|
||||
}
|
||||
Poll::Ready(Some(bytes))
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
buffer.clear();
|
||||
*reader = None;
|
||||
Poll::Ready(Some(Err(error)))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::disk::{BackgroundMemoryBudget, test_resource_units, with_optional_resource_unit};
|
||||
use futures::StreamExt;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::Duration;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
struct ObservedReader {
|
||||
polls: Arc<AtomicUsize>,
|
||||
fail: bool,
|
||||
}
|
||||
|
||||
impl AsyncRead for ObservedReader {
|
||||
fn poll_read(self: Pin<&mut Self>, _cx: &mut Context<'_>, output: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
|
||||
self.polls.fetch_add(1, Ordering::SeqCst);
|
||||
if self.fail {
|
||||
Poll::Ready(Err(io::Error::from(io::ErrorKind::PermissionDenied)))
|
||||
} else {
|
||||
output.put_slice(b"data");
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn background_frames_keep_capacity_until_the_last_slice() {
|
||||
let memory = BackgroundMemoryBudget::new();
|
||||
memory.set_sampled_limit(16, Duration::from_secs(30));
|
||||
let (resources, gates) = test_resource_units(2);
|
||||
let mut stream = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
|
||||
BackgroundReaderStream::with_capacity(io::Cursor::new(b"data"), 16)
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
gates.iter().all(|gate| gate.snapshot().active == 0),
|
||||
"the stream captures only memory ownership"
|
||||
);
|
||||
assert_eq!(memory.snapshot().used, 0, "construction does not allocate payload storage");
|
||||
let bytes = stream.next().await.expect("first frame").expect("read frame");
|
||||
assert_eq!(bytes.as_ref(), b"data");
|
||||
assert_eq!(memory.snapshot().used, 16, "charge full capacity rather than visible length");
|
||||
let slice = bytes.slice(1..3);
|
||||
memory.set_sampled_limit(0, Duration::ZERO);
|
||||
drop(bytes);
|
||||
drop(stream);
|
||||
assert_eq!(slice.as_ref(), b"at");
|
||||
assert_eq!(memory.snapshot().used, 16);
|
||||
drop(slice);
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn background_denial_happens_before_reading_and_never_falls_back() {
|
||||
for sample in [None, Some((7, Duration::from_secs(30))), Some((8, Duration::ZERO))] {
|
||||
let memory = BackgroundMemoryBudget::new();
|
||||
if let Some((limit, validity)) = sample {
|
||||
memory.set_sampled_limit(limit, validity);
|
||||
}
|
||||
let polls = Arc::new(AtomicUsize::new(0));
|
||||
let (resources, _) = test_resource_units(1);
|
||||
let mut stream = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
|
||||
BackgroundReaderStream::with_capacity(
|
||||
ObservedReader {
|
||||
polls: polls.clone(),
|
||||
fail: false,
|
||||
},
|
||||
8,
|
||||
)
|
||||
})
|
||||
.await;
|
||||
let error = stream
|
||||
.next()
|
||||
.await
|
||||
.expect("allocation failure")
|
||||
.expect_err("background allocation denied");
|
||||
assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
|
||||
assert_eq!(polls.load(Ordering::SeqCst), 0, "denial cannot consume source bytes");
|
||||
assert!(stream.next().await.is_none());
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn background_pending_read_keeps_storage_through_shrink_or_cancellation() {
|
||||
for cancelled in [false, true] {
|
||||
let memory = BackgroundMemoryBudget::new();
|
||||
memory.set_sampled_limit(16, Duration::from_secs(30));
|
||||
let (resources, _) = test_resource_units(1);
|
||||
let (mut sender, reader) = tokio::io::duplex(4);
|
||||
let mut stream = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
|
||||
BackgroundReaderStream::with_capacity(reader, 16)
|
||||
})
|
||||
.await;
|
||||
assert!(futures::poll!(stream.next()).is_pending());
|
||||
memory.set_sampled_limit(0, Duration::ZERO);
|
||||
assert_eq!(memory.snapshot().used, 16);
|
||||
let bytes = if cancelled {
|
||||
None
|
||||
} else {
|
||||
sender.write_all(b"data").await.expect("release pending read");
|
||||
Some(
|
||||
stream
|
||||
.next()
|
||||
.await
|
||||
.expect("pending frame")
|
||||
.expect("existing allocation survives shrink"),
|
||||
)
|
||||
};
|
||||
drop(stream);
|
||||
if let Some(bytes) = bytes {
|
||||
assert_eq!(bytes.as_ref(), b"data");
|
||||
assert_eq!(memory.snapshot().used, 16);
|
||||
drop(bytes);
|
||||
}
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn background_read_error_and_eof_release_storage() {
|
||||
for fail in [false, true] {
|
||||
let memory = BackgroundMemoryBudget::new();
|
||||
memory.set_sampled_limit(8, Duration::from_secs(30));
|
||||
let (resources, _) = test_resource_units(1);
|
||||
let mut stream = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
|
||||
let reader: Box<dyn AsyncRead + Unpin> = if fail {
|
||||
Box::new(ObservedReader {
|
||||
polls: Arc::new(AtomicUsize::new(0)),
|
||||
fail: true,
|
||||
})
|
||||
} else {
|
||||
Box::new(tokio::io::empty())
|
||||
};
|
||||
BackgroundReaderStream::with_capacity(reader, 8)
|
||||
})
|
||||
.await;
|
||||
if fail {
|
||||
let error = stream.next().await.expect("read failure").expect_err("preserve reader error");
|
||||
assert_eq!(error.kind(), io::ErrorKind::PermissionDenied);
|
||||
}
|
||||
assert!(stream.next().await.is_none());
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn retained_frame_blocks_new_storage_until_released() {
|
||||
let memory = BackgroundMemoryBudget::new();
|
||||
memory.set_sampled_limit(8, Duration::from_secs(30));
|
||||
let (resources, _) = test_resource_units(1);
|
||||
let mut stream = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
|
||||
BackgroundReaderStream::with_capacity(io::Cursor::new(b"firstonefollowup"), 8)
|
||||
})
|
||||
.await;
|
||||
let first = stream.next().await.expect("first frame").expect("fits exact budget");
|
||||
assert_eq!(first.as_ref(), b"firstone");
|
||||
let error = stream
|
||||
.next()
|
||||
.await
|
||||
.expect("next allocation")
|
||||
.expect_err("held frame exhausts budget");
|
||||
assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
|
||||
assert_eq!(memory.snapshot().used, 8);
|
||||
drop(stream);
|
||||
drop(first);
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn consumed_frames_keep_reusable_capacity_until_eof() {
|
||||
let memory = BackgroundMemoryBudget::new();
|
||||
memory.set_sampled_limit(8, Duration::from_secs(30));
|
||||
let (resources, _) = test_resource_units(1);
|
||||
let mut stream = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
|
||||
BackgroundReaderStream::with_capacity(io::Cursor::new(b"firstonefollowup"), 8)
|
||||
})
|
||||
.await;
|
||||
for expected in [b"firstone", b"followup"] {
|
||||
let bytes = stream.next().await.expect("next frame").expect("released previous capacity");
|
||||
assert_eq!(bytes.as_ref(), expected);
|
||||
assert_eq!(memory.snapshot().used, 8);
|
||||
drop(bytes);
|
||||
assert_eq!(memory.snapshot().used, 8, "the reusable allocation remains owned by the stream");
|
||||
}
|
||||
assert!(stream.next().await.is_none());
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn foreground_stream_and_unpolled_cancellation_preserve_existing_behavior() {
|
||||
let mut stream = BackgroundReaderStream::with_capacity(io::Cursor::new(b"foreground"), 4);
|
||||
assert!(matches!(stream.inner, ReaderState::Foreground(_)));
|
||||
let mut bytes = Vec::new();
|
||||
while let Some(chunk) = stream.next().await {
|
||||
bytes.extend_from_slice(&chunk.expect("foreground chunk"));
|
||||
}
|
||||
assert_eq!(bytes, b"foreground");
|
||||
|
||||
let memory = BackgroundMemoryBudget::new();
|
||||
let (resources, _) = test_resource_units(1);
|
||||
let stream = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
|
||||
BackgroundReaderStream::with_capacity(tokio::io::empty(), 8)
|
||||
})
|
||||
.await;
|
||||
drop(stream);
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod reuse_tests {
|
||||
use super::*;
|
||||
use bytes::Bytes;
|
||||
use futures::StreamExt;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use std::{
|
||||
io,
|
||||
pin::Pin,
|
||||
task::{Context, Poll},
|
||||
};
|
||||
use tokio::io::{AsyncRead, ReadBuf};
|
||||
|
||||
static DATA: [u8; 1024] = [0x37; 1024];
|
||||
struct ShortReader {
|
||||
offset: usize,
|
||||
length: usize,
|
||||
chunk: usize,
|
||||
fail_at: Option<usize>,
|
||||
}
|
||||
impl AsyncRead for ShortReader {
|
||||
fn poll_read(mut self: Pin<&mut Self>, _: &mut Context<'_>, out: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
|
||||
if self.fail_at == Some(self.offset) {
|
||||
return Poll::Ready(Err(io::ErrorKind::PermissionDenied.into()));
|
||||
}
|
||||
let n = (self.length - self.offset).min(self.chunk).min(out.remaining());
|
||||
out.put_slice(&DATA[self.offset..self.offset + n]);
|
||||
self.offset += n;
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
fn budget(limit: u64) -> Arc<crate::disk::BackgroundMemoryBudget> {
|
||||
let b = crate::disk::BackgroundMemoryBudget::new();
|
||||
b.set_sampled_limit(limit, Duration::from_secs(30));
|
||||
b
|
||||
}
|
||||
async fn stream<R: AsyncRead>(
|
||||
budget: &Arc<crate::disk::BackgroundMemoryBudget>,
|
||||
reader: R,
|
||||
capacity: usize,
|
||||
) -> BackgroundReaderStream<R> {
|
||||
let (resources, _) = crate::disk::test_resource_units(1);
|
||||
crate::disk::with_optional_resource_unit(Some(resources.with_memory_budget(budget.clone())), async {
|
||||
BackgroundReaderStream::with_capacity(reader, capacity)
|
||||
})
|
||||
.await
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn complete_consumed_frames_reclaim_the_same_storage_after_expiry() {
|
||||
let b = budget(64);
|
||||
let mut s = stream(&b, io::Cursor::new(&DATA[..]), 64).await;
|
||||
let mut pointer = None;
|
||||
for _ in 0..16 {
|
||||
let frame = s.next().await.expect("next full frame").expect("reclaimed backing");
|
||||
assert_eq!(frame.len(), 64);
|
||||
assert_eq!(frame.as_ref(), &DATA[..64]);
|
||||
if let Some(p) = pointer {
|
||||
assert_eq!(frame.as_ptr(), p);
|
||||
} else {
|
||||
pointer = Some(frame.as_ptr());
|
||||
}
|
||||
b.set_sampled_limit(0, Duration::ZERO);
|
||||
drop(frame);
|
||||
assert_eq!(b.snapshot().used, 64);
|
||||
}
|
||||
assert!(s.next().await.is_none());
|
||||
assert_eq!(b.snapshot().used, 0);
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn short_live_frames_use_disjoint_tail_then_reserve_separate_storage() {
|
||||
let b = budget(32);
|
||||
let mut s = stream(
|
||||
&b,
|
||||
ShortReader {
|
||||
offset: 0,
|
||||
length: 32,
|
||||
chunk: 4,
|
||||
fail_at: None,
|
||||
},
|
||||
16,
|
||||
)
|
||||
.await;
|
||||
let mut frames: Vec<Bytes> = Vec::new();
|
||||
for i in 0..4 {
|
||||
let frame = s.next().await.expect("short frame").expect("remaining initialized tail");
|
||||
if i != 0 {
|
||||
assert_eq!(frame.as_ptr(), frames[0].as_ptr().wrapping_add(i * 4));
|
||||
}
|
||||
assert_eq!(frame.as_ref(), &DATA[..4]);
|
||||
frames.push(frame);
|
||||
assert_eq!(b.snapshot().used, 16);
|
||||
}
|
||||
let next = s.next().await.expect("independent frame").expect("second backing fits");
|
||||
assert_ne!(next.as_ptr(), frames[0].as_ptr());
|
||||
assert_eq!(b.snapshot().used, 32);
|
||||
drop(frames);
|
||||
assert_eq!(b.snapshot().used, 16);
|
||||
drop(s);
|
||||
assert_eq!(b.snapshot().used, 16);
|
||||
drop(next);
|
||||
assert_eq!(b.snapshot().used, 0);
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn final_slice_prevents_reclaim_and_keeps_old_plus_new_charged() {
|
||||
let b = budget(32);
|
||||
let mut s = stream(&b, io::Cursor::new(&DATA[..32]), 16).await;
|
||||
let first = s.next().await.expect("old frame").expect("old backing");
|
||||
let last = first.slice(15..);
|
||||
drop(first);
|
||||
let next = s.next().await.expect("new frame").expect("second backing fits");
|
||||
assert_eq!(last.as_ref(), &DATA[..1]);
|
||||
assert_ne!(next.as_ptr(), last.as_ptr());
|
||||
assert_eq!(b.snapshot().used, 32);
|
||||
drop(s);
|
||||
assert_eq!(b.snapshot().used, 32);
|
||||
drop(next);
|
||||
assert_eq!(b.snapshot().used, 16);
|
||||
drop(last);
|
||||
assert_eq!(b.snapshot().used, 0);
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn expiry_uses_tail_but_rejects_new_storage_before_consuming_source() {
|
||||
let b = budget(16);
|
||||
let mut s = stream(
|
||||
&b,
|
||||
ShortReader {
|
||||
offset: 0,
|
||||
length: 32,
|
||||
chunk: 4,
|
||||
fail_at: None,
|
||||
},
|
||||
16,
|
||||
)
|
||||
.await;
|
||||
let first = s.next().await.expect("first frame").expect("initial backing");
|
||||
b.set_sampled_limit(0, Duration::ZERO);
|
||||
for _ in 0..3 {
|
||||
drop(s.next().await.expect("tail frame").expect("reuse charged storage"));
|
||||
}
|
||||
let error = s
|
||||
.next()
|
||||
.await
|
||||
.expect("new backing denied")
|
||||
.expect_err("live frame prevents reclaim");
|
||||
assert!(crate::disk::background_memory::is_memory_pressure(&error));
|
||||
assert!(s.next().await.is_none());
|
||||
assert_eq!(b.snapshot().used, 16);
|
||||
drop(first);
|
||||
assert_eq!(b.snapshot().used, 0);
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn eof_and_error_release_spare_but_keep_live_output_charged() {
|
||||
for fail_at in [None, Some(4)] {
|
||||
let b = budget(16);
|
||||
let mut s = stream(
|
||||
&b,
|
||||
ShortReader {
|
||||
offset: 0,
|
||||
length: 4,
|
||||
chunk: 4,
|
||||
fail_at,
|
||||
},
|
||||
16,
|
||||
)
|
||||
.await;
|
||||
let frame = s.next().await.expect("first short frame").expect("payload");
|
||||
match fail_at {
|
||||
None => assert!(s.next().await.is_none()),
|
||||
Some(_) => assert_eq!(
|
||||
s.next()
|
||||
.await
|
||||
.expect("read error")
|
||||
.expect_err("preserve source failure")
|
||||
.kind(),
|
||||
io::ErrorKind::PermissionDenied
|
||||
),
|
||||
}
|
||||
assert!(s.next().await.is_none());
|
||||
assert_eq!(b.snapshot().used, 16);
|
||||
assert_eq!(frame.as_ref(), &DATA[..4]);
|
||||
drop(frame);
|
||||
assert_eq!(b.snapshot().used, 0);
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn runtime_shutdown_drops_a_pending_stream_without_refunding_early() {
|
||||
for abort in [false, true] {
|
||||
let b = budget(16);
|
||||
let rt = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("runtime");
|
||||
let (_tx, reader) = tokio::io::duplex(4);
|
||||
let (began, wait) = tokio::sync::oneshot::channel();
|
||||
let memory = b.clone();
|
||||
let handle = rt.spawn(async move {
|
||||
let mut s = stream(&memory, reader, 16).await;
|
||||
assert!(futures::poll!(s.next()).is_pending());
|
||||
assert_eq!(memory.snapshot().used, 16);
|
||||
began.send(()).expect("notify pending allocation");
|
||||
let _ = s.next().await;
|
||||
});
|
||||
rt.block_on(wait).expect("stream pending");
|
||||
assert_eq!(b.snapshot().used, 16);
|
||||
if abort {
|
||||
handle.abort();
|
||||
let _ = rt.block_on(handle);
|
||||
}
|
||||
drop(rt);
|
||||
assert_eq!(b.snapshot().used, 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -17,10 +17,10 @@ use crate::cluster::rpc::http_auth::RPC_REPLAY_SCOPE_VERSION_HEADER;
|
||||
use crate::cluster::rpc::http_auth::{
|
||||
AuthenticatedPeerReplayCapabilities, RPC_AUTH_VERSION_HEADER, RPC_AUTH_VERSION_V2, RPC_BOOT_EPOCH_CHALLENGE_HEADER,
|
||||
RPC_CONTENT_SHA256_HEADER, RPC_REPLAY_CACHE_CAPABILITY_HEADER, RPC_REPLAY_CACHE_CAPABILITY_PROOF_HEADER,
|
||||
RollingMutationBodyDigest, TIMESTAMP_HEADER, internode_rpc_body_digest_strict,
|
||||
verify_tonic_peer_replay_capabilities_response,
|
||||
RollingMutationBodyDigest, TIMESTAMP_HEADER, gen_tonic_replay_scope_headers_with_admission, internode_rpc_body_digest_strict,
|
||||
rpc_background_admission, verify_tonic_peer_replay_capabilities_response,
|
||||
};
|
||||
use crate::cluster::rpc::{gen_tonic_replay_scope_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience};
|
||||
use crate::cluster::rpc::{gen_tonic_signature_headers, normalize_tonic_rpc_audience};
|
||||
#[cfg(test)]
|
||||
use crate::cluster::rpc::{tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers};
|
||||
use crate::disk::error::{DiskError, Error as DiskErrorType, RpcStatusError};
|
||||
@@ -415,7 +415,16 @@ where
|
||||
.get(RPC_CONTENT_SHA256_HEADER)
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
) {
|
||||
match gen_tonic_replay_scope_headers(audience, request.uri().path(), timestamp, content_sha256, boot_epoch) {
|
||||
match rpc_background_admission(request.headers()).and_then(|background| {
|
||||
gen_tonic_replay_scope_headers_with_admission(
|
||||
audience,
|
||||
request.uri().path(),
|
||||
timestamp,
|
||||
content_sha256,
|
||||
boot_epoch,
|
||||
background,
|
||||
)
|
||||
}) {
|
||||
Ok(headers) => request.headers_mut().extend(headers),
|
||||
Err(error) => debug!(error = %error, "could not attach replay-scoped RPC signature"),
|
||||
}
|
||||
|
||||
@@ -32,7 +32,6 @@ use crate::storage_api_contracts::internode::{
|
||||
PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_CAPABILITY_VERSION,
|
||||
};
|
||||
|
||||
use hmac::{Hmac, KeyInit, Mac};
|
||||
use http::uri::Authority;
|
||||
use http::{HeaderMap, HeaderValue, Method, Uri};
|
||||
#[cfg(test)]
|
||||
@@ -47,6 +46,7 @@ use rustfs_io_metrics::internode_metrics::{
|
||||
global_internode_metrics,
|
||||
};
|
||||
use rustfs_object_data_cache::{MemoryBasis, resolve_effective_memory};
|
||||
use rustfs_utils::crypto::{HmacSha256, KeyInit, Mac};
|
||||
use rustfs_utils::get_env_bool;
|
||||
use sha2::Digest as _;
|
||||
use sha2::Sha256;
|
||||
@@ -58,9 +58,10 @@ use time::OffsetDateTime;
|
||||
use tracing::{error, info, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
type HmacSha256 = Hmac<Sha256>;
|
||||
|
||||
const SIGNATURE_HEADER: &str = "x-rustfs-signature";
|
||||
pub const RPC_BACKGROUND_ADMISSION_HEADER: &str = "x-rustfs-rpc-background-admission";
|
||||
const RPC_BACKGROUND_ADMISSION_V1: &str = "v1";
|
||||
const RPC_BACKGROUND_ADMISSION_DOMAIN: &[u8] = b"rustfs-rpc-background-admission-v1\0";
|
||||
pub(crate) const TIMESTAMP_HEADER: &str = "x-rustfs-timestamp";
|
||||
pub(crate) const RPC_AUTH_VERSION_HEADER: &str = "x-rustfs-rpc-auth-version";
|
||||
const RPC_SIGNATURE_V2_HEADER: &str = "x-rustfs-rpc-signature-v2";
|
||||
@@ -77,6 +78,46 @@ pub(crate) const RPC_REPLAY_CACHE_CAPABILITY_HEADER: &str = "x-rustfs-rpc-replay
|
||||
pub(crate) const RPC_REPLAY_CACHE_CAPABILITY_PROOF_HEADER: &str = "x-rustfs-rpc-replay-cache-capability-proof";
|
||||
const RPC_REPLAY_SCOPE_VERSION_V3: &str = "3";
|
||||
const RPC_RESPONSE_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-response-proof-v1\0";
|
||||
|
||||
tokio::task_local! {
|
||||
static RPC_BACKGROUND_REQUEST: bool;
|
||||
}
|
||||
|
||||
pub(crate) fn current_rpc_background_request() -> bool {
|
||||
RPC_BACKGROUND_REQUEST
|
||||
.try_with(|background| *background)
|
||||
.unwrap_or_else(|_| crate::disk::has_resource_io_scope())
|
||||
}
|
||||
|
||||
/// Preserve the authenticated wire class while polling locally admitted work.
|
||||
/// This scope carries only the class, never node, disk, or memory ownership.
|
||||
pub async fn with_rpc_background_request<F: std::future::Future>(background: bool, operation: F) -> F::Output {
|
||||
RPC_BACKGROUND_REQUEST.scope(background, operation).await
|
||||
}
|
||||
|
||||
/// Parse the request class only; callers must verify the signature before using it.
|
||||
pub fn rpc_background_admission(headers: &HeaderMap) -> std::io::Result<bool> {
|
||||
let mut values = headers.get_all(RPC_BACKGROUND_ADMISSION_HEADER).iter();
|
||||
match (values.next(), values.next()) {
|
||||
(None, None) => Ok(false),
|
||||
(Some(value), None) if value == RPC_BACKGROUND_ADMISSION_V1 => Ok(true),
|
||||
_ => Err(std::io::Error::other("Invalid RPC background admission version")),
|
||||
}
|
||||
}
|
||||
|
||||
fn bind_background_admission(mac: &mut HmacSha256, background: bool) {
|
||||
if background {
|
||||
// Every auth lane binds the class, so stripping the marker or falling
|
||||
// back to an older signature cannot turn admitted work into foreground I/O.
|
||||
mac.update(RPC_BACKGROUND_ADMISSION_DOMAIN);
|
||||
}
|
||||
}
|
||||
|
||||
fn insert_background_admission(headers: &mut HeaderMap, background: bool) {
|
||||
if background {
|
||||
headers.insert(RPC_BACKGROUND_ADMISSION_HEADER, HeaderValue::from_static(RPC_BACKGROUND_ADMISSION_V1));
|
||||
}
|
||||
}
|
||||
const RPC_REPLAY_SCOPE_DOMAIN: &[u8] = b"rustfs-rpc-replay-scope-v3\0";
|
||||
const RPC_BOOT_EPOCH_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-boot-epoch-proof-v1\0";
|
||||
const RPC_REPLAY_CACHE_CAPABILITY_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-replay-cache-capability-proof-v1\0";
|
||||
@@ -519,21 +560,23 @@ fn redacted_rpc_path(url: &str) -> String {
|
||||
}
|
||||
|
||||
/// Generate HMAC-SHA256 signature for the given data
|
||||
fn generate_signature(secret: &str, url: &str, method: &Method, timestamp: i64) -> String {
|
||||
fn generate_signature(secret: &str, url: &str, method: &Method, timestamp: i64, background: bool) -> String {
|
||||
let data = signature_payload(url, method, timestamp);
|
||||
let mut mac = <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).expect("HMAC can take key of any size");
|
||||
bind_background_admission(&mut mac, background);
|
||||
mac.update(data.as_bytes());
|
||||
let result = mac.finalize();
|
||||
base64_simd::STANDARD.encode_to_string(result.into_bytes())
|
||||
}
|
||||
|
||||
fn verify_signature(secret: &str, url: &str, method: &Method, timestamp: i64, signature: &str) -> bool {
|
||||
fn verify_signature(secret: &str, url: &str, method: &Method, timestamp: i64, signature: &str, background: bool) -> bool {
|
||||
let Ok(signature) = base64_simd::STANDARD.decode_to_vec(signature) else {
|
||||
return false;
|
||||
};
|
||||
|
||||
let data = signature_payload(url, method, timestamp);
|
||||
let mut mac = <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).expect("HMAC can take key of any size");
|
||||
bind_background_admission(&mut mac, background);
|
||||
mac.update(data.as_bytes());
|
||||
mac.verify_slice(&signature).is_ok()
|
||||
}
|
||||
@@ -771,6 +814,7 @@ pub fn verify_ns_scanner_capability_with_checkpoint_coverage(
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct SignatureV2Scope<'a> {
|
||||
background: bool,
|
||||
audience: &'a str,
|
||||
service: &'a str,
|
||||
rpc_method: &'a str,
|
||||
@@ -780,6 +824,7 @@ struct SignatureV2Scope<'a> {
|
||||
}
|
||||
|
||||
fn update_signature_v2(mac: &mut HmacSha256, scope: SignatureV2Scope<'_>) {
|
||||
bind_background_admission(mac, scope.background);
|
||||
for part in [
|
||||
b"rustfs-rpc-auth-v2|".as_slice(),
|
||||
scope.audience.as_bytes(),
|
||||
@@ -818,6 +863,7 @@ fn verify_signature_v2(secret: &str, scope: SignatureV2Scope<'_>, signature: &st
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct ReplayScope<'a> {
|
||||
background: bool,
|
||||
audience: &'a str,
|
||||
path: &'a str,
|
||||
timestamp: &'a str,
|
||||
@@ -827,6 +873,7 @@ struct ReplayScope<'a> {
|
||||
}
|
||||
|
||||
fn update_replay_scope(mac: &mut HmacSha256, scope: ReplayScope<'_>) {
|
||||
bind_background_admission(mac, scope.background);
|
||||
mac.update(RPC_REPLAY_SCOPE_DOMAIN);
|
||||
for part in [
|
||||
scope.audience.as_bytes(),
|
||||
@@ -970,6 +1017,24 @@ pub fn gen_tonic_replay_scope_headers(
|
||||
timestamp: &str,
|
||||
content_sha256: &str,
|
||||
boot_epoch: Uuid,
|
||||
) -> std::io::Result<HeaderMap> {
|
||||
gen_tonic_replay_scope_headers_with_admission(
|
||||
audience,
|
||||
path,
|
||||
timestamp,
|
||||
content_sha256,
|
||||
boot_epoch,
|
||||
current_rpc_background_request(),
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn gen_tonic_replay_scope_headers_with_admission(
|
||||
audience: &str,
|
||||
path: &str,
|
||||
timestamp: &str,
|
||||
content_sha256: &str,
|
||||
boot_epoch: Uuid,
|
||||
background: bool,
|
||||
) -> std::io::Result<HeaderMap> {
|
||||
if audience.is_empty() || !path.starts_with('/') || !valid_content_sha256(content_sha256) || boot_epoch.is_nil() {
|
||||
return Err(std::io::Error::other("Invalid replay-scoped RPC signing scope"));
|
||||
@@ -983,6 +1048,7 @@ pub fn gen_tonic_replay_scope_headers(
|
||||
let signature = generate_replay_scope_signature(
|
||||
&get_shared_secret()?,
|
||||
ReplayScope {
|
||||
background,
|
||||
audience,
|
||||
path,
|
||||
timestamp,
|
||||
@@ -992,6 +1058,7 @@ pub fn gen_tonic_replay_scope_headers(
|
||||
},
|
||||
)?;
|
||||
let mut headers = HeaderMap::new();
|
||||
insert_background_admission(&mut headers, background);
|
||||
headers.insert(RPC_REPLAY_SCOPE_VERSION_HEADER, HeaderValue::from_static(RPC_REPLAY_SCOPE_VERSION_V3));
|
||||
headers.insert(
|
||||
RPC_REPLAY_SCOPE_SIGNATURE_HEADER,
|
||||
@@ -1226,10 +1293,12 @@ pub fn build_auth_headers(url: &str, method: &Method, headers: &mut HeaderMap) -
|
||||
pub fn gen_signature_headers(url: &str, method: &Method) -> std::io::Result<HeaderMap> {
|
||||
let secret = get_shared_secret()?;
|
||||
let timestamp = OffsetDateTime::now_utc().unix_timestamp();
|
||||
let background = current_rpc_background_request();
|
||||
|
||||
let signature = generate_signature(&secret, url, method, timestamp);
|
||||
let signature = generate_signature(&secret, url, method, timestamp, background);
|
||||
|
||||
let mut headers = HeaderMap::new();
|
||||
insert_background_admission(&mut headers, background);
|
||||
headers.insert(SIGNATURE_HEADER, HeaderValue::from_str(&signature).expect("Invalid header value"));
|
||||
headers.insert(
|
||||
TIMESTAMP_HEADER,
|
||||
@@ -1264,10 +1333,12 @@ pub fn gen_tonic_signature_headers(
|
||||
let timestamp_header = timestamp.to_string();
|
||||
let body_nonce = (content_sha256 != UNSIGNED_PAYLOAD).then(|| Uuid::new_v4().to_string());
|
||||
let nonce = body_nonce.as_deref().unwrap_or(UNSIGNED_PAYLOAD_NONCE);
|
||||
let legacy_signature = generate_signature(&secret, TONIC_RPC_PREFIX, &Method::GET, timestamp);
|
||||
let background = current_rpc_background_request();
|
||||
let legacy_signature = generate_signature(&secret, TONIC_RPC_PREFIX, &Method::GET, timestamp, background);
|
||||
let signature_v2 = generate_signature_v2(
|
||||
&secret,
|
||||
SignatureV2Scope {
|
||||
background,
|
||||
audience,
|
||||
service,
|
||||
rpc_method,
|
||||
@@ -1278,6 +1349,7 @@ pub fn gen_tonic_signature_headers(
|
||||
)?;
|
||||
|
||||
let mut headers = HeaderMap::new();
|
||||
insert_background_admission(&mut headers, background);
|
||||
headers.insert(SIGNATURE_HEADER, header_value(&legacy_signature, SIGNATURE_HEADER)?);
|
||||
headers.insert(TIMESTAMP_HEADER, header_value(×tamp_header, TIMESTAMP_HEADER)?);
|
||||
headers.insert(RPC_AUTH_VERSION_HEADER, HeaderValue::from_static(RPC_AUTH_VERSION_V2));
|
||||
@@ -1501,6 +1573,7 @@ fn verify_tonic_replay_scope_signature(audience: &str, path: &str, headers: &Hea
|
||||
if !verify_replay_scope_signature(
|
||||
&secret,
|
||||
ReplayScope {
|
||||
background: rpc_background_admission(headers)?,
|
||||
audience,
|
||||
path,
|
||||
timestamp,
|
||||
@@ -1515,7 +1588,13 @@ fn verify_tonic_replay_scope_signature(audience: &str, path: &str, headers: &Hea
|
||||
if boot_epoch != tonic_rpc_boot_epoch() {
|
||||
return Err(std::io::Error::other("RPC boot epoch is stale"));
|
||||
}
|
||||
check_and_record_tonic_nonce(nonce, signed_at, path)
|
||||
check_and_record_tonic_nonce(nonce, signed_at, path)?;
|
||||
// A body-bound request also carries a usable v2 signature. Consume both
|
||||
// nonces so stripping the replay-scope headers cannot authorize it again.
|
||||
if content_sha256 != UNSIGNED_PAYLOAD {
|
||||
verify_tonic_rpc_signature_with_strictness(audience, path, headers, true)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Verify gRPC authentication, preferring v2 without downgrade on malformed v2 metadata.
|
||||
@@ -1625,6 +1704,9 @@ fn verify_tonic_rpc_signature_with_strictness(
|
||||
strict: bool,
|
||||
) -> std::io::Result<()> {
|
||||
if !has_v2_auth_headers(headers) {
|
||||
if rpc_background_admission(headers)? {
|
||||
return Err(std::io::Error::other("RPC background admission requires v2 authentication"));
|
||||
}
|
||||
// RUSTFS_COMPAT_TODO(heal-rpc-auth-v2): accept old peers during rolling upgrades. Remove after the minimum
|
||||
// supported RustFS peer version sends v2 authentication on every internode gRPC request.
|
||||
if strict {
|
||||
@@ -1696,6 +1778,7 @@ fn verify_tonic_rpc_signature_with_strictness(
|
||||
if !verify_signature_v2(
|
||||
&secret,
|
||||
SignatureV2Scope {
|
||||
background: rpc_background_admission(headers)?,
|
||||
audience,
|
||||
service,
|
||||
rpc_method,
|
||||
@@ -1736,7 +1819,7 @@ pub fn verify_rpc_signature(url: &str, method: &Method, headers: &HeaderMap) ->
|
||||
// Verify signature with constant-time HMAC comparison.
|
||||
let secret = get_shared_secret()?;
|
||||
|
||||
if !verify_signature(&secret, url, method, timestamp, signature) {
|
||||
if !verify_signature(&secret, url, method, timestamp, signature, rpc_background_admission(headers)?) {
|
||||
let rpc_path = redacted_rpc_path(url);
|
||||
error!(
|
||||
rpc_path = %rpc_path,
|
||||
@@ -1828,6 +1911,98 @@ mod tests {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn background_admission_is_bound_in_http_and_both_tonic_signature_lanes() {
|
||||
ensure_test_rpc_secret();
|
||||
let secret = get_shared_secret().expect("RPC secret");
|
||||
let path = "/node_service.NodeService/ReadAll";
|
||||
for background in [false, true] {
|
||||
let (http, tonic) = with_rpc_background_request(background, async {
|
||||
(
|
||||
gen_signature_headers("/rustfs/rpc/read?disk=a", &Method::GET).expect("HTTP signature"),
|
||||
gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "ReadAll", None)
|
||||
.expect("gRPC signature"),
|
||||
)
|
||||
})
|
||||
.await;
|
||||
assert_eq!(rpc_background_admission(&http).expect("HTTP class"), background);
|
||||
assert_eq!(rpc_background_admission(&tonic).expect("gRPC class"), background);
|
||||
verify_rpc_signature("/rustfs/rpc/read?disk=a", &Method::GET, &http).expect("HTTP authentication");
|
||||
verify_tonic_rpc_signature_with_strictness("node-a:9000", path, &tonic, true).expect("v2 authentication");
|
||||
|
||||
let timestamp = tonic[TIMESTAMP_HEADER].to_str().expect("timestamp");
|
||||
let content = tonic[RPC_CONTENT_SHA256_HEADER].to_str().expect("content digest");
|
||||
assert_eq!(
|
||||
verify_signature_v2(
|
||||
&secret,
|
||||
SignatureV2Scope {
|
||||
background: false,
|
||||
audience: "node-a:9000",
|
||||
service: "node_service.NodeService",
|
||||
rpc_method: "ReadAll",
|
||||
timestamp,
|
||||
nonce: tonic[RPC_NONCE_HEADER].to_str().expect("nonce"),
|
||||
content_sha256: content,
|
||||
},
|
||||
tonic[RPC_SIGNATURE_V2_HEADER].to_str().expect("signature"),
|
||||
),
|
||||
!background,
|
||||
"an old peer cannot authenticate a background request as foreground"
|
||||
);
|
||||
|
||||
let mut replay = tonic.clone();
|
||||
// The channel can run outside the caller's task-local scope.
|
||||
replay.extend(
|
||||
gen_tonic_replay_scope_headers_with_admission(
|
||||
"node-a:9000",
|
||||
path,
|
||||
timestamp,
|
||||
content,
|
||||
tonic_rpc_boot_epoch(),
|
||||
background,
|
||||
)
|
||||
.expect("v3 signature"),
|
||||
);
|
||||
for mutation in ["strip-or-add", "unknown", "duplicate"] {
|
||||
let alter = |headers: &HeaderMap| {
|
||||
let mut changed = headers.clone();
|
||||
match mutation {
|
||||
"strip-or-add" if background => {
|
||||
changed.remove(RPC_BACKGROUND_ADMISSION_HEADER);
|
||||
}
|
||||
"strip-or-add" => insert_background_admission(&mut changed, true),
|
||||
"unknown" => {
|
||||
changed.insert(RPC_BACKGROUND_ADMISSION_HEADER, HeaderValue::from_static("v2"));
|
||||
}
|
||||
_ => {
|
||||
insert_background_admission(&mut changed, true);
|
||||
changed.append(RPC_BACKGROUND_ADMISSION_HEADER, HeaderValue::from_static("v1"));
|
||||
}
|
||||
}
|
||||
changed
|
||||
};
|
||||
assert!(verify_rpc_signature("/rustfs/rpc/read?disk=a", &Method::GET, &alter(&http)).is_err());
|
||||
assert!(verify_tonic_rpc_signature_with_strictness("node-a:9000", path, &alter(&tonic), true).is_err());
|
||||
assert!(verify_tonic_replay_scope_signature("node-a:9000", path, &alter(&replay)).is_err());
|
||||
}
|
||||
verify_tonic_replay_scope_signature("node-a:9000", path, &replay).expect("v3 authentication");
|
||||
assert!(
|
||||
verify_tonic_replay_scope_signature("node-a:9000", path, &replay).is_err(),
|
||||
"replay rejected"
|
||||
);
|
||||
|
||||
if background {
|
||||
let mut downgraded = HeaderMap::new();
|
||||
for name in [SIGNATURE_HEADER, TIMESTAMP_HEADER, RPC_BACKGROUND_ADMISSION_HEADER] {
|
||||
downgraded.insert(name, tonic[name].clone());
|
||||
}
|
||||
assert!(verify_tonic_rpc_signature_with_strictness("node-a:9000", path, &downgraded, false).is_err());
|
||||
downgraded.remove(RPC_BACKGROUND_ADMISSION_HEADER);
|
||||
assert!(verify_tonic_rpc_signature_with_strictness("node-a:9000", path, &downgraded, false).is_err());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn namespace_scanner_capability_proof_binds_challenge_and_server_epoch() {
|
||||
let secret = "test-scanner-capability-secret";
|
||||
@@ -1965,8 +2140,8 @@ mod tests {
|
||||
let method = Method::GET;
|
||||
let timestamp = 1640995200; // Fixed timestamp
|
||||
|
||||
let signature1 = generate_signature(secret, url, &method, timestamp);
|
||||
let signature2 = generate_signature(secret, url, &method, timestamp);
|
||||
let signature1 = generate_signature(secret, url, &method, timestamp, false);
|
||||
let signature2 = generate_signature(secret, url, &method, timestamp, false);
|
||||
|
||||
assert_eq!(signature1, signature2, "Same inputs should produce same signature");
|
||||
assert!(!signature1.is_empty(), "Signature should not be empty");
|
||||
@@ -1979,10 +2154,10 @@ mod tests {
|
||||
let method = Method::GET;
|
||||
let timestamp = 1640995200;
|
||||
|
||||
let signature1 = generate_signature(secret, url, &method, timestamp);
|
||||
let signature2 = generate_signature(secret, "http://different.com/api/test2", &method, timestamp);
|
||||
let signature3 = generate_signature(secret, url, &Method::POST, timestamp);
|
||||
let signature4 = generate_signature(secret, url, &method, timestamp + 1);
|
||||
let signature1 = generate_signature(secret, url, &method, timestamp, false);
|
||||
let signature2 = generate_signature(secret, "http://different.com/api/test2", &method, timestamp, false);
|
||||
let signature3 = generate_signature(secret, url, &Method::POST, timestamp, false);
|
||||
let signature4 = generate_signature(secret, url, &method, timestamp + 1, false);
|
||||
|
||||
assert_ne!(signature1, signature2, "Different URLs should produce different signatures");
|
||||
assert_ne!(signature1, signature3, "Different methods should produce different signatures");
|
||||
@@ -2088,14 +2263,14 @@ mod tests {
|
||||
let url = "http://example.com/api/test";
|
||||
let method = Method::GET;
|
||||
let timestamp = 1640995200;
|
||||
let signature = generate_signature(secret, url, &method, timestamp);
|
||||
let signature = generate_signature(secret, url, &method, timestamp, false);
|
||||
let mut tampered = base64_simd::STANDARD.decode_to_vec(&signature).unwrap();
|
||||
tampered[0] ^= 1;
|
||||
let tampered_signature = base64_simd::STANDARD.encode_to_string(tampered);
|
||||
|
||||
assert!(verify_signature(secret, url, &method, timestamp, &signature));
|
||||
assert!(!verify_signature(secret, url, &method, timestamp, &tampered_signature));
|
||||
assert!(!verify_signature(secret, url, &method, timestamp, "invalid-signature"));
|
||||
assert!(verify_signature(secret, url, &method, timestamp, &signature, false));
|
||||
assert!(!verify_signature(secret, url, &method, timestamp, &tampered_signature, false));
|
||||
assert!(!verify_signature(secret, url, &method, timestamp, "invalid-signature", false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2112,11 +2287,11 @@ mod tests {
|
||||
);
|
||||
let method = Method::GET;
|
||||
let timestamp = 1_640_995_200;
|
||||
let signature = generate_signature(secret, signed_url, &method, timestamp);
|
||||
let signature = generate_signature(secret, signed_url, &method, timestamp, false);
|
||||
|
||||
assert!(verify_signature(secret, signed_url, &method, timestamp, &signature));
|
||||
assert!(!verify_signature(secret, downgraded_url, &method, timestamp, &signature));
|
||||
assert!(!verify_signature(secret, tampered_body_digest, &method, timestamp, &signature));
|
||||
assert!(verify_signature(secret, signed_url, &method, timestamp, &signature, false));
|
||||
assert!(!verify_signature(secret, downgraded_url, &method, timestamp, &signature, false));
|
||||
assert!(!verify_signature(secret, tampered_body_digest, &method, timestamp, &signature, false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2126,7 +2301,7 @@ mod tests {
|
||||
let method = Method::GET;
|
||||
let timestamp = OffsetDateTime::now_utc().unix_timestamp();
|
||||
let secret = get_shared_secret().expect("test RPC secret should resolve");
|
||||
let expected_signature = generate_signature(&secret, url, &method, timestamp);
|
||||
let expected_signature = generate_signature(&secret, url, &method, timestamp, false);
|
||||
let invalid_signature = "invalid-signature";
|
||||
let logs = CapturedLogs::default();
|
||||
let subscriber = tracing_subscriber::fmt()
|
||||
@@ -2164,7 +2339,7 @@ mod tests {
|
||||
// Set expired timestamp (older than SIGNATURE_VALID_DURATION)
|
||||
let expired_timestamp = OffsetDateTime::now_utc().unix_timestamp() - SIGNATURE_VALID_DURATION - 10;
|
||||
let secret = get_shared_secret().expect("test RPC secret should resolve");
|
||||
let signature = generate_signature(&secret, url, &method, expired_timestamp);
|
||||
let signature = generate_signature(&secret, url, &method, expired_timestamp, false);
|
||||
|
||||
headers.insert(SIGNATURE_HEADER, HeaderValue::from_str(&signature).unwrap());
|
||||
headers.insert(TIMESTAMP_HEADER, HeaderValue::from_str(&expired_timestamp.to_string()).unwrap());
|
||||
@@ -2186,7 +2361,7 @@ mod tests {
|
||||
|
||||
let future_timestamp = OffsetDateTime::now_utc().unix_timestamp() + SIGNATURE_VALID_DURATION + 10;
|
||||
let secret = get_shared_secret().expect("test RPC secret should resolve");
|
||||
let signature = generate_signature(&secret, url, &method, future_timestamp);
|
||||
let signature = generate_signature(&secret, url, &method, future_timestamp, false);
|
||||
|
||||
headers.insert(SIGNATURE_HEADER, HeaderValue::from_str(&signature).unwrap());
|
||||
headers.insert(TIMESTAMP_HEADER, HeaderValue::from_str(&future_timestamp.to_string()).unwrap());
|
||||
@@ -2300,7 +2475,7 @@ mod tests {
|
||||
// Test timestamp just within valid duration
|
||||
let valid_timestamp = current_time - SIGNATURE_VALID_DURATION + 1;
|
||||
|
||||
let signature = generate_signature(&secret, url, &method, valid_timestamp);
|
||||
let signature = generate_signature(&secret, url, &method, valid_timestamp, false);
|
||||
|
||||
headers.insert(SIGNATURE_HEADER, HeaderValue::from_str(&signature).unwrap());
|
||||
headers.insert(TIMESTAMP_HEADER, HeaderValue::from_str(&valid_timestamp.to_string()).unwrap());
|
||||
@@ -2311,7 +2486,7 @@ mod tests {
|
||||
// Test timestamp just outside valid duration
|
||||
let mut headers = HeaderMap::new();
|
||||
let invalid_timestamp = current_time - SIGNATURE_VALID_DURATION - 15;
|
||||
let signature = generate_signature(&secret, url, &method, invalid_timestamp);
|
||||
let signature = generate_signature(&secret, url, &method, invalid_timestamp, false);
|
||||
|
||||
headers.insert(SIGNATURE_HEADER, HeaderValue::from_str(&signature).unwrap());
|
||||
headers.insert(TIMESTAMP_HEADER, HeaderValue::from_str(&invalid_timestamp.to_string()).unwrap());
|
||||
@@ -2410,6 +2585,43 @@ mod tests {
|
||||
assert_eq!(path_error.to_string(), "Invalid RPC replay scope signature");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_scope_consumes_the_body_bound_v2_nonce() {
|
||||
ensure_test_rpc_secret();
|
||||
let audience = "node-a:9000";
|
||||
let path = "/node_service.NodeService/RecoverDeleteTransaction";
|
||||
let digest = "a".repeat(64);
|
||||
for v3_first in [true, false] {
|
||||
let mut headers =
|
||||
gen_tonic_signature_headers(audience, "node_service.NodeService", "RecoverDeleteTransaction", Some(&digest))
|
||||
.expect("body-bound v2 headers should build");
|
||||
let timestamp = headers[TIMESTAMP_HEADER].to_str().expect("timestamp").to_string();
|
||||
headers.extend(
|
||||
gen_tonic_replay_scope_headers(audience, path, ×tamp, &digest, tonic_rpc_boot_epoch())
|
||||
.expect("replay-scope headers should build"),
|
||||
);
|
||||
let mut downgraded = headers.clone();
|
||||
for header in [
|
||||
RPC_REPLAY_SCOPE_VERSION_HEADER,
|
||||
RPC_REPLAY_SCOPE_SIGNATURE_HEADER,
|
||||
RPC_REPLAY_SCOPE_NONCE_HEADER,
|
||||
RPC_BOOT_EPOCH_HEADER,
|
||||
] {
|
||||
downgraded.remove(header);
|
||||
}
|
||||
let (first, replay) = if v3_first {
|
||||
(&headers, &downgraded)
|
||||
} else {
|
||||
(&downgraded, &headers)
|
||||
};
|
||||
verify_tonic_rpc_signature_with_policy(audience, path, first, true, false, false)
|
||||
.expect("the first delivery must be accepted");
|
||||
let error = verify_tonic_rpc_signature_with_policy(audience, path, replay, true, false, false)
|
||||
.expect_err("a mutation cannot replay through the other signature generation");
|
||||
assert_eq!(error.to_string(), "RPC request replay detected");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_scope_rejects_partial_metadata_and_stale_epoch_without_fallback() {
|
||||
ensure_test_rpc_secret();
|
||||
|
||||
@@ -33,7 +33,9 @@ use rustfs_config::{
|
||||
DEFAULT_INTERNODE_DATA_TRANSPORT, ENV_RUSTFS_INTERNODE_DATA_TRANSPORT, INTERNODE_DATA_TRANSPORT_TCP,
|
||||
KNOWN_INTERNODE_DATA_TRANSPORT_BACKENDS,
|
||||
};
|
||||
use rustfs_rio::{ChunkReaderBox, HttpChunkReader, HttpReader, HttpWriter};
|
||||
use rustfs_rio::{
|
||||
ChunkReaderBox, HttpChunkReader, HttpReader, HttpWriter, HttpWriterMemoryReservation, HttpWriterMemoryReserver,
|
||||
};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashMap;
|
||||
use std::future::Future;
|
||||
@@ -66,6 +68,19 @@ const PUT_FILE_CAPABILITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
const CONTENT_TYPE_JSON: &str = "application/json";
|
||||
const CONTENT_TYPE_MSGPACK: &str = "application/msgpack";
|
||||
|
||||
fn http_writer_memory_reserver() -> Option<HttpWriterMemoryReserver> {
|
||||
crate::disk::current_resource_unit()
|
||||
.and_then(|resources| resources.memory_budget())
|
||||
.map(|budget| {
|
||||
// Payloads can outlive the operation; never retain its node or disk units.
|
||||
Arc::new(move |capacity| {
|
||||
budget
|
||||
.reserve(capacity)
|
||||
.map(|reservation| Box::new(reservation) as HttpWriterMemoryReservation)
|
||||
}) as HttpWriterMemoryReserver
|
||||
})
|
||||
}
|
||||
|
||||
fn unsupported_transport_message(transport: &str) -> String {
|
||||
format!(
|
||||
"invalid {ENV_RUSTFS_INTERNODE_DATA_TRANSPORT}={transport:?}; supported values: {}",
|
||||
@@ -346,13 +361,17 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
|
||||
}
|
||||
|
||||
async fn open_write(&self, request: WriteStreamRequest) -> Result<FileWriter> {
|
||||
let memory_reserver = http_writer_memory_reserver();
|
||||
let server_epoch = self.put_file_auth_capability(&request.endpoint).await?;
|
||||
let auth_scope = server_epoch.map(|server_epoch| (Uuid::new_v4(), server_epoch));
|
||||
let url = build_put_file_stream_url(&request, auth_scope);
|
||||
let endpoint = request.endpoint;
|
||||
let mut headers = json_headers();
|
||||
build_auth_headers(&url, &Method::PUT, &mut headers)?;
|
||||
let writer = HttpWriter::new(url.clone(), Method::PUT, headers).await?;
|
||||
let writer = match memory_reserver {
|
||||
Some(reserve) => HttpWriter::new_with_memory_reserver(url.clone(), Method::PUT, headers, reserve).await?,
|
||||
None => HttpWriter::new(url.clone(), Method::PUT, headers).await?,
|
||||
};
|
||||
match auth_scope {
|
||||
Some((nonce, server_epoch)) => Ok(Box::new(PutFileAuthWriter::new(writer, url, nonce, endpoint, server_epoch))),
|
||||
None => Ok(Box::new(writer)),
|
||||
@@ -948,6 +967,193 @@ mod tests {
|
||||
use tokio::io::AsyncWriteExt;
|
||||
use tokio::sync::{Barrier, Notify};
|
||||
|
||||
fn memory_test_write_request(authenticated: bool) -> WriteStreamRequest {
|
||||
crate::runtime::sources::ensure_test_rpc_secret();
|
||||
let endpoint = format!("http://writer-memory-{}.invalid", Uuid::new_v4());
|
||||
put_file_capability_cache_entry(&endpoint).write().cached = Some(if authenticated {
|
||||
PutFileCapabilityState::V1 {
|
||||
server_epoch: Uuid::new_v4(),
|
||||
revalidate_after: Instant::now() + PUT_FILE_V1_CAPABILITY_TTL,
|
||||
}
|
||||
} else {
|
||||
PutFileCapabilityState::LegacyUntil(Instant::now() + PUT_FILE_LEGACY_CAPABILITY_TTL)
|
||||
});
|
||||
WriteStreamRequest {
|
||||
endpoint,
|
||||
disk: "disk-a".to_owned(),
|
||||
volume: "bucket".to_owned(),
|
||||
path: "part.1".to_owned(),
|
||||
append: false,
|
||||
size: 4,
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_write_memory_reserver_outlives_only_the_memory_scope() {
|
||||
use crate::disk::{BackgroundMemoryBudget, test_resource_units, with_optional_resource_unit};
|
||||
|
||||
assert!(http_writer_memory_reserver().is_none(), "foreground must use the existing writer");
|
||||
let memory = BackgroundMemoryBudget::new();
|
||||
memory.set_sampled_limit(2 * 1024 * 1024, Duration::from_secs(30));
|
||||
let (resources, gates) = test_resource_units(2);
|
||||
let request = memory_test_write_request(true);
|
||||
let endpoint = request.endpoint.clone();
|
||||
let writer = with_optional_resource_unit(
|
||||
Some(resources.with_memory_budget(memory.clone())),
|
||||
TcpHttpInternodeDataTransport.open_write(request),
|
||||
)
|
||||
.await;
|
||||
PUT_FILE_CAPABILITY_CACHE.write().remove(&endpoint);
|
||||
let mut writer = writer.expect("admitted transport writer");
|
||||
assert!(
|
||||
gates.iter().all(|gate| gate.snapshot().active == 0),
|
||||
"the callback must not retain disk units"
|
||||
);
|
||||
assert_eq!(memory.snapshot().used, 1024 * 1024);
|
||||
writer.write_all(b"data").await.expect("buffered bytes");
|
||||
writer.flush().await.expect("transfer charged allocation to queue");
|
||||
memory.set_sampled_limit(2 * 1024 * 1024, Duration::ZERO);
|
||||
writer
|
||||
.write_all(&vec![0x5a; 1024 * 1024 - 4])
|
||||
.await
|
||||
.expect("reuse the already charged tail after observation expiry");
|
||||
assert_eq!(memory.snapshot().used, 1024 * 1024, "reuse must not allocate separate storage");
|
||||
let error = writer
|
||||
.write(b"rejected")
|
||||
.await
|
||||
.expect_err("expired observation must reject new storage");
|
||||
assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
|
||||
assert!(
|
||||
crate::disk::background_memory::is_memory_pressure(&error),
|
||||
"preserve the typed memory-pressure marker"
|
||||
);
|
||||
assert_eq!(memory.snapshot().used, 1024 * 1024, "queued storage remains charged after expiry");
|
||||
drop(writer);
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_write_memory_denial_preserves_the_budget_error() {
|
||||
use crate::disk::{BackgroundMemoryBudget, test_resource_units, with_optional_resource_unit};
|
||||
|
||||
for (authenticated, sample) in [false, true].into_iter().flat_map(|authenticated| {
|
||||
[
|
||||
None,
|
||||
Some((1024 * 1024 - 1, Duration::from_secs(30))),
|
||||
Some((1024 * 1024, Duration::ZERO)),
|
||||
]
|
||||
.into_iter()
|
||||
.map(move |sample| (authenticated, sample))
|
||||
}) {
|
||||
let memory = BackgroundMemoryBudget::new();
|
||||
if let Some((limit, valid_for)) = sample {
|
||||
memory.set_sampled_limit(limit, valid_for);
|
||||
}
|
||||
let (resources, _) = test_resource_units(1);
|
||||
let request = memory_test_write_request(authenticated);
|
||||
let endpoint = request.endpoint.clone();
|
||||
let result = with_optional_resource_unit(
|
||||
Some(resources.with_memory_budget(memory.clone())),
|
||||
TcpHttpInternodeDataTransport.open_write(request),
|
||||
)
|
||||
.await;
|
||||
PUT_FILE_CAPABILITY_CACHE.write().remove(&endpoint);
|
||||
let error = match result {
|
||||
Err(Error::Io(error)) => error,
|
||||
Err(error) => panic!("unexpected transport error: {error}"),
|
||||
Ok(_) => panic!("unavailable memory must reject construction"),
|
||||
};
|
||||
assert!(crate::disk::background_memory::is_memory_pressure(&error));
|
||||
assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admitted_remote_reads_bind_receiver_admission_to_the_http_signature() {
|
||||
use crate::disk::with_background_resource_unit;
|
||||
use rustfs_concurrency::adaptive::AdaptiveAdmission;
|
||||
use std::num::NonZeroU16;
|
||||
|
||||
crate::runtime::sources::ensure_test_rpc_secret();
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("remote admission listener");
|
||||
let address = listener.local_addr().expect("remote admission address");
|
||||
let server = tokio::spawn(async move {
|
||||
let (mut socket, _) = listener.accept().await.expect("remote read connection");
|
||||
let mut bytes = Vec::new();
|
||||
while !bytes.windows(4).any(|window| window == b"\r\n\r\n") {
|
||||
let mut chunk = [0; 1024];
|
||||
let read = socket.read(&mut chunk).await.expect("remote read headers");
|
||||
assert!(read > 0 && bytes.len() < 16 * 1024, "bounded complete request headers");
|
||||
bytes.extend_from_slice(&chunk[..read]);
|
||||
}
|
||||
let request = std::str::from_utf8(&bytes).expect("HTTP request");
|
||||
let mut lines = request.split("\r\n");
|
||||
let target = lines
|
||||
.next()
|
||||
.expect("request line")
|
||||
.split_whitespace()
|
||||
.nth(1)
|
||||
.expect("request target")
|
||||
.to_owned();
|
||||
let mut headers = HeaderMap::new();
|
||||
for line in lines.take_while(|line| !line.is_empty()) {
|
||||
let (name, value) = line.split_once(':').expect("header field");
|
||||
headers.append(
|
||||
http::header::HeaderName::from_bytes(name.as_bytes()).expect("header name"),
|
||||
HeaderValue::from_str(value.trim()).expect("header value"),
|
||||
);
|
||||
}
|
||||
socket
|
||||
.write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 5\r\nconnection: close\r\n\r\nhello")
|
||||
.await
|
||||
.expect("remote read response");
|
||||
(target, headers)
|
||||
});
|
||||
let gate = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
|
||||
let resource = gate.try_acquire().expect("sender device admission");
|
||||
let mut reader = tokio::time::timeout(
|
||||
Duration::from_secs(5),
|
||||
with_background_resource_unit(
|
||||
resource,
|
||||
TcpHttpInternodeDataTransport.open_read(ReadStreamRequest {
|
||||
endpoint: format!("http://{address}"),
|
||||
disk: "disk-a".to_owned(),
|
||||
volume: "bucket".to_owned(),
|
||||
path: "part.1".to_owned(),
|
||||
offset: 0,
|
||||
length: 5,
|
||||
stall_timeout: Some(Duration::from_secs(5)),
|
||||
}),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("bounded remote request")
|
||||
.expect("remote reader");
|
||||
let mut body = Vec::new();
|
||||
reader.read_to_end(&mut body).await.expect("complete remote body");
|
||||
assert_eq!(body, b"hello");
|
||||
let (target, headers) = server.await.expect("remote request capture");
|
||||
assert_eq!(gate.snapshot().active, 0, "remote body does not retain the sender's device");
|
||||
assert_eq!(
|
||||
headers
|
||||
.get("x-rustfs-rpc-background-admission")
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some("v1"),
|
||||
"resource-controlled remote work must request receiver-side admission"
|
||||
);
|
||||
crate::cluster::rpc::verify_rpc_signature(&target, &Method::GET, &headers).expect("bound request signature");
|
||||
let mut stripped = headers;
|
||||
stripped.remove("x-rustfs-rpc-background-admission");
|
||||
assert!(
|
||||
crate::cluster::rpc::verify_rpc_signature(&target, &Method::GET, &stripped).is_err(),
|
||||
"removing background admission must invalidate the signature"
|
||||
);
|
||||
}
|
||||
|
||||
async fn wait_for_capability_flight_waiters(entry: &PutFileCapabilityCacheEntry, waiters: usize) {
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
|
||||
@@ -13,6 +13,8 @@
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) mod background_monitor;
|
||||
pub(crate) mod background_pipe;
|
||||
pub(crate) mod background_reader_stream;
|
||||
pub(crate) mod client;
|
||||
pub(crate) mod context_propagation;
|
||||
pub(crate) mod http_auth;
|
||||
@@ -33,16 +35,17 @@ pub use client::{
|
||||
// Re-exported through `api::rpc`; not every item is consumed inside this crate.
|
||||
#[allow(unused_imports)]
|
||||
pub use http_auth::{
|
||||
TONIC_RPC_PREFIX, build_auth_headers, build_put_file_auth_trailer, check_and_record_signed_rpc_nonce, gen_signature_headers,
|
||||
gen_tonic_replay_scope_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
|
||||
set_tonic_mutation_body_digest, set_tonic_rolling_canonical_body_digest, set_tonic_rolling_mutation_body_digest,
|
||||
sign_ns_scanner_capability, sign_ns_scanner_capability_with_checkpoint_coverage,
|
||||
sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, sign_tonic_rpc_response_proof,
|
||||
tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_ns_scanner_capability,
|
||||
verify_ns_scanner_capability_with_checkpoint_coverage, verify_ns_scanner_capability_with_tier_registry_generation,
|
||||
verify_put_file_auth_trailer, verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response,
|
||||
verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned,
|
||||
verify_tonic_rpc_response_proof, verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
|
||||
RPC_BACKGROUND_ADMISSION_HEADER, TONIC_RPC_PREFIX, build_auth_headers, build_put_file_auth_trailer,
|
||||
check_and_record_signed_rpc_nonce, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
|
||||
normalize_tonic_rpc_audience, rpc_background_admission, set_tonic_canonical_body_digest, set_tonic_mutation_body_digest,
|
||||
set_tonic_rolling_canonical_body_digest, set_tonic_rolling_mutation_body_digest, sign_ns_scanner_capability,
|
||||
sign_ns_scanner_capability_with_checkpoint_coverage, sign_ns_scanner_capability_with_tier_registry_generation,
|
||||
sign_put_file_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
|
||||
tonic_rpc_auth_failure_reason, verify_ns_scanner_capability, verify_ns_scanner_capability_with_checkpoint_coverage,
|
||||
verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer, verify_put_file_capability,
|
||||
verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
|
||||
verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned, verify_tonic_rpc_response_proof,
|
||||
verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;
|
||||
|
||||
@@ -2014,9 +2014,21 @@ impl PeerRestClient {
|
||||
}
|
||||
result
|
||||
})
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(Error::other(format!("load_bucket_metadata({bucket}) timed out"))));
|
||||
self.finalize_result(result).await
|
||||
.await;
|
||||
self.settle_bucket_metadata_reload(bucket, result).await
|
||||
}
|
||||
|
||||
async fn settle_bucket_metadata_reload(
|
||||
&self,
|
||||
bucket: &str,
|
||||
result: std::result::Result<Result<()>, tokio::time::error::Elapsed>,
|
||||
) -> Result<()> {
|
||||
match result {
|
||||
Ok(result) => self.finalize_result(result).await,
|
||||
// A local operation deadline does not establish that the peer is
|
||||
// unreachable. In particular, it must not gate IAM revocations.
|
||||
Err(_) => Err(Error::other(format!("load_bucket_metadata({bucket}) timed out"))),
|
||||
}
|
||||
}
|
||||
|
||||
async fn load_bucket_metadata_once(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
|
||||
@@ -4753,6 +4765,31 @@ mod tests {
|
||||
assert!(client.offline.load(Ordering::Acquire));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bucket_metadata_deadline_does_not_gate_healthy_peer_notifications() {
|
||||
let client = test_peer_client();
|
||||
let elapsed = tokio::time::timeout(std::time::Duration::ZERO, std::future::pending::<Result<()>>()).await;
|
||||
let error = client
|
||||
.settle_bucket_metadata_reload("slow-policy", elapsed)
|
||||
.await
|
||||
.expect_err("local budget expires");
|
||||
assert!(error.to_string().contains("timed out"));
|
||||
assert!(
|
||||
!client.offline.load(Ordering::Acquire),
|
||||
"local work must not suppress subsequent IAM RPCs"
|
||||
);
|
||||
client
|
||||
.settle_bucket_metadata_reload("healthy", Ok(Ok(())))
|
||||
.await
|
||||
.expect("peer reply");
|
||||
let error = client
|
||||
.settle_bucket_metadata_reload("unreachable", Ok(Err(Error::from(tonic::Status::unavailable("connection refused")))))
|
||||
.await
|
||||
.expect_err("real transport failure");
|
||||
assert!(error.to_string().contains("connection refused"));
|
||||
assert!(client.offline.load(Ordering::Acquire), "transport failures must still enter recovery");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn peer_rest_client_finalize_result_keeps_online_for_business_errors() {
|
||||
let client = test_peer_client();
|
||||
|
||||
@@ -17,6 +17,7 @@ use crate::cluster::rpc::client::{
|
||||
node_service_delete_recovery_client, node_service_time_out_client, node_service_time_out_client_for_class,
|
||||
node_service_time_out_client_no_auth,
|
||||
};
|
||||
use crate::cluster::rpc::http_auth::{current_rpc_background_request, with_rpc_background_request};
|
||||
use crate::cluster::rpc::internode_data_transport::{
|
||||
InternodeDataTransport, NsScannerCapabilityRequest, NsScannerStreamRequest, ReadStreamRequest, WalkDirStreamRequest,
|
||||
WriteStreamRequest,
|
||||
@@ -570,6 +571,7 @@ struct RetryingRemoteReader {
|
||||
reader: Option<FileReader>,
|
||||
transport: Arc<dyn InternodeDataTransport>,
|
||||
request: ReadStreamRequest,
|
||||
background: bool,
|
||||
emitted: usize,
|
||||
retried: bool,
|
||||
initial_read_timeout: Option<Duration>,
|
||||
@@ -590,6 +592,7 @@ impl RetryingRemoteReader {
|
||||
reader: Some(reader),
|
||||
transport,
|
||||
request,
|
||||
background: current_rpc_background_request(),
|
||||
emitted: 0,
|
||||
retried: false,
|
||||
initial_read_timeout,
|
||||
@@ -607,9 +610,10 @@ impl RetryingRemoteReader {
|
||||
let request = resumed_read_request(&self.request, self.emitted)?;
|
||||
let recovery_timeout = self.recovery_timeout;
|
||||
let transport = Arc::clone(&self.transport);
|
||||
self.resume = Some(AbortOnDropTask::new(tokio::spawn(async move {
|
||||
with_remote_read_recovery_timeout(recovery_timeout, transport.open_read_fresh(request)).await
|
||||
})));
|
||||
self.resume = Some(AbortOnDropTask::new(tokio::spawn(with_rpc_background_request(
|
||||
self.background,
|
||||
async move { with_remote_read_recovery_timeout(recovery_timeout, transport.open_read_fresh(request)).await },
|
||||
))));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -717,6 +721,7 @@ struct RetryingRemoteChunkReader {
|
||||
reader: Option<rustfs_rio::ChunkReaderBox>,
|
||||
transport: Arc<dyn InternodeDataTransport>,
|
||||
request: ReadStreamRequest,
|
||||
background: bool,
|
||||
emitted: usize,
|
||||
retried: bool,
|
||||
initial_read_timeout: Option<Duration>,
|
||||
@@ -737,6 +742,7 @@ impl RetryingRemoteChunkReader {
|
||||
reader: Some(reader),
|
||||
transport,
|
||||
request,
|
||||
background: current_rpc_background_request(),
|
||||
emitted: 0,
|
||||
retried: false,
|
||||
initial_read_timeout,
|
||||
@@ -754,9 +760,10 @@ impl RetryingRemoteChunkReader {
|
||||
let request = resumed_read_request(&self.request, self.emitted)?;
|
||||
let recovery_timeout = self.recovery_timeout;
|
||||
let transport = Arc::clone(&self.transport);
|
||||
self.resume = Some(AbortOnDropTask::new(tokio::spawn(async move {
|
||||
with_remote_read_recovery_timeout(recovery_timeout, transport.open_read_chunks_fresh(request)).await
|
||||
})));
|
||||
self.resume = Some(AbortOnDropTask::new(tokio::spawn(with_rpc_background_request(
|
||||
self.background,
|
||||
async move { with_remote_read_recovery_timeout(recovery_timeout, transport.open_read_chunks_fresh(request)).await },
|
||||
))));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1075,7 +1082,10 @@ impl RemoteDisk {
|
||||
if payload.len() > operation.max_wire_payload_bytes() {
|
||||
return Err(Error::other("delete recovery operation exceeds payload limit"));
|
||||
}
|
||||
let settle_timeout = if matches!(&operation, DeleteRecoveryRpcOperation::Settle { .. }) {
|
||||
let settle_timeout = if matches!(
|
||||
&operation,
|
||||
DeleteRecoveryRpcOperation::Settle { .. } | DeleteRecoveryRpcOperation::SettleBounded(_)
|
||||
) {
|
||||
Some(crate::disk::get_delete_recovery_timeout()?)
|
||||
} else {
|
||||
None
|
||||
@@ -1630,6 +1640,7 @@ impl RemoteDisk {
|
||||
let initial_probe_ok = Self::perform_connectivity_check(&addr).await.is_ok();
|
||||
if initial_probe_ok {
|
||||
health.record_operation_success(&endpoint, "connectivity_probe_success");
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
|
||||
} else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
|
||||
warn!(
|
||||
event = EVENT_REMOTE_DISK_HEALTH,
|
||||
@@ -1701,6 +1712,7 @@ impl RemoteDisk {
|
||||
// Perform basic connectivity check
|
||||
if Self::perform_connectivity_check(&addr).await.is_ok() {
|
||||
health.record_operation_success(&endpoint, "connectivity_probe_success");
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
|
||||
} else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
|
||||
warn!(
|
||||
event = EVENT_REMOTE_DISK_HEALTH,
|
||||
@@ -1771,6 +1783,7 @@ impl RemoteDisk {
|
||||
}
|
||||
_ = interval.tick() => {
|
||||
if Self::perform_recovery_probe(&addr, &endpoint).await.is_ok() {
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable(&addr);
|
||||
let became_online = health.mark_recovery_success(&endpoint, "disk_info_probe_success");
|
||||
debug!(
|
||||
event = EVENT_REMOTE_DISK_HEALTH,
|
||||
@@ -2002,6 +2015,7 @@ impl RemoteDisk {
|
||||
let operation_result = operation().await;
|
||||
if operation_result.is_ok() {
|
||||
self.health.record_operation_success(&self.endpoint, "operation_success");
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
|
||||
}
|
||||
self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
|
||||
.await;
|
||||
@@ -2016,6 +2030,7 @@ impl RemoteDisk {
|
||||
// Log success; the waiting guard balances every exit path.
|
||||
if operation_result.is_ok() {
|
||||
self.health.record_operation_success(&self.endpoint, "operation_success");
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
|
||||
}
|
||||
self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
|
||||
.await;
|
||||
@@ -2704,6 +2719,54 @@ impl RemoteDisk {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl DiskAPI for RemoteDisk {
|
||||
async fn list_multipart_index_roots(
|
||||
&self,
|
||||
bucket: &str,
|
||||
marker: Option<&str>,
|
||||
) -> Result<crate::disk::multipart_index::MultipartIndexPage> {
|
||||
use crate::disk::multipart_index::{MULTIPART_INDEX_RPC_MAX_BYTES, MultipartIndexRpcReply, MultipartIndexRpcRequest};
|
||||
let expected_disk = self
|
||||
.id
|
||||
.lock()
|
||||
.await
|
||||
.filter(|id| !id.is_nil())
|
||||
.ok_or(DiskError::DiskNotFound)?;
|
||||
let query = MultipartIndexRpcRequest::new(bucket, marker);
|
||||
let endpoint = self.endpoint.to_string();
|
||||
let canonical = query.canonical_request(&endpoint)?;
|
||||
let opts = serde_json::to_string(&query)?;
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(DiskInfoRequest {
|
||||
disk: endpoint.clone(),
|
||||
opts: opts.clone(),
|
||||
});
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
|
||||
let response = client.disk_info(request).await?.into_inner();
|
||||
if !response.success {
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
}
|
||||
if response.disk_info.len() > MULTIPART_INDEX_RPC_MAX_BYTES {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
// An old server ignores the additive query and returns ordinary DiskInfo.
|
||||
// It cannot satisfy the dedicated reply schema or the fresh challenge.
|
||||
let reply: MultipartIndexRpcReply =
|
||||
serde_json::from_str(&response.disk_info).map_err(|_| DiskError::MethodNotAllowed)?;
|
||||
reply.validate(&query, expected_disk)?;
|
||||
crate::cluster::rpc::verify_tonic_rpc_response_proof(
|
||||
&query.canonical_response(&endpoint, &reply.page)?,
|
||||
&reply.proof,
|
||||
)
|
||||
.map_err(Error::other)?;
|
||||
Ok(reply.page)
|
||||
},
|
||||
get_drive_disk_info_timeout(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn supports_guarded_delete_versions(&self) -> Result<()> {
|
||||
let response = self.guarded_delete_versions_rpc("", Vec::new(), true).await?;
|
||||
if !response.errors.is_empty() || !response.item_errors.is_empty() {
|
||||
@@ -4287,6 +4350,38 @@ impl DiskAPI for RemoteDisk {
|
||||
}
|
||||
}
|
||||
|
||||
async fn discover_delete_transactions_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
max_entries: u32,
|
||||
) -> Result<crate::disk::BoundedDeleteRecoveryDiscovery> {
|
||||
// Older peers reject this variant. Do not retry through the unbounded protocol;
|
||||
// they are reported as unsupported participants (rolling-upgrade contract).
|
||||
match self
|
||||
.recover_delete_transaction(
|
||||
volume,
|
||||
DeleteRecoveryRpcOperation::DiscoverBounded {
|
||||
object: object.to_owned(),
|
||||
max_entries,
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(DeleteRecoveryRpcReply::DiscoveredBounded(page)) => Ok(page),
|
||||
Ok(_) => Err(Error::FileCorrupt),
|
||||
Err(error)
|
||||
if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
|
||||
|| matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
|
||||
.is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
|
||||
{
|
||||
// No paginated fallback through the unbounded protocol; the peer
|
||||
// is an unsupported participant until it is upgraded.
|
||||
Err(Error::UnsupportedDisk)
|
||||
}
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
async fn discover_delete_transactions(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -4322,6 +4417,62 @@ impl DiskAPI for RemoteDisk {
|
||||
_ => Err(Error::other("unexpected delete discovery reply")),
|
||||
}
|
||||
}
|
||||
async fn settle_delete_transaction_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
request: crate::disk::BoundedDeleteRecoverySettlement,
|
||||
_namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<crate::disk::BoundedDeleteRecoverySettlementReply> {
|
||||
match self
|
||||
.recover_delete_transaction(volume, DeleteRecoveryRpcOperation::SettleBounded(request))
|
||||
.await
|
||||
{
|
||||
Ok(DeleteRecoveryRpcReply::SettledBounded(reply)) => Ok(reply),
|
||||
Ok(_) => Err(Error::FileCorrupt),
|
||||
Err(error)
|
||||
if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
|
||||
|| matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
|
||||
.is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
|
||||
{
|
||||
// No paginated fallback through the unbounded protocol; the peer
|
||||
// is an unsupported participant until it is upgraded.
|
||||
Err(Error::UnsupportedDisk)
|
||||
}
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
async fn inspect_delete_transaction_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
max_entries: u32,
|
||||
) -> Result<crate::disk::BoundedDeleteRecoveryInspection> {
|
||||
match self
|
||||
.recover_delete_transaction(
|
||||
volume,
|
||||
DeleteRecoveryRpcOperation::InspectBounded {
|
||||
object: object.to_owned(),
|
||||
transaction,
|
||||
max_entries,
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(DeleteRecoveryRpcReply::InspectedBounded(page)) => Ok(page),
|
||||
Ok(_) => Err(Error::FileCorrupt),
|
||||
Err(error)
|
||||
if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
|
||||
|| matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
|
||||
.is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
|
||||
{
|
||||
// No paginated fallback through the unbounded protocol; the peer
|
||||
// is an unsupported participant until it is upgraded.
|
||||
Err(Error::UnsupportedDisk)
|
||||
}
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
async fn inspect_delete_transaction(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -4842,6 +4993,113 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bounded_delete_discovery_lost_remote_reply_leaves_no_cursor() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
|
||||
.await
|
||||
.expect("test peer must bind");
|
||||
let directory = tempfile::tempdir().expect("temporary disk");
|
||||
let endpoint = Endpoint::try_from(directory.path().to_str().expect("disk path")).expect("endpoint");
|
||||
let disk = Arc::new(
|
||||
crate::disk::local::LocalDisk::new(&endpoint, false)
|
||||
.await
|
||||
.expect("local disk"),
|
||||
);
|
||||
disk.make_volume("bucket").await.expect("bucket");
|
||||
let prefix = directory.path().join("bucket/prefix");
|
||||
std::fs::create_dir(&prefix).expect("prefix");
|
||||
for index in 0..64 {
|
||||
std::fs::create_dir(prefix.join(format!("child-{index}"))).expect("legitimate child");
|
||||
}
|
||||
let fixture = Arc::new(TestBoundedDiscovery {
|
||||
disk: Arc::clone(&disk),
|
||||
response_gate: TestFormatReadGate::default(),
|
||||
});
|
||||
*peer.peer.bounded_discovery.lock().expect("bounded fixture") = Some(Arc::clone(&fixture));
|
||||
let remote = remote_disk_for_addr(&peer.addr).await;
|
||||
let task = tokio::spawn(async move { remote.discover_delete_transactions_bounded("bucket", "prefix", 8).await });
|
||||
tokio::time::timeout(Duration::from_secs(5), fixture.response_gate.entered.notified())
|
||||
.await
|
||||
.expect("disk completed bounded scan");
|
||||
task.abort();
|
||||
assert!(task.await.expect_err("caller cancelled").is_cancelled());
|
||||
assert_eq!(disk.public_delete_discovery_sessions_for_test(), 0);
|
||||
fixture.response_gate.release.cancel();
|
||||
peer.stop().await;
|
||||
assert_eq!(disk.public_delete_discovery_sessions_for_test(), 0);
|
||||
assert!(prefix.join("child-63").is_dir());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_index_old_disk_info_handler_is_not_a_ready_empty_index() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
|
||||
.await
|
||||
.expect("old peer must bind");
|
||||
let remote = remote_disk_for_addr(&peer.addr).await;
|
||||
remote
|
||||
.set_disk_id(Some(Uuid::new_v4()))
|
||||
.await
|
||||
.expect("expected disk identity");
|
||||
assert!(matches!(
|
||||
remote.list_multipart_index_roots("bucket", None).await,
|
||||
Err(DiskError::MethodNotAllowed)
|
||||
));
|
||||
assert_eq!(
|
||||
peer.peer.disk_info_calls.load(Ordering::Acquire),
|
||||
1,
|
||||
"exercise the actual legacy JSON handler"
|
||||
);
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bounded_delete_discovery_old_peer_rejects_without_paginated_fallback() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
|
||||
.await
|
||||
.expect("old peer must bind");
|
||||
let remote = remote_disk_for_addr(&peer.addr).await;
|
||||
assert_eq!(
|
||||
remote
|
||||
.discover_delete_transactions_bounded("bucket", "prefix", 4096)
|
||||
.await
|
||||
.expect_err("old peer cannot prove a bounded inventory"),
|
||||
DiskError::UnsupportedDisk
|
||||
);
|
||||
let transaction = Uuid::new_v4();
|
||||
assert_eq!(
|
||||
remote
|
||||
.inspect_delete_transaction_bounded("bucket", "prefix", transaction, 65536)
|
||||
.await
|
||||
.expect_err("old peer cannot inspect with a budget"),
|
||||
DiskError::UnsupportedDisk
|
||||
);
|
||||
assert_eq!(
|
||||
remote
|
||||
.settle_delete_transaction_bounded(
|
||||
"bucket",
|
||||
crate::disk::BoundedDeleteRecoverySettlement {
|
||||
object: "prefix".to_owned(),
|
||||
transaction,
|
||||
expected_fingerprint: [0; 32],
|
||||
action: DeleteRecoveryAction::PublishCommit,
|
||||
max_entries: 65536,
|
||||
},
|
||||
None
|
||||
)
|
||||
.await
|
||||
.expect_err("old peer cannot settle with a budget"),
|
||||
DiskError::UnsupportedDisk
|
||||
);
|
||||
assert_eq!(
|
||||
*peer.peer.unsupported_methods.lock().expect("observed methods"),
|
||||
vec!["/node_service.NodeService/RecoverDeleteTransaction"; 3]
|
||||
);
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_source_recovery_old_peer_rejects_without_generic_fallback() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
@@ -5734,9 +5992,16 @@ mod tests {
|
||||
format_data: Bytes,
|
||||
read_all_data: Bytes,
|
||||
next_format_read_gate: Arc<StdMutex<Option<Arc<TestFormatReadGate>>>>,
|
||||
bounded_discovery: Arc<StdMutex<Option<Arc<TestBoundedDiscovery>>>>,
|
||||
unsupported_methods: Arc<StdMutex<Vec<String>>>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct TestBoundedDiscovery {
|
||||
disk: Arc<crate::disk::local::LocalDisk>,
|
||||
response_gate: TestFormatReadGate,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct TestFormatReadGate {
|
||||
entered: tokio::sync::Notify,
|
||||
@@ -5753,6 +6018,7 @@ mod tests {
|
||||
format_data,
|
||||
read_all_data,
|
||||
next_format_read_gate: Arc::default(),
|
||||
bounded_discovery: Arc::default(),
|
||||
unsupported_methods: Arc::default(),
|
||||
}
|
||||
}
|
||||
@@ -5852,6 +6118,56 @@ mod tests {
|
||||
Ok(grpc.unary(method, request).await)
|
||||
})
|
||||
}
|
||||
"/node_service.NodeService/RecoverDeleteTransaction"
|
||||
if self.peer.bounded_discovery.lock().expect("bounded fixture").is_some() =>
|
||||
{
|
||||
#[derive(Clone)]
|
||||
struct BoundedSvc(Arc<TestBoundedDiscovery>);
|
||||
impl tonic::server::UnaryService<RecoverDeleteTransactionRequest> for BoundedSvc {
|
||||
type Response = rustfs_protos::proto_gen::node_service::RecoverDeleteTransactionResponse;
|
||||
type Future = Pin<Box<dyn Future<Output = std::result::Result<Response<Self::Response>, Status>> + Send>>;
|
||||
fn call(&mut self, request: Request<RecoverDeleteTransactionRequest>) -> Self::Future {
|
||||
let fixture = Arc::clone(&self.0);
|
||||
Box::pin(async move {
|
||||
let request = request.into_inner();
|
||||
let operation: DeleteRecoveryRpcOperation =
|
||||
rustfs_protos::decode_delete_recovery_payload(&request.operation)
|
||||
.map_err(|_| Status::invalid_argument("invalid bounded request"))?;
|
||||
let DeleteRecoveryRpcOperation::DiscoverBounded { object, max_entries } = operation else {
|
||||
return Err(Status::invalid_argument("unexpected fallback"));
|
||||
};
|
||||
let page = fixture
|
||||
.disk
|
||||
.discover_delete_transactions_bounded(&request.volume, &object, max_entries)
|
||||
.await
|
||||
.map_err(|_| Status::internal("bounded discovery failed"))?;
|
||||
// Lose the response only after the real disk operation has completed.
|
||||
fixture.response_gate.entered.notify_one();
|
||||
fixture.response_gate.release.cancelled().await;
|
||||
let result = rmp_serde::to_vec_named(&DeleteRecoveryRpcReply::DiscoveredBounded(page))
|
||||
.map_err(|_| Status::internal("encode bounded reply"))?;
|
||||
Ok(Response::new(rustfs_protos::proto_gen::node_service::RecoverDeleteTransactionResponse {
|
||||
success: true,
|
||||
protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION,
|
||||
result: result.into(),
|
||||
error: None,
|
||||
}))
|
||||
})
|
||||
}
|
||||
}
|
||||
let fixture = self
|
||||
.peer
|
||||
.bounded_discovery
|
||||
.lock()
|
||||
.expect("bounded fixture")
|
||||
.clone()
|
||||
.expect("configured fixture");
|
||||
Box::pin(async move {
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let mut grpc = tonic::server::Grpc::new(codec);
|
||||
Ok(grpc.unary(BoundedSvc(fixture), request).await)
|
||||
})
|
||||
}
|
||||
"/node_service.NodeService/ReadAll" => {
|
||||
#[derive(Clone)]
|
||||
struct ReadAllSvc(AuthenticatedReadPeer);
|
||||
@@ -7769,6 +8085,98 @@ mod tests {
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remote_read_retries_preserve_signed_admission_after_sender_scope_ends() {
|
||||
use crate::cluster::rpc::{rpc_background_admission, verify_rpc_signature};
|
||||
use crate::disk::with_background_resource_unit;
|
||||
use rustfs_concurrency::adaptive::AdaptiveAdmission;
|
||||
use std::num::NonZeroU16;
|
||||
|
||||
crate::runtime::sources::ensure_test_rpc_secret();
|
||||
for background in [false, true] {
|
||||
for chunked in [false, true] {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("retry listener");
|
||||
let address = listener.local_addr().expect("retry address");
|
||||
let server = tokio::spawn(async move {
|
||||
let (mut socket, _) = listener.accept().await.expect("fresh connection");
|
||||
let mut bytes = Vec::new();
|
||||
while !bytes.windows(4).any(|window| window == b"\r\n\r\n") {
|
||||
let mut chunk = [0; 1024];
|
||||
let read = socket.read(&mut chunk).await.expect("retry request");
|
||||
assert!(read > 0 && bytes.len() < 16 * 1024, "bounded complete headers");
|
||||
bytes.extend_from_slice(&chunk[..read]);
|
||||
}
|
||||
let request = std::str::from_utf8(&bytes).expect("HTTP request");
|
||||
let mut lines = request.split("\r\n");
|
||||
let target = lines
|
||||
.next()
|
||||
.expect("request line")
|
||||
.split_whitespace()
|
||||
.nth(1)
|
||||
.expect("target")
|
||||
.to_owned();
|
||||
let mut headers = http::HeaderMap::new();
|
||||
for line in lines.take_while(|line| !line.is_empty()) {
|
||||
let (name, value) = line.split_once(':').expect("header");
|
||||
headers.append(
|
||||
http::header::HeaderName::from_bytes(name.as_bytes()).expect("header name"),
|
||||
http::HeaderValue::from_str(value.trim()).expect("header value"),
|
||||
);
|
||||
}
|
||||
socket
|
||||
.write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 6\r\nconnection: close\r\n\r\n456789")
|
||||
.await
|
||||
.expect("retry response");
|
||||
(target, headers)
|
||||
});
|
||||
let gate = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
|
||||
let request = ReadStreamRequest {
|
||||
endpoint: format!("http://{address}"),
|
||||
..resume_request(10)
|
||||
};
|
||||
let create = async {
|
||||
let transport = Arc::new(TcpHttpInternodeDataTransport);
|
||||
if chunked {
|
||||
Box::new(RetryingRemoteChunkReader::new_with_timeouts(
|
||||
resume_step_chunk_reader(ResumeReadStep::PartialThenReset(b"0123".to_vec())),
|
||||
transport,
|
||||
request,
|
||||
None,
|
||||
None,
|
||||
)) as FileReader
|
||||
} else {
|
||||
Box::new(RetryingRemoteReader::new_with_timeouts(
|
||||
resume_step_reader(ResumeReadStep::PartialThenReset(b"0123".to_vec())),
|
||||
transport,
|
||||
request,
|
||||
None,
|
||||
None,
|
||||
)) as FileReader
|
||||
}
|
||||
};
|
||||
let mut reader = if background {
|
||||
with_background_resource_unit(gate.try_acquire().expect("sender admission"), create).await
|
||||
} else {
|
||||
create.await
|
||||
};
|
||||
assert_eq!(gate.snapshot().active, 0, "retry reader cannot retain sender disk capacity");
|
||||
let mut output = Vec::new();
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(5),
|
||||
with_rpc_background_request(!background, reader.read_to_end(&mut output)),
|
||||
)
|
||||
.await
|
||||
.expect("bounded retry")
|
||||
.expect("resumed body");
|
||||
assert_eq!(output, b"0123456789", "chunked={chunked}, background={background}");
|
||||
let (target, headers) = server.await.expect("captured retry");
|
||||
assert!(target.contains("offset=11") && target.contains("length=6"), "{target}");
|
||||
assert_eq!(rpc_background_admission(&headers).expect("admission marker"), background);
|
||||
verify_rpc_signature(&target, &http::Method::GET, &headers).expect("signed retry class");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remote_reader_resumes_from_emitted_bytes_without_duplicates() {
|
||||
let transport = Arc::new(ResumeTransport::with_read_steps(vec![ResumeReadStep::Data(b"456789".to_vec())]));
|
||||
|
||||
@@ -18,6 +18,7 @@ use crate::cluster::rpc::client::{
|
||||
use crate::cluster::rpc::set_tonic_rolling_mutation_body_digest;
|
||||
use async_trait::async_trait;
|
||||
use bytes::Bytes;
|
||||
use futures::{FutureExt, future::BoxFuture};
|
||||
use rustfs_lock::{
|
||||
LockClient, LockError, LockInfo, LockRequest, LockResponse, LockStats, LockStatus, LockType, Result,
|
||||
types::{LockId, LockMetadata, LockPriority},
|
||||
@@ -32,16 +33,17 @@ use rustfs_protos::{
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use std::future::Future;
|
||||
use std::panic::AssertUnwindSafe;
|
||||
use std::pin::Pin;
|
||||
use std::sync::{Mutex, OnceLock};
|
||||
use std::time::Duration;
|
||||
use tokio::task::JoinHandle;
|
||||
use tokio::time::{Instant, timeout};
|
||||
use tonic::service::interceptor::InterceptedService;
|
||||
use tonic::{Request, Response};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
const EVENT_NAMESPACE_LOCK_RELEASE: &str = "namespace_lock_release";
|
||||
const EVENT_REMOTE_LOCK_RPC_CLEANUP: &str = "remote_lock_rpc_cleanup";
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_REMOTE_LOCK: &str = "remote_lock";
|
||||
|
||||
@@ -49,18 +51,38 @@ fn attach_lock_mutation_body_digest<T: rustfs_protos::CanonicalMutationBody>(req
|
||||
set_tonic_rolling_mutation_body_digest(request)
|
||||
}
|
||||
|
||||
/// Work to run if an RPC that already timed out for its caller completes later.
|
||||
/// Work to run if an RPC completes after its caller stops waiting.
|
||||
type LateCompletion<T> = Option<Box<dyn FnOnce(T) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send>>;
|
||||
type LateFailure = LateCompletion<()>;
|
||||
type RpcFuture<T> = BoxFuture<'static, std::thread::Result<std::result::Result<T, tonic::Status>>>;
|
||||
|
||||
struct RpcCleanup<T> {
|
||||
late: LateCompletion<T>,
|
||||
late_failure: LateFailure,
|
||||
/// Own admission before connecting as well as while an RPC is in flight.
|
||||
struct LockRequestGuard<'a> {
|
||||
client: &'a RemoteClient,
|
||||
op: &'static str,
|
||||
admission: Option<LockRequestAdmission>,
|
||||
}
|
||||
|
||||
impl<T> RpcCleanup<T> {
|
||||
fn new(late: LateCompletion<T>, late_failure: LateFailure, admission: Option<LockRequestAdmission>) -> Self {
|
||||
impl LockRequestGuard<'_> {
|
||||
fn into_admission(mut self) -> Option<LockRequestAdmission> {
|
||||
self.admission.take()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for LockRequestGuard<'_> {
|
||||
fn drop(&mut self) {
|
||||
self.client.release_lock_request(self.op, self.admission.take());
|
||||
}
|
||||
}
|
||||
|
||||
struct RpcCleanup<'a, T> {
|
||||
late: LateCompletion<T>,
|
||||
late_failure: LateFailure,
|
||||
admission: Option<LockRequestGuard<'a>>,
|
||||
}
|
||||
|
||||
impl<'a, T> RpcCleanup<'a, T> {
|
||||
fn new(late: LateCompletion<T>, late_failure: LateFailure, admission: Option<LockRequestGuard<'a>>) -> Self {
|
||||
Self {
|
||||
late,
|
||||
late_failure,
|
||||
@@ -69,6 +91,25 @@ impl<T> RpcCleanup<T> {
|
||||
}
|
||||
}
|
||||
|
||||
/// A dropped caller hands its RPC and cleanup to the same bounded owner as a timeout.
|
||||
struct InFlightRpc<'a, T: Send + 'static> {
|
||||
client: &'a RemoteClient,
|
||||
op: &'static str,
|
||||
resource_summary: &'a str,
|
||||
future: Option<RpcFuture<T>>,
|
||||
cleanup: Option<RpcCleanup<'a, T>>,
|
||||
}
|
||||
|
||||
impl<T: Send + 'static> Drop for InFlightRpc<'_, T> {
|
||||
fn drop(&mut self) {
|
||||
if let Some(future) = self.future.take()
|
||||
&& let Some(cleanup) = self.cleanup.take()
|
||||
{
|
||||
self.client.detach_rpc(self.op, self.resource_summary, future, cleanup);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The liveness window is this many RPC deadlines: a peer that completed a
|
||||
/// lock RPC within it is slow, not gone, and keeps its channel on a timeout.
|
||||
const LOCK_RPC_LIVENESS_WINDOW_DEADLINES: u32 = 2;
|
||||
@@ -137,6 +178,9 @@ fn with_lock_peer_health<R>(addr: &str, update: impl FnOnce(&mut LockPeerChannel
|
||||
let mut peers = lock_peer_channel_health()
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if let Some(health) = peers.get_mut(addr) {
|
||||
return update(health);
|
||||
}
|
||||
update(peers.entry(addr.to_string()).or_default())
|
||||
}
|
||||
|
||||
@@ -158,6 +202,15 @@ fn reset_lock_peer_health_for_test(addr: &str) {
|
||||
.remove(addr);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
type EvictionPause = (tokio::sync::oneshot::Sender<()>, tokio::sync::oneshot::Receiver<()>);
|
||||
|
||||
#[cfg(test)]
|
||||
fn eviction_pauses_for_test() -> &'static Mutex<HashMap<String, EvictionPause>> {
|
||||
static PAUSES: OnceLock<Mutex<HashMap<String, EvictionPause>>> = OnceLock::new();
|
||||
PAUSES.get_or_init(Mutex::default)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum EvictionTrigger {
|
||||
/// The caller's deadline expired while the stream was still open.
|
||||
@@ -445,6 +498,10 @@ impl RemoteClient {
|
||||
}
|
||||
}
|
||||
});
|
||||
// A successful lock RPC over a cached or fresh channel proves the peer is
|
||||
// reachable; reconcile the internode peer-health gauge so it does not stay
|
||||
// stale when no fresh dial fires (issue #9119).
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
|
||||
}
|
||||
|
||||
fn admit_late_release(&self) -> bool {
|
||||
@@ -502,6 +559,14 @@ impl RemoteClient {
|
||||
admission
|
||||
}
|
||||
|
||||
fn admit_lock_request_guard(&self, op: &'static str) -> Option<LockRequestGuard<'_>> {
|
||||
self.admit_lock_request(op).map(|admission| LockRequestGuard {
|
||||
client: self,
|
||||
op,
|
||||
admission: Some(admission),
|
||||
})
|
||||
}
|
||||
|
||||
fn release_lock_request(&self, op: &'static str, admission: Option<LockRequestAdmission>) {
|
||||
if is_request_breaker_operation(op) && admission.is_some() {
|
||||
with_lock_peer_health(&self.addr, |health| {
|
||||
@@ -583,6 +648,17 @@ impl RemoteClient {
|
||||
trigger: EvictionTrigger,
|
||||
deadline: Duration,
|
||||
) {
|
||||
#[cfg(test)]
|
||||
let pause = eviction_pauses_for_test()
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.remove(&self.addr);
|
||||
#[cfg(test)]
|
||||
if let Some((entered, resume)) = pause {
|
||||
let _ = entered.send(());
|
||||
let _ = resume.await;
|
||||
}
|
||||
|
||||
let now = Instant::now();
|
||||
let cooldown = Self::eviction_cooldown();
|
||||
let liveness_window = Self::liveness_window(deadline);
|
||||
@@ -613,22 +689,26 @@ impl RemoteClient {
|
||||
);
|
||||
}
|
||||
|
||||
/// Keep a timed-out RPC running instead of cancelling its stream.
|
||||
/// Keep an abandoned RPC running instead of cancelling its stream.
|
||||
///
|
||||
/// Dropping the future sends `RST_STREAM`; under load those resets pile up
|
||||
/// in the server's pending-accept queue until it answers `GOAWAY
|
||||
/// too_many_resets` and kills every stream on the connection. A detached
|
||||
/// stream ends on its own within the internode RPC timeout, the number per
|
||||
/// peer is bounded, and a lock granted after its caller gave up is released.
|
||||
fn detach_timed_out_rpc<T: Send + 'static>(
|
||||
fn detach_rpc<T: Send + 'static>(
|
||||
&self,
|
||||
op: &'static str,
|
||||
resource_summary: &str,
|
||||
handle: JoinHandle<std::result::Result<T, tonic::Status>>,
|
||||
late: LateCompletion<T>,
|
||||
late_failure: LateFailure,
|
||||
admission: Option<LockRequestAdmission>,
|
||||
future: RpcFuture<T>,
|
||||
cleanup: RpcCleanup<'_, T>,
|
||||
) {
|
||||
let RpcCleanup {
|
||||
late,
|
||||
late_failure,
|
||||
admission,
|
||||
} = cleanup;
|
||||
let admission = admission.and_then(LockRequestGuard::into_admission);
|
||||
let limit = Self::detached_rpc_limit();
|
||||
let admitted = with_lock_peer_health(&self.addr, |health| {
|
||||
if health.detached_rpcs >= limit {
|
||||
@@ -639,25 +719,29 @@ impl RemoteClient {
|
||||
}
|
||||
});
|
||||
if !admitted {
|
||||
handle.abort();
|
||||
drop(future);
|
||||
self.release_lock_request(op, admission);
|
||||
if let Some(late_failure) = late_failure {
|
||||
drop(tokio::spawn(late_failure(())));
|
||||
}
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_detached(op, "aborted");
|
||||
debug!(
|
||||
event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
|
||||
state = "detached_budget_exhausted",
|
||||
addr = %self.addr,
|
||||
op,
|
||||
resource_summary,
|
||||
limit,
|
||||
"Cancelled timed-out remote lock RPC because the detached stream budget is exhausted"
|
||||
"Abandoned remote lock RPC cancelled"
|
||||
);
|
||||
return;
|
||||
}
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_detached(op, "detached");
|
||||
let addr = self.addr.clone();
|
||||
tokio::spawn(async move {
|
||||
let outcome = handle.await;
|
||||
let outcome = future.await;
|
||||
with_lock_peer_health(&addr, |health| {
|
||||
health.detached_rpcs = health.detached_rpcs.saturating_sub(1);
|
||||
if is_request_breaker_operation(op) && admission.is_some() {
|
||||
@@ -672,6 +756,7 @@ impl RemoteClient {
|
||||
with_lock_peer_health(&addr, |health| {
|
||||
health.last_success = Some(Instant::now());
|
||||
});
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "success");
|
||||
if let Some(late) = late {
|
||||
late(response).await;
|
||||
@@ -683,19 +768,31 @@ impl RemoteClient {
|
||||
}
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "error");
|
||||
debug!(
|
||||
event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
|
||||
state = "late_rpc_failed",
|
||||
addr = %addr,
|
||||
op,
|
||||
tonic_code = ?status.code(),
|
||||
tonic_message = status.message(),
|
||||
"Detached remote lock RPC failed after its caller timed out"
|
||||
"Abandoned remote lock RPC failed"
|
||||
);
|
||||
}
|
||||
Err(join_error) => {
|
||||
Err(_) => {
|
||||
if let Some(late_failure) = late_failure {
|
||||
late_failure(()).await;
|
||||
}
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "join_error");
|
||||
debug!(addr = %addr, op, error = %join_error, "Detached remote lock RPC task ended abnormally");
|
||||
debug!(
|
||||
event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
|
||||
state = "late_rpc_panicked",
|
||||
addr = %addr,
|
||||
op,
|
||||
"Abandoned remote lock RPC panicked"
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -783,16 +880,28 @@ impl RemoteClient {
|
||||
self.release_late_release();
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_late_release(outcome);
|
||||
if outcome == "released" {
|
||||
debug!(addr = %self.addr, count = original_count, "Released remote locks granted after their caller timed out");
|
||||
debug!(
|
||||
event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
|
||||
state = "late_grants_released",
|
||||
addr = %self.addr,
|
||||
count = original_count,
|
||||
"Released remote locks granted after their caller stopped waiting"
|
||||
);
|
||||
} else {
|
||||
let (log_warning, suppressed_timeouts) = self.timeout_log_decision("late_release", Self::rpc_timeout());
|
||||
if log_warning {
|
||||
warn!(
|
||||
event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
|
||||
state = "late_grants_unreleased",
|
||||
addr = %self.addr,
|
||||
count = original_count,
|
||||
outcome,
|
||||
suppressed_timeouts,
|
||||
"Could not release every remote lock granted after its caller timed out; the server lease will expire it"
|
||||
"Could not release every remote lock granted after its caller stopped waiting; the server lease will expire it"
|
||||
);
|
||||
} else {
|
||||
debug!(addr = %self.addr, count = original_count, outcome, suppressed_timeouts, "Suppressed repeated late remote lock release warning");
|
||||
@@ -806,25 +915,47 @@ impl RemoteClient {
|
||||
resource_summary: &str,
|
||||
deadline: Duration,
|
||||
future: Fut,
|
||||
cleanup: RpcCleanup<T>,
|
||||
cleanup: RpcCleanup<'_, T>,
|
||||
) -> std::result::Result<T, LockError>
|
||||
where
|
||||
Fut: Future<Output = std::result::Result<T, tonic::Status>> + Send + 'static,
|
||||
T: Send + 'static,
|
||||
{
|
||||
let RpcCleanup {
|
||||
late,
|
||||
let mut rpc = InFlightRpc {
|
||||
client: self,
|
||||
op,
|
||||
resource_summary,
|
||||
future: Some(AssertUnwindSafe(future).catch_unwind().boxed()),
|
||||
cleanup: Some(cleanup),
|
||||
};
|
||||
let outcome = match rpc.future.as_mut() {
|
||||
Some(future) => tokio::select! {
|
||||
// A response becoming ready on the deadline tick must not turn an
|
||||
// expired caller budget into a transport error and channel eviction.
|
||||
biased;
|
||||
_ = tokio::time::sleep(deadline) => None,
|
||||
outcome = future => Some(outcome),
|
||||
},
|
||||
None => return Err(LockError::internal("remote lock RPC has no in-flight future")),
|
||||
};
|
||||
if outcome.is_some() {
|
||||
rpc.future.take();
|
||||
}
|
||||
let Some(RpcCleanup {
|
||||
late: _,
|
||||
late_failure,
|
||||
admission,
|
||||
} = cleanup;
|
||||
let mut handle = tokio::spawn(future);
|
||||
match timeout(deadline, &mut handle).await {
|
||||
Ok(Ok(Ok(response))) => {
|
||||
self.record_rpc_success(op, admission);
|
||||
self.release_lock_request(op, admission);
|
||||
}) = rpc.cleanup.as_mut()
|
||||
else {
|
||||
return Err(LockError::internal("remote lock RPC has no cleanup owner"));
|
||||
};
|
||||
let token = admission.as_ref().and_then(|guard| guard.admission);
|
||||
let result = match outcome {
|
||||
Some(Ok(Ok(response))) => {
|
||||
self.record_rpc_success(op, token);
|
||||
Ok(response)
|
||||
}
|
||||
Ok(Ok(Err(err))) => {
|
||||
Some(Ok(Err(err))) => {
|
||||
let reason = err.to_string();
|
||||
if err.code() == tonic::Code::DeadlineExceeded {
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_timeout(&self.addr, op);
|
||||
@@ -857,12 +988,11 @@ impl RemoteClient {
|
||||
"Suppressed repeated remote lock RPC deadline"
|
||||
);
|
||||
}
|
||||
self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Timeout, deadline)
|
||||
.await;
|
||||
self.release_lock_request(op, admission);
|
||||
if let Some(late_failure) = late_failure {
|
||||
if let Some(late_failure) = late_failure.take() {
|
||||
drop(tokio::spawn(late_failure(())));
|
||||
}
|
||||
self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Timeout, deadline)
|
||||
.await;
|
||||
return Err(LockError::timeout(format!("remote lock RPC {op} on {}", self.addr), deadline));
|
||||
}
|
||||
// Only evict (and re-dial) the cached channel when the failure is a genuine
|
||||
@@ -923,30 +1053,32 @@ impl RemoteClient {
|
||||
);
|
||||
}
|
||||
if transport_failure {
|
||||
// Ambiguous-grant cleanup owns its work before channel eviction can suspend.
|
||||
if let Some(late_failure) = late_failure.take() {
|
||||
drop(tokio::spawn(late_failure(())));
|
||||
}
|
||||
self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Transport, deadline)
|
||||
.await;
|
||||
}
|
||||
self.release_lock_request(op, admission);
|
||||
if transport_failure && let Some(late_failure) = late_failure {
|
||||
drop(tokio::spawn(late_failure(())));
|
||||
}
|
||||
Err(LockError::internal(format!("{op} RPC failed: {reason}")))
|
||||
}
|
||||
Ok(Err(join_error)) => {
|
||||
Some(Err(_)) => {
|
||||
warn!(
|
||||
event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
|
||||
state = "rpc_panicked",
|
||||
addr = %self.addr,
|
||||
op,
|
||||
resource_summary,
|
||||
error = %join_error,
|
||||
"Remote lock RPC task ended abnormally"
|
||||
"Remote lock RPC panicked"
|
||||
);
|
||||
self.release_lock_request(op, admission);
|
||||
if let Some(late_failure) = late_failure {
|
||||
if let Some(late_failure) = late_failure.take() {
|
||||
drop(tokio::spawn(late_failure(())));
|
||||
}
|
||||
Err(LockError::internal(format!("{op} RPC task failed: {join_error}")))
|
||||
Err(LockError::internal(format!("{op} RPC future panicked")))
|
||||
}
|
||||
Err(_) => {
|
||||
None => {
|
||||
let reason = format!("RPC timed out after {deadline:?}");
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_timeout(&self.addr, op);
|
||||
self.record_request_timeout(op, deadline);
|
||||
@@ -978,12 +1110,17 @@ impl RemoteClient {
|
||||
"Suppressed repeated remote lock RPC timeout"
|
||||
);
|
||||
}
|
||||
// Continue draining the abandoned stream while channel eviction waits.
|
||||
drop(rpc);
|
||||
self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Timeout, deadline)
|
||||
.await;
|
||||
self.detach_timed_out_rpc(op, resource_summary, handle, late, late_failure, admission);
|
||||
Err(LockError::timeout(format!("remote lock RPC {op} on {}", self.addr), deadline))
|
||||
return Err(LockError::timeout(format!("remote lock RPC {op} on {}", self.addr), deadline));
|
||||
}
|
||||
};
|
||||
if rpc.future.is_none() {
|
||||
rpc.cleanup.take();
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn rpc_timeout_failure_response(request: &LockRequest, err: &LockError) -> LockResponse {
|
||||
@@ -1074,7 +1211,7 @@ impl RemoteClient {
|
||||
#[async_trait]
|
||||
impl LockClient for RemoteClient {
|
||||
async fn acquire_lock(&self, request: &LockRequest) -> Result<LockResponse> {
|
||||
let admission = match self.admit_lock_request("lock") {
|
||||
let admission = match self.admit_lock_request_guard("lock") {
|
||||
Some(admission) => admission,
|
||||
None => return Ok(Self::rpc_backoff_failure_response(request)),
|
||||
};
|
||||
@@ -1085,7 +1222,6 @@ impl LockClient for RemoteClient {
|
||||
Ok(client) => client,
|
||||
Err(err) => {
|
||||
self.record_request_timeout("lock", rpc_timeout);
|
||||
self.release_lock_request("lock", Some(admission));
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
@@ -1093,14 +1229,12 @@ impl LockClient for RemoteClient {
|
||||
let args = match serde_json::to_string(&bounded_request) {
|
||||
Ok(args) => args,
|
||||
Err(err) => {
|
||||
self.release_lock_request("lock", Some(admission));
|
||||
return Err(LockError::internal(format!("Failed to serialize request: {err}")));
|
||||
}
|
||||
};
|
||||
let mut req = Request::new(GenerallyLockRequest { args });
|
||||
req.set_timeout(rpc_timeout);
|
||||
if let Err(err) = attach_lock_mutation_body_digest(&mut req) {
|
||||
self.release_lock_request("lock", Some(admission));
|
||||
return Err(err.into());
|
||||
}
|
||||
let late = self.late_release_hook(request.lock_id.clone());
|
||||
@@ -1142,7 +1276,7 @@ impl LockClient for RemoteClient {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let admission = match self.admit_lock_request("lock_batch") {
|
||||
let admission = match self.admit_lock_request_guard("lock_batch") {
|
||||
Some(admission) => admission,
|
||||
None => return Ok(Self::rpc_backoff_failure_batch(requests)),
|
||||
};
|
||||
@@ -1156,7 +1290,6 @@ impl LockClient for RemoteClient {
|
||||
Ok(client) => client,
|
||||
Err(err) => {
|
||||
self.record_request_timeout("lock_batch", rpc_timeout);
|
||||
self.release_lock_request("lock_batch", Some(admission));
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
@@ -1170,14 +1303,12 @@ impl LockClient for RemoteClient {
|
||||
{
|
||||
Ok(args) => args,
|
||||
Err(err) => {
|
||||
self.release_lock_request("lock_batch", Some(admission));
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
let mut req = Request::new(BatchGenerallyLockRequest { args });
|
||||
req.set_timeout(rpc_timeout);
|
||||
if let Err(err) = attach_lock_mutation_body_digest(&mut req) {
|
||||
self.release_lock_request("lock_batch", Some(admission));
|
||||
return Err(err.into());
|
||||
}
|
||||
let late = self.late_release_batch_hook(requests.iter().map(|request| request.lock_id.clone()).collect());
|
||||
@@ -1315,34 +1446,6 @@ impl LockClient for RemoteClient {
|
||||
Ok(resp.success)
|
||||
}
|
||||
|
||||
async fn downgrade_committed_tail(&self, lock_id: &LockId) -> Result<bool> {
|
||||
let marker = rustfs_lock::client::COMMITTED_TAIL_CONVERSION_MARKER;
|
||||
let mut request = Self::create_unlock_request(lock_id);
|
||||
request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = request.resource.to_string();
|
||||
let mut req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&request)
|
||||
.map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
|
||||
});
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
let response = self
|
||||
.execute_rpc(
|
||||
"refresh",
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.refresh(req).await },
|
||||
RpcCleanup::new(None, None, None),
|
||||
)
|
||||
.await?
|
||||
.into_inner();
|
||||
if let Some(error) = response.error_info {
|
||||
return Err(LockError::internal(error));
|
||||
}
|
||||
// Older nodes refresh normally but return no conversion marker.
|
||||
Ok(response.success && response.lock_info.as_deref() == Some(marker))
|
||||
}
|
||||
|
||||
async fn force_release(&self, lock_id: &LockId) -> Result<bool> {
|
||||
info!("remote force_release for {}", lock_id);
|
||||
let force_request = Self::create_unlock_request(lock_id);
|
||||
@@ -1733,6 +1836,351 @@ mod tests {
|
||||
assert!(acquired_lock_ids(&lock_ids, &[]).is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_remote_client_cancelled_connection_releases_request_admission() {
|
||||
ensure_test_rpc_secret();
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind pending-handshake peer");
|
||||
let addr = format!("http://{}", listener.local_addr().expect("pending-handshake peer address"));
|
||||
let client = RemoteClient::new(addr.clone());
|
||||
let request = test_lock_request(Duration::from_secs(30));
|
||||
|
||||
for batch in [false, true] {
|
||||
reset_lock_peer_health_for_test(&addr);
|
||||
with_lock_peer_health(&addr, |health| {
|
||||
health.request_backoff_until = Some(Instant::now() - Duration::from_secs(1));
|
||||
});
|
||||
let mut acquisition = Box::pin(async {
|
||||
if batch {
|
||||
client.acquire_locks_batch(std::slice::from_ref(&request)).await.map(|_| ())
|
||||
} else {
|
||||
client.acquire_lock(&request).await.map(|_| ())
|
||||
}
|
||||
});
|
||||
std::future::poll_fn(|cx| {
|
||||
assert!(acquisition.as_mut().poll(cx).is_pending(), "peer must wait for its HTTP/2 handshake");
|
||||
std::task::Poll::Ready(())
|
||||
})
|
||||
.await;
|
||||
let pending = lock_peer_health_for_test(&addr);
|
||||
assert_eq!(pending.request_in_flight, 1, "pending connection owns one request admission");
|
||||
assert!(pending.request_probe_generation.is_some(), "the request is the half-open probe");
|
||||
|
||||
drop(acquisition);
|
||||
let cancelled = lock_peer_health_for_test(&addr);
|
||||
assert_eq!(cancelled.request_in_flight, 0, "cancellation must return request admission");
|
||||
assert!(cancelled.request_probe_generation.is_none(), "cancellation must unblock the next probe");
|
||||
let probe = client.admit_lock_request("lock").expect("cancelled peer can be probed again");
|
||||
client.release_lock_request("lock", Some(probe));
|
||||
}
|
||||
reset_lock_peer_health_for_test(&addr);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_remote_client_cancelled_rpc_runs_late_grant_cleanup() {
|
||||
let addr = "http://cancelled-rpc-late-grant";
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
let client = RemoteClient::new(addr.to_string());
|
||||
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
|
||||
let (grant_tx, grant_rx) = tokio::sync::oneshot::channel();
|
||||
let (released_tx, released_rx) = tokio::sync::oneshot::channel();
|
||||
let caller = tokio::spawn(async move {
|
||||
let late: LateCompletion<bool> = Some(Box::new(move |granted| {
|
||||
Box::pin(async move {
|
||||
released_tx.send(granted).expect("late grant observer remains alive");
|
||||
})
|
||||
}));
|
||||
client
|
||||
.execute_rpc(
|
||||
"lock",
|
||||
"bucket/object",
|
||||
Duration::from_secs(30),
|
||||
async move {
|
||||
started_tx.send(()).expect("caller observes RPC dispatch");
|
||||
Ok(grant_rx.await.expect("test decides when peer grants lock"))
|
||||
},
|
||||
RpcCleanup::new(late, None, None),
|
||||
)
|
||||
.await
|
||||
});
|
||||
timeout(Duration::from_secs(1), started_rx)
|
||||
.await
|
||||
.expect("RPC must dispatch")
|
||||
.expect("dispatch signal");
|
||||
caller.abort();
|
||||
assert!(caller.await.expect_err("caller was cancelled").is_cancelled());
|
||||
assert_eq!(
|
||||
lock_peer_health_for_test(addr).detached_rpcs,
|
||||
1,
|
||||
"cancelled RPC uses bounded detached ownership"
|
||||
);
|
||||
grant_tx.send(true).expect("cancellation must preserve the running RPC");
|
||||
assert!(
|
||||
timeout(Duration::from_secs(1), released_rx)
|
||||
.await
|
||||
.expect("late grant must be cleaned up")
|
||||
.expect("cleanup result")
|
||||
);
|
||||
assert_eq!(lock_peer_health_for_test(addr).detached_rpcs, 0, "completed cleanup returns its slot");
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_remote_client_cancelled_rpc_runs_ambiguous_failure_cleanup() {
|
||||
let addr = "http://cancelled-rpc-late-failure";
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
let client = RemoteClient::new(addr.to_string());
|
||||
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
|
||||
let (failure_tx, failure_rx) = tokio::sync::oneshot::channel();
|
||||
let (released_tx, released_rx) = tokio::sync::oneshot::channel();
|
||||
let caller = tokio::spawn(async move {
|
||||
let late_failure: LateFailure = Some(Box::new(move |()| {
|
||||
Box::pin(async move {
|
||||
released_tx.send(()).expect("ambiguous cleanup observer remains alive");
|
||||
})
|
||||
}));
|
||||
client
|
||||
.execute_rpc(
|
||||
"lock_batch",
|
||||
"bucket/object",
|
||||
Duration::from_secs(30),
|
||||
async move {
|
||||
started_tx.send(()).expect("caller observes RPC dispatch");
|
||||
failure_rx.await.expect("test decides when transport fails");
|
||||
Err::<(), _>(tonic::Status::cancelled("transport failure after caller cancellation"))
|
||||
},
|
||||
RpcCleanup::new(None, late_failure, None),
|
||||
)
|
||||
.await
|
||||
});
|
||||
timeout(Duration::from_secs(1), started_rx)
|
||||
.await
|
||||
.expect("RPC must dispatch")
|
||||
.expect("dispatch signal");
|
||||
caller.abort();
|
||||
assert!(caller.await.expect_err("caller was cancelled").is_cancelled());
|
||||
failure_tx.send(()).expect("cancellation must preserve the running RPC");
|
||||
timeout(Duration::from_secs(1), released_rx)
|
||||
.await
|
||||
.expect("ambiguous grant must be cleaned up")
|
||||
.expect("cleanup signal");
|
||||
assert_eq!(lock_peer_health_for_test(addr).detached_rpcs, 0, "failed detached RPC returns its slot");
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_remote_client_cancelled_completed_failure_runs_cleanup() {
|
||||
for (label, status) in [
|
||||
("transport", tonic::Status::cancelled("ambiguous transport failure")),
|
||||
("deadline", tonic::Status::deadline_exceeded("ambiguous deadline failure")),
|
||||
] {
|
||||
let addr = format!("http://cancelled-completed-{label}");
|
||||
reset_lock_peer_health_for_test(&addr);
|
||||
let client = RemoteClient::new(addr.clone());
|
||||
let (evicting_tx, evicting_rx) = tokio::sync::oneshot::channel();
|
||||
let (_resume_tx, resume_rx) = tokio::sync::oneshot::channel();
|
||||
eviction_pauses_for_test()
|
||||
.lock()
|
||||
.expect("eviction pause registry")
|
||||
.insert(addr.clone(), (evicting_tx, resume_rx));
|
||||
let (released_tx, released_rx) = tokio::sync::oneshot::channel();
|
||||
let caller = tokio::spawn(async move {
|
||||
let late_failure: LateFailure = Some(Box::new(move |()| {
|
||||
Box::pin(async move {
|
||||
released_tx
|
||||
.send(())
|
||||
.expect("completed-failure cleanup observer remains alive");
|
||||
})
|
||||
}));
|
||||
client
|
||||
.execute_rpc(
|
||||
"lock",
|
||||
"bucket/object",
|
||||
Duration::from_secs(30),
|
||||
async move { Err::<(), _>(status) },
|
||||
RpcCleanup::new(None, late_failure, None),
|
||||
)
|
||||
.await
|
||||
});
|
||||
timeout(Duration::from_secs(1), evicting_rx)
|
||||
.await
|
||||
.expect("completed error must reach channel eviction")
|
||||
.expect("eviction signal");
|
||||
timeout(Duration::from_secs(1), released_rx)
|
||||
.await
|
||||
.expect("cleanup must transfer before channel eviction suspends")
|
||||
.expect("cleanup signal");
|
||||
caller.abort();
|
||||
assert!(caller.await.expect_err("caller is cancelled during eviction").is_cancelled());
|
||||
assert_eq!(
|
||||
lock_peer_health_for_test(&addr).detached_rpcs,
|
||||
0,
|
||||
"completed RPC cannot consume a detached slot"
|
||||
);
|
||||
reset_lock_peer_health_for_test(&addr);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_remote_client_deadline_wins_over_same_tick_transport_timeout() {
|
||||
let addr = "http://same-tick-rpc-deadline";
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
with_lock_peer_health(addr, |health| health.last_success = Some(Instant::now()));
|
||||
let client = RemoteClient::new(addr.to_string());
|
||||
let budget = Duration::from_millis(50);
|
||||
let (released_tx, released_rx) = tokio::sync::oneshot::channel();
|
||||
let late_failure: LateFailure = Some(Box::new(move |()| {
|
||||
Box::pin(async move {
|
||||
released_tx.send(()).expect("same-tick cleanup observer remains alive");
|
||||
})
|
||||
}));
|
||||
let result = client
|
||||
.execute_rpc(
|
||||
"lock",
|
||||
"bucket/object",
|
||||
budget,
|
||||
async move {
|
||||
tokio::time::sleep(budget).await;
|
||||
Err::<(), _>(tonic::Status::cancelled("inner transport deadline expired"))
|
||||
},
|
||||
RpcCleanup::new(None, late_failure, None),
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
matches!(result, Err(LockError::Timeout { .. })),
|
||||
"expired caller budget must retain timeout classification"
|
||||
);
|
||||
let health = lock_peer_health_for_test(addr);
|
||||
assert_eq!(health.consecutive_timeouts, 1, "same-tick completion must follow timeout eviction policy");
|
||||
assert!(
|
||||
health.last_eviction.is_none(),
|
||||
"a recently serving peer keeps its channel after a caller timeout"
|
||||
);
|
||||
timeout(Duration::from_secs(1), released_rx)
|
||||
.await
|
||||
.expect("same-tick ambiguous completion must trigger cleanup")
|
||||
.expect("cleanup signal");
|
||||
assert_eq!(
|
||||
lock_peer_health_for_test(addr).detached_rpcs,
|
||||
0,
|
||||
"same-tick completion returns detached admission"
|
||||
);
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_remote_client_timeout_drains_late_grant_during_eviction() {
|
||||
let addr = "http://timeout-late-grant-during-eviction";
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
let client = RemoteClient::new(addr.to_string());
|
||||
let (evicting_tx, evicting_rx) = tokio::sync::oneshot::channel();
|
||||
let (resume_tx, resume_rx) = tokio::sync::oneshot::channel();
|
||||
eviction_pauses_for_test()
|
||||
.lock()
|
||||
.expect("eviction pause registry")
|
||||
.insert(addr.to_string(), (evicting_tx, resume_rx));
|
||||
let (grant_tx, grant_rx) = tokio::sync::oneshot::channel();
|
||||
let (released_tx, released_rx) = tokio::sync::oneshot::channel();
|
||||
let caller = tokio::spawn(async move {
|
||||
let late: LateCompletion<bool> = Some(Box::new(move |granted| {
|
||||
Box::pin(async move {
|
||||
released_tx.send(granted).expect("late grant observer remains alive");
|
||||
})
|
||||
}));
|
||||
client
|
||||
.execute_rpc(
|
||||
"lock",
|
||||
"bucket/object",
|
||||
Duration::from_millis(25),
|
||||
async move { Ok(grant_rx.await.expect("test controls the late grant")) },
|
||||
RpcCleanup::new(late, None, None),
|
||||
)
|
||||
.await
|
||||
});
|
||||
timeout(Duration::from_secs(1), evicting_rx)
|
||||
.await
|
||||
.expect("timed-out RPC must reach channel eviction")
|
||||
.expect("eviction signal");
|
||||
assert_eq!(
|
||||
lock_peer_health_for_test(addr).detached_rpcs,
|
||||
1,
|
||||
"eviction waits with a detached RPC owner"
|
||||
);
|
||||
grant_tx
|
||||
.send(true)
|
||||
.expect("timed-out stream remains available for a late grant");
|
||||
assert!(
|
||||
timeout(Duration::from_secs(1), released_rx)
|
||||
.await
|
||||
.expect("late grant must be cleaned up before eviction resumes")
|
||||
.expect("late cleanup result")
|
||||
);
|
||||
assert!(!caller.is_finished(), "channel eviction is still paused");
|
||||
assert_eq!(
|
||||
lock_peer_health_for_test(addr).detached_rpcs,
|
||||
0,
|
||||
"late completion returns detached admission"
|
||||
);
|
||||
resume_tx.send(()).expect("resume channel eviction");
|
||||
let result = caller.await.expect("timed-out caller completes without panicking");
|
||||
assert!(
|
||||
matches!(result, Err(LockError::Timeout { .. })),
|
||||
"caller preserves its RPC timeout result"
|
||||
);
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_remote_client_success_polls_rpc_on_callers_task() {
|
||||
tokio::spawn(async {
|
||||
let client = RemoteClient::new("http://direct-poll-success".to_string());
|
||||
let caller_id = tokio::task::id();
|
||||
let rpc_id = client
|
||||
.execute_rpc(
|
||||
"ping",
|
||||
"health-lock-online",
|
||||
Duration::from_secs(1),
|
||||
async { Ok(tokio::task::id()) },
|
||||
RpcCleanup::new(None, None, None),
|
||||
)
|
||||
.await
|
||||
.expect("successful RPC must return its result");
|
||||
assert_eq!(rpc_id, caller_id, "successful RPC must not allocate and schedule a separate task");
|
||||
})
|
||||
.await
|
||||
.expect("caller task must complete");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_remote_client_rpc_panic_isolated_and_cleans_up() {
|
||||
let (released_tx, released_rx) = tokio::sync::oneshot::channel();
|
||||
let caller = tokio::spawn(async move {
|
||||
let client = RemoteClient::new("http://direct-poll-panic".to_string());
|
||||
let late_failure: LateFailure = Some(Box::new(move |()| {
|
||||
Box::pin(async move {
|
||||
released_tx.send(()).expect("panic cleanup observer remains alive");
|
||||
})
|
||||
}));
|
||||
client
|
||||
.execute_rpc::<(), _>(
|
||||
"lock",
|
||||
"bucket/object",
|
||||
Duration::from_secs(1),
|
||||
async { panic!("test peer RPC panic") },
|
||||
RpcCleanup::new(None, late_failure, None),
|
||||
)
|
||||
.await
|
||||
});
|
||||
let result = caller.await.expect("RPC panic must not escape to its caller");
|
||||
assert!(
|
||||
matches!(result, Err(LockError::Internal { .. })),
|
||||
"RPC panic remains an internal lock failure"
|
||||
);
|
||||
timeout(Duration::from_secs(1), released_rx)
|
||||
.await
|
||||
.expect("panic must trigger ambiguous-grant cleanup")
|
||||
.expect("cleanup signal");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn test_remote_client_timeout_keeps_channel_of_recently_serving_peer() {
|
||||
|
||||
@@ -3506,6 +3506,7 @@ mod tests {
|
||||
replication_decision: String::new(),
|
||||
checksum: None,
|
||||
multipart_completion_replayed: false,
|
||||
backfilled_old_current_size: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
use codec::{fixture_inline_pool_meta_replicas as replicas, fixture_split_inline_pool_meta_replicas as split};
|
||||
use std::path::PathBuf;
|
||||
use tokio::fs;
|
||||
use uuid::Uuid;
|
||||
|
||||
struct PhysicalFixture {
|
||||
_directories: Vec<tempfile::TempDir>,
|
||||
@@ -628,3 +629,559 @@ async fn pool_meta_native_physical_three_plus_one_checkpoint_converges_all_shard
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async fn owner_repair_no_backups_fixture() -> PhysicalFixture {
|
||||
let fixture = fixture(0).await;
|
||||
for pool in disks(&fixture.store).await {
|
||||
for disk in pool {
|
||||
let object = object_path(&disk);
|
||||
let mut entries = fs::read_dir(&object).await.expect("fixture metadata directory");
|
||||
while let Some(entry) = entries.next_entry().await.expect("fixture directory entry") {
|
||||
if entry.file_type().await.expect("fixture entry type").is_dir() {
|
||||
fs::remove_dir_all(entry.path())
|
||||
.await
|
||||
.expect("remove task-owned fixture rollback directory");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
fixture
|
||||
}
|
||||
|
||||
async fn owner_repair_identity_heads(store: &ECStore) -> Vec<Vec<(Uuid, Bytes)>> {
|
||||
let mut all = Vec::new();
|
||||
for pool in &store.pools {
|
||||
let set = pool.get_disks_by_key(POOL_META_IDENTITY_NAME);
|
||||
let members = set
|
||||
.disks
|
||||
.read()
|
||||
.await
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(|disk| disk.expect("identity fixture disk must be online"))
|
||||
.collect::<Vec<_>>();
|
||||
let mut heads = Vec::new();
|
||||
for disk in members {
|
||||
match disk
|
||||
.pool_meta_native_recovery(Request::InspectIdentity {})
|
||||
.await
|
||||
.expect("physical identity inspection")
|
||||
{
|
||||
Response::IdentityInspected { disk_id, raw } => heads.push((disk_id, raw)),
|
||||
_ => panic!("identity inspection must return its physical head"),
|
||||
}
|
||||
}
|
||||
all.push(heads);
|
||||
}
|
||||
all
|
||||
}
|
||||
|
||||
async fn legacy_native_owner_fixture(version: u16, stale: bool) -> (PhysicalFixture, Vec<u8>) {
|
||||
let fixture = owner_repair_no_backups_fixture().await;
|
||||
let mut meta = match decode_pool_meta_replica(fixture.canonical.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
meta, committed: true, ..
|
||||
} => meta,
|
||||
_ => panic!("fixture must provide valid committed pool metadata"),
|
||||
};
|
||||
meta.version = version;
|
||||
let raw = meta
|
||||
.encode_config_data_for_v2_gate(version == POOL_META_VERSION)
|
||||
.expect("legacy fixture should encode in its original version");
|
||||
assert_eq!(u16::from_le_bytes([raw[2], raw[3]]), version);
|
||||
let all = disks(&fixture.store).await;
|
||||
let now = OffsetDateTime::now_utc();
|
||||
for pool in &all {
|
||||
let heads = replicas(&raw, now).await;
|
||||
for (index, disk) in pool.iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &heads[index])
|
||||
.await
|
||||
.expect("install exact legacy head");
|
||||
}
|
||||
}
|
||||
if stale {
|
||||
let old = replicas(&raw, now - Duration::seconds(10)).await;
|
||||
fs::write(object_path(&all[0][0]).join("xl.meta"), &old[0])
|
||||
.await
|
||||
.expect("install stale same-raw legacy head");
|
||||
}
|
||||
(fixture, raw)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_heal_v1_with_all_upgrade_gates_disabled_keeps_exact_raw() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
|
||||
],
|
||||
async {
|
||||
let (fixture, raw) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
fixture.store.heal_pool_meta_replica_under_fence(0, guard, &mut state),
|
||||
)
|
||||
.await
|
||||
.expect("legacy native heal must terminate");
|
||||
assert!(result.is_ok(), "fully online legacy native heal should succeed: {result:?}");
|
||||
drop(state);
|
||||
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
for read in &reads {
|
||||
assert!(
|
||||
matches!(&read.replica, PoolMetaReplica::Valid { raw: bytes, revision, .. }
|
||||
if bytes == &raw && revision.version == POOL_META_V1_VERSION),
|
||||
"native heal must not upgrade a V1 raw payload when both upgrade gates are disabled"
|
||||
);
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_heal_rejects_legacy_source_with_trailing_bytes_without_writes() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
|
||||
],
|
||||
async {
|
||||
for version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
|
||||
let (fixture, raw) = legacy_native_owner_fixture(version, false).await;
|
||||
let mut source_raw = raw.clone();
|
||||
// A second MessagePack value is readable by legacy decoding but
|
||||
// must not be copied as part of an unverified repair payload.
|
||||
source_raw.push(0xc0);
|
||||
assert!(
|
||||
matches!(decode_pool_meta_replica(source_raw.clone()), PoolMetaReplica::Valid { raw, .. } if raw == source_raw),
|
||||
"ordinary legacy reads must remain compatible with trailing bytes"
|
||||
);
|
||||
let all_disks = disks(&fixture.store).await;
|
||||
let source_heads = replicas(&source_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
|
||||
for (index, disk) in all_disks[0].iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
|
||||
.await
|
||||
.expect("install readable legacy source with trailing bytes");
|
||||
}
|
||||
let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
|
||||
.await
|
||||
.expect("legacy semantic selection must remain readable");
|
||||
assert_eq!(selection.legacy_raw.as_deref(), Some(source_raw.as_slice()));
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let identities = owner_repair_identity_heads(&fixture.store).await;
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
|
||||
)
|
||||
.await
|
||||
.expect("legacy native heal must terminate");
|
||||
drop(state);
|
||||
assert!(result.is_err(), "version {version}: unparsed trailing source bytes must not be republished");
|
||||
assert_eq!(snapshots(&fixture.store).await, before, "version {version}: no pool.bin head or backup may change");
|
||||
assert_eq!(
|
||||
owner_repair_identity_heads(&fixture.store).await,
|
||||
identities,
|
||||
"version {version}: no identity shard may change"
|
||||
);
|
||||
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &source_raw));
|
||||
assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw: target, .. } if target == &raw));
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_heal_mixed_v1_v2_preserves_observed_format_floor() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
|
||||
],
|
||||
async {
|
||||
for first_version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
|
||||
for target_pool in [0, 1] {
|
||||
let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
|
||||
let mut meta = match decode_pool_meta_replica(raw_v1.clone()) {
|
||||
PoolMetaReplica::Valid { meta, committed: true, .. } => meta,
|
||||
_ => panic!("initial V1 metadata must decode"),
|
||||
};
|
||||
meta.version = POOL_META_VERSION;
|
||||
let raw_v2 = meta.encode_config_data_for_v2_gate(true).expect("equivalent V2 payload must encode");
|
||||
assert_eq!(u16::from_le_bytes([raw_v1[2], raw_v1[3]]), POOL_META_V1_VERSION);
|
||||
assert_eq!(u16::from_le_bytes([raw_v2[2], raw_v2[3]]), POOL_META_VERSION);
|
||||
let semantic_v1 = match decode_pool_meta_replica(raw_v1.clone()) {
|
||||
PoolMetaReplica::Valid { canonical, .. } => canonical,
|
||||
_ => panic!("V1 payload must remain valid"),
|
||||
};
|
||||
let semantic_v2 = match decode_pool_meta_replica(raw_v2.clone()) {
|
||||
PoolMetaReplica::Valid { canonical, .. } => canonical,
|
||||
_ => panic!("V2 payload must remain valid"),
|
||||
};
|
||||
assert_eq!(semantic_v1, semantic_v2, "the two formats must carry equivalent metadata");
|
||||
let other_version = if first_version == POOL_META_V1_VERSION { POOL_META_VERSION } else { POOL_META_V1_VERSION };
|
||||
// Pool two follows pool zero; pools zero and one cover both
|
||||
// orderings and both target formats in separate runs.
|
||||
let versions = [first_version, other_version, first_version];
|
||||
let physical = disks(&fixture.store).await;
|
||||
let timestamp = OffsetDateTime::now_utc();
|
||||
for (pool_index, pool) in physical.iter().enumerate() {
|
||||
let raw = if versions[pool_index] == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
|
||||
let heads = replicas(raw, timestamp).await;
|
||||
for (index, disk) in pool.iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &heads[index]).await.expect("install mixed legacy pool head");
|
||||
}
|
||||
}
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
|
||||
.await.expect("equivalent legacy pools should be selectable");
|
||||
assert_eq!(selection.revision.version, POOL_META_VERSION, "the observed V2 format floor must be retained");
|
||||
assert_eq!(selection.canonical.as_deref(), Some(semantic_v2.as_slice()));
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
tokio::time::timeout(std::time::Duration::from_secs(30),
|
||||
fixture.store.heal_pool_meta_replica_under_fence(target_pool, guard, &mut state))
|
||||
.await.expect("mixed legacy heal must terminate")
|
||||
.expect("equivalent legacy metadata should heal without downgrading the format floor");
|
||||
drop(state);
|
||||
let after = snapshots(&fixture.store).await;
|
||||
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
for (pool_index, read) in reads.iter().enumerate() {
|
||||
let expected_version = if pool_index == target_pool { POOL_META_VERSION } else { versions[pool_index] };
|
||||
let expected_raw = if expected_version == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
|
||||
assert!(matches!(&read.replica,
|
||||
PoolMetaReplica::Valid { raw, canonical, revision, committed: true, .. }
|
||||
if raw == expected_raw && canonical == &semantic_v2 && revision.version == expected_version
|
||||
), "pool {pool_index}, first={first_version}, target={target_pool}: preserve equivalent payload and expected legacy version");
|
||||
for (index, snapshot) in after[pool_index].iter().enumerate() {
|
||||
codec::validate_inline_pool_meta_shard(snapshot.current.as_ref().expect("physical head"), index, expected_raw)
|
||||
.await.expect("every physical shard must match its selected legacy payload");
|
||||
}
|
||||
if pool_index != target_pool {
|
||||
assert_eq!(after[pool_index], before[pool_index], "healing one pool must not modify other pools");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_heal_rejects_v1_and_v3_prepare_in_either_pool_order() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
|
||||
],
|
||||
async {
|
||||
for prepare_pool in [0, 1] {
|
||||
let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
|
||||
let (legacy_meta, legacy_canonical) = match decode_pool_meta_replica(raw_v1.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
meta,
|
||||
canonical,
|
||||
revision,
|
||||
committed: true,
|
||||
..
|
||||
} if revision.version == POOL_META_V1_VERSION => (meta, canonical),
|
||||
_ => panic!("baseline V1 metadata must be committed and readable"),
|
||||
};
|
||||
let previous = PoolMetaCommittedCandidate {
|
||||
canonical: legacy_canonical,
|
||||
meta: legacy_meta.clone(),
|
||||
revision: PoolMetaRevision::legacy(POOL_META_V1_VERSION),
|
||||
};
|
||||
let mut next = legacy_meta;
|
||||
next.version = POOL_META_GENERATION_VERSION;
|
||||
next.pools[0].last_update += Duration::seconds(1);
|
||||
let revision = PoolMetaRevision {
|
||||
version: POOL_META_GENERATION_VERSION,
|
||||
cluster_id: fixture.revision.cluster_id,
|
||||
epoch: fixture.revision.epoch,
|
||||
generation: 1,
|
||||
transaction_id: Some(Uuid::new_v4()),
|
||||
};
|
||||
let pending = encode_pool_meta_v3_envelope(&next, revision, false, Some(&previous))
|
||||
.expect("V3 PREPARE with V1 predecessor must encode");
|
||||
assert!(matches!(decode_pool_meta_replica(pending.clone()),
|
||||
PoolMetaReplica::Valid { committed: false, revision: actual, .. } if actual == revision));
|
||||
let heads = replicas(&pending, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
|
||||
let physical = disks(&fixture.store).await;
|
||||
for (index, disk) in physical[prepare_pool].iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &heads[index])
|
||||
.await
|
||||
.expect("install task-owned V3 PREPARE evidence");
|
||||
}
|
||||
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(&reads[prepare_pool].replica,
|
||||
PoolMetaReplica::Valid { raw, committed: false, revision: actual, .. }
|
||||
if raw == &pending && *actual == revision));
|
||||
let other_pool = 1 - prepare_pool;
|
||||
assert!(matches!(&reads[other_pool].replica,
|
||||
PoolMetaReplica::Valid { raw, committed: true, revision: actual, .. }
|
||||
if raw == &raw_v1 && actual.version == POOL_META_V1_VERSION));
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let identities = owner_repair_identity_heads(&fixture.store).await;
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
fixture
|
||||
.store
|
||||
.heal_pool_meta_replica_under_fence(prepare_pool, guard, &mut state),
|
||||
)
|
||||
.await
|
||||
.expect("mixed V1/V3 PREPARE heal must terminate");
|
||||
drop(state);
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"pool {prepare_pool}: native heal must not overwrite a V3 PREPARE with legacy data"
|
||||
);
|
||||
assert_eq!(
|
||||
snapshots(&fixture.store).await,
|
||||
before,
|
||||
"pool {prepare_pool}: rejection must retain all physical head and backup evidence"
|
||||
);
|
||||
assert_eq!(
|
||||
owner_repair_identity_heads(&fixture.store).await,
|
||||
identities,
|
||||
"pool {prepare_pool}: rejection must not mutate identity evidence"
|
||||
);
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_heal_rejects_v1_legacy_dont_save_true_without_writes() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
|
||||
],
|
||||
async {
|
||||
let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
|
||||
let meta = match decode_pool_meta_replica(safe_v1.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
meta, committed: true, ..
|
||||
} => meta,
|
||||
_ => panic!("baseline V1 metadata must decode"),
|
||||
};
|
||||
let statuses = meta
|
||||
.pools
|
||||
.iter()
|
||||
.map(|pool| {
|
||||
assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
|
||||
LegacyPoolStatus {
|
||||
id: pool.id,
|
||||
cmd_line: pool.cmd_line.clone(),
|
||||
last_update: pool.last_update,
|
||||
decommission: None,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let legacy = LegacyPoolMeta {
|
||||
version: POOL_META_V1_VERSION,
|
||||
pools: statuses,
|
||||
dont_save: true,
|
||||
};
|
||||
let mut dangerous_raw = Vec::new();
|
||||
dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
|
||||
dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
|
||||
dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
|
||||
let old_reader: LegacyPoolMeta =
|
||||
rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode source");
|
||||
assert!(old_reader.dont_save, "the old reader would treat this source as permanently do-not-save");
|
||||
assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
|
||||
let source = match decode_pool_meta_replica(dangerous_raw.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
canonical,
|
||||
revision,
|
||||
committed: true,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(revision.version, POOL_META_V1_VERSION);
|
||||
canonical
|
||||
}
|
||||
_ => panic!("current reader must recognize the legacy source"),
|
||||
};
|
||||
assert!(matches!(decode_pool_meta_replica(safe_v1.clone()),
|
||||
PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
|
||||
if canonical == source && revision.version == POOL_META_V1_VERSION));
|
||||
let physical = disks(&fixture.store).await;
|
||||
let source_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
|
||||
for (index, disk) in physical[0].iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
|
||||
.await
|
||||
.expect("install legacy source on pool zero");
|
||||
}
|
||||
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
|
||||
assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let identities = owner_repair_identity_heads(&fixture.store).await;
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
|
||||
)
|
||||
.await
|
||||
.expect("unsafe legacy heal must terminate");
|
||||
drop(state);
|
||||
let error = result.expect_err("native heal must reject a source whose old-reader dont_save is true");
|
||||
assert!(
|
||||
error.to_string().contains("dont_save"),
|
||||
"rejection must identify the unsafe legacy field: {error}"
|
||||
);
|
||||
assert_eq!(
|
||||
snapshots(&fixture.store).await,
|
||||
before,
|
||||
"rejection must preserve every original head and backup"
|
||||
);
|
||||
assert_eq!(
|
||||
owner_repair_identity_heads(&fixture.store).await,
|
||||
identities,
|
||||
"rejection must not write identity shards"
|
||||
);
|
||||
let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw));
|
||||
assert!(
|
||||
matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1),
|
||||
"the dangerous source bytes must not be copied into the target pool"
|
||||
);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_heal_rejects_target_v1_dont_save_true_without_writes() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
|
||||
],
|
||||
async {
|
||||
let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
|
||||
let meta = match decode_pool_meta_replica(safe_v1.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
meta, committed: true, ..
|
||||
} => meta,
|
||||
_ => panic!("baseline V1 metadata must decode"),
|
||||
};
|
||||
let statuses = meta
|
||||
.pools
|
||||
.iter()
|
||||
.map(|pool| {
|
||||
assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
|
||||
LegacyPoolStatus {
|
||||
id: pool.id,
|
||||
cmd_line: pool.cmd_line.clone(),
|
||||
last_update: pool.last_update,
|
||||
decommission: None,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let legacy = LegacyPoolMeta {
|
||||
version: POOL_META_V1_VERSION,
|
||||
pools: statuses,
|
||||
dont_save: true,
|
||||
};
|
||||
let mut dangerous_raw = Vec::new();
|
||||
dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
|
||||
dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
|
||||
dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
|
||||
let old_reader: LegacyPoolMeta =
|
||||
rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode target");
|
||||
assert!(old_reader.dont_save, "the old reader interprets the target as do-not-save");
|
||||
assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
|
||||
let source_canonical = match decode_pool_meta_replica(safe_v1.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
canonical,
|
||||
revision,
|
||||
committed: true,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(revision.version, POOL_META_V1_VERSION);
|
||||
canonical
|
||||
}
|
||||
_ => panic!("safe pool-zero source must decode"),
|
||||
};
|
||||
assert!(
|
||||
matches!(decode_pool_meta_replica(dangerous_raw.clone()),
|
||||
PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
|
||||
if canonical == source_canonical && revision.version == POOL_META_V1_VERSION),
|
||||
"the dangerous target must remain semantically equivalent to the safe source"
|
||||
);
|
||||
let physical = disks(&fixture.store).await;
|
||||
let target_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
|
||||
for (index, disk) in physical[1].iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &target_heads[index])
|
||||
.await
|
||||
.expect("install legacy dangerous target head");
|
||||
}
|
||||
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
|
||||
assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let identities = owner_repair_identity_heads(&fixture.store).await;
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
|
||||
)
|
||||
.await
|
||||
.expect("unsafe legacy target heal must terminate");
|
||||
drop(state);
|
||||
let error = result.expect_err("native heal must reject a target whose old-reader dont_save is true");
|
||||
assert!(
|
||||
error.to_string().contains("dont_save"),
|
||||
"rejection must identify the unsafe legacy field: {error}"
|
||||
);
|
||||
assert_eq!(
|
||||
snapshots(&fixture.store).await,
|
||||
before,
|
||||
"rejection must preserve all physical heads and backups"
|
||||
);
|
||||
assert_eq!(
|
||||
owner_repair_identity_heads(&fixture.store).await,
|
||||
identities,
|
||||
"rejection must not write identity shards"
|
||||
);
|
||||
let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1));
|
||||
assert!(
|
||||
matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw),
|
||||
"unsafe target evidence must not be overwritten by the safe source"
|
||||
);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
@@ -391,7 +391,7 @@ pub(super) async fn fixture_inline_pool_meta_replicas(payload: &[u8], modified:
|
||||
writer.shutdown().await.expect("fixture bitrot writer should finish");
|
||||
let mut info = base.clone();
|
||||
info.erasure.index = *coding_index;
|
||||
info.data = Some(Bytes::from(writer.into_inline_data().expect("fixture writer should retain inline bytes")));
|
||||
info.data = Some(writer.into_inline_data().expect("fixture writer should retain inline bytes"));
|
||||
let mut meta = FileMeta::new();
|
||||
meta.add_version(info).expect("test inline FileInfo should serialize");
|
||||
raw.push(Bytes::from(meta.marshal_msg().expect("test xl.meta should serialize")));
|
||||
|
||||
@@ -94,7 +94,6 @@ use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::fmt::Display;
|
||||
#[cfg(test)]
|
||||
use std::io::Cursor;
|
||||
use std::io::Write;
|
||||
use std::path::PathBuf;
|
||||
@@ -3720,33 +3719,45 @@ fn is_decommission_copy_quorum_deferral(err: &Error) -> bool {
|
||||
data_movement::data_movement_stage_source_as::<DecommissionCopyQuorumFailure>(err).is_some()
|
||||
}
|
||||
|
||||
/// The storage error of a version copy that kept timing out on a namespace
|
||||
/// lock through its in-place retries, kept typed so the entry failure policy
|
||||
/// defers it.
|
||||
/// A copy whose object namespace lock remained unavailable through its
|
||||
/// in-place retries, retained as a typed deferral.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("{0}")]
|
||||
struct DecommissionCopyLockTimeout(#[source] Error);
|
||||
struct DecommissionCopyLockUnavailable(#[source] Error);
|
||||
|
||||
fn is_decommission_copy_lock_timeout_error(err: &Error) -> bool {
|
||||
let err = data_movement::data_movement_stage_source(err).unwrap_or(err);
|
||||
matches!(err, Error::Lock(rustfs_lock::LockError::Timeout { .. }))
|
||||
fn is_decommission_copy_lock_unavailable_error(err: &Error, bucket: &str, object: &str) -> bool {
|
||||
let source = data_movement::data_movement_stage_source(err).unwrap_or(err);
|
||||
match source {
|
||||
Error::Lock(rustfs_lock::LockError::Timeout { .. }) => true,
|
||||
Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "read",
|
||||
bucket: lock_bucket,
|
||||
object: lock_object,
|
||||
..
|
||||
} => {
|
||||
data_movement::data_movement_error_is_stage(err, "prepare_put_object")
|
||||
&& lock_bucket == bucket
|
||||
&& lock_object == object
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// A version copy that timed out acquiring a namespace lock leaves the source
|
||||
/// A version copy that could not acquire its namespace lock leaves the source
|
||||
/// in place; the next attempt copies the version again or finds an equivalent
|
||||
/// committed target. The lock is held by another mutation of the key or, when
|
||||
/// this node restarted mid-copy, by its previous incarnation until that lock's
|
||||
/// lease expires on the peers. Either clears by waiting, so the entry is
|
||||
/// deferred instead of being counted as a permanent per-object failure.
|
||||
fn decommission_copy_lock_timeout_deferral(stage: &str, bucket: &str, object: &str, err: Error) -> Error {
|
||||
fn decommission_copy_lock_unavailable_deferral(stage: &str, bucket: &str, object: &str, err: Error) -> Error {
|
||||
data_movement::data_movement_context_error(
|
||||
format!("decommission {stage} timed out on a namespace lock for {bucket}/{object}: {err}"),
|
||||
DecommissionCopyLockTimeout(err),
|
||||
format!("decommission {stage} could not acquire a namespace lock for {bucket}/{object}: {err}"),
|
||||
DecommissionCopyLockUnavailable(err),
|
||||
)
|
||||
}
|
||||
|
||||
fn is_decommission_copy_lock_timeout_deferral(err: &Error) -> bool {
|
||||
data_movement::data_movement_stage_source_as::<DecommissionCopyLockTimeout>(err).is_some()
|
||||
fn is_decommission_copy_lock_unavailable_deferral(err: &Error) -> bool {
|
||||
data_movement::data_movement_stage_source_as::<DecommissionCopyLockUnavailable>(err).is_some()
|
||||
}
|
||||
|
||||
/// The deferral of a version copy whose in-place retries ended on `error`, if
|
||||
@@ -3756,8 +3767,8 @@ fn take_decommission_copy_deferral(stage: &str, bucket: &str, object: &str, erro
|
||||
return Some(decommission_copy_quorum_deferral(stage, bucket, object, err));
|
||||
}
|
||||
error
|
||||
.take_if(|err| is_decommission_copy_lock_timeout_error(err))
|
||||
.map(|err| decommission_copy_lock_timeout_deferral(stage, bucket, object, err))
|
||||
.take_if(|err| is_decommission_copy_lock_unavailable_error(err, bucket, object))
|
||||
.map(|err| decommission_copy_lock_unavailable_deferral(stage, bucket, object, err))
|
||||
}
|
||||
|
||||
fn resolve_decommission_entry_reload_result(result: Result<()>, bucket: &str, object_name: &str) -> Result<()> {
|
||||
@@ -5822,13 +5833,13 @@ enum DecommissionEntryOutcome {
|
||||
/// entry is done and must not be counted as a failure or escalated. A quorum
|
||||
/// failure of the source cleanup is deferred too: the copied source is still
|
||||
/// in place and the next attempt cleans it up again. So is a version copy that
|
||||
/// lost drive quorum or timed out on a namespace lock through its in-place
|
||||
/// lost drive quorum or could not acquire an object namespace lock through its in-place
|
||||
/// retries: the source still holds the version and the next attempt copies it
|
||||
/// again. Every other failure stays terminal.
|
||||
fn resolve_decommission_entry_failure(err: Error) -> Result<DecommissionEntryOutcome> {
|
||||
if is_decommission_source_cleanup_retryable_error(&err)
|
||||
|| is_decommission_copy_quorum_deferral(&err)
|
||||
|| is_decommission_copy_lock_timeout_deferral(&err)
|
||||
|| is_decommission_copy_lock_unavailable_deferral(&err)
|
||||
{
|
||||
return Ok(DecommissionEntryOutcome::Deferred(err));
|
||||
}
|
||||
@@ -6827,11 +6838,47 @@ impl Drop for PoolMetaTransactionArm {
|
||||
}
|
||||
}
|
||||
|
||||
fn ensure_legacy_pool_meta_republish_safe(data: &[u8]) -> Result<()> {
|
||||
if data.len() >= 4
|
||||
&& LittleEndian::read_u16(&data[2..4]) == POOL_META_V1_VERSION
|
||||
// The selection already validates the payload. The runtime-only
|
||||
// flag can only appear in the legacy three-field sequence or map.
|
||||
&& rmp::decode::read_array_len(&mut &data[4..])
|
||||
.or_else(|_| rmp::decode::read_map_len(&mut &data[4..]))
|
||||
.ok()
|
||||
== Some(3)
|
||||
&& let Ok(legacy) = rmp_serde::from_slice::<LegacyPoolMeta>(&data[4..])
|
||||
&& legacy.dont_save
|
||||
{
|
||||
return Err(Error::other("pool metadata legacy source contains dont_save=true"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Ordinary legacy reads retain their compatibility behavior, but bytes that
|
||||
// will be copied to another pool must not contain an unparsed extension.
|
||||
fn ensure_legacy_pool_meta_payload_complete(data: &[u8]) -> Result<()> {
|
||||
let payload = data
|
||||
.get(4..)
|
||||
.ok_or_else(|| Error::other("pool metadata legacy source is too short"))?;
|
||||
let mut decoder = rmp_serde::Deserializer::new(Cursor::new(payload));
|
||||
serde::de::IgnoredAny::deserialize(&mut decoder).map_err(|_| Error::other("pool metadata legacy source is invalid"))?;
|
||||
let payload_len = u64::try_from(payload.len()).map_err(|_| Error::other("pool metadata legacy source is too large"))?;
|
||||
if decoder.get_ref().position() != payload_len {
|
||||
return Err(Error::other("pool metadata legacy source has trailing bytes"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct PoolMetaSelection {
|
||||
meta: PoolMeta,
|
||||
revision: PoolMetaRevision,
|
||||
canonical: Option<Vec<u8>>,
|
||||
// Legacy semantic comparison canonicalizes to V2; a heal must instead
|
||||
// preserve verified raw bytes at the highest observed format floor.
|
||||
legacy_raw: Option<Vec<u8>>,
|
||||
legacy_has_unsafe_flag: bool,
|
||||
replica_state: PoolMetaReplicaState,
|
||||
cas_tokens: Vec<PoolMetaCasToken>,
|
||||
absent: bool,
|
||||
@@ -7222,6 +7269,8 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
|
||||
meta: PoolMeta::default(),
|
||||
revision: PoolMetaRevision::legacy(0),
|
||||
canonical: None,
|
||||
legacy_raw: None,
|
||||
legacy_has_unsafe_flag: false,
|
||||
replica_state: PoolMetaReplicaState {
|
||||
needs_repair: false,
|
||||
repair_write_safe: true,
|
||||
@@ -7276,6 +7325,8 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
|
||||
meta: selected.meta.clone(),
|
||||
revision: selected.revision,
|
||||
canonical: Some(selected.canonical.clone()),
|
||||
legacy_raw: None,
|
||||
legacy_has_unsafe_flag: false,
|
||||
replica_state: PoolMetaReplicaState {
|
||||
needs_repair,
|
||||
repair_write_safe,
|
||||
@@ -7289,6 +7340,10 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
|
||||
|
||||
// Legacy V1/V2 has no durable generation. Pool zero remains the commit
|
||||
// record; divergent backups without it are ambiguous and fail closed.
|
||||
let legacy_has_unsafe_flag = committed
|
||||
.iter()
|
||||
.any(|(_, raw, _)| ensure_legacy_pool_meta_republish_safe(raw).is_err());
|
||||
let mut floor_raw = None;
|
||||
let mut selected: Option<(usize, Vec<u8>, PoolMetaCommittedCandidate)> = None;
|
||||
for (idx, raw, candidate) in committed {
|
||||
if let Some((selected_idx, selected_raw, selected_candidate)) = selected.as_ref() {
|
||||
@@ -7302,19 +7357,36 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
|
||||
}
|
||||
} else {
|
||||
needs_repair |= selected_raw != &raw;
|
||||
if floor_raw.is_none()
|
||||
&& candidate.revision.version == observed_version
|
||||
&& raw
|
||||
.get(2..4)
|
||||
.is_some_and(|version| LittleEndian::read_u16(version) == observed_version)
|
||||
{
|
||||
floor_raw = Some(raw);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
selected = Some((idx, raw, candidate));
|
||||
}
|
||||
}
|
||||
let (_, _, mut selected) =
|
||||
let (_, selected_raw, mut selected) =
|
||||
selected.ok_or_else(|| Error::other("pool metadata recovery required: no valid legacy replica is available"))?;
|
||||
if selected.revision.version == observed_version
|
||||
&& selected_raw
|
||||
.get(2..4)
|
||||
.is_some_and(|version| LittleEndian::read_u16(version) == observed_version)
|
||||
{
|
||||
floor_raw = Some(selected_raw);
|
||||
}
|
||||
selected.meta.version = observed_version;
|
||||
selected.revision.version = observed_version;
|
||||
Ok(PoolMetaSelection {
|
||||
meta: selected.meta,
|
||||
revision: selected.revision,
|
||||
canonical: Some(selected.canonical),
|
||||
legacy_raw: floor_raw,
|
||||
legacy_has_unsafe_flag,
|
||||
replica_state: PoolMetaReplicaState {
|
||||
needs_repair,
|
||||
repair_write_safe,
|
||||
@@ -14534,6 +14606,23 @@ impl ECStore {
|
||||
.canonical
|
||||
.as_ref()
|
||||
.ok_or_else(|| Error::other("pool metadata heal has no committed source"))?;
|
||||
if selection.generation_protocol_observed && !selection.revision.is_generation_protocol() {
|
||||
return Err(Error::other("pool metadata heal requires recovery of the observed generation protocol"));
|
||||
}
|
||||
if selection.legacy_has_unsafe_flag {
|
||||
return Err(Error::other("pool metadata legacy replica contains dont_save=true"));
|
||||
}
|
||||
let repair_data = if selection.revision.is_generation_protocol() {
|
||||
canonical.clone()
|
||||
} else {
|
||||
let raw = selection
|
||||
.legacy_raw
|
||||
.clone()
|
||||
.ok_or_else(|| Error::other("pool metadata heal cannot prove the legacy format floor"))?;
|
||||
ensure_legacy_pool_meta_payload_complete(&raw)?;
|
||||
raw
|
||||
};
|
||||
ensure_legacy_pool_meta_republish_safe(&repair_data)?;
|
||||
let pool = self
|
||||
.pools
|
||||
.get(pool_index)
|
||||
@@ -14544,10 +14633,10 @@ impl ECStore {
|
||||
.get(pool_index)
|
||||
.ok_or_else(|| Error::other("pool metadata heal has no target revision"))?;
|
||||
let mut arm = state.arm_transaction();
|
||||
save_pool_meta_object_cas(pool.clone(), POOL_META_NAME, canonical.clone(), token, &fence, "native_heal", &mut arm)
|
||||
save_pool_meta_object_cas(pool.clone(), POOL_META_NAME, repair_data.clone(), token, &fence, "native_heal", &mut arm)
|
||||
.await?;
|
||||
let (confirmed, _) = read_config_no_lock_preserve_empty_with_metadata(pool, POOL_META_NAME).await?;
|
||||
if confirmed != *canonical {
|
||||
if confirmed != repair_data {
|
||||
return Err(Error::other("pool metadata native heal readback diverged"));
|
||||
}
|
||||
fence.ensure_held()?;
|
||||
@@ -28353,7 +28442,7 @@ mod tests {
|
||||
"one version must commit before the next version times out"
|
||||
);
|
||||
assert!(
|
||||
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_timeout_deferral(err)),
|
||||
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_unavailable_deferral(err)),
|
||||
"{outcome:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -28387,6 +28476,88 @@ mod tests {
|
||||
assert_eq!(actual, b"written after versioning was enabled");
|
||||
}
|
||||
|
||||
/// A target lookup that loses its read-lock quorum must retain the source
|
||||
/// and resume an entry after an earlier version has already committed.
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn decommission_defers_target_lookup_read_lock_quorum_and_resumes() {
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
|
||||
let fixture = NullVersionDecommission::new(NullVersionTop::NewerVersion).await;
|
||||
let object = fixture.object;
|
||||
let attempted = Arc::new(AtomicUsize::new(0));
|
||||
let fault = DecommissionTestFaultGuard::install_with_error(
|
||||
Arc::new({
|
||||
let attempted = Arc::clone(&attempted);
|
||||
move |stage, _, faulted, _, success| {
|
||||
stage == "before_target_commit"
|
||||
&& faulted == object
|
||||
&& success
|
||||
&& attempted.fetch_add(1, Ordering::SeqCst) > 0
|
||||
}
|
||||
}),
|
||||
Some(Arc::new(|_, bucket, object| {
|
||||
data_movement::data_movement_stage_error_for_test(
|
||||
"decommission_object",
|
||||
"prepare_put_object",
|
||||
bucket,
|
||||
object,
|
||||
Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "read",
|
||||
bucket: bucket.to_owned(),
|
||||
object: object.to_owned(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
},
|
||||
)
|
||||
})),
|
||||
);
|
||||
let (outcome, _) = fixture.run_entry().await;
|
||||
assert!(
|
||||
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_unavailable_deferral(err)),
|
||||
"{outcome:?}"
|
||||
);
|
||||
fixture.assert_source_retained().await;
|
||||
assert_eq!(
|
||||
fixture
|
||||
.target_versions()
|
||||
.await
|
||||
.expect("first version committed")
|
||||
.versions
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
fixture.store.pool_meta.read().await.pools[0]
|
||||
.decommission
|
||||
.as_ref()
|
||||
.expect("active decommission")
|
||||
.items_decommission_failed,
|
||||
0
|
||||
);
|
||||
|
||||
drop(fault);
|
||||
let (outcome, _) = fixture.run_entry().await;
|
||||
assert!(matches!(outcome, DecommissionEntryOutcome::Complete), "{outcome:?}");
|
||||
fixture.assert_moved().await;
|
||||
let mut reader = fixture.store.pools[1]
|
||||
.get_object_reader(
|
||||
fixture.bucket,
|
||||
fixture.object,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
version_id: Some(fixture.seeded.versions[0].version_id.expect("newer version").to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("the newer version must remain readable");
|
||||
let mut actual = Vec::new();
|
||||
reader.stream.read_to_end(&mut actual).await.expect("read the newer version");
|
||||
assert_eq!(actual, b"written after versioning was enabled");
|
||||
}
|
||||
|
||||
/// A namespace lock still held on the target key, as by a copy whose node
|
||||
/// was killed mid-write, makes target selection fail with a typed
|
||||
/// object-lock timeout. A real held lock, not an injected error, must defer
|
||||
@@ -28429,7 +28600,7 @@ mod tests {
|
||||
|
||||
let (outcome, _) = fixture.run_entry().await;
|
||||
assert!(
|
||||
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_timeout_deferral(err)),
|
||||
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_unavailable_deferral(err)),
|
||||
"{outcome:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -28838,17 +29009,17 @@ mod tests {
|
||||
timeout(),
|
||||
);
|
||||
for err in [timeout(), prepare] {
|
||||
assert!(is_decommission_copy_lock_timeout_error(&err), "{err}");
|
||||
assert!(is_decommission_copy_lock_unavailable_error(&err, "bucket-a", "object-a"), "{err}");
|
||||
assert!(!is_decommission_copy_quorum_error(&err), "{err}");
|
||||
let mut error = Some(err);
|
||||
let deferred = take_decommission_copy_deferral("migrate_object", "bucket-a", "object-a", &mut error)
|
||||
.expect("a lock timeout copy must be deferred");
|
||||
assert!(error.is_none());
|
||||
assert!(is_decommission_copy_lock_timeout_deferral(&deferred));
|
||||
assert!(is_decommission_copy_lock_unavailable_deferral(&deferred));
|
||||
assert!(!is_decommission_copy_quorum_deferral(&deferred));
|
||||
let failure = data_movement::data_movement_stage_source_as::<DecommissionCopyLockTimeout>(&deferred)
|
||||
let failure = data_movement::data_movement_stage_source_as::<DecommissionCopyLockUnavailable>(&deferred)
|
||||
.expect("the deferred error must keep its lock-timeout origin");
|
||||
assert!(is_decommission_copy_lock_timeout_error(&failure.0));
|
||||
assert!(is_decommission_copy_lock_unavailable_error(&failure.0, "bucket-a", "object-a"));
|
||||
assert!(matches!(
|
||||
resolve_decommission_entry_failure(deferred),
|
||||
Ok(DecommissionEntryOutcome::Deferred(_))
|
||||
@@ -28873,6 +29044,43 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decommission_copy_defers_only_the_matching_object_read_lock_quorum() {
|
||||
let quorum = |mode, bucket: &str, object: &str| Error::NamespaceLockQuorumUnavailable {
|
||||
mode,
|
||||
bucket: bucket.to_owned(),
|
||||
object: object.to_owned(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
};
|
||||
let wrap = |stage, err| {
|
||||
data_movement::data_movement_stage_error_for_test("decommission_object", stage, "bucket-a", "object-a", err)
|
||||
};
|
||||
let mut error = Some(wrap("prepare_put_object", quorum("read", "bucket-a", "object-a")));
|
||||
let deferred = take_decommission_copy_deferral("migrate_object", "bucket-a", "object-a", &mut error)
|
||||
.expect("matching read-lock quorum must defer");
|
||||
assert!(error.is_none());
|
||||
assert!(matches!(
|
||||
resolve_decommission_entry_failure(deferred),
|
||||
Ok(DecommissionEntryOutcome::Deferred(_))
|
||||
));
|
||||
for err in [
|
||||
quorum("read", "bucket-a", "object-a"),
|
||||
wrap("source_recheck", quorum("read", "bucket-a", "object-a")),
|
||||
wrap("prepare_put_object", quorum("write", "bucket-a", "object-a")),
|
||||
wrap(
|
||||
"prepare_put_object",
|
||||
quorum("exact_delete_capacity_reconciliation", "bucket-a", "object-a"),
|
||||
),
|
||||
wrap("prepare_put_object", quorum("read", "other-bucket", "object-a")),
|
||||
wrap("prepare_put_object", quorum("read", "bucket-a", "other-object")),
|
||||
] {
|
||||
let mut error = Some(err);
|
||||
assert!(take_decommission_copy_deferral("migrate_object", "bucket-a", "object-a", &mut error).is_none());
|
||||
assert!(error.is_some(), "unrelated lock failures must stay terminal");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decommission_target_capacity_error_accepts_wrapped_capacity_errors() {
|
||||
let disk_full = Error::other(format!("decommission_object: put_object failed for bucket/object: {}", Error::DiskFull));
|
||||
@@ -29351,6 +29559,53 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_republish_guard_checks_only_three_field_sequence_or_map() {
|
||||
let frame = |payload: Vec<u8>| {
|
||||
let mut data = Vec::new();
|
||||
data.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
|
||||
data.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
|
||||
data.extend_from_slice(&payload);
|
||||
data
|
||||
};
|
||||
let two_fields = PersistedPoolMetaV1 {
|
||||
version: POOL_META_V1_VERSION,
|
||||
pools: Vec::new(),
|
||||
};
|
||||
for payload in [rmp_serde::to_vec(&two_fields), rmp_serde::to_vec_named(&two_fields)] {
|
||||
let data = frame(payload.expect("two-field V1 payload must serialize"));
|
||||
assert!(matches!(decode_pool_meta_replica(data.clone()), PoolMetaReplica::Valid { .. }));
|
||||
assert!(ensure_legacy_pool_meta_republish_safe(&data).is_ok());
|
||||
assert!(ensure_legacy_pool_meta_payload_complete(&data).is_ok());
|
||||
let mut trailing = data;
|
||||
trailing.push(0xc0);
|
||||
assert!(ensure_legacy_pool_meta_payload_complete(&trailing).is_err());
|
||||
}
|
||||
for dont_save in [false, true] {
|
||||
let legacy = LegacyPoolMeta {
|
||||
version: POOL_META_V1_VERSION,
|
||||
pools: Vec::new(),
|
||||
dont_save,
|
||||
};
|
||||
for payload in [rmp_serde::to_vec(&legacy), rmp_serde::to_vec_named(&legacy)] {
|
||||
let data = frame(payload.expect("three-field V1 payload must serialize"));
|
||||
assert!(matches!(decode_pool_meta_replica(data.clone()), PoolMetaReplica::Valid { .. }));
|
||||
assert_eq!(ensure_legacy_pool_meta_republish_safe(&data).is_err(), dont_save);
|
||||
assert!(ensure_legacy_pool_meta_payload_complete(&data).is_ok());
|
||||
let mut trailing = data;
|
||||
trailing.push(0xc0);
|
||||
assert!(ensure_legacy_pool_meta_payload_complete(&trailing).is_err());
|
||||
}
|
||||
}
|
||||
// This guard is only called for already-decoded replicas; malformed
|
||||
// containers must fail selection rather than becoming repair sources.
|
||||
let truncated = frame(vec![0x93, 0x01]);
|
||||
assert!(matches!(
|
||||
decode_pool_meta_replica(truncated),
|
||||
PoolMetaReplica::Corrupt(_) | PoolMetaReplica::Incompatible(_)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pool_meta_decode_supports_legacy_payload() {
|
||||
let start_time = OffsetDateTime::now_utc();
|
||||
|
||||
@@ -317,8 +317,9 @@ impl Sets {
|
||||
&self.ctx
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn stop_endpoint_monitor_for_test(&self) {
|
||||
/// Stop endpoint reconnection and wait for any in-flight pass before a test owns disk probes.
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub async fn stop_endpoint_monitor_for_test(&self) {
|
||||
if let Some(exit_signal) = &self.exit_signal {
|
||||
let _ = exit_signal.send(());
|
||||
// Wait for an in-flight reconnect pass as well as its receiver to exit.
|
||||
|
||||
@@ -84,7 +84,7 @@ pub(crate) enum CrashPoint {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) use armed::{arm, disarm, should_crash_at};
|
||||
pub(crate) use armed::{arm, disarm, is_armed, should_crash_at};
|
||||
|
||||
/// Production no-op: the compiler inlines this to `false`, so armed crash points
|
||||
/// exist only in `#[cfg(test)]` builds and never add a branch to real commits.
|
||||
@@ -94,6 +94,12 @@ pub(crate) fn should_crash_at(_point: CrashPoint, _key: &str) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
#[inline(always)]
|
||||
pub(crate) fn is_armed(_point: CrashPoint, _key: &str) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod armed {
|
||||
use super::CrashPoint;
|
||||
@@ -132,6 +138,10 @@ mod armed {
|
||||
guard().retain(|(p, k)| !(*p == point && k == key));
|
||||
}
|
||||
|
||||
pub(crate) fn is_armed(point: CrashPoint, key: &str) -> bool {
|
||||
guard().iter().any(|(p, k)| *p == point && k == key)
|
||||
}
|
||||
|
||||
/// Returns `true` (consuming the arm) iff a crash is armed for exactly this
|
||||
/// `point` and `key`.
|
||||
pub(crate) fn should_crash_at(point: CrashPoint, key: &str) -> bool {
|
||||
|
||||
@@ -605,6 +605,7 @@ fn resolve_data_movement_abort_result(
|
||||
#[derive(Debug)]
|
||||
struct DataMovementStageError {
|
||||
rendered: String,
|
||||
stage: Option<String>,
|
||||
source: Box<dyn std::error::Error + Send + Sync>,
|
||||
}
|
||||
|
||||
@@ -626,6 +627,7 @@ where
|
||||
{
|
||||
Error::other(DataMovementStageError {
|
||||
rendered,
|
||||
stage: None,
|
||||
source: Box::new(err),
|
||||
})
|
||||
}
|
||||
@@ -634,7 +636,21 @@ fn data_movement_stage_error<E>(op_label: &str, stage: &str, bucket: &str, objec
|
||||
where
|
||||
E: std::error::Error + Send + Sync + 'static,
|
||||
{
|
||||
data_movement_context_error(format!("{op_label}: {stage} failed for {bucket}/{object}: {err}"), err)
|
||||
Error::other(DataMovementStageError {
|
||||
rendered: format!("{op_label}: {stage} failed for {bucket}/{object}: {err}"),
|
||||
stage: Some(stage.to_owned()),
|
||||
source: Box::new(err),
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn data_movement_error_is_stage(err: &Error, stage: &str) -> bool {
|
||||
let Error::Io(io_err) = err else {
|
||||
return false;
|
||||
};
|
||||
io_err
|
||||
.get_ref()
|
||||
.and_then(|source| source.downcast_ref::<DataMovementStageError>())
|
||||
.is_some_and(|source| source.stage.as_deref() == Some(stage))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -624,58 +624,58 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn cleanup_dispatch_retains_resource_units_without_inheriting_disk_policy() {
|
||||
use crate::disk::io_admission::{current_disk_io_context, with_background_disk_io, with_background_resource_unit};
|
||||
use rustfs_concurrency::adaptive::AdaptiveAdmission;
|
||||
use std::num::NonZeroU16;
|
||||
use crate::disk::io_admission::{current_disk_io_context, test_resource_units, with_background_disk_io};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
for mode in 0..4 {
|
||||
let runtime = Arc::new(CleanupRuntime::new(config(1, 1, 1)).expect("cleanup runtime"));
|
||||
let worker_runtime = runtime.clone();
|
||||
let gate = AdaptiveAdmission::new(NonZeroU16::new(1).expect("one unit"), 1);
|
||||
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
|
||||
let work = async move {
|
||||
started_tx
|
||||
.send((current_resource_unit().is_some(), current_disk_io_context().is_none()))
|
||||
.expect("report worker context");
|
||||
release_rx.await.expect("release owned worker");
|
||||
Ok::<_, super::super::error::DiskError>(())
|
||||
};
|
||||
let task = tokio::spawn(with_background_disk_io(
|
||||
CancellationToken::new(),
|
||||
with_background_resource_unit(gate.acquire().await, async move {
|
||||
match mode {
|
||||
0 => worker_runtime
|
||||
.disk_operation(work, DiskAdmission::Request)
|
||||
.await
|
||||
.expect("disk operation"),
|
||||
1 => worker_runtime.gc_operation(move |_| work).await.expect("GC operation"),
|
||||
2 => {
|
||||
let reservation = worker_runtime.reserve().expect("cleanup reservation");
|
||||
run(Some(reservation), async { work.await.expect("cleanup operation") }).await;
|
||||
for physical_count in [1, 2] {
|
||||
for mode in 0..4 {
|
||||
let runtime = Arc::new(CleanupRuntime::new(config(1, 1, 1)).expect("cleanup runtime"));
|
||||
let worker_runtime = runtime.clone();
|
||||
let (unit, gates) = test_resource_units(physical_count);
|
||||
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
|
||||
let work = async move {
|
||||
started_tx
|
||||
.send((current_resource_unit().is_some(), current_disk_io_context().is_none()))
|
||||
.expect("report worker context");
|
||||
release_rx.await.expect("release owned worker");
|
||||
Ok::<_, super::super::error::DiskError>(())
|
||||
};
|
||||
let task = tokio::spawn(with_background_disk_io(
|
||||
CancellationToken::new(),
|
||||
with_optional_resource_unit(Some(unit), async move {
|
||||
match mode {
|
||||
0 => worker_runtime
|
||||
.disk_operation(work, DiskAdmission::Request)
|
||||
.await
|
||||
.expect("disk operation"),
|
||||
1 => worker_runtime.gc_operation(move |_| work).await.expect("GC operation"),
|
||||
2 => {
|
||||
let reservation = worker_runtime.reserve().expect("cleanup reservation");
|
||||
run(Some(reservation), async { work.await.expect("cleanup operation") }).await;
|
||||
}
|
||||
_ => spawn(None, work)
|
||||
.await
|
||||
.expect("unisolated worker")
|
||||
.expect("unisolated operation"),
|
||||
}
|
||||
_ => spawn(None, work)
|
||||
.await
|
||||
.expect("unisolated worker")
|
||||
.expect("unisolated operation"),
|
||||
}),
|
||||
));
|
||||
let (inherited, original_policy) = deadline(started_rx).await.expect("owned work starts");
|
||||
task.abort();
|
||||
assert!(task.await.expect_err("waiter cancelled").is_cancelled());
|
||||
let held: Vec<_> = gates.iter().map(|gate| gate.snapshot().active).collect();
|
||||
release_tx.send(()).expect("owned work survives cancellation");
|
||||
deadline(async {
|
||||
while gates.iter().any(|gate| gate.snapshot().active != 0) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
}),
|
||||
));
|
||||
let (inherited, original_policy) = deadline(started_rx).await.expect("owned work starts");
|
||||
task.abort();
|
||||
assert!(task.await.expect_err("waiter cancelled").is_cancelled());
|
||||
let held = gate.snapshot().active;
|
||||
release_tx.send(()).expect("owned work survives cancellation");
|
||||
deadline(async {
|
||||
while gate.snapshot().active != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
assert!(inherited, "mode {mode} lost resource ownership");
|
||||
assert!(original_policy, "mode {mode} changed the detached worker's disk policy");
|
||||
assert_eq!(held, 1, "mode {mode} released a live worker's unit");
|
||||
})
|
||||
.await;
|
||||
assert!(inherited, "mode {mode} lost resource ownership");
|
||||
assert!(original_policy, "mode {mode} changed the detached worker's disk policy");
|
||||
assert_eq!(held, vec![1; physical_count], "mode {mode} released a live worker's unit");
|
||||
}
|
||||
}
|
||||
}
|
||||
fn config(pending: usize, workers: usize, blocking_threads: usize) -> Config {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -172,6 +172,8 @@ pub enum DiskError {
|
||||
|
||||
#[error("guarded batch deletion is not supported by this disk")]
|
||||
GuardedDeleteUnsupported,
|
||||
#[error("foreground delete recovery work limit reached")]
|
||||
DeleteRecoveryWorkLimit,
|
||||
|
||||
#[error("erasure write quorum")]
|
||||
ErasureWriteQuorum,
|
||||
@@ -770,6 +772,7 @@ impl Clone for DiskError {
|
||||
DiskError::MethodNotAllowed => DiskError::MethodNotAllowed,
|
||||
DiskError::ConditionalDeleteConflict => DiskError::ConditionalDeleteConflict,
|
||||
DiskError::GuardedDeleteUnsupported => DiskError::GuardedDeleteUnsupported,
|
||||
DiskError::DeleteRecoveryWorkLimit => DiskError::DeleteRecoveryWorkLimit,
|
||||
DiskError::ErasureWriteQuorum => DiskError::ErasureWriteQuorum,
|
||||
DiskError::ErasureReadQuorum => DiskError::ErasureReadQuorum,
|
||||
DiskError::ShortWrite => DiskError::ShortWrite,
|
||||
@@ -829,6 +832,7 @@ impl DiskError {
|
||||
DiskError::RemoteClientUnavailable(_) => 0x2B,
|
||||
DiskError::ConditionalDeleteConflict => 0x2C,
|
||||
DiskError::GuardedDeleteUnsupported => 0x2D,
|
||||
DiskError::DeleteRecoveryWorkLimit => 0x2E,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -879,6 +883,7 @@ impl DiskError {
|
||||
0x2B => Some(DiskError::RemoteClientUnavailable(String::new())),
|
||||
0x2C => Some(DiskError::ConditionalDeleteConflict),
|
||||
0x2D => Some(DiskError::GuardedDeleteUnsupported),
|
||||
0x2E => Some(DiskError::DeleteRecoveryWorkLimit),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -1662,6 +1667,7 @@ mod tests {
|
||||
for (error, code) in [
|
||||
(DiskError::ConditionalDeleteConflict, 0x2C),
|
||||
(DiskError::GuardedDeleteUnsupported, 0x2D),
|
||||
(DiskError::DeleteRecoveryWorkLimit, 0x2E),
|
||||
] {
|
||||
assert_eq!(error.to_u32(), code);
|
||||
assert_eq!(DiskError::from_u32(code), Some(error.clone()));
|
||||
|
||||
@@ -5,6 +5,7 @@ use super::{FdCache, FdCacheEntry, FdKey};
|
||||
use std::fs::File;
|
||||
use std::os::unix::fs::FileExt;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
|
||||
fn key(volume: &str, path: &str, direct: bool) -> FdKey {
|
||||
FdKey {
|
||||
@@ -52,7 +53,11 @@ async fn uring_fd_cache_exact_invalidation_removes_both_modes_and_preserves_othe
|
||||
async fn uring_fd_cache_exact_invalidation_bumps_one_generation_and_rejects_old_epoch_inserts() {
|
||||
let (_directory, file) = fixture();
|
||||
let cache = FdCache::new();
|
||||
let entry = Arc::new(FdCacheEntry { file, len: 3 });
|
||||
let entry = Arc::new(FdCacheEntry {
|
||||
file,
|
||||
len: 3,
|
||||
invalidated: AtomicBool::new(false),
|
||||
});
|
||||
let old_generation = cache.generation();
|
||||
cache.invalidate_exact("v", "part").await;
|
||||
let current_generation = cache.generation();
|
||||
@@ -125,7 +130,7 @@ async fn uring_fd_cache_volume_and_clear_invalidation_cover_both_modes() {
|
||||
cache.insert(key("v", "part", direct), Arc::clone(&file)).await;
|
||||
assert!(
|
||||
cache.get(&key("v", "part", direct)).await.is_some(),
|
||||
"new entries must survive the earlier volume predicate"
|
||||
"new entries must survive the earlier volume invalidation"
|
||||
);
|
||||
}
|
||||
cache.clear();
|
||||
@@ -164,6 +169,7 @@ async fn uring_fd_cache_invalidation_keeps_borrowed_old_inode_alive_and_allows_r
|
||||
let replacement = Arc::new(FdCacheEntry {
|
||||
file: Arc::new(File::open(directory.path().join("part")).expect("open replacement inode")),
|
||||
len: 11,
|
||||
invalidated: AtomicBool::new(false),
|
||||
});
|
||||
for direct in [false, true] {
|
||||
let key = key("v", "part", direct);
|
||||
@@ -177,3 +183,459 @@ async fn uring_fd_cache_invalidation_keeps_borrowed_old_inode_alive_and_allows_r
|
||||
assert_eq!(fresh.len, 11);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fd_cache_tracker_skips_unmatched_invalidations_and_handles_deep_prefixes() {
|
||||
let (_directory, file) = fixture();
|
||||
let mut cache = FdCache::new();
|
||||
|
||||
// Unmatched invalidations on an empty cache must not spawn the maintenance task.
|
||||
cache.invalidate_under("v", "missing/prefix");
|
||||
cache.invalidate_volume("v");
|
||||
cache.invalidate_exact("v", "missing/prefix/part.1").await;
|
||||
cache.clear();
|
||||
assert!(
|
||||
cache.maintenance_task.get().is_none(),
|
||||
"empty-cache invalidations must not spawn maintenance"
|
||||
);
|
||||
|
||||
// Populate a normal entry and verify matched vs unmatched filter gates directly,
|
||||
// and ensure unmatched invalidations do not enqueue pending removals.
|
||||
let cached_key = key("v", "obj-1/0198a3f0/part.1", false);
|
||||
cache.insert(cached_key.clone(), Arc::clone(&file)).await;
|
||||
assert_eq!(cache.tracker.live_entries(), 1);
|
||||
assert!(cache.tracker.may_match_volume("v"));
|
||||
assert!(!cache.tracker.may_match_volume("other-volume"));
|
||||
assert!(cache.tracker.may_match_prefix("v", "obj-1"));
|
||||
assert!(cache.tracker.may_match_prefix("v", "obj-1/0198a3f0"));
|
||||
assert!(!cache.tracker.may_match_prefix("v", "obj-2/0198a3f0"));
|
||||
assert!(cache.tracker.may_match_exact("v", "obj-1/0198a3f0/part.1"));
|
||||
assert!(!cache.tracker.may_match_exact("v", "obj-1/0198a3f0/xl.meta"));
|
||||
|
||||
cache.invalidate_volume("other-volume");
|
||||
cache.invalidate_under("v", "obj-2/0198a3f0");
|
||||
cache.invalidate_exact("v", "obj-1/0198a3f0/xl.meta").await;
|
||||
assert!(
|
||||
!cache
|
||||
.tracker
|
||||
.has_pending_invalidations
|
||||
.load(std::sync::atomic::Ordering::SeqCst),
|
||||
"unmatched invalidations must not enqueue maintenance removals"
|
||||
);
|
||||
assert!(cache.get(&cached_key).await.is_some());
|
||||
|
||||
// Deeply nested path (> FD_CACHE_MAX_INDEXED_PREFIXES slashes) must fall back
|
||||
// to per-volume unindexed_counts without degrading other volumes.
|
||||
let other_vol_key = key("v2", "shallow/part.1", false);
|
||||
cache.insert(other_vol_key.clone(), Arc::clone(&file)).await;
|
||||
let deep_path = "c1/c2/c3/c4/c5/c6/c7/c8/c9/c10/c11/c12/c13/c14/part.1";
|
||||
let deep_key = key("v", deep_path, false);
|
||||
cache.insert(deep_key.clone(), Arc::clone(&file)).await;
|
||||
assert_eq!(
|
||||
cache
|
||||
.tracker
|
||||
.unindexed_counts
|
||||
.iter()
|
||||
.map(|c| c.load(std::sync::atomic::Ordering::Acquire))
|
||||
.sum::<u32>(),
|
||||
1
|
||||
);
|
||||
assert!(
|
||||
!cache.tracker.may_match_prefix("v2", "unmatched/prefix"),
|
||||
"deep key in volume v must not disable prefix filtering in volume v2"
|
||||
);
|
||||
cache.invalidate_under("v", "c1/c2/c3/c4/c5/c6/c7/c8/c9/c10/c11/c12/c13");
|
||||
assert!(cache.get(&deep_key).await.is_none());
|
||||
cache.invalidate_volume("v2");
|
||||
assert!(cache.get(&other_vol_key).await.is_none());
|
||||
|
||||
// Replacing an invalidated key before maintenance drains pending_invalidations
|
||||
// must release the old descriptor immediately and keep the replacement live
|
||||
// even when a strong reference to the old CachedFdEntry shell survives until
|
||||
// maintenance runs its Arc::ptr_eq check.
|
||||
let old_file = Arc::new(tempfile::tempfile().expect("create replaced descriptor"));
|
||||
let old_weak = Arc::downgrade(&old_file);
|
||||
cache.insert(cached_key.clone(), old_file).await;
|
||||
let held_old_shell = cache.cache.get(&cached_key).expect("cached shell must exist");
|
||||
cache.invalidate_under("v", "obj-1/0198a3f0");
|
||||
assert!(cache.get(&cached_key).await.is_none());
|
||||
cache.insert(cached_key.clone(), Arc::clone(&file)).await;
|
||||
assert!(
|
||||
old_weak.upgrade().is_none(),
|
||||
"replacing an entry must retire and drop the previous descriptor immediately"
|
||||
);
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
while cache
|
||||
.tracker
|
||||
.has_pending_invalidations
|
||||
.load(std::sync::atomic::Ordering::SeqCst)
|
||||
{
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("maintenance should drain pending invalidations");
|
||||
drop(held_old_shell);
|
||||
assert!(
|
||||
cache.get(&cached_key).await.is_some(),
|
||||
"stale pending invalidation must not evict a newer replacement"
|
||||
);
|
||||
assert_eq!(
|
||||
cache
|
||||
.tracker
|
||||
.unindexed_counts
|
||||
.iter()
|
||||
.map(|c| c.load(std::sync::atomic::Ordering::Acquire))
|
||||
.sum::<u32>(),
|
||||
0
|
||||
);
|
||||
|
||||
// Dropping a cache while a live non-invalidated entry is present (even if a
|
||||
// cloned Moka handle outlives FdCache and the maintenance task is aborted)
|
||||
// must retire the descriptor and release FdCacheTracker without cycles.
|
||||
let live_drop_file = Arc::new(tempfile::tempfile().expect("create live drop descriptor"));
|
||||
let live_drop_weak = Arc::downgrade(&live_drop_file);
|
||||
let shutdown_file = Arc::new(tempfile::tempfile().expect("create shutdown descriptor"));
|
||||
let shutdown_weak = Arc::downgrade(&shutdown_file);
|
||||
let tracker_weak = Arc::downgrade(&cache.tracker);
|
||||
let cloned_moka = cache.cache.clone();
|
||||
cache.insert(key("v", "live-on-drop/part.1", false), live_drop_file).await;
|
||||
cache.insert(key("v", "shutdown/part.1", false), shutdown_file).await;
|
||||
cache.invalidate_under("v", "shutdown");
|
||||
if let Some(handle) = cache.maintenance_task.take() {
|
||||
handle.abort();
|
||||
let _ = handle.await;
|
||||
}
|
||||
drop(cache);
|
||||
assert!(
|
||||
live_drop_weak.upgrade().is_none(),
|
||||
"Drop for FdCache must retire live descriptors even if a cloned Moka handle survives"
|
||||
);
|
||||
assert!(
|
||||
shutdown_weak.upgrade().is_none(),
|
||||
"dropping cache when maintenance is aborted must still release descriptors"
|
||||
);
|
||||
assert!(
|
||||
tracker_weak.upgrade().is_none(),
|
||||
"dropping cache must release FdCacheTracker without reference cycles"
|
||||
);
|
||||
drop(cloned_moka);
|
||||
|
||||
// Short-TTL expired entries (including mixed expired and fresh entries) must
|
||||
// be retired by invalidate_under and clear() even though Moka's iterator skips
|
||||
// expired entries.
|
||||
let ttl_cache = FdCache::with_ttl(std::time::Duration::from_millis(15));
|
||||
let expired_file = Arc::new(tempfile::tempfile().expect("create short-ttl descriptor"));
|
||||
let expired_weak = Arc::downgrade(&expired_file);
|
||||
ttl_cache.insert(key("v", "ttl-obj/part.1", false), expired_file).await;
|
||||
tokio::time::sleep(std::time::Duration::from_millis(30)).await;
|
||||
let fresh_file = Arc::new(tempfile::tempfile().expect("create fresh descriptor"));
|
||||
let fresh_weak = Arc::downgrade(&fresh_file);
|
||||
ttl_cache.insert(key("v", "ttl-obj/part.2", false), fresh_file).await;
|
||||
ttl_cache.invalidate_under("v", "ttl-obj");
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
while expired_weak.upgrade().is_some() || fresh_weak.upgrade().is_some() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("invalidate_under must retire both expired and fresh matching descriptors");
|
||||
assert_eq!(ttl_cache.tracker.live_entries(), 0);
|
||||
|
||||
let clear_expired_file = Arc::new(tempfile::tempfile().expect("create clear short-ttl descriptor"));
|
||||
let clear_expired_weak = Arc::downgrade(&clear_expired_file);
|
||||
ttl_cache
|
||||
.insert(key("v", "ttl-clear/part.1", false), clear_expired_file)
|
||||
.await;
|
||||
tokio::time::sleep(std::time::Duration::from_millis(30)).await;
|
||||
let clear_live_file = Arc::new(tempfile::tempfile().expect("create clear live descriptor"));
|
||||
let clear_live_weak = Arc::downgrade(&clear_live_file);
|
||||
ttl_cache.insert(key("v", "ttl-clear/part.2", false), clear_live_file).await;
|
||||
ttl_cache.clear();
|
||||
assert!(
|
||||
clear_expired_weak.upgrade().is_none() && clear_live_weak.upgrade().is_none(),
|
||||
"clear() must synchronously retire both TTL-expired and live descriptors"
|
||||
);
|
||||
assert_eq!(ttl_cache.tracker.live_entries(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fd_cache_lockstep_oracle_equivalence_and_counter_conservation() {
|
||||
use std::collections::HashMap;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
let (_directory, file) = fixture();
|
||||
let cache = FdCache::with_ttl(std::time::Duration::from_secs(3600));
|
||||
let mut oracle: HashMap<FdKey, Arc<FdCacheEntry>> = HashMap::new();
|
||||
|
||||
let volumes = ["vol-a", "vol-b", "vol-c", ".rustfs.sys/multipart"];
|
||||
let paths = [
|
||||
"a/b",
|
||||
"a/b/part.1",
|
||||
"a/b/dir/part.2",
|
||||
"a/bc",
|
||||
"a/bc/part.1",
|
||||
"prefix/obj-0001.bin/0198a3f0/part.1",
|
||||
"prefix/obj-0002.bin/0198a3f0/part.1",
|
||||
"d1/d2/d3/d4/d5/d6/d7/d8/d9/d10/d11/d12/d13/part.1",
|
||||
];
|
||||
let prefixes = [
|
||||
"a",
|
||||
"a/",
|
||||
"a/b",
|
||||
"a/b/",
|
||||
"a/b/dir",
|
||||
"a/bc",
|
||||
"prefix",
|
||||
"prefix/obj-0001.bin",
|
||||
"prefix/obj-0001.bin/0198a3f0",
|
||||
"d1/d2/d3/d4/d5/c_missing",
|
||||
"d1/d2/d3/d4/d5/d6/d7/d8/d9/d10/d11/d12/d13",
|
||||
"nonexistent/prefix",
|
||||
];
|
||||
|
||||
let mut candidate_keys = Vec::new();
|
||||
for v in volumes {
|
||||
for p in paths {
|
||||
for direct in [false, true] {
|
||||
candidate_keys.push(key(v, p, direct));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut rng = 0x9e37_79b9_7f4a_7c15u64;
|
||||
let mut next_u64 = || {
|
||||
rng ^= rng << 13;
|
||||
rng ^= rng >> 7;
|
||||
rng ^= rng << 17;
|
||||
rng
|
||||
};
|
||||
|
||||
for step in 0..1000 {
|
||||
let op = next_u64() % 100;
|
||||
if op < 45 {
|
||||
// Insert or replace (either fresh or stale generation)
|
||||
let k = candidate_keys[(next_u64() as usize) % candidate_keys.len()].clone();
|
||||
let entry = Arc::new(FdCacheEntry {
|
||||
file: Arc::clone(&file),
|
||||
len: step,
|
||||
invalidated: AtomicBool::new(false),
|
||||
});
|
||||
let cur_gen = cache.generation();
|
||||
let stale = (next_u64() % 5) == 0 && cur_gen > 0;
|
||||
let gen_to_use = if stale { cur_gen - 1 } else { cur_gen };
|
||||
cache.insert_if_fresh(k.clone(), Arc::clone(&entry), gen_to_use).await;
|
||||
if !stale {
|
||||
oracle.insert(k, entry);
|
||||
}
|
||||
} else if op < 65 {
|
||||
// Exact invalidation
|
||||
let v = volumes[(next_u64() as usize) % volumes.len()];
|
||||
let p = paths[(next_u64() as usize) % paths.len()];
|
||||
cache.invalidate_exact(v, p).await;
|
||||
oracle.remove(&key(v, p, false));
|
||||
oracle.remove(&key(v, p, true));
|
||||
} else if op < 88 {
|
||||
// Prefix invalidation
|
||||
let v = volumes[(next_u64() as usize) % volumes.len()];
|
||||
let raw_prefix = prefixes[(next_u64() as usize) % prefixes.len()];
|
||||
cache.invalidate_under(v, raw_prefix);
|
||||
let trimmed = raw_prefix.trim_end_matches('/');
|
||||
oracle.retain(|k, _| {
|
||||
!(k.volume == v && (k.path == trimmed || k.path.strip_prefix(trimmed).is_some_and(|r| r.starts_with('/'))))
|
||||
});
|
||||
} else if op < 97 {
|
||||
// Volume invalidation
|
||||
let v = volumes[(next_u64() as usize) % volumes.len()];
|
||||
cache.invalidate_volume(v);
|
||||
oracle.retain(|k, _| k.volume != v);
|
||||
} else {
|
||||
// Full clear
|
||||
cache.clear();
|
||||
oracle.clear();
|
||||
}
|
||||
|
||||
// Verify lockstep equivalence against the reference oracle after every operation.
|
||||
for k in &candidate_keys {
|
||||
let got = cache.get(k).await;
|
||||
let expected = oracle.get(k);
|
||||
match (got, expected) {
|
||||
(Some(g), Some(e)) => assert!(
|
||||
Arc::ptr_eq(&g, e),
|
||||
"step {step}: pointer mismatch for key ({}, {}, {})",
|
||||
k.volume,
|
||||
k.path,
|
||||
k.direct
|
||||
),
|
||||
(None, None) => {}
|
||||
(g, e) => panic!(
|
||||
"step {step}: oracle mismatch for key ({}, {}, {}): got={}, expected={}",
|
||||
k.volume,
|
||||
k.path,
|
||||
k.direct,
|
||||
g.is_some(),
|
||||
e.is_some()
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Clear and wait for maintenance to drain all entries, then verify exact counter conservation.
|
||||
cache.clear();
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
while cache.tracker.live_entries() != 0 || cache.tracker.has_pending_invalidations.load(Ordering::SeqCst) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("clear must drain all live entries and pending invalidations");
|
||||
|
||||
assert_eq!(cache.tracker.live_entries(), 0);
|
||||
assert!(cache.tracker.unindexed_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
|
||||
assert!(cache.tracker.volume_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
|
||||
assert!(cache.tracker.exact_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
|
||||
assert!(cache.tracker.prefix_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
|
||||
assert!(cache.tracker.prefix_shards.iter().all(|s| s.lock().is_empty()));
|
||||
assert!(cache.tracker.entry_shards.iter().all(|s| s.lock().is_empty()));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn fd_cache_concurrent_insert_and_invalidation_stress() {
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
let (_directory, file) = fixture();
|
||||
let cache = Arc::new(FdCache::new());
|
||||
|
||||
// Deterministic Dekker-partner interleaving check: pause `CachedFdEntry::new`
|
||||
// at `entry_shards[shard].lock()` after it has passed the initial generation
|
||||
// check and incremented `volume_counts`, bump generation via an unmatched
|
||||
// invalidation, and verify the post-`new()` recheck refuses publication and
|
||||
// preserves the existing entry.
|
||||
let seam_key = key("v-seam", "obj-1/part.1", false);
|
||||
cache.insert(seam_key.clone(), Arc::clone(&file)).await;
|
||||
let initial_entry = cache.get(&seam_key).await.expect("initial entry must exist");
|
||||
let (vol_hash, vol_slot) = super::fd_cache_volume_hash_and_slot("v-seam");
|
||||
let entry_shard = (vol_hash as usize) & (super::FD_CACHE_PREFIX_SHARDS - 1);
|
||||
let stale_gen = cache.generation();
|
||||
let stale_entry = Arc::new(FdCacheEntry {
|
||||
file: Arc::clone(&file),
|
||||
len: 999_999,
|
||||
invalidated: AtomicBool::new(false),
|
||||
});
|
||||
let blocked_insert = {
|
||||
let hold = cache.tracker.entry_shards[entry_shard].lock();
|
||||
let cache_clone = Arc::clone(&cache);
|
||||
let seam_key_clone = seam_key.clone();
|
||||
let task = tokio::spawn(async move {
|
||||
cache_clone.insert_if_fresh(seam_key_clone, stale_entry, stale_gen).await;
|
||||
});
|
||||
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
|
||||
while cache.tracker.volume_counts[usize::from(vol_slot)].load(Ordering::SeqCst) < 2 {
|
||||
assert!(
|
||||
std::time::Instant::now() < deadline,
|
||||
"insert never reached CachedFdEntry::new with the previous entry still live"
|
||||
);
|
||||
std::thread::yield_now();
|
||||
}
|
||||
cache.invalidate_under("v-seam", "unmatched-prefix");
|
||||
drop(hold);
|
||||
task
|
||||
};
|
||||
blocked_insert.await.expect("blocked insert task should complete");
|
||||
let preserved = cache
|
||||
.get(&seam_key)
|
||||
.await
|
||||
.expect("refused stale insert must not retire a valid previous entry");
|
||||
assert!(Arc::ptr_eq(&preserved, &initial_entry));
|
||||
cache.invalidate_volume("v-seam");
|
||||
|
||||
let floor_v1 = Arc::new(AtomicU64::new(0));
|
||||
let floor_v2 = Arc::new(AtomicU64::new(0));
|
||||
let mut handles = Vec::new();
|
||||
|
||||
for t in 0..8usize {
|
||||
let cache = Arc::clone(&cache);
|
||||
let file = Arc::clone(&file);
|
||||
let floor_v1 = Arc::clone(&floor_v1);
|
||||
let floor_v2 = Arc::clone(&floor_v2);
|
||||
handles.push(tokio::spawn(async move {
|
||||
for i in 0..250u64 {
|
||||
let is_v1 = (t + i as usize).is_multiple_of(2);
|
||||
let vol = if is_v1 { "v1" } else { "v2" };
|
||||
let path = format!("dir-{}/obj-{}/part.1", t % 4, i % 16);
|
||||
let k = key(vol, &path, i.is_multiple_of(2));
|
||||
let g = cache.generation();
|
||||
let entry = Arc::new(FdCacheEntry {
|
||||
file: Arc::clone(&file),
|
||||
len: g,
|
||||
invalidated: AtomicBool::new(false),
|
||||
});
|
||||
cache.insert_if_fresh(k.clone(), entry, g).await;
|
||||
let min_gen = if is_v1 {
|
||||
floor_v1.load(Ordering::SeqCst)
|
||||
} else {
|
||||
floor_v2.load(Ordering::SeqCst)
|
||||
};
|
||||
if let Some(hit) = cache.get(&k).await {
|
||||
assert!(
|
||||
hit.len >= min_gen,
|
||||
"stale descriptor served after volume invalidation: opened_gen={}, floor_gen={min_gen}",
|
||||
hit.len
|
||||
);
|
||||
}
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
for t in 0..4usize {
|
||||
let cache = Arc::clone(&cache);
|
||||
let floor_v1 = Arc::clone(&floor_v1);
|
||||
let floor_v2 = Arc::clone(&floor_v2);
|
||||
handles.push(tokio::spawn(async move {
|
||||
for i in 0..250u64 {
|
||||
let is_v1 = (t + i as usize).is_multiple_of(2);
|
||||
let vol = if is_v1 { "v1" } else { "v2" };
|
||||
match i % 4 {
|
||||
0 => cache.invalidate_under(vol, &format!("dir-{}/obj-{}", t % 4, i % 16)),
|
||||
1 => {
|
||||
let p = format!("dir-{}/obj-{}/part.1", t % 4, i % 16);
|
||||
cache.invalidate_exact(vol, &p).await;
|
||||
}
|
||||
2 => {
|
||||
let before = cache.generation();
|
||||
cache.invalidate_volume(vol);
|
||||
let floor = before + 1;
|
||||
if is_v1 {
|
||||
floor_v1.fetch_max(floor, Ordering::SeqCst);
|
||||
} else {
|
||||
floor_v2.fetch_max(floor, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
_ => cache.invalidate_under("uncached-vol", "uncached/prefix"),
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
for h in handles {
|
||||
h.await.expect("worker task should succeed");
|
||||
}
|
||||
|
||||
cache.clear();
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
while cache.tracker.live_entries() != 0 || cache.tracker.has_pending_invalidations.load(Ordering::SeqCst) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("clear after concurrent stress must drain all live entries");
|
||||
|
||||
assert_eq!(cache.tracker.live_entries(), 0);
|
||||
assert!(cache.tracker.unindexed_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
|
||||
assert!(cache.tracker.volume_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
|
||||
assert!(cache.tracker.exact_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
|
||||
assert!(cache.tracker.prefix_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
|
||||
assert!(cache.tracker.prefix_shards.iter().all(|s| s.lock().is_empty()));
|
||||
assert!(cache.tracker.entry_shards.iter().all(|s| s.lock().is_empty()));
|
||||
}
|
||||
|
||||
+132
-62
@@ -16,40 +16,15 @@ use std::{
|
||||
collections::HashMap,
|
||||
fs::Metadata,
|
||||
path::{Path, PathBuf},
|
||||
sync::{Arc, Mutex, OnceLock},
|
||||
sync::Mutex,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use tokio::{
|
||||
fs::{self, File},
|
||||
io,
|
||||
};
|
||||
use tokio::{fs::File, io};
|
||||
|
||||
static READONLY_OPTIONS: OnceLock<Arc<fs::OpenOptions>> = OnceLock::new();
|
||||
static WRITEONLY_OPTIONS: OnceLock<Arc<fs::OpenOptions>> = OnceLock::new();
|
||||
static READWRITE_OPTIONS: OnceLock<Arc<fs::OpenOptions>> = OnceLock::new();
|
||||
|
||||
fn get_readonly_options() -> &'static Arc<fs::OpenOptions> {
|
||||
READONLY_OPTIONS.get_or_init(|| {
|
||||
let mut opts = fs::OpenOptions::new();
|
||||
opts.read(true);
|
||||
Arc::new(opts)
|
||||
})
|
||||
}
|
||||
|
||||
fn get_writeonly_options() -> &'static Arc<fs::OpenOptions> {
|
||||
WRITEONLY_OPTIONS.get_or_init(|| {
|
||||
let mut opts = fs::OpenOptions::new();
|
||||
opts.write(true);
|
||||
Arc::new(opts)
|
||||
})
|
||||
}
|
||||
|
||||
fn get_readwrite_options() -> &'static Arc<fs::OpenOptions> {
|
||||
READWRITE_OPTIONS.get_or_init(|| {
|
||||
let mut opts = fs::OpenOptions::new();
|
||||
opts.read(true).write(true);
|
||||
Arc::new(opts)
|
||||
})
|
||||
async fn native_io<T: Send + 'static>(operation: impl FnOnce() -> io::Result<T> + Send + 'static) -> io::Result<T> {
|
||||
super::io_admission::spawn_blocking_with_resource_ownership(operation)
|
||||
.await
|
||||
.map_err(|_| io::Error::other("background task failed"))?
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
@@ -117,33 +92,32 @@ pub const O_APPEND: FileMode = 0x00400;
|
||||
// create_new: bool,
|
||||
|
||||
pub async fn open_file(path: impl AsRef<Path>, mode: FileMode) -> io::Result<File> {
|
||||
let base_opts = match mode & (O_RDONLY | O_WRONLY | O_RDWR) {
|
||||
O_RDONLY => get_readonly_options(),
|
||||
O_WRONLY => get_writeonly_options(),
|
||||
O_RDWR => get_readwrite_options(),
|
||||
_ => get_readonly_options(),
|
||||
};
|
||||
|
||||
if (mode & (O_CREATE | O_APPEND | O_TRUNC)) != 0 {
|
||||
let mut opts = (**base_opts).clone();
|
||||
if mode & O_CREATE != 0 {
|
||||
opts.create(true);
|
||||
let path = path.as_ref().to_path_buf();
|
||||
native_io(move || {
|
||||
let mut options = std::fs::OpenOptions::new();
|
||||
match mode & (O_RDONLY | O_WRONLY | O_RDWR) {
|
||||
O_WRONLY => options.write(true),
|
||||
O_RDWR => options.read(true).write(true),
|
||||
_ => options.read(true),
|
||||
}
|
||||
if mode & O_APPEND != 0 {
|
||||
opts.append(true);
|
||||
.create(mode & O_CREATE != 0)
|
||||
.append(mode & O_APPEND != 0)
|
||||
.truncate(mode & O_TRUNC != 0);
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
if mode == O_RDONLY {
|
||||
super::os::prepared_publication_test_hooks::run(
|
||||
super::os::prepared_publication_test_hooks::Stage::NativeReadOpen,
|
||||
&path,
|
||||
);
|
||||
}
|
||||
if mode & O_TRUNC != 0 {
|
||||
opts.truncate(true);
|
||||
}
|
||||
opts.open(path.as_ref()).await
|
||||
} else {
|
||||
base_opts.open(path.as_ref()).await
|
||||
}
|
||||
options.open(path).map(File::from_std)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn access(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
fs::metadata(path).await?;
|
||||
Ok(())
|
||||
let path = path.as_ref().to_path_buf();
|
||||
native_io(move || std::fs::metadata(path).map(|_| ())).await
|
||||
}
|
||||
|
||||
pub fn access_std(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
@@ -152,7 +126,8 @@ pub fn access_std(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
}
|
||||
|
||||
pub async fn lstat(path: impl AsRef<Path>) -> io::Result<Metadata> {
|
||||
fs::symlink_metadata(path).await
|
||||
let path = path.as_ref().to_path_buf();
|
||||
native_io(move || std::fs::symlink_metadata(path)).await
|
||||
}
|
||||
|
||||
pub fn lstat_std(path: impl AsRef<Path>) -> io::Result<Metadata> {
|
||||
@@ -170,7 +145,7 @@ pub fn create_dir_all_std(path: &Path) -> io::Result<()> {
|
||||
|
||||
pub async fn make_dir_all(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
let path = path.as_ref().to_path_buf();
|
||||
tokio::task::spawn_blocking(move || create_dir_all_std(&path)).await?
|
||||
native_io(move || create_dir_all_std(&path)).await
|
||||
}
|
||||
|
||||
fn is_dir_error(e: &io::Error) -> bool {
|
||||
@@ -183,9 +158,13 @@ fn is_dir_error(e: &io::Error) -> bool {
|
||||
#[tracing::instrument(level = "debug", skip_all)]
|
||||
pub async fn remove(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
// Try remove_file first; fall back to remove_dir if it's a directory
|
||||
match fs::remove_file(path.as_ref()).await {
|
||||
let file_path = path.as_ref().to_path_buf();
|
||||
match native_io(move || std::fs::remove_file(file_path)).await {
|
||||
Ok(()) => Ok(()),
|
||||
Err(e) if is_dir_error(&e) => fs::remove_dir(path.as_ref()).await,
|
||||
Err(e) if is_dir_error(&e) => {
|
||||
let path = path.as_ref().to_path_buf();
|
||||
native_io(move || std::fs::remove_dir(path)).await
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
@@ -193,9 +172,13 @@ pub async fn remove(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub async fn remove_all(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
// Try remove_file first; fall back to remove_dir_all if it's a directory
|
||||
match fs::remove_file(path.as_ref()).await {
|
||||
let file_path = path.as_ref().to_path_buf();
|
||||
match native_io(move || std::fs::remove_file(file_path)).await {
|
||||
Ok(()) => Ok(()),
|
||||
Err(e) if is_dir_error(&e) => fs::remove_dir_all(path.as_ref()).await,
|
||||
Err(e) if is_dir_error(&e) => {
|
||||
let path = path.as_ref().to_path_buf();
|
||||
native_io(move || std::fs::remove_dir_all(path)).await
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
@@ -222,11 +205,14 @@ pub fn remove_all_std(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
|
||||
#[cfg(test)]
|
||||
pub async fn mkdir(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
fs::create_dir(path.as_ref()).await
|
||||
let path = path.as_ref().to_path_buf();
|
||||
native_io(move || std::fs::create_dir(path)).await
|
||||
}
|
||||
|
||||
pub async fn rename(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()> {
|
||||
fs::rename(from, to).await
|
||||
let from = from.as_ref().to_path_buf();
|
||||
let to = to.as_ref().to_path_buf();
|
||||
native_io(move || std::fs::rename(from, to)).await
|
||||
}
|
||||
|
||||
#[cfg(any(not(windows), test))]
|
||||
@@ -289,7 +275,7 @@ pub async fn cached_access(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
}
|
||||
|
||||
let path_buf = path.to_path_buf();
|
||||
let result = fs::metadata(&path_buf).await;
|
||||
let result = access(&path_buf).await;
|
||||
|
||||
match &result {
|
||||
Ok(_) => BUCKET_EXISTENCE_CACHE.record(path_buf, true),
|
||||
@@ -304,6 +290,7 @@ pub async fn cached_access(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
}
|
||||
|
||||
pub fn invalidate_bucket_cache(path: impl AsRef<Path>) {
|
||||
crate::bucket::metadata_sys::bump_local_meta_mutation_gen();
|
||||
BUCKET_EXISTENCE_CACHE.invalidate(path.as_ref());
|
||||
}
|
||||
|
||||
@@ -313,6 +300,89 @@ mod tests {
|
||||
use tempfile::TempDir;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
#[test]
|
||||
fn cancelled_filesystem_jobs_retain_physical_resources() {
|
||||
use crate::disk::io_admission::{test_resource_units, with_optional_resource_unit};
|
||||
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.max_blocking_threads(1)
|
||||
.build()
|
||||
.expect("single native worker");
|
||||
runtime.block_on(async {
|
||||
let mut observations = Vec::new();
|
||||
for count in 1..=2 {
|
||||
for operation in [
|
||||
"access",
|
||||
"cached_access",
|
||||
"lstat",
|
||||
"mkdir",
|
||||
"make_dir_all",
|
||||
"remove",
|
||||
"remove_all",
|
||||
"rename",
|
||||
"open",
|
||||
] {
|
||||
let dir = TempDir::new().expect("filesystem ownership fixture");
|
||||
let source = dir.path().join("source");
|
||||
let target = dir.path().join("target");
|
||||
match operation {
|
||||
"mkdir" | "make_dir_all" | "open" => {}
|
||||
"remove_all" => {
|
||||
std::fs::create_dir(&source).expect("recursive removal directory");
|
||||
std::fs::write(source.join("child"), b"payload").expect("recursive removal child");
|
||||
}
|
||||
_ => std::fs::write(&source, b"payload").expect("existing file"),
|
||||
}
|
||||
let (resources, gates) = test_resource_units(count);
|
||||
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel();
|
||||
let blocker = tokio::task::spawn_blocking(move || {
|
||||
let _ = started_tx.send(());
|
||||
let _ = release_rx.recv();
|
||||
});
|
||||
started_rx.await.expect("native worker occupied");
|
||||
let mut future = Box::pin(with_optional_resource_unit(Some(resources), async {
|
||||
match operation {
|
||||
"access" => access(&source).await,
|
||||
"cached_access" => cached_access(&source).await,
|
||||
"lstat" => lstat(&source).await.map(|_| ()),
|
||||
"mkdir" => mkdir(&source).await,
|
||||
"make_dir_all" => make_dir_all(source.join("child")).await,
|
||||
"remove" => remove(&source).await,
|
||||
"remove_all" => remove_all(&source).await,
|
||||
"rename" => rename(&source, &target).await,
|
||||
"open" => open_file(&source, O_CREATE | O_WRONLY).await.map(|_| ()),
|
||||
_ => unreachable!("fixed operation list"),
|
||||
}
|
||||
}));
|
||||
assert!(futures::poll!(&mut future).is_pending(), "{operation}: native job queued");
|
||||
drop(future);
|
||||
let held: Vec<_> = gates.iter().map(|gate| gate.snapshot().active).collect();
|
||||
release_tx.send(()).expect("release native worker before assertions");
|
||||
blocker.await.expect("native blocker completed");
|
||||
tokio::task::spawn_blocking(|| ()).await.expect("native queue drained");
|
||||
assert!(gates.iter().all(|gate| gate.snapshot().active == 0));
|
||||
match operation {
|
||||
"mkdir" => assert!(source.is_dir()),
|
||||
"make_dir_all" => assert!(source.join("child").is_dir()),
|
||||
"open" => assert!(source.is_file()),
|
||||
"remove" => assert!(!source.exists()),
|
||||
"rename" => {
|
||||
assert!(!source.exists());
|
||||
assert_eq!(std::fs::read(&target).expect("renamed file"), b"payload");
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
observations.push((count, operation, held));
|
||||
}
|
||||
}
|
||||
for (count, operation, held) in observations {
|
||||
assert_eq!(held, vec![1; count], "{operation}: queued native job keeps its physical resources");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_file_mode_constants() {
|
||||
assert_eq!(O_RDONLY, 0x00000);
|
||||
|
||||
@@ -14,6 +14,18 @@
|
||||
|
||||
//! Per-local-disk foreground-priority admission for scanner I/O.
|
||||
|
||||
pub(crate) mod memory;
|
||||
mod physical_identity;
|
||||
mod physical_owners;
|
||||
mod physical_sampler;
|
||||
|
||||
pub use memory::BackgroundMemoryBudget;
|
||||
|
||||
pub(crate) use physical_sampler::PhysicalHealBinding;
|
||||
pub use physical_sampler::{
|
||||
PhysicalDiskBinding, PhysicalDiskPlan, PhysicalDiskSample, PhysicalDiskSampler, PhysicalResourcePlan,
|
||||
};
|
||||
|
||||
use std::collections::VecDeque;
|
||||
use std::future::Future;
|
||||
use std::io;
|
||||
@@ -25,18 +37,118 @@ use std::time::Duration;
|
||||
|
||||
use metrics::{counter, gauge, histogram};
|
||||
use rustfs_concurrency::adaptive::AdaptivePermit;
|
||||
use rustfs_concurrency::adaptive::permit_set::AdaptivePermitSet;
|
||||
use rustfs_rio::ChunkReader;
|
||||
use tokio::io::{AsyncRead, ReadBuf};
|
||||
use tokio::sync::Notify;
|
||||
use tokio::time::Instant;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
pub(crate) mod remote;
|
||||
pub use remote::{RemoteBackgroundIo, with_remote_background_io};
|
||||
|
||||
const BACKGROUND_FOREGROUND_BURST: u32 = 8;
|
||||
const BACKGROUND_MAX_WAIT: Duration = Duration::from_millis(100);
|
||||
|
||||
tokio::task_local! {
|
||||
static DISK_IO_CONTEXT: DiskIoContext;
|
||||
static RESOURCE_IO_UNIT: AdaptivePermit;
|
||||
static RESOURCE_IO_UNIT: ResourceIoUnit;
|
||||
}
|
||||
|
||||
/// Native I/O retains the physical subset selected before storage work. A
|
||||
/// layered disk may touch several leaves; unrelated devices stay outside it.
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct ResourceIoUnit {
|
||||
permits: ResourceIoPermits,
|
||||
memory_budget: Option<Arc<BackgroundMemoryBudget>>,
|
||||
#[cfg(target_os = "linux")]
|
||||
_mount_lease: Option<Arc<std::fs::File>>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
enum ResourceIoPermits {
|
||||
Single(AdaptivePermit),
|
||||
Set(AdaptivePermitSet),
|
||||
}
|
||||
|
||||
impl Clone for ResourceIoPermits {
|
||||
fn clone(&self) -> Self {
|
||||
match self {
|
||||
Self::Single(permit) => Self::Single(permit.clone()),
|
||||
Self::Set(permits) => Self::Set(permits.clone()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<AdaptivePermit> for ResourceIoUnit {
|
||||
fn from(permit: AdaptivePermit) -> Self {
|
||||
Self {
|
||||
permits: ResourceIoPermits::Single(permit),
|
||||
memory_budget: BackgroundMemoryBudget::current(),
|
||||
#[cfg(target_os = "linux")]
|
||||
_mount_lease: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<AdaptivePermitSet> for ResourceIoUnit {
|
||||
fn from(permits: AdaptivePermitSet) -> Self {
|
||||
Self {
|
||||
permits: ResourceIoPermits::Set(permits),
|
||||
memory_budget: BackgroundMemoryBudget::current(),
|
||||
#[cfg(target_os = "linux")]
|
||||
_mount_lease: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ResourceIoUnit {
|
||||
pub(crate) fn memory_budget(&self) -> Option<Arc<BackgroundMemoryBudget>> {
|
||||
self.memory_budget.clone()
|
||||
}
|
||||
|
||||
pub(crate) fn with_permits(mut self, permits: AdaptivePermitSet) -> Self {
|
||||
self.permits = ResourceIoPermits::Set(permits);
|
||||
self
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn with_memory_budget(mut self, budget: Arc<BackgroundMemoryBudget>) -> Self {
|
||||
self.memory_budget = Some(budget);
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn as_set(&self) -> Option<&AdaptivePermitSet> {
|
||||
match &self.permits {
|
||||
ResourceIoPermits::Single(_) => None,
|
||||
ResourceIoPermits::Set(permits) => Some(permits),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
pub(crate) fn with_mount_lease(mut self, mount_lease: Arc<std::fs::File>) -> Self {
|
||||
// The selected disk's proc-fd paths must survive native jobs after its
|
||||
// async wrapper retires. Clones carry the same pinned mount generation.
|
||||
self._mount_lease = Some(mount_lease);
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn test_resource_units(count: usize) -> (ResourceIoUnit, Vec<Arc<rustfs_concurrency::adaptive::AdaptiveAdmission>>) {
|
||||
use rustfs_concurrency::adaptive::{AdaptiveAdmission, permit_set::AdaptivePermitSetOwner};
|
||||
use std::num::NonZeroU16;
|
||||
|
||||
assert!((1..=2).contains(&count), "test single and layered disks");
|
||||
let gates: Vec<_> = (0..count).map(|_| AdaptiveAdmission::new(NonZeroU16::MIN, 1)).collect();
|
||||
let mut admitted: Vec<_> = gates.iter().map(|gate| gate.try_acquire().expect("test resource")).collect();
|
||||
let unit = if count == 1 {
|
||||
ResourceIoUnit::from(admitted.pop().expect("single resource"))
|
||||
} else {
|
||||
let owner = AdaptivePermitSetOwner::new(NonZeroU16::MIN, gates.clone()).expect("layered resource");
|
||||
ResourceIoUnit::from(owner.capture(&admitted.iter().collect::<Vec<_>>()).expect("admitted subset"))
|
||||
};
|
||||
(unit, gates)
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
@@ -51,7 +163,7 @@ pub(crate) struct DiskIoContext {
|
||||
pub(crate) cancellation: CancellationToken,
|
||||
pub(crate) admission: Option<Arc<DiskIoAdmission>>,
|
||||
pub(crate) permit: Option<DiskIoPermit>,
|
||||
pub(crate) resource_unit: Option<AdaptivePermit>,
|
||||
pub(crate) resource_unit: Option<ResourceIoUnit>,
|
||||
}
|
||||
|
||||
/// Mark disk I/O issued by `future` as background work for local-disk admission.
|
||||
@@ -78,24 +190,46 @@ pub async fn with_background_disk_io<F: Future>(cancellation: CancellationToken,
|
||||
/// of a stalled native operation. Escaping buffers need their own memory leases.
|
||||
#[doc(hidden)]
|
||||
pub async fn with_background_resource_unit<F: Future>(unit: AdaptivePermit, future: F) -> F::Output {
|
||||
RESOURCE_IO_UNIT.scope(unit, future).await
|
||||
RESOURCE_IO_UNIT.scope(ResourceIoUnit::from(unit), future).await
|
||||
}
|
||||
|
||||
pub(crate) fn current_resource_unit() -> Option<AdaptivePermit> {
|
||||
RESOURCE_IO_UNIT.try_with(Clone::clone).ok().or_else(|| {
|
||||
DISK_IO_CONTEXT
|
||||
.try_with(|context| context.resource_unit.clone())
|
||||
.ok()
|
||||
.flatten()
|
||||
})
|
||||
/// Retain only the previously admitted physical leaves touched by this I/O.
|
||||
/// Projection of owned units never waits and may run under storage locks.
|
||||
/// Acquire the planned resources before locking; auxiliary leaves discovered
|
||||
/// later must use nonblocking admission. Do not pass unrelated parent resources
|
||||
/// to a native job that touches only some of its disks.
|
||||
#[doc(hidden)]
|
||||
pub async fn with_background_resource_set<F: Future>(units: AdaptivePermitSet, future: F) -> F::Output {
|
||||
RESOURCE_IO_UNIT.scope(ResourceIoUnit::from(units), future).await
|
||||
}
|
||||
|
||||
pub(crate) fn current_resource_unit() -> Option<ResourceIoUnit> {
|
||||
RESOURCE_IO_UNIT
|
||||
.try_with(Clone::clone)
|
||||
.ok()
|
||||
.or_else(|| {
|
||||
DISK_IO_CONTEXT
|
||||
.try_with(|context| context.resource_unit.clone())
|
||||
.ok()
|
||||
.flatten()
|
||||
})
|
||||
.or_else(remote::resources)
|
||||
}
|
||||
|
||||
pub(crate) fn has_resource_io_scope() -> bool {
|
||||
RESOURCE_IO_UNIT.try_with(|_| ()).is_ok()
|
||||
|| DISK_IO_CONTEXT
|
||||
.try_with(|context| context.resource_unit.is_some())
|
||||
.unwrap_or(false)
|
||||
|| remote::active()
|
||||
}
|
||||
|
||||
/// Transfer only resource ownership into an existing spawned task. Disk
|
||||
/// classification/permits are not inherited: changing those can introduce a
|
||||
/// wait behind a disk permit still held by the parent.
|
||||
pub(crate) async fn with_optional_resource_unit<F: Future>(unit: Option<AdaptivePermit>, future: F) -> F::Output {
|
||||
pub(crate) async fn with_optional_resource_unit<F: Future>(unit: Option<ResourceIoUnit>, future: F) -> F::Output {
|
||||
match unit {
|
||||
Some(unit) => with_background_resource_unit(unit, future).await,
|
||||
Some(unit) => RESOURCE_IO_UNIT.scope(unit, future).await,
|
||||
None => future.await,
|
||||
}
|
||||
}
|
||||
@@ -233,7 +367,7 @@ impl DiskIoAdmission {
|
||||
self.foreground_with_resource(current_resource_unit())
|
||||
}
|
||||
|
||||
fn foreground_with_resource(self: &Arc<Self>, resource_unit: Option<AdaptivePermit>) -> DiskIoPermit {
|
||||
fn foreground_with_resource(self: &Arc<Self>, resource_unit: Option<ResourceIoUnit>) -> DiskIoPermit {
|
||||
self.foreground_active.fetch_add(1, Ordering::AcqRel);
|
||||
if self.background_waiters.load(Ordering::Acquire) > 0 {
|
||||
self.record_state();
|
||||
@@ -254,7 +388,7 @@ impl DiskIoAdmission {
|
||||
async fn background_with_resource(
|
||||
self: &Arc<Self>,
|
||||
cancellation: &CancellationToken,
|
||||
resource_unit: Option<AdaptivePermit>,
|
||||
resource_unit: Option<ResourceIoUnit>,
|
||||
) -> std::io::Result<DiskIoPermit> {
|
||||
let started_at = Instant::now();
|
||||
let ticket = {
|
||||
@@ -429,7 +563,7 @@ impl DiskIoPermit {
|
||||
struct DiskIoPermitLease {
|
||||
admission: Arc<DiskIoAdmission>,
|
||||
class: DiskIoClass,
|
||||
resource_unit: Option<AdaptivePermit>,
|
||||
resource_unit: Option<ResourceIoUnit>,
|
||||
}
|
||||
|
||||
pub(crate) fn spawn_blocking_with_disk_io_permit<T, F>(
|
||||
@@ -450,6 +584,18 @@ where
|
||||
})
|
||||
}
|
||||
|
||||
/// Run native filesystem work with the current resource ownership. Cancelling
|
||||
/// the async waiter cannot refund a unit while the native operation still runs.
|
||||
/// This does not acquire capacity or change foreground/background policy.
|
||||
#[doc(hidden)]
|
||||
pub fn spawn_blocking_with_resource_ownership<T, F>(operation: F) -> tokio::task::JoinHandle<T>
|
||||
where
|
||||
T: Send + 'static,
|
||||
F: FnOnce() -> T + Send + 'static,
|
||||
{
|
||||
spawn_blocking_with_disk_io_permit(None, operation)
|
||||
}
|
||||
|
||||
type AdmissionWait = Pin<Box<dyn Future<Output = io::Result<DiskIoPermit>> + Send>>;
|
||||
|
||||
impl Drop for DiskIoPermitLease {
|
||||
@@ -595,6 +741,63 @@ mod tests {
|
||||
use std::num::NonZeroU16;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
|
||||
#[tokio::test]
|
||||
async fn physical_subset_survives_native_cancellation_without_retaining_unrelated_disks() {
|
||||
use rustfs_concurrency::adaptive::permit_set::AdaptivePermitSetOwner;
|
||||
|
||||
for disk_permit in [false, true] {
|
||||
let maximum = NonZeroU16::new(1).expect("one physical unit");
|
||||
let first = AdaptiveAdmission::new(maximum, 1);
|
||||
let second = AdaptiveAdmission::new(maximum, 1);
|
||||
let unrelated = AdaptiveAdmission::new(maximum, 1);
|
||||
let owner = AdaptivePermitSetOwner::new(maximum, vec![first.clone(), second.clone()]).expect("layered disk");
|
||||
let a = first.try_acquire().expect("first backing device");
|
||||
let b = second.try_acquire().expect("second backing device");
|
||||
let healthy = unrelated.try_acquire().expect("another disk in the parent operation");
|
||||
let subset = owner.capture(&[&a, &b]).expect("capture before storage work");
|
||||
drop((a, b, owner));
|
||||
let disk = DiskIoAdmission::new("physical-subset-native");
|
||||
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
|
||||
let task = tokio::spawn(async move {
|
||||
let _healthy = healthy;
|
||||
with_background_resource_set(subset, async {
|
||||
let permit = if disk_permit {
|
||||
Some(disk.background(&CancellationToken::new()).await.expect("logical disk permit"))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
spawn_blocking_with_disk_io_permit(permit, move || {
|
||||
started_tx.send(()).expect("native I/O started");
|
||||
release_rx.blocking_recv().expect("release native I/O");
|
||||
})
|
||||
.await
|
||||
.expect("native I/O completed");
|
||||
})
|
||||
.await;
|
||||
});
|
||||
started_rx.await.expect("native worker retains subset");
|
||||
task.abort();
|
||||
assert!(task.await.expect_err("cancel async waiter").is_cancelled());
|
||||
let active = [first.snapshot().active, second.snapshot().active, unrelated.snapshot().active];
|
||||
let healthy_available = unrelated.try_acquire().is_some();
|
||||
first.set_limit(0);
|
||||
second.set_limit(0);
|
||||
release_tx.send(()).expect("finish native work");
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while first.snapshot().active != 0 || second.snapshot().active != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("both backing devices are returned after native completion");
|
||||
assert_eq!(active, [1, 1, 0], "only the native I/O's physical subset remains charged");
|
||||
assert!(healthy_available, "cancellation frees the independent disk immediately");
|
||||
assert!(first.try_acquire().is_none());
|
||||
assert!(second.try_acquire().is_none(), "old completions cannot reopen shrunken owners");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn resource_unit_outlives_cancelled_scope_until_native_io_finishes() {
|
||||
for permit_mode in 0..4 {
|
||||
|
||||
@@ -0,0 +1,547 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use bytes::Bytes;
|
||||
use std::io;
|
||||
use std::sync::{Arc, Mutex, OnceLock};
|
||||
use std::time::Duration;
|
||||
use tokio::time::Instant;
|
||||
|
||||
static BACKGROUND_MEMORY: OnceLock<Arc<BackgroundMemoryBudget>> = OnceLock::new();
|
||||
|
||||
/// Process-wide payload storage retained by admitted background I/O. The
|
||||
/// existing resource sampler publishes limits; this owner starts no workers.
|
||||
#[derive(Debug)]
|
||||
pub struct BackgroundMemoryBudget {
|
||||
// This short critical section only changes counters and a deadline. It
|
||||
// never allocates, calls storage, or acquires another lock while held.
|
||||
state: Mutex<State>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct State {
|
||||
limit: u64,
|
||||
used: u64,
|
||||
valid_until: Instant,
|
||||
}
|
||||
|
||||
/// A coherent view of the leased cap and retained payload storage.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct BackgroundMemorySnapshot {
|
||||
/// Zero when the latest resource observation has expired.
|
||||
pub limit: u64,
|
||||
/// Existing storage remains charged even after the cap shrinks below it.
|
||||
pub used: u64,
|
||||
}
|
||||
|
||||
impl BackgroundMemoryBudget {
|
||||
/// Start closed until the existing resource sampler publishes a limit.
|
||||
pub fn new() -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
state: Mutex::new(State {
|
||||
limit: 0,
|
||||
used: 0,
|
||||
valid_until: Instant::now(),
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
/// Install the sole process owner before starting adaptive background work.
|
||||
/// Returns false if an owner was already installed.
|
||||
pub fn install(self: &Arc<Self>) -> bool {
|
||||
BACKGROUND_MEMORY.set(self.clone()).is_ok()
|
||||
}
|
||||
|
||||
/// Return only the installed process memory policy; no node or disk ownership.
|
||||
/// No policy is installed in the default, non-adaptive startup path.
|
||||
pub fn current() -> Option<Arc<Self>> {
|
||||
BACKGROUND_MEMORY.get().cloned()
|
||||
}
|
||||
|
||||
/// Update admission without revoking storage retained by native tasks or
|
||||
/// readers. Missing observations must publish zero capacity.
|
||||
pub fn set_sampled_limit(&self, limit: u64, valid_for: Duration) {
|
||||
let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
state.limit = limit;
|
||||
let now = Instant::now();
|
||||
state.valid_until = now.checked_add(valid_for).unwrap_or(now);
|
||||
}
|
||||
|
||||
/// Read the cap and retained byte count under the same short lock.
|
||||
pub fn snapshot(&self) -> BackgroundMemorySnapshot {
|
||||
let state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
BackgroundMemorySnapshot {
|
||||
limit: if Instant::now() < state.valid_until { state.limit } else { 0 },
|
||||
used: state.used,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn reserve(self: &Arc<Self>, bytes: usize) -> io::Result<Reservation> {
|
||||
let bytes = u64::try_from(bytes).map_err(|_| io::Error::from(io::ErrorKind::OutOfMemory))?;
|
||||
let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let used = state
|
||||
.used
|
||||
.checked_add(bytes)
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::WouldBlock, BackgroundMemoryExhausted))?;
|
||||
if bytes != 0 && (Instant::now() >= state.valid_until || used > state.limit) {
|
||||
return Err(io::Error::new(io::ErrorKind::WouldBlock, BackgroundMemoryExhausted));
|
||||
}
|
||||
state.used = used;
|
||||
Ok(Reservation {
|
||||
owner: self.clone(),
|
||||
bytes,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct BackgroundMemoryExhausted;
|
||||
|
||||
impl std::fmt::Display for BackgroundMemoryExhausted {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter.write_str("background buffer memory budget exhausted")
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for BackgroundMemoryExhausted {}
|
||||
|
||||
pub(crate) fn is_memory_pressure(error: &io::Error) -> bool {
|
||||
error.kind() == io::ErrorKind::OutOfMemory || error.get_ref().is_some_and(|inner| inner.is::<BackgroundMemoryExhausted>())
|
||||
}
|
||||
|
||||
pub(crate) struct Reservation {
|
||||
owner: Arc<BackgroundMemoryBudget>,
|
||||
bytes: u64,
|
||||
}
|
||||
|
||||
impl Reservation {
|
||||
pub(crate) fn belongs_to(&self, budget: &Arc<BackgroundMemoryBudget>) -> bool {
|
||||
Arc::ptr_eq(&self.owner, budget)
|
||||
}
|
||||
|
||||
pub(crate) fn wrap(bytes: Bytes, reservation: Option<Self>) -> Bytes {
|
||||
if bytes.is_empty() {
|
||||
drop(bytes);
|
||||
drop(reservation);
|
||||
return Bytes::new();
|
||||
}
|
||||
match reservation {
|
||||
Some(reservation) => Bytes::from_owner(BackgroundBytes {
|
||||
bytes,
|
||||
_reservation: reservation,
|
||||
}),
|
||||
None => bytes,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct BackgroundBytes {
|
||||
// Free the payload before refunding it, including after the last slice.
|
||||
bytes: Bytes,
|
||||
_reservation: Reservation,
|
||||
}
|
||||
|
||||
impl AsRef<[u8]> for BackgroundBytes {
|
||||
fn as_ref(&self) -> &[u8] {
|
||||
&self.bytes
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Reservation {
|
||||
fn drop(&mut self) {
|
||||
self.owner
|
||||
.state
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.used -= self.bytes;
|
||||
}
|
||||
}
|
||||
|
||||
/// Reusable initialized payload storage. The reservation is embedded in the
|
||||
/// existing native job state, with no per-read owner allocation or payload copy.
|
||||
pub(crate) struct BackgroundBuffer {
|
||||
// Field order releases the allocation before refunding its reservation.
|
||||
bytes: Box<[u8]>,
|
||||
reservation: Option<Reservation>,
|
||||
budget: Option<Arc<BackgroundMemoryBudget>>,
|
||||
}
|
||||
|
||||
impl BackgroundBuffer {
|
||||
pub(crate) fn new(budget: Option<Arc<BackgroundMemoryBudget>>) -> Self {
|
||||
Self {
|
||||
bytes: Box::default(),
|
||||
reservation: None,
|
||||
budget,
|
||||
}
|
||||
}
|
||||
|
||||
/// Call only after the previous contents have been consumed. Growth charges
|
||||
/// both allocations until the old buffer is freed, including after shrink.
|
||||
pub(crate) fn ensure_len(&mut self, length: usize) -> io::Result<()> {
|
||||
self.grow(length, 0)
|
||||
}
|
||||
|
||||
/// Preserve an initialized prefix when replacing storage. Callers keep
|
||||
/// `preserve` within both the existing allocation and the requested length.
|
||||
pub(crate) fn grow(&mut self, length: usize, preserve: usize) -> io::Result<()> {
|
||||
if self.bytes.len() >= length {
|
||||
return Ok(());
|
||||
}
|
||||
let reservation = self.budget.as_ref().map(|budget| budget.reserve(length)).transpose()?;
|
||||
let mut bytes = Vec::new();
|
||||
bytes
|
||||
.try_reserve_exact(length)
|
||||
.map_err(|_| io::Error::from(io::ErrorKind::OutOfMemory))?;
|
||||
bytes.resize(length, 0);
|
||||
bytes[..preserve].copy_from_slice(&self.bytes[..preserve]);
|
||||
self.bytes = bytes.into_boxed_slice();
|
||||
self.reservation = reservation;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Metadata is only a size hint: concurrent growth still reads through EOF.
|
||||
/// Every replacement is charged before copying the retained prefix.
|
||||
pub(crate) fn read_to_end(mut self, reader: &mut impl io::Read, hint: usize) -> io::Result<Bytes> {
|
||||
self.ensure_len(hint)?;
|
||||
let mut filled = 0;
|
||||
loop {
|
||||
if filled == self.bytes.len() {
|
||||
// Probe EOF without forcing a larger allocation for an exact fit.
|
||||
let mut probe = [0];
|
||||
match reader.read(&mut probe) {
|
||||
Ok(0) => break,
|
||||
Ok(_) => {
|
||||
let length = self.bytes.len().checked_mul(2).ok_or(io::ErrorKind::OutOfMemory)?.max(32);
|
||||
self.grow(length, filled)?;
|
||||
self.bytes[filled] = probe[0];
|
||||
filled += 1;
|
||||
}
|
||||
Err(error) if error.kind() == io::ErrorKind::Interrupted => continue,
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
match reader.read(&mut self.bytes[filled..]) {
|
||||
Ok(0) => break,
|
||||
Ok(length) => filled += length,
|
||||
Err(error) if error.kind() == io::ErrorKind::Interrupted => continue,
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
Ok(self.take_bytes(filled))
|
||||
}
|
||||
|
||||
pub(crate) fn take_bytes(&mut self, length: usize) -> Bytes {
|
||||
let bytes = std::mem::take(&mut self.bytes);
|
||||
let reservation = self.reservation.take();
|
||||
let mut bytes = Bytes::from(bytes.into_vec());
|
||||
bytes.truncate(length);
|
||||
Reservation::wrap(bytes, reservation)
|
||||
}
|
||||
|
||||
pub(crate) fn memory_budget(&self) -> Option<Arc<BackgroundMemoryBudget>> {
|
||||
self.budget.clone()
|
||||
}
|
||||
|
||||
pub(crate) fn as_mut_slice(&mut self) -> &mut [u8] {
|
||||
&mut self.bytes
|
||||
}
|
||||
|
||||
pub(crate) fn as_slice(&self) -> &[u8] {
|
||||
&self.bytes
|
||||
}
|
||||
}
|
||||
|
||||
/// Bounded encoded-frame storage for serializer output and raw read staging.
|
||||
/// Both paths reserve the actual allocation before initializing its bytes.
|
||||
pub struct BackgroundEncodeBuffer {
|
||||
buffer: BackgroundBuffer,
|
||||
filled: usize,
|
||||
maximum: usize,
|
||||
}
|
||||
|
||||
impl BackgroundEncodeBuffer {
|
||||
pub fn new(memory: Arc<BackgroundMemoryBudget>, maximum: usize) -> Self {
|
||||
Self {
|
||||
buffer: BackgroundBuffer::new(Some(memory)),
|
||||
filled: 0,
|
||||
maximum,
|
||||
}
|
||||
}
|
||||
|
||||
/// Borrow initialized raw-frame staging; decode only after a successful read.
|
||||
/// Staging is not encoded output: `into_bytes` remains empty for this path.
|
||||
pub fn prepare_read(&mut self, length: usize) -> io::Result<&mut [u8]> {
|
||||
if length > self.maximum {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"background encoded frame exceeds its size limit",
|
||||
));
|
||||
}
|
||||
self.buffer.ensure_len(length)?;
|
||||
self.filled = 0;
|
||||
Ok(&mut self.buffer.as_mut_slice()[..length])
|
||||
}
|
||||
|
||||
pub fn into_bytes(mut self) -> Bytes {
|
||||
self.buffer.take_bytes(self.filled)
|
||||
}
|
||||
}
|
||||
|
||||
impl io::Write for BackgroundEncodeBuffer {
|
||||
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
|
||||
let required = self.filled.checked_add(bytes.len()).ok_or(io::ErrorKind::OutOfMemory)?;
|
||||
if required > self.maximum {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"background encoded frame exceeds its size limit",
|
||||
));
|
||||
}
|
||||
if required > self.buffer.as_slice().len() {
|
||||
let capacity = self
|
||||
.buffer
|
||||
.as_slice()
|
||||
.len()
|
||||
.saturating_mul(2)
|
||||
.max(32)
|
||||
.max(required)
|
||||
.min(self.maximum);
|
||||
self.buffer.grow(capacity, self.filled)?;
|
||||
}
|
||||
self.buffer.as_mut_slice()[self.filled..required].copy_from_slice(bytes);
|
||||
self.filled = required;
|
||||
Ok(bytes.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn encoded_buffer_growth_retains_both_allocations_and_the_last_slice() {
|
||||
use std::io::Write;
|
||||
|
||||
let owner = BackgroundMemoryBudget::new();
|
||||
owner.set_sampled_limit(95, Duration::from_secs(30));
|
||||
let mut buffer = BackgroundEncodeBuffer::new(owner.clone(), 64);
|
||||
buffer.write_all(&[7; 32]).expect("initial allocation fits");
|
||||
assert_eq!(owner.snapshot().used, 32);
|
||||
|
||||
let error = buffer.write_all(&[8]).expect_err("old 32 plus replacement 64 exceed 95");
|
||||
assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
|
||||
assert!(is_memory_pressure(&error));
|
||||
assert_eq!(buffer.filled, 32);
|
||||
assert_eq!(buffer.buffer.as_slice(), &[7; 32]);
|
||||
assert_eq!(owner.snapshot().used, 32);
|
||||
|
||||
owner.set_sampled_limit(96, Duration::from_secs(30));
|
||||
buffer.write_all(&[8]).expect("old 32 plus replacement 64 fit 96");
|
||||
assert_eq!(owner.snapshot().used, 64);
|
||||
let bytes = buffer.into_bytes();
|
||||
assert_eq!(&bytes[..32], &[7; 32]);
|
||||
assert_eq!(bytes[32], 8);
|
||||
assert_eq!(bytes.len(), 33);
|
||||
let slice = bytes.slice(32..);
|
||||
drop(bytes);
|
||||
assert_eq!(slice.as_ref(), &[8]);
|
||||
assert_eq!(owner.snapshot().used, 64);
|
||||
drop(slice);
|
||||
assert_eq!(owner.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encoded_buffer_checks_its_limit_before_reserving_storage() {
|
||||
use std::io::Write;
|
||||
|
||||
let owner = BackgroundMemoryBudget::new();
|
||||
for maximum in [0, 3] {
|
||||
let mut buffer = BackgroundEncodeBuffer::new(owner.clone(), maximum);
|
||||
buffer
|
||||
.write_all(&[])
|
||||
.expect("empty input needs no storage, including at limit zero");
|
||||
let error = buffer
|
||||
.write_all(&[7; 4])
|
||||
.expect_err("oversized input must fail before memory admission");
|
||||
assert_eq!(error.kind(), io::ErrorKind::InvalidData);
|
||||
assert!(buffer.buffer.as_slice().is_empty());
|
||||
assert!(buffer.into_bytes().is_empty());
|
||||
assert_eq!(owner.snapshot().used, 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn returned_bytes_keep_the_allocation_charged_through_slices_and_shrink() {
|
||||
let owner = BackgroundMemoryBudget::new();
|
||||
owner.set_sampled_limit(8, Duration::from_secs(30));
|
||||
let bytes = BackgroundBuffer::new(Some(owner.clone()))
|
||||
.read_to_end(&mut &b"abcdefgh"[..], 8)
|
||||
.expect("exact allocation fits without an EOF growth allocation");
|
||||
assert_eq!(bytes.as_ref(), b"abcdefgh");
|
||||
assert_eq!(owner.snapshot().used, 8);
|
||||
let slice = bytes.slice(2..6);
|
||||
owner.set_sampled_limit(0, Duration::ZERO);
|
||||
drop(bytes);
|
||||
assert_eq!(slice.as_ref(), b"cdef");
|
||||
assert_eq!(owner.snapshot().used, 8, "the final slice still owns the complete allocation");
|
||||
assert!(owner.reserve(1).is_err());
|
||||
drop(slice);
|
||||
assert_eq!(owner.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn whole_file_size_hint_never_truncates_growth_or_exposes_unused_bytes() {
|
||||
for (hint, retained) in [(0, 32), (4, 32), (8, 8), (16, 16)] {
|
||||
let owner = BackgroundMemoryBudget::new();
|
||||
owner.set_sampled_limit(36, Duration::from_secs(30));
|
||||
let bytes = BackgroundBuffer::new(Some(owner.clone()))
|
||||
.read_to_end(&mut &b"abcdefgh"[..], hint)
|
||||
.expect("read through EOF independently of the metadata hint");
|
||||
assert_eq!(bytes.as_ref(), b"abcdefgh", "hint={hint}");
|
||||
assert_eq!(owner.snapshot().used, retained, "charge allocated capacity, not visible length");
|
||||
drop(bytes);
|
||||
assert_eq!(owner.snapshot().used, 0);
|
||||
}
|
||||
let owner = BackgroundMemoryBudget::new();
|
||||
owner.set_sampled_limit(16, Duration::from_secs(30));
|
||||
let bytes = BackgroundBuffer::new(Some(owner.clone()))
|
||||
.read_to_end(&mut std::io::empty(), 16)
|
||||
.expect("file shrank to empty");
|
||||
assert!(bytes.is_empty());
|
||||
assert_eq!(owner.snapshot().used, 0, "empty results free unused storage before refund");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preserving_growth_keeps_existing_payload_on_denial_and_reuse() {
|
||||
let owner = BackgroundMemoryBudget::new();
|
||||
owner.set_sampled_limit(11, Duration::from_secs(30));
|
||||
let mut buffer = BackgroundBuffer::new(Some(owner.clone()));
|
||||
buffer.ensure_len(4).expect("allocate initial payload");
|
||||
buffer.as_mut_slice().copy_from_slice(b"data");
|
||||
|
||||
let error = buffer
|
||||
.grow(8, 4)
|
||||
.expect_err("old and replacement allocations need twelve bytes");
|
||||
assert!(is_memory_pressure(&error));
|
||||
assert_eq!(buffer.as_slice(), b"data");
|
||||
assert_eq!(owner.snapshot().used, 4);
|
||||
|
||||
owner.set_sampled_limit(12, Duration::from_secs(30));
|
||||
buffer.grow(8, 4).expect("preserve the payload after admission");
|
||||
assert_eq!(buffer.as_slice(), b"data\0\0\0\0");
|
||||
assert_eq!(owner.snapshot().used, 8);
|
||||
owner.set_sampled_limit(0, Duration::ZERO);
|
||||
buffer.grow(8, 4).expect("reuse retained storage after expiry");
|
||||
assert_eq!(buffer.as_slice(), b"data\0\0\0\0");
|
||||
|
||||
let bytes = buffer.take_bytes(4);
|
||||
drop(buffer);
|
||||
assert_eq!(bytes.as_ref(), b"data");
|
||||
assert_eq!(owner.snapshot().used, 8);
|
||||
drop(bytes);
|
||||
assert_eq!(owner.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn whole_file_growth_denial_and_source_failure_refund_storage() {
|
||||
let owner = BackgroundMemoryBudget::new();
|
||||
owner.set_sampled_limit(35, Duration::from_secs(30));
|
||||
let error = BackgroundBuffer::new(Some(owner.clone()))
|
||||
.read_to_end(&mut &b"abcdefgh"[..], 4)
|
||||
.expect_err("old four plus new thirty-two bytes exceed capacity");
|
||||
assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
|
||||
assert_eq!(owner.snapshot().used, 0);
|
||||
|
||||
struct FailedRead(u8);
|
||||
impl io::Read for FailedRead {
|
||||
fn read(&mut self, bytes: &mut [u8]) -> io::Result<usize> {
|
||||
self.0 += 1;
|
||||
match self.0 {
|
||||
1 => Err(io::ErrorKind::Interrupted.into()),
|
||||
2 => {
|
||||
bytes[..2].copy_from_slice(b"ab");
|
||||
Ok(2)
|
||||
}
|
||||
_ => Err(io::ErrorKind::PermissionDenied.into()),
|
||||
}
|
||||
}
|
||||
}
|
||||
let error = BackgroundBuffer::new(Some(owner.clone()))
|
||||
.read_to_end(&mut FailedRead(0), 8)
|
||||
.expect_err("failure after partial output is not successful EOF");
|
||||
assert_eq!(error.kind(), io::ErrorKind::PermissionDenied);
|
||||
assert_eq!(owner.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn shrink_and_expiry_keep_retained_storage_charged() {
|
||||
let owner = BackgroundMemoryBudget::new();
|
||||
assert!(owner.reserve(1).is_err());
|
||||
owner.set_sampled_limit(16, Duration::from_secs(3));
|
||||
let mut buffer = BackgroundBuffer::new(Some(owner.clone()));
|
||||
buffer.ensure_len(8).expect("initial allocation");
|
||||
let pointer = buffer.as_slice().as_ptr();
|
||||
owner.set_sampled_limit(4, Duration::from_secs(3));
|
||||
buffer.ensure_len(8).expect("existing storage can be reused after shrink");
|
||||
assert_eq!(buffer.as_slice().as_ptr(), pointer);
|
||||
assert_eq!(owner.snapshot(), BackgroundMemorySnapshot { limit: 4, used: 8 });
|
||||
assert_eq!(buffer.ensure_len(9).expect_err("growth must be denied").kind(), io::ErrorKind::WouldBlock);
|
||||
assert_eq!(owner.snapshot().used, 8, "failed growth preserves the existing reservation");
|
||||
tokio::time::advance(Duration::from_secs(3)).await;
|
||||
assert_eq!(owner.snapshot(), BackgroundMemorySnapshot { limit: 0, used: 8 });
|
||||
assert!(owner.reserve(1).is_err(), "an expired sampler cannot grant memory");
|
||||
drop(buffer);
|
||||
assert_eq!(owner.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn growth_accounts_for_both_allocations_and_refunds_failed_attempts() {
|
||||
let owner = BackgroundMemoryBudget::new();
|
||||
owner.set_sampled_limit(12, Duration::from_secs(3));
|
||||
let mut buffer = BackgroundBuffer::new(Some(owner.clone()));
|
||||
buffer.ensure_len(4).expect("first allocation");
|
||||
buffer.as_mut_slice().copy_from_slice(b"data");
|
||||
assert!(buffer.ensure_len(9).is_err(), "four old plus nine new bytes exceed the cap");
|
||||
assert_eq!(buffer.as_slice(), b"data");
|
||||
buffer.ensure_len(8).expect("four old plus eight new bytes fit");
|
||||
assert_eq!(owner.snapshot().used, 8);
|
||||
assert_eq!(buffer.as_slice(), &[0; 8]);
|
||||
drop(buffer);
|
||||
assert_eq!(owner.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn concurrent_reservations_share_one_cap() {
|
||||
let owner = BackgroundMemoryBudget::new();
|
||||
owner.set_sampled_limit(1024, Duration::from_secs(30));
|
||||
let ready = Arc::new(std::sync::Barrier::new(17));
|
||||
let release = Arc::new(std::sync::Barrier::new(17));
|
||||
// Start every worker before the first barrier; lazy spawning would deadlock.
|
||||
let mut workers = Vec::with_capacity(16);
|
||||
for _ in 0..16 {
|
||||
let owner = owner.clone();
|
||||
let ready = ready.clone();
|
||||
let release = release.clone();
|
||||
workers.push(std::thread::spawn(move || {
|
||||
let reservation = owner.reserve(128).ok();
|
||||
ready.wait();
|
||||
release.wait();
|
||||
reservation.is_some()
|
||||
}));
|
||||
}
|
||||
ready.wait();
|
||||
let retained = owner.snapshot().used;
|
||||
release.wait();
|
||||
let granted = workers
|
||||
.into_iter()
|
||||
.map(|worker| worker.join().expect("reservation worker"))
|
||||
.filter(|granted| *granted)
|
||||
.count();
|
||||
assert_eq!(retained, 1024);
|
||||
assert_eq!(granted, 8);
|
||||
assert_eq!(owner.snapshot().used, 0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,443 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::fs::File;
|
||||
use std::io::{self, Read};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
const MAX_TOPOLOGY_NODES: usize = 256;
|
||||
const MAX_TOPOLOGY_DEPTH: usize = 32;
|
||||
const MAX_ATTRIBUTE_BYTES: u64 = 128;
|
||||
|
||||
/// Kernel device numbers can be reused after removal. `diskseq` distinguishes
|
||||
/// those incarnations; this identity does not authorize storage mutations.
|
||||
#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
|
||||
pub(super) struct PhysicalDeviceIdentity {
|
||||
pub(super) major: u32,
|
||||
pub(super) minor: u32,
|
||||
pub(super) generation: u64,
|
||||
}
|
||||
|
||||
fn invalid(message: &'static str) -> io::Error {
|
||||
io::Error::new(io::ErrorKind::InvalidData, message)
|
||||
}
|
||||
|
||||
pub(super) fn attribute(path: &Path) -> io::Result<String> {
|
||||
let mut value = String::new();
|
||||
File::open(path)?.take(MAX_ATTRIBUTE_BYTES + 1).read_to_string(&mut value)?;
|
||||
if value.len() > MAX_ATTRIBUTE_BYTES as usize {
|
||||
return Err(invalid("physical device attribute exceeds its bound"));
|
||||
}
|
||||
Ok(value.trim().to_owned())
|
||||
}
|
||||
|
||||
fn number(value: &str) -> io::Result<u64> {
|
||||
if value.is_empty() || !value.bytes().all(|byte| byte.is_ascii_digit()) {
|
||||
return Err(invalid("invalid physical device number"));
|
||||
}
|
||||
value.parse().map_err(|_| invalid("physical device number overflow"))
|
||||
}
|
||||
|
||||
fn identity(path: &Path) -> io::Result<PhysicalDeviceIdentity> {
|
||||
let generation = number(&attribute(&path.join("diskseq"))?)?;
|
||||
if generation == 0 {
|
||||
return Err(invalid("physical device generation is missing"));
|
||||
}
|
||||
let device = attribute(&path.join("dev"))?;
|
||||
let (major, minor) = device
|
||||
.split_once(':')
|
||||
.ok_or_else(|| invalid("invalid physical device identity"))?;
|
||||
let major = u32::try_from(number(major)?).map_err(|_| invalid("physical major number overflow"))?;
|
||||
let minor = u32::try_from(number(minor)?).map_err(|_| invalid("physical minor number overflow"))?;
|
||||
if number(&attribute(&path.join("diskseq"))?)? != generation {
|
||||
return Err(invalid("physical device changed during discovery"));
|
||||
}
|
||||
Ok(PhysicalDeviceIdentity {
|
||||
major,
|
||||
minor,
|
||||
generation,
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Debug, Eq, PartialEq)]
|
||||
enum NodeTopology {
|
||||
Partition {
|
||||
device: String,
|
||||
number: u64,
|
||||
parent: PathBuf,
|
||||
},
|
||||
Whole {
|
||||
identity: PhysicalDeviceIdentity,
|
||||
children: Vec<PathBuf>,
|
||||
},
|
||||
}
|
||||
|
||||
fn canonical_beneath(path: &Path, root: &Path) -> io::Result<PathBuf> {
|
||||
let path = path.canonicalize()?;
|
||||
if !path.starts_with(root) {
|
||||
return Err(invalid("physical device topology escapes sysfs"));
|
||||
}
|
||||
Ok(path)
|
||||
}
|
||||
|
||||
fn node_topology(path: &Path, root: &Path) -> io::Result<NodeTopology> {
|
||||
if canonical_beneath(path, root)? != path {
|
||||
return Err(invalid("physical device path changed during discovery"));
|
||||
}
|
||||
match attribute(&path.join("partition")) {
|
||||
Ok(partition) => {
|
||||
let number = number(&partition)?;
|
||||
if number == 0 {
|
||||
return Err(invalid("physical partition number is missing"));
|
||||
}
|
||||
let parent = path.parent().ok_or_else(|| invalid("physical partition has no parent"))?;
|
||||
return Ok(NodeTopology::Partition {
|
||||
device: attribute(&path.join("dev"))?,
|
||||
number,
|
||||
parent: canonical_beneath(parent, root)?,
|
||||
});
|
||||
}
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => {}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
let identity = identity(path)?;
|
||||
let mut children = Vec::new();
|
||||
for entry in std::fs::read_dir(path.join("slaves"))? {
|
||||
if children.len() >= MAX_TOPOLOGY_NODES {
|
||||
return Err(invalid("physical device topology fanout exceeds its bound"));
|
||||
}
|
||||
children.push(canonical_beneath(&entry?.path(), root)?);
|
||||
}
|
||||
children.sort_unstable();
|
||||
children.dedup();
|
||||
Ok(NodeTopology::Whole { identity, children })
|
||||
}
|
||||
|
||||
struct TopologySnapshot {
|
||||
root: PathBuf,
|
||||
entry: PathBuf,
|
||||
first: PathBuf,
|
||||
nodes: BTreeMap<PathBuf, NodeTopology>,
|
||||
leaves: BTreeSet<PhysicalDeviceIdentity>,
|
||||
}
|
||||
|
||||
impl TopologySnapshot {
|
||||
fn discover(sysfs_root: &Path, major: u64, minor: u64) -> io::Result<Self> {
|
||||
let root = sysfs_root.canonicalize()?;
|
||||
let entry = root.join(format!("dev/block/{major}:{minor}"));
|
||||
let first = canonical_beneath(&entry, &root)?;
|
||||
if attribute(&first.join("dev"))? != format!("{major}:{minor}") {
|
||||
return Err(invalid("physical device entry does not match its device number"));
|
||||
}
|
||||
let mut stack = vec![(first.clone(), 0, false)];
|
||||
let mut active = BTreeSet::new();
|
||||
let mut nodes = BTreeMap::new();
|
||||
let mut leaves = BTreeMap::<PhysicalDeviceIdentity, PathBuf>::new();
|
||||
while let Some((path, depth, exiting)) = stack.pop() {
|
||||
if exiting {
|
||||
active.remove(&path);
|
||||
continue;
|
||||
}
|
||||
if depth > MAX_TOPOLOGY_DEPTH {
|
||||
return Err(invalid("physical device topology is too deep"));
|
||||
}
|
||||
if active.contains(&path) {
|
||||
return Err(invalid("physical device topology contains a cycle"));
|
||||
}
|
||||
if nodes.contains_key(&path) {
|
||||
continue;
|
||||
}
|
||||
if nodes.len() >= MAX_TOPOLOGY_NODES {
|
||||
return Err(invalid("physical device topology exceeds its node bound"));
|
||||
}
|
||||
let topology = node_topology(&path, &root)?;
|
||||
active.insert(path.clone());
|
||||
stack.push((path.clone(), depth, true));
|
||||
match &topology {
|
||||
NodeTopology::Partition { parent, .. } => stack.push((parent.clone(), depth + 1, false)),
|
||||
NodeTopology::Whole { identity, children } => {
|
||||
if stack.len() + children.len() > MAX_TOPOLOGY_NODES * 2 {
|
||||
return Err(invalid("physical device topology fanout exceeds its bound"));
|
||||
}
|
||||
for child in children {
|
||||
stack.push((child.clone(), depth + 1, false));
|
||||
}
|
||||
if children.is_empty()
|
||||
&& let Some(previous) = leaves.insert(*identity, path.clone())
|
||||
&& previous != path
|
||||
{
|
||||
return Err(invalid("physical device identity is ambiguous"));
|
||||
}
|
||||
}
|
||||
}
|
||||
nodes.insert(path, topology);
|
||||
}
|
||||
if leaves.is_empty() {
|
||||
return Err(invalid("physical device topology has no leaves"));
|
||||
}
|
||||
Ok(Self {
|
||||
root,
|
||||
entry,
|
||||
first,
|
||||
nodes,
|
||||
leaves: leaves.into_keys().collect(),
|
||||
})
|
||||
}
|
||||
|
||||
fn validate(&self) -> io::Result<()> {
|
||||
if canonical_beneath(&self.entry, &self.root)? != self.first {
|
||||
return Err(invalid("physical device entry changed during discovery"));
|
||||
}
|
||||
for (path, expected) in &self.nodes {
|
||||
if &node_topology(path, &self.root)? != expected {
|
||||
return Err(invalid("physical device topology changed during discovery"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve a bounded sysfs topology once, before installing an admission owner.
|
||||
/// Missing topology or generation evidence is unknown, never a name-based owner.
|
||||
/// Validate intermediate mappings too: stable leaves alone do not prove that a
|
||||
/// device-mapper table still points to those leaves. This is an observation, not
|
||||
/// a kernel topology lock; the admission owner must expire and refresh it.
|
||||
pub(super) fn resolve_physical_devices(sysfs_root: &Path, major: u64, minor: u64) -> io::Result<Vec<PhysicalDeviceIdentity>> {
|
||||
let snapshot = TopologySnapshot::discover(sysfs_root, major, minor)?;
|
||||
snapshot.validate()?;
|
||||
Ok(snapshot.leaves.into_iter().collect())
|
||||
}
|
||||
|
||||
#[cfg(all(test, unix))]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::os::unix::fs::symlink;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
struct Fixture(PathBuf);
|
||||
|
||||
impl Fixture {
|
||||
fn new() -> Self {
|
||||
static NEXT: AtomicU64 = AtomicU64::new(0);
|
||||
let stamp = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.expect("clock")
|
||||
.as_nanos();
|
||||
let root = std::env::temp_dir().join(format!(
|
||||
"physical-identity-{}-{stamp}-{}",
|
||||
std::process::id(),
|
||||
NEXT.fetch_add(1, Ordering::Relaxed)
|
||||
));
|
||||
std::fs::create_dir(&root).expect("exclusive test root");
|
||||
std::fs::create_dir_all(root.join("sys/dev/block")).expect("sysfs fixture");
|
||||
Self(root)
|
||||
}
|
||||
|
||||
fn root(&self) -> PathBuf {
|
||||
self.0.join("sys")
|
||||
}
|
||||
|
||||
fn device(&self, name: &str, major: u32, minor: u32, generation: u64) -> PathBuf {
|
||||
let path = self.root().join("devices").join(name);
|
||||
std::fs::create_dir_all(path.join("slaves")).expect("device fixture");
|
||||
std::fs::write(path.join("dev"), format!("{major}:{minor}\n")).expect("device number");
|
||||
std::fs::write(path.join("diskseq"), format!("{generation}\n")).expect("device generation");
|
||||
symlink(&path, self.root().join(format!("dev/block/{major}:{minor}"))).expect("sysfs link");
|
||||
path
|
||||
}
|
||||
|
||||
fn link(&self, parent: &Path, child: &Path, name: &str) {
|
||||
symlink(child, parent.join("slaves").join(name)).expect("slave link");
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Fixture {
|
||||
fn drop(&mut self) {
|
||||
std::fs::remove_dir_all(&self.0).expect("remove owned fixture");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn partitions_share_the_leaf_generation_and_replacement_changes_identity() {
|
||||
let fixture = Fixture::new();
|
||||
let leaf = fixture.device("sda", 8, 0, 41);
|
||||
let first = fixture.device("sda/sda1", 8, 1, 41);
|
||||
let second = fixture.device("sda/sda2", 8, 2, 41);
|
||||
std::fs::write(first.join("partition"), "1\n").expect("first partition");
|
||||
std::fs::write(second.join("partition"), "2\n").expect("second partition");
|
||||
let old = resolve_physical_devices(&fixture.root(), 8, 1).expect("first leaf");
|
||||
assert_eq!(old, resolve_physical_devices(&fixture.root(), 8, 2).expect("alias leaf"));
|
||||
assert_eq!(
|
||||
old,
|
||||
vec![PhysicalDeviceIdentity {
|
||||
major: 8,
|
||||
minor: 0,
|
||||
generation: 41
|
||||
}]
|
||||
);
|
||||
std::fs::write(leaf.join("diskseq"), "42\n").expect("replacement generation");
|
||||
let new = resolve_physical_devices(&fixture.root(), 8, 1).expect("replacement leaf");
|
||||
assert_ne!(old, new);
|
||||
assert_eq!(new[0].generation, 42);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn overlapping_stacks_deduplicate_shared_physical_leaves() {
|
||||
let fixture = Fixture::new();
|
||||
let a = fixture.device("a", 8, 0, 1);
|
||||
let b = fixture.device("b", 8, 16, 2);
|
||||
let inner = fixture.device("dm-0", 253, 0, 3);
|
||||
let outer = fixture.device("dm-1", 253, 1, 4);
|
||||
fixture.link(&inner, &a, "a");
|
||||
fixture.link(&inner, &b, "b");
|
||||
fixture.link(&outer, &inner, "dm-0");
|
||||
fixture.link(&outer, &a, "a-again");
|
||||
assert_eq!(
|
||||
resolve_physical_devices(&fixture.root(), 253, 0).expect("inner"),
|
||||
resolve_physical_devices(&fixture.root(), 253, 1).expect("outer")
|
||||
);
|
||||
assert_eq!(resolve_physical_devices(&fixture.root(), 253, 1).expect("leaves").len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_or_invalid_generation_never_falls_back_to_a_device_name() {
|
||||
let fixture = Fixture::new();
|
||||
let leaf = fixture.device("a", 8, 0, 1);
|
||||
for value in ["0", "-1", "18446744073709551616", "+1"] {
|
||||
std::fs::write(leaf.join("diskseq"), value).expect("bad generation");
|
||||
assert!(resolve_physical_devices(&fixture.root(), 8, 0).is_err(), "accepted {value}");
|
||||
}
|
||||
std::fs::remove_file(leaf.join("diskseq")).expect("missing generation");
|
||||
assert_eq!(
|
||||
resolve_physical_devices(&fixture.root(), 8, 0)
|
||||
.expect_err("unknown generation")
|
||||
.kind(),
|
||||
io::ErrorKind::NotFound
|
||||
);
|
||||
assert!(resolve_physical_devices(&fixture.root(), 99, 99).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn topology_cycles_and_escaping_links_are_rejected() {
|
||||
let fixture = Fixture::new();
|
||||
let a = fixture.device("a", 8, 0, 1);
|
||||
let b = fixture.device("b", 8, 16, 2);
|
||||
fixture.link(&a, &b, "b");
|
||||
fixture.link(&b, &a, "a");
|
||||
assert!(
|
||||
resolve_physical_devices(&fixture.root(), 8, 0)
|
||||
.expect_err("cycle")
|
||||
.to_string()
|
||||
.contains("cycle")
|
||||
);
|
||||
std::fs::remove_file(b.join("slaves/a")).expect("remove cycle edge");
|
||||
fixture.link(&b, &fixture.0, "outside");
|
||||
assert!(
|
||||
resolve_physical_devices(&fixture.root(), 8, 0)
|
||||
.expect_err("escape")
|
||||
.to_string()
|
||||
.contains("escapes")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn intermediate_mapping_changes_are_rejected_even_when_old_leaves_survive() {
|
||||
let fixture = Fixture::new();
|
||||
let a = fixture.device("a", 8, 0, 1);
|
||||
let b = fixture.device("b", 8, 16, 2);
|
||||
let mapper = fixture.device("dm-0", 253, 0, 3);
|
||||
fixture.link(&mapper, &a, "backing");
|
||||
let snapshot = TopologySnapshot::discover(&fixture.root(), 253, 0).expect("initial mapping");
|
||||
snapshot.validate().expect("unchanged topology");
|
||||
std::fs::remove_file(mapper.join("slaves/backing")).expect("replace mapping");
|
||||
fixture.link(&mapper, &b, "backing");
|
||||
assert_eq!(identity(&a).expect("old leaf still exists").generation, 1);
|
||||
assert!(snapshot.validate().is_err(), "stable old leaves cannot certify a changed mapper");
|
||||
let current = resolve_physical_devices(&fixture.root(), 253, 0).expect("new complete mapping");
|
||||
assert_eq!(current, vec![identity(&b).expect("new leaf")]);
|
||||
let snapshot = TopologySnapshot::discover(&fixture.root(), 253, 0).expect("new mapping");
|
||||
std::fs::write(mapper.join("diskseq"), "4\n").expect("replace mapper incarnation");
|
||||
assert!(
|
||||
snapshot.validate().is_err(),
|
||||
"an intermediate generation change also invalidates discovery"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn entry_and_partition_changes_are_not_hidden_by_an_unchanged_parent() {
|
||||
let fixture = Fixture::new();
|
||||
fixture.device("a", 8, 0, 1);
|
||||
let partition = fixture.device("a/p1", 8, 1, 1);
|
||||
std::fs::write(partition.join("partition"), "1\n").expect("partition index");
|
||||
let snapshot = TopologySnapshot::discover(&fixture.root(), 8, 1).expect("partition topology");
|
||||
std::fs::write(partition.join("partition"), "2\n").expect("changed partition");
|
||||
assert!(snapshot.validate().is_err());
|
||||
let b = fixture.device("b", 8, 16, 2);
|
||||
let snapshot = TopologySnapshot::discover(&fixture.root(), 8, 0).expect("first entry");
|
||||
let entry = fixture.root().join("dev/block/8:0");
|
||||
std::fs::remove_file(&entry).expect("replace entry");
|
||||
symlink(&b, &entry).expect("changed entry target");
|
||||
assert!(snapshot.validate().is_err());
|
||||
assert!(
|
||||
resolve_physical_devices(&fixture.root(), 8, 0).is_err(),
|
||||
"entry device numbers must match"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_slave_directory_is_unknown_not_a_physical_leaf() {
|
||||
let fixture = Fixture::new();
|
||||
let mapper = fixture.device("dm-0", 253, 0, 1);
|
||||
std::fs::remove_dir(mapper.join("slaves")).expect("missing topology evidence");
|
||||
assert!(resolve_physical_devices(&fixture.root(), 253, 0).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn topology_depth_and_attribute_sizes_are_bounded() {
|
||||
let fixture = Fixture::new();
|
||||
let first = fixture.device("d0", 253, 0, 1);
|
||||
let mut previous = first.clone();
|
||||
for index in 1..=MAX_TOPOLOGY_DEPTH + 1 {
|
||||
let child = fixture.device(&format!("d{index}"), 253, u32::try_from(index).expect("bounded index"), 1);
|
||||
fixture.link(&previous, &child, "next");
|
||||
previous = child;
|
||||
}
|
||||
assert!(
|
||||
resolve_physical_devices(&fixture.root(), 253, 0)
|
||||
.expect_err("depth bound")
|
||||
.to_string()
|
||||
.contains("deep")
|
||||
);
|
||||
std::fs::remove_file(first.join("slaves/next")).expect("make first node a leaf");
|
||||
std::fs::write(first.join("diskseq"), "1".repeat(129)).expect("oversized attribute");
|
||||
assert!(
|
||||
resolve_physical_devices(&fixture.root(), 253, 0)
|
||||
.expect_err("attribute bound")
|
||||
.to_string()
|
||||
.contains("attribute")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exact_node_bound_passes_and_one_more_node_is_rejected() {
|
||||
let fixture = Fixture::new();
|
||||
let root = fixture.device("root", 253, 0, 1);
|
||||
for index in 0..MAX_TOPOLOGY_NODES - 1 {
|
||||
let leaf = fixture.device(&format!("leaf-{index}"), 8, u32::try_from(index).expect("bounded index"), 2);
|
||||
fixture.link(&root, &leaf, &format!("leaf-{index}"));
|
||||
}
|
||||
assert_eq!(
|
||||
resolve_physical_devices(&fixture.root(), 253, 0)
|
||||
.expect("exact node bound")
|
||||
.len(),
|
||||
MAX_TOPOLOGY_NODES - 1
|
||||
);
|
||||
let extra = fixture.device("extra", 9, 0, 3);
|
||||
fixture.link(&root, &extra, "extra");
|
||||
assert!(
|
||||
resolve_physical_devices(&fixture.root(), 253, 0).is_err(),
|
||||
"reject a graph above the node bound"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,268 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::io;
|
||||
use std::num::NonZeroU16;
|
||||
use std::sync::{Arc, Weak};
|
||||
use std::time::Duration;
|
||||
|
||||
use rustfs_concurrency::adaptive::AdaptiveAdmission;
|
||||
#[cfg(test)]
|
||||
use rustfs_concurrency::adaptive::{AdaptiveAdmissionSnapshot, AdaptivePermit};
|
||||
|
||||
use super::physical_identity::PhysicalDeviceIdentity;
|
||||
|
||||
const MAX_PHYSICAL_OWNERS: usize = 256;
|
||||
|
||||
/// Resolve aliases outside object work. The sampling owner serializes changes;
|
||||
/// attached handles and native permits never consult this registry.
|
||||
pub(super) struct PhysicalOwnerRegistry {
|
||||
maximum: NonZeroU16,
|
||||
owners: BTreeMap<PhysicalDeviceIdentity, Weak<AdaptiveAdmission>>,
|
||||
}
|
||||
|
||||
/// One attachment to an existing physical capacity owner, not a reservation.
|
||||
/// The gate itself owns permit slots, so native permit clones keep a registry
|
||||
/// entry alive even after the last logical disk attachment disappears.
|
||||
pub(super) struct PhysicalDeviceAdmission {
|
||||
#[cfg(test)]
|
||||
identity: PhysicalDeviceIdentity,
|
||||
gate: Arc<AdaptiveAdmission>,
|
||||
}
|
||||
|
||||
impl PhysicalOwnerRegistry {
|
||||
pub(super) fn new(maximum: NonZeroU16) -> Self {
|
||||
Self {
|
||||
maximum,
|
||||
owners: BTreeMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Old topology descriptors may still reference an unused gate. Close it
|
||||
/// before reclaiming its registry slot; only admitted I/O keeps an absent
|
||||
/// incarnation alive, and its final release still reaches the same owner.
|
||||
pub(super) fn retire_absent(&mut self, current: &BTreeSet<PhysicalDeviceIdentity>) {
|
||||
self.owners.retain(|identity, owner| {
|
||||
let Some(gate) = owner.upgrade() else {
|
||||
return false;
|
||||
};
|
||||
if current.contains(identity) {
|
||||
return true;
|
||||
}
|
||||
gate.set_sampled_limit(0, Duration::ZERO);
|
||||
gate.snapshot().active != 0
|
||||
});
|
||||
}
|
||||
|
||||
pub(super) fn bind(&mut self, identities: &[PhysicalDeviceIdentity]) -> io::Result<Vec<PhysicalDeviceAdmission>> {
|
||||
if identities.is_empty() || identities.len() > MAX_PHYSICAL_OWNERS || identities.iter().any(|id| id.generation == 0) {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid physical capacity identities"));
|
||||
}
|
||||
let identities = identities.iter().copied().collect::<BTreeSet<_>>();
|
||||
self.owners.retain(|_, owner| owner.strong_count() != 0);
|
||||
let new_owners = identities
|
||||
.iter()
|
||||
.filter(|identity| !self.owners.contains_key(identity))
|
||||
.count();
|
||||
if new_owners > MAX_PHYSICAL_OWNERS - self.owners.len() {
|
||||
return Err(io::Error::new(io::ErrorKind::WouldBlock, "physical capacity registry is full"));
|
||||
}
|
||||
Ok(identities
|
||||
.into_iter()
|
||||
.map(|identity| {
|
||||
let gate = self.owners.get(&identity).and_then(Weak::upgrade).unwrap_or_else(|| {
|
||||
let gate = AdaptiveAdmission::new(self.maximum, 0);
|
||||
self.owners.insert(identity, Arc::downgrade(&gate));
|
||||
gate
|
||||
});
|
||||
PhysicalDeviceAdmission {
|
||||
#[cfg(test)]
|
||||
identity,
|
||||
gate,
|
||||
}
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Only the sampling owner publishes capacity. New or expired identities
|
||||
/// remain closed until a fresh observation establishes a bounded limit.
|
||||
#[cfg(test)]
|
||||
pub(super) fn publish(&mut self, identity: PhysicalDeviceIdentity, limit: u32, valid_for: Duration) -> bool {
|
||||
let Some(gate) = self.owners.get(&identity).and_then(Weak::upgrade) else {
|
||||
return false;
|
||||
};
|
||||
gate.set_sampled_limit(limit, valid_for);
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
impl PhysicalDeviceAdmission {
|
||||
pub(super) fn gate(&self) -> &Arc<AdaptiveAdmission> {
|
||||
&self.gate
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn identity(&self) -> PhysicalDeviceIdentity {
|
||||
self.identity
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn snapshot(&self) -> AdaptiveAdmissionSnapshot {
|
||||
self.gate.snapshot()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn try_acquire(&self) -> Option<AdaptivePermit> {
|
||||
self.gate.try_acquire()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn device(minor: u32, generation: u64) -> PhysicalDeviceIdentity {
|
||||
PhysicalDeviceIdentity {
|
||||
major: 259,
|
||||
minor,
|
||||
generation,
|
||||
}
|
||||
}
|
||||
|
||||
fn registry() -> PhysicalOwnerRegistry {
|
||||
PhysicalOwnerRegistry::new(NonZeroU16::new(2).expect("two units"))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn aliases_and_overlapping_stacks_share_one_physical_capacity() {
|
||||
let mut registry = registry();
|
||||
let a = device(1, 11);
|
||||
let b = device(2, 12);
|
||||
let single = registry.bind(&[a]).expect("single device");
|
||||
let stack = registry.bind(&[b, a, a]).expect("overlapping stack");
|
||||
assert_eq!(stack.len(), 2, "duplicates cannot reserve additional units");
|
||||
assert_eq!(stack[0].identity(), a);
|
||||
assert!(single[0].try_acquire().is_none(), "unobserved capacity is closed");
|
||||
assert!(registry.publish(a, 1, Duration::from_secs(60)));
|
||||
let held = single[0].try_acquire().expect("physical unit");
|
||||
assert!(stack[0].try_acquire().is_none(), "the alias cannot double the capacity");
|
||||
assert_eq!(stack[0].snapshot().active, 1);
|
||||
drop(held);
|
||||
assert!(stack[0].try_acquire().is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_tail_keeps_the_owner_alive_after_all_attachments_are_gone() {
|
||||
let mut registry = registry();
|
||||
let id = device(1, 11);
|
||||
let attachment = registry.bind(&[id]).expect("logical disk");
|
||||
assert!(registry.publish(id, 1, Duration::from_secs(60)));
|
||||
let waiter = attachment[0].try_acquire().expect("admitted work");
|
||||
let native = waiter.clone();
|
||||
drop(waiter);
|
||||
drop(attachment);
|
||||
let reconnected = registry.bind(&[id]).expect("reconnect during native I/O");
|
||||
assert_eq!(reconnected[0].snapshot().active, 1);
|
||||
assert!(reconnected[0].try_acquire().is_none());
|
||||
drop(native);
|
||||
assert!(reconnected[0].try_acquire().is_some(), "only the final native release returns capacity");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replacement_generation_does_not_receive_old_native_refunds() {
|
||||
let mut registry = registry();
|
||||
let old_id = device(1, 11);
|
||||
let new_id = device(1, 12);
|
||||
let old = registry.bind(&[old_id]).expect("old incarnation");
|
||||
assert!(registry.publish(old_id, 1, Duration::from_secs(60)));
|
||||
let native = old[0].try_acquire().expect("old native I/O");
|
||||
drop(old);
|
||||
let new = registry.bind(&[new_id]).expect("replacement incarnation");
|
||||
assert_eq!(new[0].snapshot().active, 0);
|
||||
assert!(new[0].try_acquire().is_none());
|
||||
assert!(registry.publish(new_id, 1, Duration::from_secs(60)));
|
||||
let current = new[0].try_acquire().expect("new work");
|
||||
drop(native);
|
||||
assert_eq!(new[0].snapshot().active, 1, "old completion cannot refund the replacement");
|
||||
assert!(new[0].try_acquire().is_none());
|
||||
drop(current);
|
||||
assert_eq!(new[0].snapshot().active, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shrink_and_expiry_preserve_active_units_and_leave_other_disks_available() {
|
||||
let mut registry = registry();
|
||||
let a = device(1, 11);
|
||||
let b = device(2, 12);
|
||||
let disks = registry.bind(&[a, b]).expect("two physical disks");
|
||||
assert!(registry.publish(a, 2, Duration::from_secs(60)));
|
||||
assert!(registry.publish(b, 1, Duration::from_secs(60)));
|
||||
let first = disks[0].try_acquire().expect("first unit");
|
||||
let second = disks[0].try_acquire().expect("second unit");
|
||||
assert!(registry.publish(a, 1, Duration::from_secs(60)));
|
||||
assert_eq!(disks[0].snapshot().active, 2);
|
||||
assert!(disks[0].try_acquire().is_none());
|
||||
assert!(disks[1].try_acquire().is_some(), "a stalled disk cannot consume another disk's capacity");
|
||||
assert!(registry.publish(a, 2, Duration::ZERO));
|
||||
assert_eq!(disks[0].snapshot().limit, 0);
|
||||
assert_eq!(disks[0].snapshot().active, 2);
|
||||
drop((first, second));
|
||||
assert!(disks[0].try_acquire().is_none(), "expired observations do not reopen after completion");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registry_bound_preserves_live_native_owners_and_reclaims_only_dead_entries() {
|
||||
let mut registry = registry();
|
||||
let ids = (0..MAX_PHYSICAL_OWNERS)
|
||||
.map(|minor| device(u32::try_from(minor).expect("bounded minor"), 1))
|
||||
.collect::<Vec<_>>();
|
||||
let attachments = registry.bind(&ids).expect("exact registry bound");
|
||||
let extra = device(999, 1);
|
||||
assert!(matches!(registry.bind(&[extra]), Err(error) if error.kind() == io::ErrorKind::WouldBlock));
|
||||
assert_eq!(registry.owners.len(), MAX_PHYSICAL_OWNERS, "failed binding must not add partial owners");
|
||||
assert!(registry.publish(ids[0], 1, Duration::from_secs(60)));
|
||||
let native = attachments[0].try_acquire().expect("native tail");
|
||||
drop(attachments);
|
||||
let replacement = registry.bind(&[extra]).expect("dead owners reclaimed");
|
||||
assert_eq!(registry.owners.len(), 2, "the native tail still owns its original entry");
|
||||
let original = registry.bind(&[ids[0]]).expect("original native owner");
|
||||
assert_eq!(original[0].snapshot().active, 1);
|
||||
drop((native, original, replacement));
|
||||
assert!(!registry.publish(extra, 1, Duration::from_secs(60)), "a dead owner cannot be refreshed");
|
||||
assert!(registry.bind(&[]).is_err());
|
||||
assert!(registry.bind(&[device(1, 0)]).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retirement_reclaims_idle_descriptors_but_keeps_active_native_incarnations() {
|
||||
let mut registry = registry();
|
||||
let ids = (0..MAX_PHYSICAL_OWNERS)
|
||||
.map(|minor| device(u32::try_from(minor).expect("bounded minor"), 1))
|
||||
.collect::<Vec<_>>();
|
||||
let descriptors = registry.bind(&ids).expect("full registry");
|
||||
assert!(registry.publish(ids[0], 1, Duration::from_secs(60)));
|
||||
let native = descriptors[0].try_acquire().expect("active old incarnation");
|
||||
let mut current = ids[2..].iter().copied().collect::<BTreeSet<_>>();
|
||||
registry.retire_absent(¤t);
|
||||
assert_eq!(registry.owners.len(), MAX_PHYSICAL_OWNERS - 1);
|
||||
assert_eq!(descriptors[0].snapshot().active, 1);
|
||||
assert!(descriptors[0].try_acquire().is_none());
|
||||
assert!(descriptors[1].try_acquire().is_none());
|
||||
let first = device(0, 2);
|
||||
let second = device(1, 2);
|
||||
let replacement = registry.bind(&[first]).expect("idle descriptor was reclaimed");
|
||||
current.insert(first);
|
||||
assert!(matches!(registry.bind(&[second]), Err(error) if error.kind() == io::ErrorKind::WouldBlock));
|
||||
drop(native);
|
||||
registry.retire_absent(¤t);
|
||||
let next = registry.bind(&[second]).expect("final native release permits reclamation");
|
||||
assert_eq!(replacement[0].snapshot().active, 0);
|
||||
assert_eq!(next[0].snapshot().active, 0);
|
||||
assert_eq!(descriptors[0].snapshot().active, 0);
|
||||
assert!(
|
||||
!registry.publish(ids[0], 1, Duration::from_secs(60)),
|
||||
"retired identity cannot be refreshed"
|
||||
);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,309 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::{PhysicalDiskBinding, ResourceIoUnit};
|
||||
use rustfs_concurrency::adaptive::{
|
||||
NodeAdmission,
|
||||
permit_set::{NodeResourceCompletion, NodeResourcePermit},
|
||||
};
|
||||
use std::cell::{Cell, RefCell};
|
||||
use std::future::Future;
|
||||
use std::io;
|
||||
use std::sync::Arc;
|
||||
|
||||
tokio::task_local! {
|
||||
static REMOTE_RESOURCES: RemoteBackgroundIo;
|
||||
}
|
||||
|
||||
/// Receiver-owned admission for an authenticated background RPC. The owner must
|
||||
/// be the same node controller used by local Scanner and Heal work.
|
||||
pub struct RemoteBackgroundIo {
|
||||
node: Arc<NodeAdmission>,
|
||||
admitted: RefCell<Option<NodeResourcePermit>>,
|
||||
progressed: Cell<bool>,
|
||||
finished: Cell<bool>,
|
||||
}
|
||||
|
||||
impl RemoteBackgroundIo {
|
||||
pub fn new(node: Arc<NodeAdmission>) -> Self {
|
||||
Self {
|
||||
node,
|
||||
admitted: RefCell::new(None),
|
||||
progressed: Cell::new(false),
|
||||
finished: Cell::new(false),
|
||||
}
|
||||
}
|
||||
|
||||
/// Move the request's reservations into its existing stream producer.
|
||||
/// Only physical subsets may be cloned into native I/O; never its node unit.
|
||||
pub fn take_current() -> Option<Self> {
|
||||
REMOTE_RESOURCES
|
||||
.try_with(|context| Self {
|
||||
node: context.node.clone(),
|
||||
admitted: RefCell::new(context.admitted.take()),
|
||||
progressed: Cell::new(context.progressed.replace(false)),
|
||||
finished: Cell::new(false),
|
||||
})
|
||||
.ok()
|
||||
}
|
||||
|
||||
/// Defer a response's progress without keeping async admission alive while
|
||||
/// its body streams. The reader retains its own native resource ownership.
|
||||
pub fn take_current_completion() -> Option<NodeResourceCompletion> {
|
||||
REMOTE_RESOURCES
|
||||
.try_with(|context| {
|
||||
let admitted = context.admitted.take();
|
||||
if context.progressed.replace(false) {
|
||||
admitted.map(NodeResourcePermit::into_completion)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.ok()
|
||||
.flatten()
|
||||
}
|
||||
|
||||
/// A failed response or cancelled producer must not credit earlier phases.
|
||||
pub fn discard_current_progress() {
|
||||
let _ = REMOTE_RESOURCES.try_with(|context| context.progressed.set(false));
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RemoteBackgroundIo {
|
||||
fn drop(&mut self) {
|
||||
if self.finished.get()
|
||||
&& self.progressed.get()
|
||||
&& let Some(permit) = self.admitted.get_mut().take()
|
||||
{
|
||||
permit.complete();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Scope a verified request or its stream producer. An interrupted request
|
||||
/// returns node capacity while native jobs retain their selected physical units.
|
||||
pub async fn with_remote_background_io<F: Future>(context: Option<RemoteBackgroundIo>, operation: F) -> F::Output {
|
||||
match context {
|
||||
Some(context) => {
|
||||
REMOTE_RESOURCES
|
||||
.scope(context, async {
|
||||
let result = operation.await;
|
||||
REMOTE_RESOURCES.with(|context| context.finished.set(true));
|
||||
result
|
||||
})
|
||||
.await
|
||||
}
|
||||
None => operation.await,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn active() -> bool {
|
||||
REMOTE_RESOURCES.try_with(|_| ()).is_ok()
|
||||
}
|
||||
|
||||
pub(crate) fn resources() -> Option<ResourceIoUnit> {
|
||||
REMOTE_RESOURCES
|
||||
.try_with(|context| {
|
||||
context
|
||||
.admitted
|
||||
.borrow()
|
||||
.as_ref()
|
||||
.map(|permit| ResourceIoUnit::from(permit.resources().clone()))
|
||||
})
|
||||
.ok()
|
||||
.flatten()
|
||||
}
|
||||
|
||||
pub(crate) fn record_progress() {
|
||||
let _ = REMOTE_RESOURCES.try_with(|context| context.progressed.set(true));
|
||||
}
|
||||
|
||||
pub(crate) fn admit(binding: &PhysicalDiskBinding) -> io::Result<()> {
|
||||
REMOTE_RESOURCES
|
||||
.try_with(|context| {
|
||||
let deferred = || io::Error::from(io::ErrorKind::WouldBlock);
|
||||
let plan = binding.plan().ok_or_else(deferred)?;
|
||||
let mut admitted = context.admitted.borrow_mut();
|
||||
if let Some(permit) = admitted.as_ref() {
|
||||
// Reuse the request's receipt for append/read/rename phases at
|
||||
// capacity one. Additional attachments never wait under locks.
|
||||
let native = plan.capture_or_acquire_native(permit.resources()).map_err(|_| deferred())?;
|
||||
drop(native);
|
||||
if !plan.is_current() {
|
||||
return Err(deferred());
|
||||
}
|
||||
} else {
|
||||
*admitted = Some(plan.try_acquire(&context.node).ok_or_else(deferred)?);
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap_or(Ok(()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rustfs_concurrency::adaptive::{
|
||||
AdaptiveAdmission, FeedbackController, FeedbackObservation, ResourcePressure, permit_set::AdaptivePermitSetOwner,
|
||||
};
|
||||
use std::num::NonZeroU16;
|
||||
|
||||
fn node() -> Arc<NodeAdmission> {
|
||||
let node = NodeAdmission::new(NonZeroU16::MIN);
|
||||
node.update(
|
||||
&mut FeedbackController::new(),
|
||||
Some(FeedbackObservation {
|
||||
generation: 1,
|
||||
max_concurrency: 1,
|
||||
pressure: ResourcePressure::Unknown,
|
||||
completed_units: None,
|
||||
}),
|
||||
);
|
||||
node
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn receiver_reuses_one_receipt_across_phases_and_releases_only_native_subsets() {
|
||||
let node = node();
|
||||
let a = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
|
||||
let b = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
|
||||
let shared = AdaptivePermitSetOwner::new(NonZeroU16::MIN, vec![a.clone(), b.clone()]).expect("device catalog");
|
||||
let first = PhysicalDiskBinding::default();
|
||||
let second = PhysicalDiskBinding::default();
|
||||
first
|
||||
.set_test_shared_capacity(shared.clone(), vec![a.clone()])
|
||||
.expect("device A");
|
||||
second.set_test_shared_capacity(shared, vec![b.clone()]).expect("device B");
|
||||
|
||||
let native = with_remote_background_io(Some(RemoteBackgroundIo::new(node.clone())), async {
|
||||
assert!(resources().is_none(), "no physical work before target selection");
|
||||
admit(&first).expect("first phase");
|
||||
let admitted = resources().expect("admitted resources");
|
||||
let native = first
|
||||
.plan()
|
||||
.expect("current plan")
|
||||
.capture_native(admitted.as_set().expect("resource set"))
|
||||
.expect("escaped native read");
|
||||
admit(&first).expect("second phase reuses capacity while the first reader lives");
|
||||
admit(&second).expect("additional selected device");
|
||||
assert_eq!([node.snapshot().active, a.snapshot().active, b.snapshot().active], [1, 1, 1]);
|
||||
record_progress();
|
||||
native
|
||||
})
|
||||
.await;
|
||||
assert_eq!([node.snapshot().active, a.snapshot().active, b.snapshot().active], [0, 1, 0]);
|
||||
assert_eq!([node.completed(), a.completed(), b.completed()], [Some(1), Some(1), Some(1)]);
|
||||
drop(native);
|
||||
assert_eq!(a.snapshot().active, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn receiver_cancellation_returns_node_but_not_escaped_native_capacity() {
|
||||
let node = node();
|
||||
let gate = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
|
||||
let binding = PhysicalDiskBinding::default();
|
||||
binding.set_test_capacity(vec![gate.clone()]).expect("device plan");
|
||||
let (sender, receiver) = tokio::sync::oneshot::channel();
|
||||
let task = tokio::spawn(with_remote_background_io(Some(RemoteBackgroundIo::new(node.clone())), async move {
|
||||
admit(&binding).expect("receiver admission");
|
||||
let admitted = resources().expect("receiver resources");
|
||||
let native = binding
|
||||
.plan()
|
||||
.expect("plan")
|
||||
.capture_native(admitted.as_set().expect("set"))
|
||||
.expect("native subset");
|
||||
sender.send(native).expect("native owner receiver");
|
||||
record_progress();
|
||||
std::future::pending::<()>().await;
|
||||
}));
|
||||
let native = receiver.await.expect("native read ownership");
|
||||
task.abort();
|
||||
assert!(task.await.expect_err("cancel request").is_cancelled());
|
||||
assert_eq!([node.snapshot().active, gate.snapshot().active], [0, 1]);
|
||||
assert_eq!([node.completed(), gate.completed()], [Some(0), Some(0)]);
|
||||
drop(native);
|
||||
assert_eq!(gate.snapshot().active, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn receiver_defers_stream_progress_without_retaining_node_capacity() {
|
||||
for (progressed, succeeds) in [(false, true), (true, false), (true, true)] {
|
||||
let node = node();
|
||||
let gate = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
|
||||
let binding = PhysicalDiskBinding::default();
|
||||
binding.set_test_capacity(vec![gate.clone()]).expect("device plan");
|
||||
let completion = with_remote_background_io(Some(RemoteBackgroundIo::new(node.clone())), async {
|
||||
admit(&binding).expect("receiver admission");
|
||||
if progressed {
|
||||
record_progress();
|
||||
}
|
||||
let completion = RemoteBackgroundIo::take_current_completion();
|
||||
assert!(RemoteBackgroundIo::take_current_completion().is_none(), "single progress owner");
|
||||
assert!(resources().is_none());
|
||||
completion
|
||||
})
|
||||
.await;
|
||||
assert_eq!([node.snapshot().active, gate.snapshot().active], [0, 0]);
|
||||
assert_eq!([node.completed(), gate.completed()], [Some(0), Some(0)]);
|
||||
assert_eq!(completion.is_some(), progressed);
|
||||
if succeeds && let Some(completion) = completion {
|
||||
completion.complete();
|
||||
}
|
||||
let expected = u64::from(progressed && succeeds);
|
||||
assert_eq!([node.completed(), gate.completed()], [Some(expected), Some(expected)]);
|
||||
}
|
||||
assert!(
|
||||
RemoteBackgroundIo::take_current_completion().is_none(),
|
||||
"foreground has no progress owner"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn receiver_failed_completion_does_not_credit_successful_preparation() {
|
||||
let node = node();
|
||||
let gate = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
|
||||
let binding = PhysicalDiskBinding::default();
|
||||
binding.set_test_capacity(vec![gate.clone()]).expect("device plan");
|
||||
with_remote_background_io(Some(RemoteBackgroundIo::new(node.clone())), async {
|
||||
admit(&binding).expect("receiver admission");
|
||||
record_progress();
|
||||
RemoteBackgroundIo::discard_current_progress();
|
||||
})
|
||||
.await;
|
||||
assert_eq!([node.snapshot().active, gate.snapshot().active], [0, 0]);
|
||||
assert_eq!([node.completed(), gate.completed()], [Some(0), Some(0)]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn receiver_stream_transfer_moves_node_ownership_and_defers_unavailable_devices() {
|
||||
let node = node();
|
||||
let gate = AdaptiveAdmission::new(NonZeroU16::MIN, 0);
|
||||
let binding = PhysicalDiskBinding::default();
|
||||
binding.set_test_capacity(vec![gate.clone()]).expect("device plan");
|
||||
let context = with_remote_background_io(Some(RemoteBackgroundIo::new(node.clone())), async {
|
||||
assert_eq!(admit(&binding).expect_err("closed device").kind(), io::ErrorKind::WouldBlock);
|
||||
assert_eq!([node.snapshot().active, gate.snapshot().active], [0, 0]);
|
||||
gate.set_limit(1);
|
||||
admit(&binding).expect("opened device");
|
||||
let context = RemoteBackgroundIo::take_current().expect("stream context");
|
||||
assert!(resources().is_none(), "request cannot keep a second node owner");
|
||||
context
|
||||
})
|
||||
.await;
|
||||
assert_eq!([node.snapshot().active, gate.snapshot().active], [1, 1]);
|
||||
assert_eq!(node.completed(), Some(0));
|
||||
with_remote_background_io(Some(context), async {
|
||||
admit(&binding).expect("stream reuses the request's receipt");
|
||||
binding
|
||||
.set_test_capacity(vec![AdaptiveAdmission::new(NonZeroU16::MIN, 1)])
|
||||
.expect("replace physical device");
|
||||
assert_eq!(
|
||||
admit(&binding).expect_err("old receipt cannot cross topology").kind(),
|
||||
io::ErrorKind::WouldBlock
|
||||
);
|
||||
})
|
||||
.await;
|
||||
assert_eq!([node.snapshot().active, gate.snapshot().active], [0, 0]);
|
||||
assert_eq!(node.completed(), Some(0), "admission alone is not completed work");
|
||||
}
|
||||
}
|
||||
+5678
-545
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user