Compare commits

..

23 Commits

Author SHA1 Message Date
overtrue a6b8ef1048 test: trace rename tail stages 2026-10-30 11:15:10 +08:00
overtrue 9518a5d1e2 fix(lock): reject force-released acquisitions before registration 2026-10-30 11:10:46 +08:00
overtrue 2bcc4895ae test(storage): wait for stable internal fixture snapshots 2026-10-30 10:53:14 +08:00
overtrue f36154a8aa fix(lock): serialize force unlock with tail conversion 2026-10-30 09:35:29 +08:00
overtrue 21cb36c8be Merge remote-tracking branch 'origin/main' into overtrue/fix-heal-lock-ci 2026-10-30 09:27:10 +08:00
overtrue e76648b5e3 fix(lock): allow reads while committed rename tails drain 2026-10-30 09:27:05 +08:00
overtrue 5cc296314e test(heal): wait for metadata write tail 2026-10-29 22:40:38 +08:00
overtrue 01d43729aa fix(heal): retain deep scan retry for read repair 2026-10-29 21:30:41 +08:00
overtrue 0a32ad9109 Merge remote-tracking branch 'origin/main' into overtrue/fix-heal-lock-ci 2026-10-29 21:16:34 +08:00
overtrue d0b26195c2 fix(heal): allow reads during read repair reconstruction 2026-10-29 21:16:34 +08:00
overtrue 7a740fe5e7 fix(lock): retain quorum error on final timeout 2026-10-29 09:13:54 +08:00
overtrue 2aaef5451d Merge remote-tracking branch 'origin/main' into overtrue/fix-heal-lock-ci 2026-10-29 09:04:36 +08:00
overtrue a213ce597e chore: merge main into heal lock fixes 2026-10-29 08:22:08 +08:00
overtrue c749cf9da5 chore: merge main into heal lock fixes 2026-10-29 06:50:01 +08:00
overtrue 1ab0612c9c chore: merge main into heal lock fixes 2026-10-29 01:55:22 +08:00
overtrue 6538f5ca36 test(heal): isolate multi-pool crash recovery 2026-10-28 21:57:29 +08:00
overtrue 7040cb65c1 Merge remote-tracking branch 'origin/main' into overtrue/fix-heal-lock-ci 2026-10-28 21:55:29 +08:00
overtrue 5427bd8f84 Merge branch 'main' of https://github.com/rustfs/rustfs into overtrue/fix-heal-lock-ci 2026-10-28 20:49:27 +08:00
overtrue 3c2b54aeb5 fix(lock): retain quorum failures across deadline retries 2026-10-28 07:53:19 +08:00
overtrue 0de223d8e6 test(lock): use virtual time for retry budget 2026-10-28 06:56:33 +08:00
overtrue 351bd12031 fix(lock): use remaining acquire budget for retries 2026-10-28 06:55:04 +08:00
overtrue a2381c8421 test(heal): isolate the direct repair fallback 2026-10-28 06:33:30 +08:00
overtrue 5eee702e88 test(heal): isolate durable repair during root recovery 2026-10-28 06:31:40 +08:00
689 changed files with 10277 additions and 108268 deletions
-1
View File
@@ -18,7 +18,6 @@ These implementation, style, and quality rules apply to all contributions, regar
- Do not include agent, tool, contributor, account, or organization names in branch names.
- Push to the user-requested remote or the repository's configured push remote. Do not hard-code or infer a remote from an account name.
- Check free space before artifact-heavy builds, tests, coverage, or downloads. Re-check before a broad gate when space is tight.
- Before handoff, remove the Cargo target directories and temporary worktrees the task created unless they stay intentionally or hold unmerged work. Identify build output by Cargo's markers (`.rustc_info.json`, `CACHEDIR.TAG`), not by directory name: `target` is also a source directory here.
- Remove only task-owned temporary/build artifacts. Never delete another task's worktree or uncommitted data.
- At handoff, mention disk or cleanup details only when they affected execution or artifacts/worktrees remain intentionally.
- For Docker verification, ensure the tested image contains the intended binary. `buildx build` without `--load` leaves the image in the build cache; use the Makefile's loaded image and rebuild without cache only when stale artifacts are evidenced.
+1 -1
View File
@@ -1,6 +1,6 @@
{
"schema_version": 1,
"rollout": "enforced",
"rollout": "shadow",
"core_packages": ["rustfs", "rustfs-ecstore", "rustfs-filemeta", "rustfs-iam", "rustfs-policy"],
"connect_packages": [
"rustfs-credentials", "rustfs-config", "rustfs-common", "rustfs-crypto",
+2 -2
View File
@@ -1,2 +1,2 @@
sha256-linux=348fa5eed9ae58962c06b33283ff4f15122cff97bce9b4eb1e3ffcb227264b0b
sha256-darwin=348fa5eed9ae58962c06b33283ff4f15122cff97bce9b4eb1e3ffcb227264b0b
sha256-linux=eb33067d90ee66c0a97e8f3658e46c152e47f897d2fbfdc6cb5ad93cfa55414b
sha256-darwin=eb33067d90ee66c0a97e8f3658e46c152e47f897d2fbfdc6cb5ad93cfa55414b
+2 -2
View File
@@ -1,2 +1,2 @@
sha256-darwin=219ac11e720b3f54dfee4d5d5e95171785ba159b2fe01db3037a8f5061c1ad32
sha256-linux=2d34698cc8ec52d5b6c37ac612919a1d21f36cd56a1af539a7522a4eea518e26
sha256-darwin=67251cb7d9978d38e899268cac4d7898297ff7e5b5c48c9c1546dc760b93568d
sha256-linux=673da76c2e5710c4622fe2fa32d2b297730895769487fcf1152ee1c2366f32f4
+2 -2
View File
@@ -1,2 +1,2 @@
sha256-darwin=e84e04208f19dc0596b10d4d1b799ce43d6cc431415902f9c5458aa8232bc2d8
sha256-linux=1f9c7add72c44d1c12ef4ac050bb7917431be657c0c90b29f70c3df56f8bbae3
sha256-darwin=3814cdde8cacf4187e18188d342ed70931cc32e268e42e5bb438d565bb4d0f81
sha256-linux=663025a80273579f95ae3f49af02c0afcbfce94df09701c360a13e607b7637db
+1 -1
View File
@@ -1 +1 @@
sha256=5bdd4c2eadbc41625ae3184fff34062e000d328108436259b2b7c4fe2531ecaf
sha256=e9dc896774122a304e2b8b90f28c82d4d4e0af8ded36a82b147b30945ec994e3
-1
View File
@@ -61,7 +61,6 @@ script-tests: ## Run shell script tests
$(RUSTFS_PYTHON_BIN) ./scripts/ci_gate.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/check_connect_release_boundary.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_test_artifact.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_test_shards.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_merge_queue.py
$(RUSTFS_PYTHON_BIN) -m unittest discover -s scripts/ci -p 'test_*.py'
$(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test
+4 -10
View File
@@ -234,7 +234,7 @@ test-group = 'embedded-test-ports'
# processes cannot exhaust those budgets. Keep deadlines and assertions intact.
# The short HTTP fixtures are tracked in rustfs/backlog#2714.
[[profile.default.overrides]]
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|cleanup_accepts_missing_object_and_still_deletes_the_bucket|cleanup_diagnostics_identify_each_disconnected_transport_stage|cleanup_diagnostics_identify_each_failed_http_stage_without_private_context|cleanup_diagnostics_identify_timeouts_without_resetting_the_shared_deadline|oversized_response_is_rejected_without_buffering_it)$/)))'
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))'
threads-required = "num-test-threads"
# This cancellation probe hashes the serving executable before acquiring its
@@ -548,7 +548,7 @@ filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnost
test-group = 'embedded-test-ports'
[[profile.ci.overrides]]
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|cleanup_accepts_missing_object_and_still_deletes_the_bucket|cleanup_diagnostics_identify_each_disconnected_transport_stage|cleanup_diagnostics_identify_each_failed_http_stage_without_private_context|cleanup_diagnostics_identify_timeouts_without_resetting_the_shared_deadline|oversized_response_is_rejected_without_buffering_it)$/)))'
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))'
threads-required = "num-test-threads"
# Preserve the cancellation probe's startup budget and assertions in CI too.
@@ -701,7 +701,7 @@ test-group = 'ecstore-serial-flaky'
[profile.e2e-smoke]
default-filter = """
package(e2e_test) & (
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|multipart_upload_id|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat|data_usage)_test::|^fake_s3_target::/)
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat|data_usage)_test::|^fake_s3_target::/)
| test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/)
| test(/^reliant::lifecycle::/)
| test(/^conditional_delete_objects_test::/)
@@ -827,7 +827,7 @@ slow-timeout = { period = "120s", terminate-after = 12, grace-period = "10s" }
# ---------------------------------------------------------------------------
# e2e-distributed profile — 4-node 4-disk Actions suite
# ---------------------------------------------------------------------------
# Nightly and manual-dispatch lane owned by
# Storage-sensitive PR / nightly / dispatch lane owned by
# .github/workflows/e2e-distributed.yml.
# Each case starts four rustfs processes (and for site replication, two
# clusters). Upgrade cases also require RUSTFS_UPGRADE_SOURCE_BINARY.
@@ -976,12 +976,6 @@ test-group = 'e2e-vault'
filter = 'package(e2e_test) & (test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter)$/) | test(=reliant::tiering::test_hermetic_transition_restore_failure_expiry_and_retry))'
threads-required = "num-test-threads"
# The mc fixture uploads hundreds of objects concurrently before measuring listing.
# Reserve the runner so other server fixtures do not starve its disk operations.
[[profile.e2e-full.overrides]]
filter = 'package(e2e_test) & test(=mc_mirror_small_bucket_test::test_mc_mirror_small_bucket_completes_without_list_timeout)'
threads-required = "num-test-threads"
# Fixed, small contract for every executable server change. These existing
# scenarios exercise write/read, anonymous denial and retained bytes on restart.
[profile.core-contract]
-77
View File
@@ -1,77 +0,0 @@
{
"schema_version": 1,
"plan_id": "rustfs-release-validation",
"description": "Versioned certification plan for every public RustFS release exit (rustfs/backlog#2662, R01). preview, prerelease and stable all require this explicit plan. Scripts/release/release_gate.py fail-closes publication on any missing, duplicate, stale, substituted or non-PASS required item.",
"release_types": ["preview", "prerelease", "stable"],
"targets": [
{"os": "linux", "arch": "x86_64", "abi": "musl"},
{"os": "linux", "arch": "aarch64", "abi": "musl"},
{"os": "linux", "arch": "x86_64", "abi": "gnu"},
{"os": "linux", "arch": "aarch64", "abi": "gnu"},
{"os": "macos", "arch": "aarch64", "abi": "none"},
{"os": "windows", "arch": "x86_64", "abi": "msvc"}
],
"targets_basis": "Platform matrix of .github/workflows/build.yml (prepare-platform-matrix). gnu and macOS targets build with the pyroscope feature; musl and Windows targets build with no extra features.",
"features": ["pyroscope"],
"trusted_workflows": [
{"repository": "rustfs/s3chaos", "workflow_id": "release-gate.yml"}
],
"trusted_workflows_basis": "The shared evidence envelope producer lives in rustfs/s3chaos (G01, fault/release_gate.rs) and its plan/live split workflow is .github/workflows/release-gate.yml there. Only results from these identities may enter a release certification; PR-uploaded JSON never gains release authority. R02 must extend this list when additional verifier workflows land, never widen it to untrusted sources.",
"lanes": [
{
"lane": "identity",
"harness_repository": "rustfs/s3chaos",
"harness_sha": "9a48f55c1d6d58ce63a8fb1ea7317e024f32f11d",
"harness_basis": "Fixed G01 evidence-envelope producer revision (rustfs/s3chaos PR #102 head recorded in rustfs/backlog#2658). R02 re-pins this when the live verifier ships.",
"timeout_seconds": 1800,
"environment_capabilities": ["process-exec"],
"cases": ["artifact-identity", "startup"],
"applies_to": {
"release_types": ["preview", "prerelease", "stable"],
"platforms": "all"
},
"applies_basis": "Artifact identity and startup checks are mandatory for every release type, preview included, on every shipped target (rustfs/backlog#2662 frozen design item 5)."
},
{
"lane": "upgrade",
"harness_repository": "rustfs/s3chaos",
"harness_sha": "9a48f55c1d6d58ce63a8fb1ea7317e024f32f11d",
"harness_basis": "Same fixed envelope producer as the identity lane; the workflow SHA and the product build SHA are bound separately and need not be equal.",
"timeout_seconds": 5400,
"environment_capabilities": ["process-exec", "previous-release-artifact"],
"cases": [
"direct-upgrade",
"mixed-version-upgrade",
"bucket-config-upgrade",
"bucket-config-rollback",
"multipart-layout-upgrade",
"multipart-layout-baseline"
],
"cases_basis": "Case IDs of the pinned previous-release matrix in .github/workflows/e2e-upgrade.yml (source.version/asset/sha256 and the case list mapped there).",
"applies_to": {
"release_types": ["prerelease", "stable"],
"platforms": [{"os": "linux", "arch": "x86_64", "abi": "gnu"}]
},
"applies_basis": "preview releases use the pre-declared smaller set (identity lane only). The upgrade lane is only exercised on linux-x86_64-gnu, the platform of the pinned source assets."
}
],
"upgrade_sources": {
"basis": "Exactly the two previous-release sources pinned in .github/workflows/e2e-upgrade.yml are exercised today (source.version / source.asset / source.sha256). No other N-1/N-2 compatibility is claimed, and these digests are not inferred from tags.",
"sources": [
{
"version": "1.0.1",
"role": "N-1",
"artifact_name": "rustfs-linux-x86_64-gnu-v1.0.1.zip",
"artifact_sha256": "9742e183cb0dfb0d48fa909797af572fe47b7f827569036baa4156e0d3958ee5"
},
{
"version": "1.0.0-rc.5",
"role": "N-2",
"artifact_name": "rustfs-linux-x86_64-gnu-v1.0.0-rc.5.zip",
"artifact_sha256": "3ee8df71e8edcfada533be452c4135868f697bc515460ae97b027313eade7a3d"
}
],
"rollback_supported": false,
"rollback_basis": "Only the bucket-config-rollback case (rollback_to_previous_release_reads_current_bucket_metadata) is exercised. Full data-plane rollback is not certified and is never a default claim."
}
}
+3 -3
View File
@@ -22,9 +22,9 @@ The following files in the project root are essential for Docker operations:
| File | Description | Usage |
| :--- | :--- | :--- |
| **`docker-buildx.sh`** | **Multi-Arch Build Script**<br>Automates building and pushing Docker images for `amd64` and `arm64`. Supports release and dev channels. | `./docker-buildx.sh --push` |
| **`Dockerfile`** | **Production Image (Alpine)**<br>Lightweight image using musl libc. Downloads verified RustFS and `rc` binaries from GitHub Releases and bundles `/etc/rustfs/bootstrap`. | `docker build -t rustfs:latest .` |
| **`Dockerfile.glibc`** | **Production Image (Ubuntu)**<br>Standard image using glibc. Includes the bundled bootstrap configuration. | `docker build -f Dockerfile.glibc .` |
| **`Dockerfile.source`** | **Development Image**<br>Builds RustFS from source code and bundles `rc` and the default bootstrap configuration. | `docker build -f Dockerfile.source .` |
| **`Dockerfile`** | **Production Image (Alpine)**<br>Lightweight image using musl libc. Downloads pre-built binaries from GitHub Releases. | `docker build -t rustfs:latest .` |
| **`Dockerfile.glibc`** | **Production Image (Ubuntu)**<br>Standard image using glibc. Useful if you need specific dynamic libraries. | `docker build -f Dockerfile.glibc .` |
| **`Dockerfile.source`** | **Development Image**<br>Builds RustFS from source code. Includes build tools. Ideal for local development and CI. | `docker build -f Dockerfile.source .` |
### Docker Compose Configurations
-2
View File
@@ -32,8 +32,6 @@ updates:
- "overtrue"
- "majinghe"
ignore:
- dependency-name: "pulsar"
versions: [ "6.9.x" ]
- dependency-name: "object_store"
versions: [ "0.13.x" ]
- dependency-name: "libunftp"
+5 -28
View File
@@ -296,32 +296,6 @@
"origin": "baseline",
"alert_contract": "scheduled-validation"
},
{
"workflow": ".github/workflows/rustfs-validation-sweep.yml",
"name": "RustFS Validation Sweep",
"roles": [
"periodic",
"manual"
],
"events": [
"schedule",
"workflow_dispatch"
],
"cadence": "nightly",
"timezone": "UTC",
"schedule": [
"30 19 * * *"
],
"owner": "@overtrue",
"policy": "active",
"monitor": "workflow",
"max_age_hours": 60,
"result_owner": "rustfs-validation-sweep.yml",
"disposition": "canonical validation cases executed in rotating batches",
"failure_owner": "workflow",
"origin": "baseline",
"alert_contract": "scheduled-validation"
},
{
"workflow": ".github/workflows/btf-prototype-reference.yml",
"name": "BTF prototype reference",
@@ -769,10 +743,12 @@
"workflow": ".github/workflows/e2e-distributed.yml",
"name": "e2e-distributed",
"roles": [
"pr",
"periodic",
"manual"
],
"events": [
"pull_request",
"workflow_dispatch",
"schedule"
],
@@ -847,11 +823,12 @@
"workflow": ".github/workflows/e2e-upgrade.yml",
"name": "Upgrade Compatibility",
"roles": [
"pr",
"periodic",
"manual",
"release"
"manual"
],
"events": [
"pull_request",
"push",
"schedule",
"workflow_dispatch"
+5 -311
View File
@@ -814,18 +814,13 @@ jobs:
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
export AWS_DEFAULT_REGION="auto"
# Determine upload path based on build type. Release and prerelease
# builds only stage their bytes here: the official
# artifacts/rustfs/release/ path is written solely by the
# certification-gated promote-release-artifacts job, from these same
# staged objects. Development builds keep the uncertified dev channel
# and never touch official or latest channels.
# Determine upload path based on build type
if [[ "$BUILD_TYPE" == "development" ]]; then
R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/dev/"
echo "📤 Uploading development build to R2 dev directory"
else
R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
echo "📤 Staging release build for certification-gated promotion"
R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/release/"
echo "📤 Uploading release build to R2 release directory"
fi
# Upload all rustfs zip files to R2 using S3-compatible API
@@ -874,221 +869,6 @@ jobs:
echo "This run does not trigger Docker publishing."
fi
# Aggregate the parent candidate batch. This job always collects results and
# retains its evidence, including on failure, and every publication job
# below requires its success. Missing, stale, substituted or non-PASS
# evidence fails closed here: only the trusted verification sources
# (rustfs/s3chaos release-gate envelopes, rustfs/backlog#2658) can satisfy
# it, so a PR-uploaded JSON set never gains release authority.
release-certification:
name: Release Certification
needs: [ build-check, build-rustfs ]
if: ${{ !cancelled() && needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') }}
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
timeout-minutes: 30
permissions:
contents: read
actions: read
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Download all build artifacts
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
path: ./artifacts
pattern: rustfs-*
merge-multiple: true
- name: Write candidate and collect trusted validation results
env:
# Cross-repo read of the trusted verification run. Falls back to the
# workflow token, which cannot read rustfs/s3chaos; a missing token
# therefore fail-closes instead of certifying.
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
PRODUCT_COMMIT: ${{ github.sha }}
VERSION: ${{ needs.build-check.outputs.version }}
BUILD_TYPE: ${{ needs.build-check.outputs.build_type }}
FEATURES: ${{ needs.build-check.outputs.build_type == 'development' && '' || 'pyroscope' }}
shell: bash
run: |
set -euo pipefail
mkdir -p results release-gate-output
PLAN_SHA256=$(python3 -c "import hashlib,json,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('.config/release-validation-plan.json'))))")
python3 - "$PLAN_SHA256" > candidate.json <<'PY'
import hashlib, json, os, sys, zipfile
plan_sha256 = sys.argv[1]
profile = {"release": "stable", "prerelease": "prerelease", "preview": "preview"}.get(os.environ["BUILD_TYPE"])
if profile is None:
raise SystemExit("release certification only covers preview, prerelease and stable builds")
def sha256(path):
digest = hashlib.sha256()
with open(path, "rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
deliverables = []
for name in sorted(os.listdir("artifacts")):
if not name.startswith("rustfs-") or not name.endswith(".zip"):
continue
path = os.path.join("artifacts", name)
members = zipfile.ZipFile(path).namelist()
binary_name = "rustfs.exe" if "rustfs.exe" in members else "rustfs"
with zipfile.ZipFile(path) as archive:
binary_bytes = archive.read(binary_name)
import tempfile
with tempfile.NamedTemporaryFile() as handle:
handle.write(binary_bytes)
handle.flush()
binary_sha = sha256(handle.name)
deliverables.append({
"kind": "zip",
"name": name,
"location": "https://github.com/rustfs/rustfs/releases/download/{}/{}".format(os.environ["VERSION"], name),
"sha256": sha256(path),
"size": os.path.getsize(path),
"binary_sha256": binary_sha,
})
if not deliverables:
raise SystemExit("no built deliverables found to certify")
candidate = {
"schema_version": 1,
"product_commit": os.environ["PRODUCT_COMMIT"],
"version": os.environ["VERSION"],
"release_profile": profile,
"features": [f for f in os.environ.get("FEATURES", "").split(",") if f],
"plan_sha256": plan_sha256,
"validation_batch": {"run_id": os.environ["GITHUB_RUN_ID"], "run_attempt": int(os.environ["GITHUB_RUN_ATTEMPT"])},
"deliverables": deliverables,
}
json.dump(candidate, sys.stdout, indent=2, sort_keys=True)
print()
PY
# Trusted verification evidence only: the release-gate envelopes of
# rustfs/s3chaos for this exact product commit. Absence blocks below.
TRUSTED_REPO="rustfs/s3chaos"
TRUSTED_WORKFLOW="release-gate.yml"
TRUSTED_RUN=$(gh api "repos/${TRUSTED_REPO}/actions/workflows/${TRUSTED_WORKFLOW}/runs?head_sha=${PRODUCT_COMMIT}&status=success&per_page=1" --jq '.workflow_runs[0].id // empty' || true)
if [[ -z "$TRUSTED_RUN" ]]; then
echo "❌ No trusted verification run of ${TRUSTED_REPO}/${TRUSTED_WORKFLOW} for ${PRODUCT_COMMIT}; refusing to certify" >&2
else
gh run download "$TRUSTED_RUN" -R "$TRUSTED_REPO" -n release-validation-evidence -D results || \
echo "❌ Trusted verification run ${TRUSTED_RUN} published no release-validation-evidence artifact" >&2
fi
python3 scripts/release/release_gate.py validate \
--plan .config/release-validation-plan.json \
--candidate candidate.json \
--results results \
--output release-gate-output
- name: Upload certification evidence
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: release-certification
path: |
candidate.json
results/
release-gate-output/
if-no-files-found: warn
retention-days: 30
# The only writer of the official R2 release path. Promotion re-checks the
# exact bytes about to be published against the passed certification list
# and copies the staged objects byte-for-byte: no rebuild-after-verify.
promote-release-artifacts:
name: Promote Release Artifacts
needs: [ build-check, build-rustfs, release-certification ]
# Target-channel lock for the official R2 release path.
concurrency:
group: release-channel-r2-release
cancel-in-progress: false
if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
runs-on: ubuntu-latest
timeout-minutes: 30
permissions:
contents: read
actions: read
env:
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
R2_BUCKET: ${{ secrets.R2_BUCKET }}
AWS_EC2_METADATA_DISABLED: true
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Download build and certification artifacts
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
path: ./artifacts
pattern: rustfs-*
merge-multiple: true
- name: Download certification evidence
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: release-certification
path: ./certification
- name: Verify and promote staged release artifacts
shell: bash
run: |
set -euo pipefail
if [[ -z "$R2_ACCESS_KEY_ID" || -z "$R2_SECRET_ACCESS_KEY" || -z "$R2_ENDPOINT" || -z "$R2_BUCKET" ]]; then
echo "❌ R2 credentials missing; refusing to promote to the official channel" >&2
exit 1
fi
for zip_file in artifacts/rustfs-*.zip; do
[[ -f "$zip_file" ]] || continue
name="$(basename "$zip_file")"
python3 scripts/release/release_gate.py verify-promotion \
--plan .config/release-validation-plan.json \
--candidate certification/candidate.json \
--summary certification/release-gate-output \
--artifact "$zip_file" \
--artifact-name "$name"
done
if ! command -v aws >/dev/null 2>&1; then
sudo apt-get update && sudo apt-get install -y awscli
fi
export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
export AWS_DEFAULT_REGION="auto"
STAGING_PATH="s3://${R2_BUCKET}/artifacts/rustfs/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
RELEASE_PATH="s3://${R2_BUCKET}/artifacts/rustfs/release/"
for zip_file in artifacts/rustfs-*.zip; do
[[ -f "$zip_file" ]] || continue
name="$(basename "$zip_file")"
staged="$(mktemp)"
aws s3 cp "${STAGING_PATH}${name}" "$staged" --endpoint-url "$R2_ENDPOINT" --only-show-errors
staged_sha="$(sha256sum "$staged" | awk '{print $1}')"
built_sha="$(sha256sum "$zip_file" | awk '{print $1}')"
if [[ "$staged_sha" != "$built_sha" ]]; then
echo "❌ Staged object ${name} does not match the verified bytes" >&2
exit 1
fi
aws s3 cp "$staged" "${RELEASE_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
rm -f "$staged"
echo "✅ Promoted ${name} (${built_sha})"
done
# Create GitHub Release for every valid release tag, including previews
create-release:
name: Create GitHub Release
@@ -1248,10 +1028,6 @@ jobs:
update-latest-version:
name: Update Latest Version
needs: [ build-check, publish-release ]
# Target-channel lock for the latest.json version pointer.
concurrency:
group: release-channel-oss-latest
cancel-in-progress: false
if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
runs-on: ubuntu-latest
timeout-minutes: 30
@@ -1292,45 +1068,6 @@ jobs:
rm -rf "$OSSUTIL_DIR" "$OSSUTIL_ZIP"
chmod +x "$OSSUTIL_BIN"
# Candidate/pointer ordering: an older candidate must never overwrite
# a newer pointer, even if its promotion job finishes later.
CURRENT_LATEST="${RUNNER_TEMP}/current-latest.json"
if "$OSSUTIL_BIN" cp oss://rustfs-version/latest.json "$CURRENT_LATEST" 2>/dev/null; then
python3 - "$CURRENT_LATEST" "$VERSION" <<'PY'
import json
import re
import sys
def key(version):
match = re.match(r"^(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?$", version)
if not match:
raise SystemExit(3)
core = tuple(int(part) for part in match.groups()[:3])
prerelease = match.group(4)
if prerelease is None:
return core + (1,)
return core + (0, tuple(int(p) if p.isdigit() else p for p in re.split(r"[.-]", prerelease)))
try:
current = json.load(open(sys.argv[1], encoding="utf-8")).get("version", "")
proposed = sys.argv[2]
if current and key(current) > key(proposed):
raise SystemExit(1)
except SystemExit:
raise
except (OSError, ValueError, TypeError):
raise SystemExit(3)
PY
guard_status=$?
if [[ "$guard_status" -eq 1 ]]; then
echo "Current latest.json points at a newer version; refusing to downgrade" >&2
exit 0
elif [[ "$guard_status" -ne 0 ]]; then
echo "Existing latest.json is unparsable; refusing to overwrite it blindly" >&2
exit 1
fi
fi
# Create latest.json
cat > latest.json << EOF
{
@@ -1347,59 +1084,16 @@ jobs:
echo "✅ Updated latest.json for ${RELEASE_TYPE} release $VERSION"
# Publish release (remove draft status). Requires the release-certification
# aggregate: without a complete legal success there is no public write.
# Publish release (remove draft status)
publish-release:
name: Publish Release
needs: [ build-check, create-release, upload-release-assets, release-certification ]
# Target-channel lock: concurrent releases serialize on this channel so an
# older candidate cannot interleave its publish with a newer one.
concurrency:
group: release-channel-github-release
cancel-in-progress: false
needs: [ build-check, create-release, upload-release-assets ]
if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'preview' || needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
timeout-minutes: 30
permissions:
contents: write
actions: read
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Download certification evidence
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: release-certification
path: ./certification
- name: Verify release assets against certification
env:
GH_TOKEN: ${{ github.token }}
shell: bash
run: |
set -euo pipefail
TAG="${{ needs.build-check.outputs.version }}"
mkdir -p "$RUNNER_TEMP/promotion-assets"
gh release download "$TAG" --dir "$RUNNER_TEMP/promotion-assets" --pattern '*.zip'
found=0
for zip_file in "$RUNNER_TEMP"/promotion-assets/*.zip; do
[[ -f "$zip_file" ]] || continue
found=1
python3 scripts/release/release_gate.py verify-promotion \
--plan .config/release-validation-plan.json \
--candidate certification/candidate.json \
--summary certification/release-gate-output \
--artifact "$zip_file" \
--artifact-name "$(basename "$zip_file")"
done
if [[ "$found" -ne 1 ]]; then
echo "No release zip assets to verify against the certification" >&2
exit 1
fi
- name: Publish release
env:
GH_TOKEN: ${{ github.token }}
+2 -2
View File
@@ -96,7 +96,7 @@ env:
RUST_BACKTRACE: 1
jobs:
# Readers: workspace-lint-and-build, workspace-tests (workspace and ILM), build-rustfs-debug-binary,
# Readers: test-and-lint (workspace and ILM), build-rustfs-debug-binary,
# e2e-tests, e2e-full.
warm-ci-dev:
name: Warm ci-dev
@@ -146,7 +146,7 @@ jobs:
if-no-files-found: error
# --all-targets covers the test binaries nextest builds, including
# e2e_test, which the workspace run excludes. The second build adds
# e2e_test, which test-and-lint's own run excludes. The second build adds
# the e2e-test-hooks feature resolution that build-rustfs-debug-binary uses
# and that no lint lane enables.
- name: Build ci-dev superset
+49 -209
View File
@@ -44,11 +44,6 @@ on:
schedule:
- cron: "11 0 * * *" # Nightly 00:11 UTC, after the 22:41 UTC cache-warm run
workflow_dispatch:
inputs:
rehearse_workspace_shards:
description: Compare one full execution with at most two resource-safe shards of the same archive
type: boolean
default: false
permissions:
contents: read
@@ -92,7 +87,6 @@ jobs:
connect: ${{ steps.scope.outputs.connect || 'true' }}
packages: ${{ steps.scope.outputs.packages }}
full_validation: ${{ steps.scope.outputs.full_validation || 'true' }}
rehearse_shards: ${{ steps.scope.outputs.rehearse_shards || 'false' }}
steps:
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
@@ -103,7 +97,6 @@ jobs:
env:
CI_BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.event.before }}
CI_EVENT_ACTION: ${{ github.event.action }}
CI_REHEARSE_SHARDS: ${{ inputs.rehearse_workspace_shards && 'true' || 'false' }}
run: |
if [[ "$GITHUB_EVENT_NAME" != "pull_request" ]]; then
python3 -I scripts/ci_gate.py select
@@ -113,9 +106,9 @@ jobs:
printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
echo "Base CI policy unavailable; running the full matrix."
fi
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=3$' "$GITHUB_OUTPUT"; then
printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'rehearse_shards=false' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
echo "Base policy does not cover this workflow contract; retaining the full matrix."
if ! grep -q '^plan=' "$GITHUB_OUTPUT"; then
printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' >> "$GITHUB_OUTPUT"
echo "Legacy policy has no verification plan; retaining the full matrix."
fi
- name: Retain verification plan
if: steps.scope.outputs.plan != ''
@@ -160,7 +153,7 @@ jobs:
- name: Run shared quick checks
uses: ./.github/actions/quick-checks
workspace-lint-and-build:
test-and-lint:
name: Workspace Lint and Build
if: contains(fromJSON('["full", "e2e"]'), needs.classify-changes.outputs.mode) && (github.event_name != 'pull_request' || github.event.action != 'closed')
needs: [ quick-checks, classify-changes ]
@@ -193,13 +186,13 @@ jobs:
- name: Prepare test evidence
run: |
mkdir -p artifacts/workspace
mkdir -p artifacts/test-and-lint
{
echo "run_id=${GITHUB_RUN_ID}"
echo "job=${GITHUB_JOB}"
echo "runner=${RUNNER_NAME}"
echo "started_at=$(date --utc --iso-8601=seconds)"
} > artifacts/workspace/run-metadata.txt
} > artifacts/test-and-lint/run-metadata.txt
# Clippy gates every event, pull requests included. When it ran only after
# merge, a lint regression merged freely and then failed every main run
@@ -224,8 +217,7 @@ jobs:
- name: Upload workspace test binaries
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: workspace-test-binaries
overwrite: true
name: workspace-test-binaries-${{ github.run_attempt }}
path: ${{ runner.temp }}/workspace-tests.tar.zst*
compression-level: 0
if-no-files-found: error
@@ -233,11 +225,11 @@ jobs:
- name: Run documentation tests
run: |
mkdir -p artifacts/workspace
mkdir -p artifacts/test-and-lint
set +e
timeout --verbose --signal=TERM --kill-after=30s 15m \
python3 scripts/ci_test_artifact.py cargo doc --configuration workspace \
2>&1 | tee artifacts/workspace/doctest.log
2>&1 | tee artifacts/test-and-lint/doctest.log
status=${PIPESTATUS[0]}
{
echo "command=cargo test --all --exclude e2e_test --doc"
@@ -246,7 +238,7 @@ jobs:
echo
echo "Remaining test-related processes:"
pgrep -af 'cargo|rustdoc|target/.*/deps/' || true
} > artifacts/workspace/doctest-diagnostics.txt
} > artifacts/test-and-lint/doctest-diagnostics.txt
exit "${status}"
- name: Upload test reports and diagnostics
@@ -255,7 +247,7 @@ jobs:
with:
name: junit-test-and-lint-${{ github.run_number }}-${{ github.run_attempt }}
path: |
artifacts/workspace
artifacts/test-and-lint
retention-days: 3
if-no-files-found: error
@@ -268,13 +260,11 @@ jobs:
workspace-tests:
name: Workspace tests
needs: [ workspace-lint-and-build, classify-changes ]
needs: [ test-and-lint, classify-changes ]
# The archive retains compile-time fixture paths and the large test set
# requires the same runner image until the hosted-runtime pilot passes.
runs-on: sm-standard-4
timeout-minutes: 90
outputs:
workspace_shards: ${{ steps.shards.outputs.workspace_shards || '[]' }}
env:
CI_PACKAGES: ${{ needs.classify-changes.outputs.packages }}
steps:
@@ -286,21 +276,21 @@ jobs:
run: chmod go-w "$(realpath "$HOME")"
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: workspace-test-binaries
name: workspace-test-binaries-${{ github.run_attempt }}
path: ${{ runner.temp }}
- name: Verify workspace archive identity
run: python3 scripts/ci_test_artifact.py verify --configuration workspace --archive "$RUNNER_TEMP/workspace-tests.tar.zst"
- name: Extract and list workspace tests
run: |
mkdir -p artifacts/workspace
mkdir -p artifacts/test-and-lint
cargo nextest list --profile ci --archive-file "$RUNNER_TEMP/workspace-tests.tar.zst" --workspace-remap "$GITHUB_WORKSPACE" --extract-to "$GITHUB_WORKSPACE" --extract-overwrite --message-format json \
> artifacts/workspace/core-test-listing.json
python3 scripts/check_test_wiring.py --check-core artifacts/workspace/core-test-listing.json
> artifacts/test-and-lint/core-test-listing.json
python3 scripts/check_test_wiring.py --check-core artifacts/test-and-lint/core-test-listing.json
- name: Run nextest tests
run: |
mkdir -p artifacts/workspace
mkdir -p artifacts/test-and-lint
rm -f target/nextest/ci/junit.xml
./scripts/ci/resource_sampler.sh start nextest
trap './scripts/ci/resource_sampler.sh stop' EXIT
@@ -309,7 +299,7 @@ jobs:
cargo nextest run --profile ci --cargo-metadata target/nextest/cargo-metadata.json --binaries-metadata target/nextest/binaries-metadata.json \
--workspace-remap "$GITHUB_WORKSPACE" --target-dir-remap "$GITHUB_WORKSPACE/target" \
--status-level all --final-status-level all \
2>&1 | tee artifacts/workspace/nextest.log
2>&1 | tee artifacts/test-and-lint/nextest.log
status=${PIPESTATUS[0]}
{
echo "command=workspace archive execution"
@@ -321,13 +311,13 @@ jobs:
echo
echo "Kernel OOM / kill events:"
dmesg -T 2>/dev/null | grep -iE 'oom|out of memory|killed process' | tail -20 || true
} > artifacts/workspace/nextest-diagnostics.txt
} > artifacts/test-and-lint/nextest-diagnostics.txt
exit "${status}"
- name: Verify workspace execution completeness
run: |
python3 scripts/ci_test_artifact.py evidence artifacts/workspace/core-test-listing.json target/nextest/ci/junit.xml
./scripts/check_migration_gate_count.sh evidence artifacts/workspace/core-test-listing.json target/nextest/ci/junit.xml
python3 scripts/ci_test_artifact.py evidence artifacts/test-and-lint/core-test-listing.json target/nextest/ci/junit.xml
./scripts/check_migration_gate_count.sh evidence artifacts/test-and-lint/core-test-listing.json target/nextest/ci/junit.xml
- name: Run ignored ILM integration tests serially
if: needs.classify-changes.outputs.mode == 'full'
@@ -358,98 +348,14 @@ jobs:
name: workspace-test-evidence-${{ github.run_attempt }}
path: |
target/nextest/ci/junit.xml
artifacts/workspace
artifacts/test-and-lint
artifacts/ilm-integration
retention-days: 3
if-no-files-found: error
- name: Prepare bounded shards from this candidate's observed durations
id: shards
if: needs.classify-changes.outputs.rehearse_shards == 'true'
run: |
python3 scripts/ci_test_shards.py prepare \
--listing artifacts/workspace/core-test-listing.json \
--junit target/nextest/ci/junit.xml --output artifacts/workspace-shard-plan
- name: Upload same-run shard plan
if: needs.classify-changes.outputs.rehearse_shards == 'true'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: workspace-shard-plan-${{ github.run_attempt }}
path: artifacts/workspace-shard-plan
retention-days: 3
if-no-files-found: error
workspace-shards:
name: Workspace shards (${{ matrix.shard }})
if: needs.classify-changes.outputs.rehearse_shards == 'true'
needs: [classify-changes, workspace-tests]
# Reuse the producer image and checkout path for the full RustFS archive.
runs-on: sm-standard-4
timeout-minutes: 90
strategy:
fail-fast: false
max-parallel: 2
matrix:
shard: ${{ fromJSON(needs.workspace-tests.outputs.workspace_shards || '[]') }}
env:
CI_PACKAGES: ${{ needs.classify-changes.outputs.packages }}
steps:
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- uses: ./.github/actions/test-runtime
- name: Protect Connect test home
run: chmod go-w "$(realpath "$HOME")"
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: workspace-test-binaries-${{ github.run_attempt }}
path: ${{ runner.temp }}
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: workspace-shard-plan-${{ github.run_attempt }}
path: artifacts/workspace-shard-plan
- name: Verify and execute this shard without compilation
env:
SHARD_INDEX: ${{ matrix.shard }}
run: |
python3 scripts/ci_test_artifact.py verify --configuration workspace --archive "$RUNNER_TEMP/workspace-tests.tar.zst"
python3 scripts/ci_test_shards.py execute --plan artifacts/workspace-shard-plan \
--archive "$RUNNER_TEMP/workspace-tests.tar.zst" --index "$SHARD_INDEX" --output artifacts/workspace-shard-result
- name: Retain shard evidence including failures
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: workspace-shard-result-${{ matrix.shard }}-${{ github.run_attempt }}
path: artifacts/workspace-shard-result
retention-days: 3
if-no-files-found: error
workspace-shard-evidence:
name: Workspace shard completeness
if: always() && needs.classify-changes.outputs.rehearse_shards == 'true'
needs: [classify-changes, workspace-tests, workspace-shards]
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: workspace-shard-plan-${{ github.run_attempt }}
path: artifacts/workspace-shard-plan
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
pattern: workspace-shard-result-*-${{ github.run_attempt }}
path: artifacts/workspace-shards
- name: Require the exact union of successful shard executions
run: |
python3 scripts/ci_test_shards.py verify --plan artifacts/workspace-shard-plan --evidence artifacts/workspace-shards
./scripts/check_migration_gate_count.sh evidence artifacts/workspace-shard-plan/listing.json artifacts/workspace-shards/combined.xml
connect-feature-boundaries:
name: Connect Feature Boundaries
if: needs.classify-changes.outputs.mode == 'full' && needs.classify-changes.outputs.connect != 'false' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
if: needs.classify-changes.outputs.mode == 'full' && needs.classify-changes.outputs.connect != 'false' && (github.event_name != 'pull_request' || github.event.action != 'closed')
needs: [ quick-checks, classify-changes ]
runs-on: sm-standard-4
timeout-minutes: 90
@@ -497,72 +403,13 @@ jobs:
- name: Check offline enrollment E2E root boundary
run: ./scripts/check_offline_enrollment_e2e.sh
fips-provider:
name: FIPS provider boundaries
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
needs: [ quick-checks, classify-changes ]
# This builds the RustFS library test target, which exceeds hosted-runner disk capacity.
runs-on: sm-standard-4
timeout-minutes: 90
strategy:
fail-fast: false
matrix:
mode: [default, fips]
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
CARGO_BUILD_JOBS: "2"
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Setup Rust environment
uses: ./.github/actions/setup
with:
rust-version: stable
cache-shared-key: ci-fips-provider
cache-save-if: 'false'
install-build-packaging-tools: 'false'
- name: Install FIPS build tools
if: matrix.mode == 'fips'
run: sudo apt-get install -y cmake golang clang libclang-dev
- name: Verify provider installation
shell: bash
run: |
args=()
if [[ '${{ matrix.mode }}' == 'fips' ]]; then
args+=(--features fips)
fi
cargo nextest run --locked -p rustfs-tls-runtime --lib "${args[@]}" --no-tests fail -E 'test(provider_policy_isolated)'
cargo nextest run --locked -p rustfs-s3-client --lib "${args[@]}" --no-tests fail -E 'test(overflowing_closed_range_is_rejected_before_network_io)'
cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(validate_pulsar)'
cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(webhook_provider_policy_isolated)'
if [[ '${{ matrix.mode }}' == 'fips' ]]; then
cargo nextest run --locked -p rustfs-protocols --lib --features ftps,webdav,fips --no-tests fail -E 'test(protocols_reject_incompatible_host_provider)'
cargo nextest run --locked -p rustfs-targets --lib --features fips --no-tests fail -E 'test(fips_postgres_rejects_incompatible_host_provider)'
cargo nextest run --locked -p rustfs-s3-client --lib --features fips --no-tests fail -E 'test(fips_transition_rejects_incompatible_host_provider)'
fi
tests=(provider_installation_isolated prepared_generation_shares_crypto_and_serves_sni_with_mtls)
if [[ '${{ matrix.mode }}' == 'fips' ]]; then
tests+=(fips_provider_builds_fips_tls_configs fips_provider_tls_handshake_accepts_and_rejects_algorithms)
fi
for test_name in "${tests[@]}"; do
cargo nextest run --locked -p rustfs --lib "${args[@]}" --no-tests fail -E "test($test_name)"
done
cargo clippy --locked -p rustfs --lib "${args[@]}" -- -D warnings
test-and-lint-rio-v2:
name: Test and Lint (rio-v2)
# Specialized boundary, hardware and alternate-feature lanes run only on
# the nightly schedule and manual dispatch. They do not gate a merge: the
# nightly run reports a regression within a day, and repeating these builds
# on every PR revision and merge group bought no earlier signal worth the
# runner occupancy. Workspace lint and tests, the core contract, the debug
# build, E2E smoke and the S3 lanes remain the pull-request gate.
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
# Alternate feature configurations run on the nightly schedule, merge
# groups and manual dispatch. The default workspace remains the PR gate and
# the per-push main baseline: repeating four feature builds on every merge
# bought nothing the nightly run does not report within a day.
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["merge_group", "schedule", "workflow_dispatch"]'), github.event_name)
needs: [ quick-checks, classify-changes ]
runs-on: sm-standard-4
timeout-minutes: 90
@@ -599,12 +446,12 @@ jobs:
# --profile ci so the quarantine list (and its junit flaky markers)
# covers this leg too; the default profile is the local no-retry
# profile and silently ignored quarantined flakes here (rustfs#6703).
mkdir -p artifacts/workspace
mkdir -p artifacts/test-and-lint
rm -f target/nextest/ci/junit.xml
./scripts/ci/resource_sampler.sh start nextest-rio-v2
trap './scripts/ci/resource_sampler.sh stop' EXIT
cargo nextest run --profile ci -p rustfs -p rustfs-ecstore --features rio-v2 \
2>&1 | tee artifacts/workspace/nextest.log
2>&1 | tee artifacts/test-and-lint/nextest.log
cargo test -p rustfs --doc --features rio-v2
- name: Upload rio-v2 test reports and diagnostics
@@ -614,14 +461,14 @@ jobs:
name: junit-test-and-lint-rio-v2-${{ github.run_number }}-${{ github.run_attempt }}
path: |
target/nextest/ci/junit.xml
artifacts/workspace
artifacts/test-and-lint
retention-days: 3
if-no-files-found: error
test-and-lint-protocols:
name: "Test and Lint (${{ matrix.features.name }})"
# Same cadence as test-and-lint-rio-v2 above.
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["merge_group", "schedule", "workflow_dispatch"]'), github.event_name)
needs: [ quick-checks, classify-changes ]
runs-on: sm-standard-4
timeout-minutes: 90
@@ -690,7 +537,7 @@ jobs:
path: |
target/nextest/ci/junit.xml
artifacts/protocol-tests
artifacts/workspace
artifacts/test-and-lint
retention-days: 3
if-no-files-found: warn
@@ -766,8 +613,7 @@ jobs:
- name: Upload debug binary
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: rustfs-debug-binary
overwrite: true
name: rustfs-debug-binary-${{ github.run_attempt }}
path: |
target/debug/rustfs
target/debug/rustfs.e2e.json
@@ -786,8 +632,7 @@ jobs:
- name: Upload E2E test binaries
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: e2e-test-binaries
overwrite: true
name: e2e-test-binaries-${{ github.run_attempt }}
path: ${{ runner.temp }}/e2e-tests.tar.zst*
compression-level: 0
if-no-files-found: error
@@ -826,8 +671,7 @@ jobs:
- name: Upload debug binary
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: rustfs-debug-binary-rio-v2
overwrite: true
name: rustfs-debug-binary-rio-v2-${{ github.run_attempt }}
path: |
target/debug/rustfs
target/debug/rustfs.e2e.json
@@ -842,7 +686,7 @@ jobs:
# job had neither, so each closed/merged PR really ran the whole io_uring
# suite (measured 4m17s / 7m19s / 7m31s on runs 30678272341 / 30678117601 /
# 30662728539) and kept the cancellation run in progress for minutes.
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
needs: [ quick-checks, classify-changes ]
# GitHub-hosted ubuntu-latest runs a recent kernel with io_uring and, unlike
# a container, applies no seccomp filter that would block io_uring_setup — so
@@ -923,11 +767,11 @@ jobs:
- uses: ./.github/actions/test-runtime
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: rustfs-debug-binary
name: rustfs-debug-binary-${{ github.run_attempt }}
path: target/debug
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: e2e-test-binaries
name: e2e-test-binaries-${{ github.run_attempt }}
path: ${{ runner.temp }}
- name: Run core storage and authorization contract
env:
@@ -968,7 +812,7 @@ jobs:
- name: Download E2E test binaries
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: e2e-test-binaries
name: e2e-test-binaries-${{ github.run_attempt }}
path: ${{ runner.temp }}
- name: Verify E2E archive identity
@@ -992,7 +836,7 @@ jobs:
- name: Download debug binary
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: rustfs-debug-binary
name: rustfs-debug-binary-${{ github.run_attempt }}
path: target/debug
- name: Make binary executable
@@ -1037,7 +881,7 @@ jobs:
path: |
${{ runner.temp }}/rustfs-e2e-smoke-logs/
${{ runner.temp }}/rustfs-e2e-smoke-list.json
artifacts/workspace
artifacts/test-and-lint
if-no-files-found: warn
- name: Upload e2e smoke JUnit report
@@ -1104,7 +948,7 @@ jobs:
- name: Download E2E test binaries
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: e2e-test-binaries
name: e2e-test-binaries-${{ github.run_attempt }}
path: ${{ runner.temp }}
- name: Verify E2E archive identity
@@ -1166,7 +1010,7 @@ jobs:
- name: Download debug binary
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: rustfs-debug-binary
name: rustfs-debug-binary-${{ github.run_attempt }}
path: target/debug
- name: Make binary executable
@@ -1278,7 +1122,7 @@ jobs:
- name: Download debug binary
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: rustfs-debug-binary-rio-v2
name: rustfs-debug-binary-rio-v2-${{ github.run_attempt }}
path: target/debug
- name: Make binary executable
@@ -1329,7 +1173,7 @@ jobs:
- name: Download debug binary
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: rustfs-debug-binary
name: rustfs-debug-binary-${{ github.run_attempt }}
path: target/debug
- name: Make binary executable
@@ -1403,7 +1247,7 @@ jobs:
- name: Download debug binary
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: rustfs-debug-binary
name: rustfs-debug-binary-${{ github.run_attempt }}
path: target/debug
- name: Make binary executable
@@ -1448,13 +1292,10 @@ jobs:
- classify-changes
- typos
- quick-checks
- workspace-lint-and-build
- test-and-lint
- workspace-tests
- workspace-shards
- workspace-shard-evidence
- core-contract
- connect-feature-boundaries
- fips-provider
- test-and-lint-rio-v2
- test-and-lint-protocols
- build-rustfs-debug-binary
@@ -1482,11 +1323,10 @@ jobs:
needs:
- classify-changes
- connect-feature-boundaries
- fips-provider
- required-checks
- typos
- quick-checks
- workspace-lint-and-build
- test-and-lint
- workspace-tests
- core-contract
- test-and-lint-rio-v2
+1 -1
View File
@@ -95,7 +95,7 @@ jobs:
- name: Run CLA Bot
if: github.event_name != 'merge_group'
uses: overtrue/cla-bot@f054da99d5777e47e1af9933737b9c0c5584be8d # v0.1.0
uses: overtrue/cla-bot@5e5929846a60cfb42b16d0d38e135ef67d94fc88 # v0.0.9
with:
github-token: ${{ github.token }}
registry-token: ${{ steps.registry-token.outputs.token }}
+2 -182
View File
@@ -329,75 +329,13 @@ jobs:
echo " - Create latest: $create_latest"
echo " - Source ref: $source_ref"
# Consume the trusted release certification for the exact version this
# workflow wraps and verify the certified input digests. A manual dispatch
# without that certification fails here and pushes no official tag; only
# results from the trusted verification sources carry release authority.
release-certification:
name: Release Certification
needs: build-check
if: needs.build-check.outputs.should_build == 'true'
runs-on: ubuntu-latest
timeout-minutes: 30
permissions:
contents: read
actions: read
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
ref: ${{ needs.build-check.outputs.source_ref }}
- name: Download and verify release certification
env:
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
VERSION: ${{ needs.build-check.outputs.version }}
shell: bash
run: |
set -euo pipefail
mkdir -p certification
run_id="$TRIGGERING_RUN_ID"
if [[ -z "$run_id" ]]; then
run_id="$(gh api --method GET \
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
-f branch="$VERSION" -f status=success -F per_page=1 \
--jq '.workflow_runs[0].id // empty' || true)"
fi
if [[ -z "$run_id" ]]; then
echo "No certified Build and Release run found for ${VERSION}; failing closed" >&2
exit 1
fi
if ! gh run download "$run_id" -n release-certification -D certification; then
echo "Build run ${run_id} published no release-certification artifact; failing closed" >&2
exit 1
fi
python3 scripts/release/release_gate.py verify-promotion \
--plan .config/release-validation-plan.json \
--candidate certification/candidate.json \
--summary certification/release-gate-output \
--artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
--artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
# Input digest binding: every zip this image wraps must be the
# certified byte-for-byte content, never a mutable latest pointer.
while IFS=$'\t' read -r name location expected; do
echo "Verifying certified input ${name}"
actual="$(curl -fsSL "$location" | sha256sum | awk '{print $1}')"
if [[ "$actual" != "$expected" ]]; then
echo "Certified input ${name} does not match its recorded digest" >&2
exit 1
fi
done < <(jq -r '.deliverables[] | select(.name | contains("linux")) | [.name, .location, .sha256] | @tsv' certification/candidate.json)
# Build multi-arch Docker images
# Strategy: Build images using pre-built binaries from dl.rustfs.com
# Supports both release and dev channel binaries based on build context
# Only runs when should_build is true (which includes workflow success check)
build-docker:
name: Build Docker Images
needs: [ build-check, release-certification ]
needs: build-check
if: needs.build-check.outputs.should_build == 'true'
runs-on: ubuntu-latest
timeout-minutes: 60
@@ -505,14 +443,6 @@ jobs:
# Output tags
echo "tags=$TAGS" >> "$GITHUB_OUTPUT"
# Staging identity for this run's promotion: official tags are only
# created after the staged image verifies against the certification,
# and they point at this same digest (no rebuild-after-verify).
STAGING_TAG="staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${VARIANT_SUFFIX}"
STAGING_TAGS="${{ env.REGISTRY_DOCKERHUB }}:${STAGING_TAG},${{ env.REGISTRY_GHCR }}:${STAGING_TAG},${{ env.REGISTRY_QUAY }}:${STAGING_TAG}"
echo "staging_tags=$STAGING_TAGS" >> "$GITHUB_OUTPUT"
echo "staging_tag=$STAGING_TAG" >> "$GITHUB_OUTPUT"
# Generate labels
LABELS="org.opencontainers.image.title=RustFS"
LABELS="$LABELS,org.opencontainers.image.description=RustFS distributed object storage system"
@@ -537,9 +467,7 @@ jobs:
file: ${{ matrix.file }}
platforms: ${{ env.DOCKER_PLATFORMS }}
push: ${{ needs.build-check.outputs.should_push == 'true' }}
# Push to staging tags only: official tags are created by the
# certification-gated promotion step below from this same digest.
tags: ${{ steps.meta.outputs.staging_tags }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
# No layer cache. This build compiles nothing — it downloads a
# release zip and runs apk/apt — so the cache could only save the
@@ -565,114 +493,6 @@ jobs:
no-cache: false
pull: true
# A later-built wrapper: the staged image must complete this platform's
# own binary identity verification before any official tag is written.
# It may not inherit an unverified package's conclusion.
- name: Verify staged image against certification
if: needs.build-check.outputs.should_push == 'true'
env:
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
VERSION: ${{ needs.build-check.outputs.version }}
shell: bash
run: |
set -euo pipefail
mkdir -p certification
run_id="$TRIGGERING_RUN_ID"
if [[ -z "$run_id" ]]; then
run_id="$(gh api --method GET \
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
-f branch="$VERSION" -f status=success -F per_page=1 \
--jq '.workflow_runs[0].id // empty' || true)"
fi
gh run download "$run_id" -n release-certification -D certification
STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
case "${{ matrix.variant }}" in
musl) libc="musl" ;;
glibc) libc="gnu" ;;
esac
for platform in linux/amd64 linux/arm64; do
case "$platform" in
linux/amd64) arch_sub="x86_64" ;;
linux/arm64) arch_sub="aarch64" ;;
esac
docker pull --platform "$platform" "$STAGING_REF"
container="$(docker create --platform "$platform" "$STAGING_REF")"
docker cp "$container:/usr/bin/rustfs" "$RUNNER_TEMP/staged-rustfs"
docker rm "$container" >/dev/null
actual="$(sha256sum "$RUNNER_TEMP/staged-rustfs" | awk '{print $1}')"
expected="$(jq -r --arg needle "linux-${arch_sub}-${libc}" \
'[.deliverables[] | select(.name | contains($needle)) | .binary_sha256][0] // empty' \
certification/candidate.json)"
if [[ -z "$expected" || "$actual" != "$expected" ]]; then
echo "Staged image binary for ${platform} does not match the certified binary digest" >&2
exit 1
fi
python3 scripts/release/release_gate.py verify-promotion \
--plan .config/release-validation-plan.json \
--candidate certification/candidate.json \
--summary certification/release-gate-output \
--artifact "$RUNNER_TEMP/staged-rustfs" \
--artifact-role binary
done
- name: Promote staged image to official tags
if: needs.build-check.outputs.should_push == 'true'
shell: bash
run: |
set -euo pipefail
STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
IFS=',' read -r -a official_tags <<< "${{ steps.meta.outputs.tags }}"
for tag in "${official_tags[@]}"; do
docker buildx imagetools create -t "$tag" "$STAGING_REF"
echo "Promoted $tag -> $STAGING_REF"
done
# Downstream consumers (Helm) verify this record instead of inheriting
# an unverified wrapper's conclusion.
- name: Record image promotion
if: needs.build-check.outputs.should_push == 'true'
shell: bash
env:
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
VERSION: ${{ needs.build-check.outputs.version }}
run: |
set -euo pipefail
mkdir -p certification
run_id="$TRIGGERING_RUN_ID"
if [[ -z "$run_id" ]]; then
run_id="$(gh api --method GET \
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
-f branch="$VERSION" -f status=success -F per_page=1 \
--jq '.workflow_runs[0].id // empty' || true)"
fi
gh run download "$run_id" -n release-certification -D certification
STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
digest="$(docker buildx imagetools inspect "$STAGING_REF" --format '{{json .Manifest}}' | jq -r '.digest')"
candidate_sha="$(python3 -c "import json,hashlib,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('certification/candidate.json'))))")"
jq -n \
--arg version "$VERSION" \
--arg digest "$digest" \
--arg staging_ref "$STAGING_REF" \
--arg candidate_sha256 "$candidate_sha" \
--arg tags "${{ steps.meta.outputs.tags }}" \
--slurpfile candidate certification/candidate.json \
'{schema_version: 1, version: $version, image_digest: $digest, staging_ref: $staging_ref,
promoted_tags: ($tags | split(",")), candidate_sha256: $candidate_sha256,
binary_sha256s: ($candidate[0].deliverables | map(.binary_sha256) | unique)}' \
> image-promotion-record.json
- name: Upload image promotion record
if: needs.build-check.outputs.should_push == 'true'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: image-promotion-record${{ matrix.suffix }}
path: image-promotion-record.json
if-no-files-found: error
retention-days: 30
# Note: Manifest creation is no longer needed as we only build one variant
# Multi-arch manifests are automatically created by docker/build-push-action
+28 -2
View File
@@ -17,8 +17,8 @@
# Each selected test starts a real localhost cluster via
# `RustFSTestClusterEnvironment` (4 processes; 4 drives per node unless the
# case is a two-site 4-node 1-drive pair or a 4-node upgrade). Membership is
# `[profile.e2e-distributed]` in `.config/nextest.toml`. The nightly schedule
# and manual dispatches execute the same fail-closed suite.
# `[profile.e2e-distributed]` in `.config/nextest.toml`. Storage-sensitive PRs,
# nightly runs, and manual dispatches all execute the same fail-closed suite.
# Upgrade cases download the same pinned previous release as e2e-upgrade.yml.
#
# Isolated pool filesystems: expand/decommission/rebalance cases require
@@ -33,6 +33,32 @@
name: e2e-distributed
on:
pull_request:
paths:
- "Cargo.lock"
- "Cargo.toml"
- ".config/nextest.toml"
- ".config/e2e-distributed-selection.txt"
- ".github/workflows/e2e-distributed.yml"
- "crates/audit/**"
- "crates/common/**"
- "crates/config/**"
- "crates/e2e_test/**"
- "crates/ecstore/**"
- "crates/filemeta/**"
- "crates/heal/**"
- "crates/iam/**"
- "crates/lock/**"
- "crates/madmin/**"
- "crates/notify/**"
- "crates/replication/**"
- "crates/s3-client/**"
- "crates/s3-ops/**"
- "crates/s3-types/**"
- "crates/scanner/**"
- "crates/storage-api/**"
- "crates/utils/**"
- "rustfs/**"
workflow_dispatch:
inputs:
filter:
+18 -7
View File
@@ -12,19 +12,30 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# Single-node upgrade and rollback lane.
#
# The weekly Monday schedule, version tags, and workflow_dispatch run this
# suite. Distributed upgrade cases stay on the nightly e2e-distributed lane.
name: Upgrade Compatibility
on:
pull_request:
paths:
- ".github/workflows/e2e-upgrade.yml"
- "crates/e2e_test/src/common.rs"
- "crates/e2e_test/src/distributed/**"
- "crates/e2e_test/src/fake_s3_target/**"
- "crates/e2e_test/src/lib.rs"
- "crates/e2e_test/src/replication_extension_test.rs"
- "crates/e2e_test/src/rpc_fault_proxy.rs"
- "crates/e2e_test/src/upgrade_compatibility_test.rs"
- "crates/ecstore/**"
- "crates/filemeta/**"
- "crates/heal/**"
- "crates/kms/**"
- "crates/storage-api/**"
- "rustfs/**"
- "Cargo.lock"
push:
tags:
- "[0-9]*.[0-9]*.[0-9]*"
schedule:
# 03:17 UTC every Monday.
- cron: "17 3 * * 1"
workflow_dispatch:
@@ -33,7 +44,7 @@ permissions:
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: false
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
env:
CARGO_TERM_COLOR: always
-93
View File
@@ -30,102 +30,9 @@ permissions:
contents: read
jobs:
# Consume the same trusted release certification as every other publication
# exit and verify the image promotion record of the Docker workflow. The
# chart wraps no binary of its own, so it may only reference an image whose
# platform verification already passed; it must not inherit an unverified
# package's conclusion. A run without that evidence fails closed here and
# never reaches the chart repository.
release-certification:
runs-on: ubuntu-latest
timeout-minutes: 30
permissions:
contents: read
actions: read
outputs:
version: ${{ steps.version.outputs.raw_tag }}
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Normalize release version
id: version
env:
RAW_INPUT: ${{ github.event.inputs.version }}
RAW_BRANCH: ${{ github.event.workflow_run.head_branch }}
run: |
set -euo pipefail
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
RAW="$RAW_INPUT"
else
RAW="$RAW_BRANCH"
fi
case "$RAW" in
refs/tags/*) RAW_TAG="${RAW#refs/tags/}" ;;
*) RAW_TAG="$RAW" ;;
esac
echo "raw_tag=$RAW_TAG" >> "$GITHUB_OUTPUT"
- name: Download and verify release certification and image record
env:
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
RAW_TAG: ${{ steps.version.outputs.raw_tag }}
shell: bash
run: |
set -euo pipefail
mkdir -p certification image-record
build_run="$TRIGGERING_RUN_ID"
if [[ -z "$build_run" ]]; then
build_run="$(gh api --method GET \
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
-f branch="$RAW_TAG" -f status=success -F per_page=1 \
--jq '.workflow_runs[0].id // empty' || true)"
fi
if [[ -z "$build_run" ]]; then
echo "No certified Build and Release run found for ${RAW_TAG}; failing closed" >&2
exit 1
fi
if ! gh run download "$build_run" -n release-certification -D certification; then
echo "Build run ${build_run} published no release-certification artifact; failing closed" >&2
exit 1
fi
python3 scripts/release/release_gate.py verify-promotion \
--plan .config/release-validation-plan.json \
--candidate certification/candidate.json \
--summary certification/release-gate-output \
--artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
--artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
docker_run="$(gh api --method GET \
"repos/${GITHUB_REPOSITORY}/actions/workflows/docker.yml/runs" \
-f branch="$RAW_TAG" -f status=success -F per_page=1 \
--jq '.workflow_runs[0].id // empty' || true)"
if [[ -z "$docker_run" ]] || ! gh run download "$docker_run" -n image-promotion-record -D image-record; then
echo "No image promotion record for ${RAW_TAG}; a chart may not wrap an unverified image" >&2
exit 1
fi
candidate_sha="$(python3 -c "import json,hashlib,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('certification/candidate.json'))))")"
record_sha="$(jq -r '.candidate_sha256 // empty' image-record/image-promotion-record.json)"
if [[ "$record_sha" != "$candidate_sha" ]]; then
echo "Image promotion record does not belong to this certification" >&2
exit 1
fi
overlap="$(jq -r --slurpfile candidate certification/candidate.json \
'[.binary_sha256s[]] - ($candidate[0].deliverables | map(.binary_sha256)) | length' \
image-record/image-promotion-record.json)"
if [[ "$overlap" -ne 0 ]]; then
echo "Image promotion record carries binary digests outside the certification" >&2
exit 1
fi
build-helm-package:
runs-on: ubuntu-latest
timeout-minutes: 30
needs: [ release-certification ]
if: |
(github.event_name == 'workflow_dispatch' && !contains(github.event.inputs.version, '-preview')) ||
(
+7 -279
View File
@@ -87,206 +87,18 @@ jobs:
install-build-packaging-tools: 'false'
install-test-tools: 'false'
- name: Download static console assets
shell: bash
env:
GITHUB_TOKEN: ${{ github.token }}
run: |
# Packaging tools are off on this lane, so setup does not install unzip.
if ! command -v unzip >/dev/null 2>&1; then
sudo apt-get update -qq
sudo apt-get install -y -qq unzip
fi
if ! command -v unzip >/dev/null 2>&1; then
echo "unzip is required to extract console assets" >&2
exit 1
fi
mkdir -p ./rustfs/static
verify_sha256() {
local expected="$1"
local file="$2"
local actual=""
local actual_line=""
if command -v sha256sum >/dev/null 2>&1; then
actual_line=$(sha256sum -- "$file") || return 1
elif command -v shasum >/dev/null 2>&1; then
actual_line=$(shasum -a 256 -- "$file") || return 1
else
echo "No SHA-256 verification tool found" >&2
return 1
fi
actual="${actual_line%%[[:space:]]*}"
if [ "$actual" = "$expected" ]; then
echo "SHA256 verified OK: $actual"
return 0
else
echo "SHA256 mismatch: expected=$expected actual=$actual" >&2
return 1
fi
}
download_console_assets() {
local curl_bin="curl"
local python_bin="python3"
local console_api="https://api.github.com/repos/rustfs/console/releases/latest"
local console_json="console-release.json"
local console_url
local console_sha256
local curl_auth_args=()
console_tag=""
chmod +w ./rustfs/static/LICENSE || true
if ! command -v "$python_bin" >/dev/null 2>&1; then
python_bin="python"
fi
if ! command -v "$python_bin" >/dev/null 2>&1; then
echo "No Python interpreter found for release metadata parsing" >&2
return 1
fi
if [[ -n "${GITHUB_TOKEN:-}" ]]; then
curl_auth_args=(-H "Authorization: Bearer ${GITHUB_TOKEN}" -H "X-GitHub-Api-Version: 2022-11-28")
fi
"$curl_bin" "${curl_auth_args[@]}" --fail -L "$console_api" \
-o "$console_json" --retry 3 --retry-delay 5 --max-time 300 || return 1
read -r console_tag console_url console_sha256 < <("$python_bin" - "$console_json" <<'PY'
import json
import re
import sys
with open(sys.argv[1], encoding="utf-8") as handle:
release = json.load(handle)
tag = release.get("tag_name", "") or "unknown"
for asset in release.get("assets", []):
name = asset.get("name", "")
digest = asset.get("digest", "")
url = asset.get("browser_download_url", "")
if name.startswith("rustfs-console-") and name.endswith(".zip") and digest.startswith("sha256:") and url:
sha256 = digest.split(":", 1)[1]
if not re.fullmatch(r"[0-9a-fA-F]{64}", sha256):
raise SystemExit(f"console zip asset has invalid sha256 digest: {sha256}")
sys.stdout.buffer.write(f"{tag} {url} {sha256}\n".encode("utf-8"))
break
else:
raise SystemExit("no console zip asset with sha256 digest found")
PY
) || return 1
if [[ -z "$console_url" || -z "$console_sha256" ]]; then
echo "Console release metadata is missing URL or SHA-256 digest" >&2
return 1
fi
echo "Console release: ${console_tag}"
echo "Downloading console asset: ${console_url}"
"$curl_bin" --fail -L "$console_url" -o console.zip --retry 3 --retry-delay 5 --max-time 300 || return 1
verify_sha256 "$console_sha256" console.zip || return 2
unzip -o console.zip -d ./rustfs/static || return 2
}
if download_console_assets; then
rm -f console.zip console-release.json
else
status=$?
rm -f console.zip console-release.json
if [[ "$status" -eq 2 ]]; then
echo "Console asset integrity verification failed" >&2
fi
echo "Failed to download verified console assets" >&2
exit 1
fi
if [[ ! -s ./rustfs/static/index.html ]]; then
echo "Console asset archive is missing static/index.html" >&2
exit 1
fi
asset_count=$(find ./rustfs/static -type f | wc -l | tr -d '[:space:]')
echo "Console assets ready: version=${console_tag:-unknown}, ${asset_count} files extracted to ./rustfs/static"
- name: Build RustFS
# Pin the glibc baseline (`<target>.<glibc>` for cargo-zigbuild) so
# nightlies run on older distros such as RHEL/Rocky 9 (glibc 2.34); #8062.
shell: bash
run: |
# Force rebuild by touching build.rs so a restored cache cannot
# reuse a rustfs artifact compiled before these console assets existed.
touch rustfs/build.rs
cargo zigbuild --release --locked --target x86_64-unknown-linux-gnu.2.31 -p rustfs --bin rustfs
run: cargo zigbuild --release --locked --target x86_64-unknown-linux-gnu.2.31 -p rustfs --bins
- name: Verify glibc symbol baseline
run: |
python3 scripts/release/check_glibc.py 2.31 \
target/x86_64-unknown-linux-gnu/release/rustfs
target/x86_64-unknown-linux-gnu/release/rustfs \
target/x86_64-unknown-linux-gnu/release/rustfs-cli
target/x86_64-unknown-linux-gnu/release/rustfs --version | grep -Fx 'build target : x86_64-unknown-linux-gnu'
- name: Verify console
shell: bash
run: |
set -euo pipefail
find_free_port() {
python3 - <<'PY'
import socket
with socket.socket() as sock:
sock.bind(("127.0.0.1", 0))
print(sock.getsockname()[1])
PY
}
binary="target/x86_64-unknown-linux-gnu/release/rustfs"
data_dir="$(mktemp -d)"
log_file="${RUNNER_TEMP}/rustfs-console-smoke.log"
server_pid=""
trap '
if [[ -n "${server_pid:-}" ]]; then
kill "$server_pid" 2>/dev/null || true
wait "$server_pid" 2>/dev/null || true
fi
rm -rf "$data_dir"
' EXIT
if [[ ! -x "$binary" ]]; then
echo "rustfs binary not found: ${binary}" >&2
exit 1
fi
api_port="$(find_free_port)"
console_port="$(find_free_port)"
console_url="http://127.0.0.1:${console_port}/rustfs/console/"
"$binary" server \
--address "127.0.0.1:${api_port}" \
--console-enable \
--console-address "127.0.0.1:${console_port}" \
--access-key console-smoke \
--secret-key console-smoke-secret \
"$data_dir" >"$log_file" 2>&1 &
server_pid=$!
response=""
for _ in {1..40}; do
response="$(curl --silent --output /dev/null --write-out '%{http_code} %{content_type}' "$console_url" || true)"
if [[ "$response" == 200\ text/html* ]]; then
exit 0
fi
sleep 0.25
done
echo "Console endpoint did not return 200 text/html: $response" >&2
cat "$log_file"
exit 1
- name: Build DEB package
id: deb
shell: bash
@@ -534,12 +346,12 @@ jobs:
path: ${{ steps.publish.outputs.candidate_file }}
if-no-files-found: error
# Publish the deb/rpm pair to the auto-testing repo's `assets` branch so
# engineers can download and install the nightly directly. The branch is
# a single-commit orphan rewritten on every build, which keeps the repo
# small while the latest files stay reachable at stable raw URLs.
# Publish the deb/rpm pair as assets of the rolling `nightly` release on
# rustfs/auto-testing (see scripts/release/publish_nightly_assets.sh).
# The dispatch below runs only after this upload and the immutable R2
# candidate both succeed. It does not replace rustfs-functional-chain.yml,
# which still starts from this workflow's workflow_run completion.
# kms-vault-lane and kms-vault-ha-failover stay parallel with this job.
- name: Publish packages to auto-testing release assets
env:
ASSETS_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
@@ -548,90 +360,6 @@ jobs:
DEB_DATE: ${{ steps.deb.outputs.deb_date }}
BUILD_REF: ${{ env.NIGHTLY_BUILD_REF }}
run: bash scripts/release/publish_nightly_assets.sh
# Cross-repo repository_dispatch cannot use GITHUB_TOKEN. The secret is a
# PAT or GitHub App token for rustfs/auto-testing: classic `repo` scope,
# or fine-grained Contents write (the permission GitHub checks for
# POST /repos/{owner}/{repo}/dispatches). A dispatch failure fails this
# step, and therefore this job, so the scheduled failure alert fires.
# GitHub allows at most 10 top-level client_payload properties, so the
# schema 2 candidate object is nested under `candidate` and the release
# fields sit beside it.
- name: Dispatch nightly candidate to auto-testing
if: ${{ steps.publish.outputs.candidate_file != '' }}
env:
DISPATCH_TOKEN: ${{ secrets.AUTO_TESTING_DISPATCH_TOKEN }}
CANDIDATE_FILE: ${{ steps.publish.outputs.candidate_file }}
DEB_DATE: ${{ steps.deb.outputs.deb_date }}
SOURCE_REPO: ${{ github.repository }}
shell: bash
run: |
set -euo pipefail
: "${DISPATCH_TOKEN:?AUTO_TESTING_DISPATCH_TOKEN is required to dispatch rustfs/auto-testing}"
: "${CANDIDATE_FILE:?nightly candidate manifest is required}"
: "${DEB_DATE:?DEB_DATE is required}"
[[ "${DEB_DATE}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]
[[ "${SOURCE_REPO}" == "rustfs/rustfs" ]]
payload="${RUNNER_TEMP}/nightly-candidate-dispatch-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
jq -n --slurpfile candidate "${CANDIDATE_FILE}" \
--arg event_type "nightly-candidate-ready" \
--arg source_repo "${SOURCE_REPO}" \
--arg latest_alias_url "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb" \
--arg auto_testing_release "https://github.com/rustfs/auto-testing/releases/tag/nightly" \
--arg channel "nightly" \
--arg deb_date "${DEB_DATE}" \
--arg nightly_workflow ".github/workflows/nightly-gnu.yml" \
'{
event_type: $event_type,
source_repo: $source_repo,
latest_alias_url: $latest_alias_url,
auto_testing_release: $auto_testing_release,
channel: $channel,
deb_date: $deb_date,
nightly_workflow: $nightly_workflow,
candidate: $candidate[0]
}' > "${payload}"
if ! jq -e '
.event_type == "nightly-candidate-ready"
and .source_repo == "rustfs/rustfs"
and .channel == "nightly"
and .deb_date == env.DEB_DATE
and .nightly_workflow == ".github/workflows/nightly-gnu.yml"
and .latest_alias_url == "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb"
and .auto_testing_release == "https://github.com/rustfs/auto-testing/releases/tag/nightly"
and .candidate.schema == 2
and ((.candidate | keys | sort) == ["build_run_attempt","build_run_id","package_sha256","package_url","schema","source_ref","source_sha","workflow_sha"])
and ((keys | sort) == ["auto_testing_release","candidate","channel","deb_date","event_type","latest_alias_url","nightly_workflow","source_repo"])
' "${payload}" >/dev/null; then
echo "nightly dispatch payload does not match the nightly-candidate-ready contract" >&2
exit 1
fi
body="${RUNNER_TEMP}/nightly-candidate-dispatch-body-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
jq -n --slurpfile client_payload "${payload}" \
'{event_type: "nightly-candidate-ready", client_payload: $client_payload[0]}' > "${body}"
response="${RUNNER_TEMP}/nightly-candidate-dispatch-response-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.txt"
status="$(curl -sS --retry 3 --connect-timeout 15 --max-time 60 \
-o "${response}" -w '%{http_code}' \
-X POST \
-H "Authorization: Bearer ${DISPATCH_TOKEN}" \
-H "Accept: application/vnd.github+json" \
-H "X-GitHub-Api-Version: 2022-11-28" \
-H "Content-Type: application/json" \
--data-binary @"${body}" \
https://api.github.com/repos/rustfs/auto-testing/dispatches)" || {
echo "repository_dispatch request to rustfs/auto-testing failed before an HTTP status" >&2
exit 1
}
if [[ "${status}" != "204" ]]; then
echo "repository_dispatch to rustfs/auto-testing failed: HTTP ${status}" >&2
if [[ -s "${response}" ]]; then
cat "${response}" >&2
fi
exit 1
fi
echo "dispatched nightly-candidate-ready to rustfs/auto-testing (deb_date=${DEB_DATE}, run=${GITHUB_RUN_ID}, attempt=${GITHUB_RUN_ATTEMPT})"
# Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
#
# RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and
+14 -210
View File
@@ -225,57 +225,10 @@ jobs:
echo " Build run ID: $BUILD_RUN_ID"
echo " Build run number: $RUN_NUMBER"
# Consume the trusted release certification of the resolved build run and
# verify the certified input digests before any package is wrapped. A run
# without that certification fails closed here; packages built later must
# complete their own verification and cannot inherit an unverified
# conclusion.
release-certification:
name: Release Certification
needs: resolve
if: needs.resolve.result == 'success'
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
timeout-minutes: 30
permissions:
contents: read
actions: read
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Download and verify release certification
env:
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
BUILD_RUN_ID: ${{ needs.resolve.outputs.build_run_id }}
BUILD_TYPE: ${{ needs.resolve.outputs.build_type }}
shell: bash
run: |
set -euo pipefail
mkdir -p certification
if [[ "$BUILD_TYPE" == "development" ]]; then
# Development artifacts are explicitly uncertified and never touch
# official or latest channels; only the staging/dev paths remain.
jq -n '{schema_version: 1, certified: false, reason: "development artifacts are explicitly uncertified"}' \
> certification/development-uncertified.json
exit 0
fi
if ! gh run download "$BUILD_RUN_ID" -n release-certification -D certification; then
echo "Build run ${BUILD_RUN_ID} published no release-certification artifact; failing closed" >&2
exit 1
fi
python3 scripts/release/release_gate.py verify-promotion \
--plan .config/release-validation-plan.json \
--candidate certification/candidate.json \
--summary certification/release-gate-output \
--artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
--artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
# Build DEB and RPM packages for each architecture and libc variant
package:
name: Package (${{ matrix.arch }}-${{ matrix.libc }})
needs: [ resolve, release-certification ]
needs: resolve
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
timeout-minutes: 30
strategy:
@@ -541,55 +494,6 @@ jobs:
echo "rpm_file=$RPM_FILE" >> "$GITHUB_OUTPUT"
echo "✅ RPM built: $RPM_FILE"
- name: Download certification evidence
if: needs.resolve.outputs.build_type != 'development'
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: release-certification
path: ./certification
# Later-built wrapper verification: the binary inside each package must
# match the certified binary digest for this platform before anything is
# uploaded, and the package itself stays on staging until promotion.
- name: Verify wrapped binary against certification
if: needs.resolve.outputs.build_type != 'development'
shell: bash
env:
DEB_FILE: ${{ steps.deb.outputs.deb_file }}
RPM_FILE: ${{ steps.rpm.outputs.rpm_file }}
ARTIFACT_NAME: ${{ matrix.artifact_name }}
run: |
set -euo pipefail
expected="$(jq -r --arg needle "$ARTIFACT_NAME" \
'[.deliverables[] | select(.name | contains($needle)) | .binary_sha256][0] // empty' \
certification/candidate.json)"
if [[ -z "$expected" ]]; then
echo "No certified binary digest recorded for ${ARTIFACT_NAME}" >&2
exit 1
fi
if ! command -v rpm2cpio >/dev/null 2>&1; then
sudo apt-get update && sudo apt-get install -y rpm2cpio
fi
if ! command -v cpio >/dev/null 2>&1; then
sudo apt-get update && sudo apt-get install -y cpio
fi
for package_file in "$DEB_FILE" "$RPM_FILE"; do
case "$package_file" in
*.deb)
dpkg-deb --fsys-tarfile "$package_file" | tar -xOf - ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
;;
*.rpm)
rpm2cpio "$package_file" | cpio -i --to-stdout ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
;;
esac
python3 scripts/release/release_gate.py verify-promotion \
--plan .config/release-validation-plan.json \
--candidate certification/candidate.json \
--summary certification/release-gate-output \
--artifact "$RUNNER_TEMP/wrapped-rustfs" \
--artifact-role binary
done
- name: Upload packages to artifacts
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
@@ -627,14 +531,10 @@ jobs:
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
export AWS_DEFAULT_REGION="auto"
# Build-phase writes stay on staging for release types: the official
# packages/release and packages/latest paths are written only by the
# certification-gated promote-packages job, from these same staged
# objects. Development packages keep the uncertified dev channel.
if [[ "$BUILD_TYPE" == "development" ]]; then
R2_PREFIX="artifacts/rustfs/packages/dev"
else
R2_PREFIX="artifacts/rustfs/packages/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}"
R2_PREFIX="artifacts/rustfs/packages/release"
fi
R2_PATH="s3://${R2_BUCKET}/${R2_PREFIX}/"
@@ -649,6 +549,18 @@ jobs:
echo "✅ Upload complete"
# Also upload as latest for release/prerelease
if [[ "$BUILD_TYPE" == "release" || "$BUILD_TYPE" == "prerelease" ]]; then
LATEST_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/latest/"
for f in "$DEB_FILE" "$RPM_FILE"; do
if [[ -n "$f" && -f "$f" ]]; then
echo "Uploading latest: $(basename "$f")"
aws s3 cp "$f" "$LATEST_PATH" --endpoint-url "$R2_ENDPOINT" --only-show-errors
fi
done
echo "✅ Latest packages updated"
fi
- name: Upload packages to GitHub Release
if: needs.resolve.outputs.tag != ''
env:
@@ -702,114 +614,6 @@ jobs:
echo "✅ GitHub release assets updated"
# The only writer of the official packages/release and packages/latest R2
# paths. Promotion re-checks each package's wrapped binary against the
# passed certification list and copies the staged objects byte-for-byte:
# no rebuild-after-verify, no mutable latest resolution downstream.
promote-packages:
name: Promote Packages
needs: [ resolve, package, release-certification ]
# Target-channel lock for the official package paths.
concurrency:
group: release-channel-r2-packages
cancel-in-progress: false
if: needs.resolve.outputs.tag != '' && (needs.resolve.outputs.build_type == 'release' || needs.resolve.outputs.build_type == 'prerelease')
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
timeout-minutes: 30
permissions:
contents: read
actions: read
env:
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
R2_BUCKET: ${{ secrets.R2_BUCKET }}
AWS_EC2_METADATA_DISABLED: true
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Download packages and certification
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
path: ./packages
pattern: packages-*
merge-multiple: true
- name: Download certification evidence
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: release-certification
path: ./certification
- name: Verify and promote staged packages
shell: bash
run: |
set -euo pipefail
if [[ -z "$R2_ACCESS_KEY_ID" || -z "$R2_SECRET_ACCESS_KEY" || -z "$R2_ENDPOINT" || -z "$R2_BUCKET" ]]; then
echo "❌ R2 credentials missing; refusing to promote to the official channel" >&2
exit 1
fi
STAGING_PREFIX="s3://${R2_BUCKET}/artifacts/rustfs/packages/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
RELEASE_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/release/"
LATEST_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/latest/"
if ! command -v aws >/dev/null 2>&1; then
sudo apt-get update && sudo apt-get install -y awscli
fi
export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
export AWS_DEFAULT_REGION="auto"
if ! command -v rpm2cpio >/dev/null 2>&1; then
sudo apt-get install -y rpm2cpio
fi
if ! command -v cpio >/dev/null 2>&1; then
sudo apt-get install -y cpio
fi
promoted=0
for package_file in packages/*.deb packages/*.rpm; do
[[ -f "$package_file" ]] || continue
name="$(basename "$package_file")"
case "$package_file" in
*.deb)
dpkg-deb --fsys-tarfile "$package_file" | tar -xOf - ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
;;
*.rpm)
rpm2cpio "$package_file" | cpio -i --to-stdout ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
;;
esac
python3 scripts/release/release_gate.py verify-promotion \
--plan .config/release-validation-plan.json \
--candidate certification/candidate.json \
--summary certification/release-gate-output \
--artifact "$RUNNER_TEMP/wrapped-rustfs" \
--artifact-role binary
staged="$(mktemp)"
aws s3 cp "${STAGING_PREFIX}${name}" "$staged" --endpoint-url "$R2_ENDPOINT" --only-show-errors
staged_sha="$(sha256sum "$staged" | awk '{print $1}')"
built_sha="$(sha256sum "$package_file" | awk '{print $1}')"
if [[ "$staged_sha" != "$built_sha" ]]; then
echo "❌ Staged package ${name} does not match the verified bytes" >&2
exit 1
fi
aws s3 cp "$staged" "${RELEASE_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
aws s3 cp "$staged" "${LATEST_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
rm -f "$staged"
promoted=$((promoted + 1))
echo "✅ Promoted ${name} (${built_sha})"
done
if [[ "$promoted" -eq 0 ]]; then
echo "No staged packages found to promote" >&2
exit 1
fi
# Summary
summary:
name: Summary
@@ -197,21 +197,21 @@ jobs:
jq -e --arg testing_sha "${AUTO_TESTING_SHA}" '
.schema_version == 1 and
.auto_testing_sha == $testing_sha and
.selected_scenarios == ["A", "B", "C", "C2", "D", "E", "F"] and
.selected_scenarios == ["A", "B", "C", "C2", "D", "E"] and
.complete == true and
.counts.expected == 54 and
.counts.seen == 54 and
.counts.expected == 44 and
.counts.seen == 44 and
.counts.harness_errors == 0 and
(.expected_cases | length) == 54 and
(.expected_cases | unique | length) == 54 and
(.seen_cases | length) == 54 and
(.seen_cases | unique | length) == 54 and
(.expected_cases | length) == 44 and
(.expected_cases | unique | length) == 44 and
(.seen_cases | length) == 44 and
(.seen_cases | unique | length) == 44 and
(.seen_cases | sort) == (.expected_cases | sort) and
(.missing_cases | length) == 0 and
(.harness_errors | length) == 0 and
(.cases | length) == 54 and
(.cases | length) == 44 and
([.cases[].case_id] | sort) == (.expected_cases | sort) and
([.cases[].case_id] | unique | length) == 54 and
([.cases[].case_id] | unique | length) == 44 and
all(.cases[]; .verdict == "pass" or .verdict == "known-divergence" or .verdict == "UNEXPECTED") and
.counts.unexpected == ([.cases[] | select(.verdict == "UNEXPECTED")] | length) and
.counts.known_divergence == ([.cases[] | select(.verdict == "known-divergence")] | length)
+6 -14
View File
@@ -281,20 +281,12 @@ jobs:
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
PERFORMANCE_OPTIONAL: ${{ contains(fromJSON(steps.selection.outputs.optional_suites), 'performance') }}
run: |
# TEMPORARY: the pf-testing environment is broken, so the
# performance lane is disabled until it is repaired. Forcing
# offline keeps the chain on its designed skip path: the lane job
# is skipped, aggregate receives --allow-skipped performance
# (required=false in scheduled-validations.json), and the report
# records the optional-lane skip with its reason.
# Restore the probe below once the environment is healthy again.
# if python3 scripts/check_functional_runners.py pf-testing; then
# echo "performance_ready=online" >> "$GITHUB_OUTPUT"
# else
# echo "performance_ready=offline" >> "$GITHUB_OUTPUT"
# test "$PERFORMANCE_OPTIONAL" = true
# fi
echo "performance_ready=offline" >> "$GITHUB_OUTPUT"
if python3 scripts/check_functional_runners.py pf-testing; then
echo "performance_ready=online" >> "$GITHUB_OUTPUT"
else
echo "performance_ready=offline" >> "$GITHUB_OUTPUT"
test "$PERFORMANCE_OPTIONAL" = true
fi
validation:
if: ${{ github.event_name == 'workflow_dispatch' && !inputs.validation_runner_inspect_only && inputs.validation_retain_evidence_run_id == '' && inputs.validation_package_sha == '' && (inputs.validation_case != '' || inputs.validation_tools_only || inputs.validation_native_tools || inputs.validation_shared_tools || inputs.validation_manager_tools || inputs.validation_cleanup_run_id != '' || inputs.validation_cleanup_retained_evidence) }}
+2 -3
View File
@@ -55,8 +55,7 @@ env:
jobs:
heal-test:
runs-on: smoke-testing
# 90-minute heal run (see the step below) plus setup, report and cleanup.
timeout-minutes: 120
timeout-minutes: 60
# Standalone manual run, or one link of the nightly functional chain
# (storage -> heal -> pool). Pool expansion no longer re-runs heal.
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
@@ -155,7 +154,7 @@ jobs:
./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
- name: Run heal test (write -> outage -> heal -> verify)
timeout-minutes: 90
timeout-minutes: 45
id: test
# Collect reports and backlog evidence before the final suite verdict.
continue-on-error: true
@@ -1,457 +0,0 @@
# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
name: RustFS Validation Sweep
# Executes the migrated canonical validation cases (rustfs/auto-testing#113)
# on the shared functional fleet. Everything with a lane executor is swept in
# rotating nightly batches so the whole set is covered roughly weekly; a
# single case or one module can be dispatched manually. That is the
# implemented catalog (66 today) plus the eight planned cases whose dedicated
# executors already exist and have live PASS evidence - fresh nightly
# evidence feeds their acceptance review. heal.* cases stay on the
# interactive validation flow: they require reviewed native/tool collectors
# built by separate validation-tools runs before fleet execution.
on:
schedule:
# After the nightly functional chain settles (chain starts ~16:48 UTC and
# runs for about two hours).
- cron: "30 19 * * *"
workflow_dispatch:
inputs:
case:
description: 'Single canonical validation case id (overrides module and rotation), e.g. pool.decommission'
type: string
required: false
default: ''
module:
description: 'Restrict the sweep to one catalog module, e.g. bucket-replication'
type: string
required: false
default: ''
limit:
description: 'Maximum cases per sweep (0 = run everything selected)'
type: number
required: false
default: 12
build_run_id:
description: 'Successful nightly package run; empty resolves the latest'
type: string
required: false
default: ''
build_run_attempt:
description: 'Attempt of that package run'
type: string
required: false
default: '1'
permissions:
contents: read
concurrency:
group: rustfs-validation-sweep
cancel-in-progress: false
env:
RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES || 'vm000 vm001 vm002 vm003' }}
RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER || 'azureuser' }}
RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
jobs:
sweep:
if: github.repository == 'rustfs/rustfs'
runs-on: smoke-testing
timeout-minutes: 150
steps:
- name: Checkout chain tooling
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Resolve nightly candidate
id: candidate
env:
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
BUILD_RUN_ID: ${{ inputs.build_run_id }}
BUILD_RUN_ATTEMPT: ${{ inputs.build_run_attempt }}
CHAIN_OUTPUT: ${{ runner.temp }}/validation-sweep-candidate.json
run: |
set -euo pipefail
# Scheduled runs carry no inputs and the resolver only derives a
# build from a workflow_run payload (which a schedule cannot
# provide), so resolve the newest successful nightly build
# explicitly before invoking it.
if [ -z "${BUILD_RUN_ID:-}" ]; then
LATEST="$(gh api "repos/rustfs/rustfs/actions/workflows/nightly-gnu.yml/runs?branch=main&status=success&per_page=1" --jq '.workflow_runs[0]')"
BUILD_RUN_ID="$(printf '%s' "${LATEST}" | jq -r '.id')"
BUILD_RUN_ATTEMPT="$(printf '%s' "${LATEST}" | jq -r '.run_attempt')"
export BUILD_RUN_ID BUILD_RUN_ATTEMPT
echo "scheduled sweep pinned to nightly ${BUILD_RUN_ID} attempt ${BUILD_RUN_ATTEMPT}"
fi
python3 scripts/resolve_functional_candidate.py
python3 - <<'PY'
import json
import os
from pathlib import Path
root = Path(os.environ["RUNNER_TEMP"]) / "validation-sweep"
root.mkdir(parents=True, exist_ok=True)
chain = json.loads((Path(os.environ["RUNNER_TEMP"]) / "validation-sweep-candidate.json").read_text())
manifest = chain["candidate"]["manifest"]
(root / "candidate.json").write_text(json.dumps({
"package_url": manifest["package_url"],
"package_sha256": manifest["package_sha256"],
"source_sha": manifest["source_sha"],
}, sort_keys=True) + "\n")
with open(os.environ["GITHUB_OUTPUT"], "a") as output:
output.write("testing_sha=" + chain["testing_sha"] + "\n")
output.write("package_url=" + manifest["package_url"] + "\n")
output.write("source_sha=" + manifest["source_sha"] + "\n")
PY
- name: Checkout auto-testing scripts
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
repository: rustfs/auto-testing
ref: ${{ steps.candidate.outputs.testing_sha }}
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
path: auto-testing
persist-credentials: false
- name: Prepare validation Python runtime
# The pool / ilm / replication validation runners require a managed
# Python, exactly like the chain's validation job provides.
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.12"
- name: Install validation Python requirements
# Case scripts call python3 and import boto3 at startup. Install the
# functional chain's client into a per-run env so that interpreter
# sees it without changing the runner's shared Python.
run: |
set -euo pipefail
validation_python="${RUNNER_TEMP}/rustfs-validation-python-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
test ! -e "${validation_python}"
python3 -m venv "${validation_python}"
"${validation_python}/bin/python" -m pip install \
-r auto-testing/scripts/validation_ilm/requirements.txt
printf '%s\n' "${validation_python}/bin" >> "${GITHUB_PATH}"
- name: Select sweep cases
id: select
env:
SELECTED_CASE: ${{ inputs.case }}
SELECTED_MODULE: ${{ inputs.module }}
CASE_LIMIT: ${{ inputs.limit }}
SWEEP_EVENT: ${{ github.event_name }}
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
run: |
set -euo pipefail
python3 - <<'PY'
import datetime
import json
import os
from pathlib import Path
sweep_root = Path(os.environ["SWEEP_ROOT"])
catalog = json.loads(Path("auto-testing/validation-cases.json").read_text())
def sweepable(case):
status = case.get("source_status")
if status == "implemented":
return True
# The remaining planned non-heal cases route to the lane
# executors and already have live PASS evidence; sweeping them
# nightly produces fresh evidence for their acceptance review.
return status == "planned" and not case["id"].startswith("heal.")
selected = [(case["id"], module["id"]) for module in catalog["modules"]
for case in module["cases"]
if sweepable(case)]
if not selected:
raise SystemExit("catalog has no sweepable cases")
one_case = os.environ.get("SELECTED_CASE", "").strip()
module_filter = os.environ.get("SELECTED_MODULE", "").strip()
known = {case_id for case_id, _ in selected}
modules = {module_id for _, module_id in selected}
if one_case:
if one_case not in known:
raise SystemExit(f"unknown case or no lane executor for it: {one_case}")
selected = [(one_case, dict(selected)[one_case])]
elif module_filter:
if module_filter not in modules:
raise SystemExit(f"unknown module or no implemented cases: {module_filter}")
selected = [pair for pair in selected if pair[1] == module_filter]
elif os.environ.get("SWEEP_EVENT") == "schedule":
# Stateless rotation: every night take one window of the catalog
# so a full sweep needs len(selected)/limit nights and no state
# has to be carried between runs.
limit = max(1, int(os.environ.get("CASE_LIMIT") or 12))
day = datetime.datetime.now(datetime.timezone.utc).timetuple().tm_yday
start = (day * limit) % len(selected)
selected = (selected[start:] + selected[:start])[:limit]
limit = int(os.environ.get("CASE_LIMIT") or 0)
if one_case or module_filter or (not os.environ.get("SWEEP_EVENT") == "schedule"):
if limit > 0:
selected = selected[:limit]
(sweep_root / "sweep-cases.txt").write_text("".join(f"{case_id}\n" for case_id, _ in selected))
with open(os.environ["GITHUB_OUTPUT"], "a") as output:
output.write(f"count={len(selected)}\n")
print(f"selected {len(selected)} case(s):", ", ".join(c for c, _ in selected))
PY
- name: Admit clean shared environment
id: environment
timeout-minutes: 5
env:
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-validation-sweep.json
run: python3 scripts/functional_environment.py enter --suite heal
- name: Run validation cases
id: run_cases
timeout-minutes: 120
env:
AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
run: |
set -uo pipefail
mkdir -p "${SWEEP_ROOT}/results"
: > "${SWEEP_ROOT}/results.txt"
while read -r CASE_ID; do
[ -n "${CASE_ID}" ] || continue
echo "::group::case ${CASE_ID}"
rm -rf "${SWEEP_ROOT}/run"
if ./auto-testing/scripts/run_validation_case.sh \
"${CASE_ID}" "${SWEEP_ROOT}/candidate.json" \
"${AUTO_TESTING_SHA}" "${SWEEP_ROOT}/run" \
> "${SWEEP_ROOT}/results/${CASE_ID//\//_}.log" 2>&1; then
printf '%s\tPASS\n' "${CASE_ID}" >> "${SWEEP_ROOT}/results.txt"
echo "case ${CASE_ID}: PASS"
else
printf '%s\tFAIL\n' "${CASE_ID}" >> "${SWEEP_ROOT}/results.txt"
echo "case ${CASE_ID}: FAIL (log retained in the sweep artifact)"
fi
if [ -f "${SWEEP_ROOT}/run/report.md" ]; then
cp "${SWEEP_ROOT}/run/report.md" "${SWEEP_ROOT}/results/${CASE_ID//\//_}.report.md" 2>/dev/null || true
fi
echo "::endgroup::"
# Fleet hygiene between cases: the case runners set up their own
# topology, this purge guarantees the next case starts clean even
# when the previous one died mid-run.
for node in ${RUSTFS_NODES}; do
# -n keeps ssh from consuming the case list this loop reads from stdin
ssh -n -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
${SUDO} systemctl stop rustfs 2>/dev/null || true
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
${SUDO} dpkg -P rustfs
fi
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
${SUDO} rm -f /etc/default/rustfs
' || echo "inter-case cleanup failed on ${node} (continuing)"
done
done < "${SWEEP_ROOT}/sweep-cases.txt"
echo "sweep finished: $(wc -l < "${SWEEP_ROOT}/results.txt") case(s) executed"
- name: Generate sweep report
if: ${{ always() && steps.environment.outcome == 'success' }}
env:
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
PACKAGE_URL: ${{ steps.candidate.outputs.package_url }}
SOURCE_SHA: ${{ steps.candidate.outputs.source_sha }}
AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
run: |
set -euo pipefail
python3 - <<'PY'
import json
import os
from pathlib import Path
root = Path(os.environ["SWEEP_ROOT"])
catalog = json.loads(Path("auto-testing/validation-cases.json").read_text())
module_of = {case["id"]: module["id"] for module in catalog["modules"] for case in module["cases"]}
rows = []
for line in (root / "results.txt").read_text().splitlines():
if not line:
continue
case_id, status = line.split("\t")
rows.append((case_id, module_of.get(case_id, "?"), status))
passed = sum(1 for _, _, s in rows if s == "PASS")
failed = len(rows) - passed
run_url = (f"{os.environ.get('GITHUB_SERVER_URL', 'https://github.com')}/"
f"{os.environ.get('GITHUB_REPOSITORY', 'rustfs/rustfs')}/actions/runs/"
f"{os.environ.get('GITHUB_RUN_ID', '')}")
report = [
"# RustFS validation sweep report", "",
f"- Run: {run_url}",
f"- Attempt: {os.environ.get('GITHUB_RUN_ATTEMPT', '')}",
f"- Trigger: {os.environ.get('GITHUB_EVENT_NAME', '')}",
f"- Package: {os.environ.get('PACKAGE_URL', 'nightly (R2 latest)')}",
f"- Source SHA: {os.environ.get('SOURCE_SHA', '')}",
f"- auto-testing: {os.environ.get('AUTO_TESTING_SHA', '')}", "",
"Canonical validation cases (rustfs/auto-testing#113) executed by the sweep, grouped by catalog module.", "",
]
# Group the executed cases by catalog module (the major category:
# pool lifecycle, heal, ILM expiration, bucket/site replication, kms,
# data services, identity, s3 compatibility). Per-module subtotals
# deliberately use "- Subtotal:" so the dashboard overview regexes
# (which anchor on "- Total:") keep reading the grand totals below.
by_module = {}
module_order = []
for case_id, module, status in rows:
if module not in by_module:
by_module[module] = []
module_order.append(module)
by_module[module].append((case_id, status))
report += ["## Case Summary", ""]
for module in module_order:
cases = by_module[module]
mod_pass = sum(1 for _, st in cases if st == "PASS")
mod_fail = len(cases) - mod_pass
report += [f"### {module} ({len(cases)})", "",
"| Case | Status |", "| --- | --- |"]
report += [f"| {case_id} | {status} |" for case_id, status in cases]
report += ["", f"- Subtotal: {mod_pass} passed, {mod_fail} failed", ""]
report += [
"## Totals", "",
f"- Total: {len(rows)}",
f"- PASS: {passed}",
f"- FAIL: {failed}",
"- UNSUPPORTED: 0", "",
"Failing cases keep this workflow red only as a report; the sweep artifact",
"retains every case log and per-case report for diagnosis.", "",
]
(root / "report.md").write_text("\n".join(report))
(root / "cases.md").write_text("\n".join(f"| {c} | {m} | {s} |" for c, m, s in rows) + "\n")
PY
cat "${SWEEP_ROOT}/report.md" >> "${GITHUB_STEP_SUMMARY}"
- name: Upload functional report to dashboard
if: ${{ always() && steps.environment.outcome == 'success' }}
timeout-minutes: 2
continue-on-error: true
env:
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
REPORT_FILE: ${{ runner.temp }}/validation-sweep/report.md
SUITE: validation
run: |
set -euo pipefail
if [ -z "${GH_TOKEN:-}" ]; then
echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
exit 0
fi
DATE="$(date -u +%Y-%m-%d)"
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
B64_FILE="$(mktemp)"
python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
if [ -n "${SHA}" ]; then
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
'{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
else
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
'{message:$msg, content:($content|rtrimstr("\n"))}' \
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
fi
echo "dashboard updated: ${REPORT_PATH}"
- name: Retain sweep evidence
if: ${{ always() }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: rustfs-validation-sweep-${{ github.run_id }}-${{ github.run_attempt }}
path: |
${{ runner.temp }}/validation-sweep/report.md
${{ runner.temp }}/validation-sweep/results.txt
${{ runner.temp }}/validation-sweep/sweep-cases.txt
${{ runner.temp }}/validation-sweep/candidate.json
${{ runner.temp }}/validation-sweep/results/
${{ runner.temp }}/validation-sweep/run/
# warn (not error): an early infra failure legitimately leaves no
# per-case evidence, and the failing step is already red.
if-no-files-found: warn
retention-days: 30
- name: Cleanup environment (after)
id: environment_cleanup
timeout-minutes: 5
if: ${{ always() && steps.environment.outcome == 'success' }}
run: |
set -uo pipefail
cleanup_status=0
for node in ${RUSTFS_NODES}; do
ssh -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
set -euo pipefail
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
${SUDO} systemctl stop rustfs 2>/dev/null || true
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
${SUDO} dpkg -P rustfs
fi
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
${SUDO} rm -f /etc/default/rustfs
' || cleanup_status=$?
done
exit "${cleanup_status}"
- name: Release environment only after successful cleanup
if: ${{ always() && steps.environment.outcome == 'success' }}
timeout-minutes: 5
env:
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-validation-sweep.json
CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
run: python3 scripts/functional_environment.py finish --suite heal --cleanup-outcome "$CLEANUP_OUTCOME"
- name: Notify on failure
if: failure()
run: echo "RustFS validation sweep failed (harness/environment breakdown); see the sweep artifact for case logs."
- name: Enforce sweep result
if: ${{ always() && steps.environment.outcome == 'success' }}
env:
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
run: |
set -euo pipefail
if grep -q $'\tFAIL$' "${SWEEP_ROOT}/results.txt"; then
echo "validation sweep has failing cases:"
grep $'\tFAIL$' "${SWEEP_ROOT}/results.txt"
exit 1
fi
alert-on-failure:
name: Alert on scheduled failure
# A scheduled sweep failure usually means a canonical validation case
# regressed or the fleet environment broke; surface it as a tracking
# issue instead of leaving it in the Actions tab.
needs: [sweep]
if: github.repository == 'rustfs/rustfs' && (always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure'))
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: read
issues: write
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.0
with:
persist-credentials: false
- name: Open or update failure-tracking issue
uses: ./.github/actions/schedule-failure-issue
with:
github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}
-26
View File
@@ -107,32 +107,6 @@ jobs:
shell: pwsh
run: cargo test -p rustfs-ecstore --lib multipart_owned_record_parent_barriers_ -- --nocapture
# Native multipart completion publishes its payload from a different
# parent than its metadata, and abort renames under a precondition; both
# need their own Windows handle discipline.
- name: Test native multipart complete and abort publication
shell: pwsh
run: >-
cargo test -p rustfs-ecstore --lib --
native_abort_missing_state_with_actual_owned_records_never_becomes_legacy_or_missing
native_abort_pending_intent_is_not_a_delete_certificate_and_raw_cleanup_preserves_exact_evidence
--nocapture
# Owned-undo reclaim removes a directory it pinned, copy heal persists
# fault bytes through a handle, and metadata reads meet a directory.
- name: Test native multipart reclaim, copy heal, and directory reads
shell: pwsh
run: >-
cargo test -p rustfs-ecstore --lib --
native_multipart_owned_undo
native_multipart_copy_heal
orphan_multipart_cleanup_rejects_metadata_directory_on_one_disk
--nocapture
- name: Test directory removal fallback
shell: pwsh
run: "cargo test -p rustfs-ecstore --lib disk::fs::tests::test_remove -- --nocapture"
- name: Test Windows path conversion
shell: pwsh
run: "cargo test -p rustfs-utils --no-default-features --features path --lib dunce:: -- --nocapture"
-1
View File
@@ -26,7 +26,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- IAM snapshot/deletion replay, target-assigned delete-marker purges, timestamp ordering, and best-effort peer broadcast now close the control-plane gaps found by the R6 review ([#7195](https://github.com/rustfs/rustfs/pull/7195)).
- A `set-remote-target?update=true&creds=true` request on a site-replication peer's bucket target is now rejected with `400 InvalidRequest`, naming the target and why it counts as peer-owned. It used to return success while keeping the stored endpoint, credentials, and TLS settings, so an endpoint edit from the Console's bucket replication form appeared to apply but did not. Change a peer's address with a site-replication edit instead; see "Changing a site's address" in `docs/operations/site-replication-operations.md`. The `sync`, `bandwidth`, `proxy`, and `path` update groups are unchanged.
- Site-replication edits that omit the site name, such as `mc admin replicate update --deployment-id <ID> --endpoint <URL>`, now keep the stored name on the editing site and on every receiver instead of renaming the site after the new endpoint host. No persisted format changes.
- The persisted replication MRF journal is now replayed every 5 minutes while the server runs, not only at startup. Delete-marker purges whose targets were unreachable, and replications that overflowed the worker queues, now converge within one replay interval after the target returns instead of waiting for a restart. A target that stays down is re-attempted once per retained entry per interval across the cluster: the start time of the last pass is shared through a small `config/replication/mrf.bin.replay` object, so nodes do not each replay once per interval. Older versions ignore that object; existing persisted formats are unchanged.
- Upgrade and rollback: upgrade every node in one site consecutively and verify reconciliation before moving to the next site; do not intentionally run a site mixed-version. Target-version ledger keys are harmless on rollback, although old code cannot use their routing. Before rolling back past [#7307](https://github.com/rustfs/rustfs/pull/7307), drain or repair every pending version purge: older code can free a retained version's data directory before its remote purge is acknowledged. See `docs/operations/site-replication-operations.md`.
### Security
Generated
+234 -449
View File
File diff suppressed because it is too large Load Diff
+22 -20
View File
@@ -147,7 +147,7 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.2" }
async-channel = "2.5.0"
async_zip = { default-features = false, version = "0.0.19" }
mysql_async = { default-features = false, version = "0.37.1" }
async-compression = { version = "0.4.52" }
async-compression = { version = "0.4.51" }
async-trait = "0.1.92"
async-nats = { version = "0.51.0", default-features = false }
axum = "0.8.9"
@@ -168,16 +168,16 @@ http-body-util = "0.1.5"
memchr = "2.8.3"
minlz = "1.2.3"
reqwest = "0.13.5"
rustfs-kafka-async = { version = "1.4.1" }
rustfs-kafka-async = { version = "1.4.0" }
socket2 = { version = "0.6.5" }
tokio = { version = "1.54.1" }
tokio-rustls = { default-features = false, version = "0.26.6" }
tokio-stream = { version = "0.1.19" }
tokio-test = "0.4.6"
tokio-util = { version = "0.7.20" }
tonic = { version = "0.14.9" }
tonic-prost = { version = "0.14.9" }
tonic-prost-build = { version = "0.14.9" }
tonic = { version = "0.14.8" }
tonic-prost = { version = "0.14.8" }
tonic-prost-build = { version = "0.14.8" }
tower = { version = "0.5.3" }
tower-http = { version = "0.7.1" }
h3 = "0.0.8"
@@ -191,7 +191,6 @@ quinn = { version = "0.11.12", default-features = false, features = [
# Serialization and Data Formats
apache-avro = { version = "0.22.0", features = ["snappy", "zstandard"] }
bytes = { version = "1.12.1" }
csv-core = "0.1.13"
bytesize = "2.7.0"
byteorder = "1.5.0"
flatbuffers = "25.12.19"
@@ -213,13 +212,12 @@ serde_urlencoded = "0.7.1"
# releases.
aes-gcm = { version = "0.11.1" }
argon2 = { version = "0.6.0" }
aws-lc-rs = { version = "1.19.0", default-features = false }
blake2 = "0.11.0"
chacha20poly1305 = { version = "0.11.0" }
crc-fast = "1.10.0"
hmac = { version = "0.13.0" }
jsonwebtoken = { version = "11.1.0" }
openidconnect = { default-features = false, version = "4.1.0" }
openidconnect = { default-features = false, version = "4.0" }
pbkdf2 = "0.13.0"
p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
rsa = { version = "=0.10.0-rc.18" }
@@ -236,7 +234,7 @@ proptest = "1"
# Time and Date
chrono = { version = "0.4.45" }
humantime = "2.4.0"
jiff = { version = "0.2.39" }
jiff = { version = "0.2.38" }
time = { version = "0.3.55" }
# Database
@@ -255,9 +253,9 @@ atoi = "3.1.1"
atomic_enum = "0.3.0"
aws-config = { version = "1.13.0" }
aws-credential-types = { version = "1.3.1" }
aws-sdk-kms = { default-features = false, version = "1.129.0" }
aws-sdk-s3 = { default-features = false, version = "1.157.0" }
aws-sdk-sts = { default-features = false, version = "1.124.0" }
aws-sdk-kms = { default-features = false, version = "1.128.0" }
aws-sdk-s3 = { default-features = false, version = "1.156.0" }
aws-sdk-sts = { default-features = false, version = "1.123.0" }
aws-smithy-async = { version = "1.3.1" }
aws-smithy-http-client = { default-features = false, version = "1.5.2" }
aws-smithy-runtime-api = { version = "1.19.2" }
@@ -321,7 +319,7 @@ rustfs-erasure-codec = { version = "9.0.0" }
reed-solomon-simd = "3.1.0"
regex = { version = "1.13.1" }
rumqttc = { package = "rumqttc-next", version = "0.34.0" }
redis = { version = "1.8.1" }
redis = { version = "1.8.0" }
rustify = { version = "0.7", default-features = false }
rustix = { version = "1.1.6" }
rust-embed = { version = "8.13.0" }
@@ -336,10 +334,10 @@ compact_str = "0.10.0"
snap = "1.1.2"
starshard = { version = "2.4.0" }
strum = { version = "0.28.0" }
sysinfo = "0.40.1"
sysinfo = "0.40.0"
temp-env = "0.3.6"
tempfile = "3.27.0"
test-case = "3.4.1"
test-case = "3.4.0"
thiserror = "2.0.21"
tracing = { version = "0.1.44" }
tracing-appender = "0.2.5"
@@ -358,13 +356,13 @@ winapi-util = "0.1.11"
windows = { version = "0.62.2" }
windows-sys = "0.61.2"
xxhash-rust = { version = "0.8.19" }
zip = "9.3.0"
zip = "9.1.3"
zstd = "0.14.3"
# Observability and Metrics
metrics = "0.24.6"
metrics-util = "0.20"
dial9 = { version = "0.5.7", default-features = false }
dial9 = { version = "0.5.6", default-features = false }
opentelemetry = { version = "0.33.1" }
opentelemetry-appender-tracing = { version = "0.33.1" }
opentelemetry-otlp = { version = "0.33.1" }
@@ -379,16 +377,16 @@ libunftp = { version = "0.24.0" }
unftp-core = "0.1.0"
suppaftp = { version = "12.2.1" }
rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
russh = { version = "0.65.1" }
russh = { version = "0.65.0" }
russh-sftp = "3.0.1"
# WebDAV
dav-server = "0.11.0"
# Performance Analysis and Memory Profiling
rustfs-mimalloc = { version = "0.7.3" }
rustfs-mimalloc = { version = "0.7.2" }
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
hotpath = { version = "0.30.3", default-features = false }
hotpath = { version = "0.30.1", default-features = false }
# Snapshot testing for output format regression detection
insta = { version = "1.49" }
@@ -429,3 +427,7 @@ strip = "none"
# Pin upstream fixes until compatible crates.io releases are available.
[patch.crates-io]
zstd = { path = "crates/zstd-compat" }
# Preserve RustFS OIDC flows while migrating the old elliptic-curve dependency chain.
# Upstream: openidconnect-rs#244 and oauth2-rs#352.
openidconnect = { git = "https://github.com/ramosbugs/openidconnect-rs", rev = "2f77ee73bc4554e1cb6279adb386005429fb4e41" }
oauth2 = { git = "https://github.com/ramosbugs/oauth2-rs", rev = "89585f4a280afd7b285eaf1c9be0068874e03d22" }
+3 -11
View File
@@ -16,7 +16,6 @@ FROM alpine:3.24.2 AS build
ARG TARGETARCH
ARG RELEASE=latest
ARG RC_RELEASE=latest
RUN apk add --no-cache ca-certificates curl jq unzip
WORKDIR /build
@@ -70,9 +69,6 @@ RUN set -eux; \
chmod +x /build/rustfs; \
rm -rf rustfs.zip /build/.tmp || true
COPY scripts/download_rc.sh /usr/local/bin/download_rc.sh
RUN sh /usr/local/bin/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /build/rc
FROM alpine:3.24.2
@@ -99,18 +95,14 @@ RUN apk upgrade --no-cache && \
ca-certificates \
coreutils \
curl \
jq \
tzdata \
&& test "$(TZ=Asia/Kolkata date +%z)" = "+0530"
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
COPY --from=build /build/rustfs /usr/bin/rustfs
COPY --from=build /build/rc /usr/bin/rc
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
COPY bootstrap/ /etc/rustfs/bootstrap/
COPY entrypoint.sh /entrypoint.sh
RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
RUN chmod +x /usr/bin/rustfs /entrypoint.sh
RUN addgroup -g 10001 -S rustfs && \
adduser -u 10001 -G rustfs -S rustfs -D && \
@@ -132,6 +124,6 @@ VOLUME ["/data"]
USER rustfs
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
ENTRYPOINT ["/entrypoint.sh"]
CMD ["rustfs"]
+4 -13
View File
@@ -12,10 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
FROM rust:1.99.1-trixie
ARG TARGETARCH
ARG RC_RELEASE=latest
FROM rust:1.97.1-trixie
RUN set -eux; \
export DEBIAN_FRONTEND=noninteractive; \
@@ -24,7 +21,6 @@ RUN set -eux; \
build-essential \
ca-certificates \
curl \
jq \
git \
pkg-config \
libssl-dev \
@@ -40,7 +36,6 @@ COPY . .
RUN ./scripts/static.sh
RUN cargo run --bin gproto
RUN cargo build --release --locked --bin rustfs
RUN sh /usr/src/rustfs/scripts/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /usr/local/bin/rc
RUN set -eux; \
groupadd -g 10001 rustfs; \
@@ -55,12 +50,8 @@ RUN set -eux; \
chown -R rustfs:rustfs /data /logs /app; \
chmod 0750 /data /logs
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
COPY bootstrap/ /etc/rustfs/bootstrap/
RUN install -m 0755 /usr/src/rustfs/target/release/rustfs /usr/bin/rustfs && \
install -m 0755 /usr/local/bin/rc /usr/bin/rc && \
chmod +x /etc/rustfs/entrypoint.sh && \
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
COPY entrypoint.sh /entrypoint.sh
RUN install -m 0755 /usr/src/rustfs/target/release/rustfs /usr/bin/rustfs && chmod +x /entrypoint.sh
ENV RUSTFS_VOLUMES="/data" \
RUST_LOG="warn" \
@@ -72,5 +63,5 @@ ENV RUSTFS_VOLUMES="/data" \
EXPOSE 9000 9001
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
ENTRYPOINT ["/entrypoint.sh"]
CMD ["/usr/bin/rustfs"]
+3 -11
View File
@@ -16,7 +16,6 @@ FROM ubuntu:26.04 AS build
ARG TARGETARCH
ARG RELEASE=latest
ARG RC_RELEASE=latest
RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates \
@@ -77,9 +76,6 @@ RUN set -eux; \
chmod +x /build/rustfs; \
rm -rf rustfs.zip /build/.tmp || true
COPY scripts/download_rc.sh /usr/local/bin/download_rc.sh
RUN sh /usr/local/bin/download_rc.sh "$TARGETARCH" gnu "$RC_RELEASE" /build/rc
FROM ubuntu:26.04
ARG RELEASE=latest
@@ -103,18 +99,14 @@ RUN apt-get update && apt-get upgrade -y \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
ca-certificates \
curl \
jq \
tzdata \
&& test "$(TZ=Asia/Kolkata date +%z)" = "+0530" \
&& rm -rf /var/lib/apt/lists/*
COPY --from=build /build/rustfs /usr/bin/rustfs
COPY --from=build /build/rc /usr/bin/rc
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
COPY bootstrap/ /etc/rustfs/bootstrap/
COPY entrypoint.sh /entrypoint.sh
RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
RUN chmod +x /usr/bin/rustfs /entrypoint.sh
RUN groupadd -g 10001 rustfs && \
useradd -u 10001 -g rustfs -m -s /sbin/nologin rustfs && \
@@ -139,6 +131,6 @@ VOLUME ["/data"]
USER rustfs
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
ENTRYPOINT ["/entrypoint.sh"]
CMD ["rustfs"]
+7 -19
View File
@@ -28,19 +28,17 @@ ARG TARGETPLATFORM
ARG BUILDPLATFORM
ARG TARGETARCH
ARG RUSTFS_BUILD_FEATURES=""
ARG RC_RELEASE=latest
# -----------------------------
# Build stage
# -----------------------------
FROM rust:1.99.1-trixie AS builder
FROM rust:1.97.1-trixie AS builder
# Re-declare args after FROM
ARG TARGETPLATFORM
ARG BUILDPLATFORM
ARG TARGETARCH
ARG RUSTFS_BUILD_FEATURES
ARG RC_RELEASE
# Debug: print platforms
RUN echo "Build info -> BUILDPLATFORM=${BUILDPLATFORM}, TARGETPLATFORM=${TARGETPLATFORM}"
@@ -54,7 +52,6 @@ RUN set -eux; \
build-essential \
ca-certificates \
curl \
jq \
git \
pkg-config \
libssl-dev \
@@ -169,8 +166,6 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
;; \
esac
RUN sh /usr/src/rustfs/scripts/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /usr/local/bin/rc
# -----------------------------
# Development stage (keeps toolchain)
# -----------------------------
@@ -204,13 +199,10 @@ ENV RUSTFS_VOLUMES="/data" \
# We rely on cargo run to build and run
EXPOSE 9000 9001
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
COPY bootstrap/ /etc/rustfs/bootstrap/
RUN install -m 0755 /usr/local/bin/rc /usr/bin/rc && \
chmod +x /etc/rustfs/entrypoint.sh && \
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
COPY entrypoint.sh /entrypoint.sh
RUN chmod +x /entrypoint.sh
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
ENTRYPOINT ["/entrypoint.sh"]
CMD ["cargo", "run", "--bin", "rustfs", "--"]
# -----------------------------
@@ -235,7 +227,6 @@ RUN set -eux; \
apt-get install -y --no-install-recommends \
ca-certificates \
curl \
jq \
tzdata \
coreutils; \
rm -rf /var/lib/apt/lists/*
@@ -257,11 +248,8 @@ RUN set -eux; \
# Copy the freshly built binary and the entrypoint
COPY --from=builder /usr/local/bin/rustfs /usr/bin/rustfs
COPY --from=builder /usr/local/bin/rc /usr/bin/rc
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
COPY bootstrap/ /etc/rustfs/bootstrap/
RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
COPY entrypoint.sh /entrypoint.sh
RUN chmod +x /usr/bin/rustfs /entrypoint.sh
# Default environment (override in docker run/compose as needed)
ENV RUSTFS_VOLUMES="/data" \
@@ -275,5 +263,5 @@ EXPOSE 9000
VOLUME ["/data"]
# Keep root here; entrypoint will drop privileges using chroot --userspec
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
ENTRYPOINT ["/entrypoint.sh"]
CMD ["/usr/bin/rustfs"]
-21
View File
@@ -1,21 +0,0 @@
# RustFS image bootstrap
The official image bundles this directory at `/etc/rustfs/bootstrap`. The default `buckets.json` is an empty JSON array, so the image starts RustFS directly and does not require a bootstrap endpoint or bootstrap credentials.
To create buckets, replace `buckets.json` in a derived image with a JSON array of bucket names. The entrypoint creates each bucket if it does not exist and applies `public-access-block.json`, whose default blocks all four public access paths. Edit that file in the derived image only when public access is an explicit requirement.
Optional per-bucket files use the bucket name as the filename without an extension:
| Directory | File format | Applied operation |
| --- | --- | --- |
| `policies/` | S3 bucket policy JSON | `rc bucket anonymous set-json` |
| `versioning/` | JSON boolean (`true` or `false`) | Enable or suspend bucket versioning |
| `lifecycle/` | `rc bucket lifecycle rule import` JSON | Replace lifecycle rules |
| `cors/` | JSON or XML accepted by `rc bucket cors set` | Replace CORS rules |
| `notifications/` | JSON array of `{ "arn": "...", "events": ["..."] }` | Add or replace event rules by ARN (prefix/suffix filters are not currently accepted) |
Every per-bucket file must correspond to a bucket listed in `buckets.json`. Applying these files is repeatable: bucket creation ignores existing buckets, while each configuration operation replaces or reconciles that bucket's requested state.
When `buckets.json` is non-empty, set `RUSTFS_BOOTSTRAP_ENDPOINT` to the RustFS S3 endpoint reachable from the container, for example `http://rustfs:9000` for a Compose service or the internal service DNS name in Kubernetes. The entrypoint waits on that endpoint's `/health/ready`; it never assumes `localhost`. `RUSTFS_BOOTSTRAP_TIMEOUT` controls the readiness wait in seconds (default `120`, maximum `3600`). `RUSTFS_BOOTSTRAP_DIR` can point to a different in-image configuration directory; it defaults to `/etc/rustfs/bootstrap`. Set `RUSTFS_ACCESS_KEY` and `RUSTFS_SECRET_KEY`, or their `_FILE` variants, for bootstrap authentication. The endpoint and credentials are read at runtime and are not baked into the image.
RustFS built-in IAM policies remain unchanged; the default bootstrap installs no custom bucket policies, lifecycle rules, versioning, CORS rules, or notification rules. The bundled RustFS CLI is resolved from a published release and its release checksum is verified. Set the Docker build argument `RC_RELEASE` to a version to pin it. Build the RustFS image after a CLI release containing the `bucket public-access-block` commands is available. For Kubernetes, the configured service must route traffic to the starting pod before its readiness probe succeeds (for example, with `publishNotReadyAddresses: true`); otherwise the service endpoint cannot reach the pod while the entrypoint waits for readiness.
-1
View File
@@ -1 +0,0 @@
[]
-1
View File
@@ -1 +0,0 @@
-1
View File
@@ -1 +0,0 @@
-1
View File
@@ -1 +0,0 @@
-1
View File
@@ -1 +0,0 @@
-6
View File
@@ -1,6 +0,0 @@
{
"block_public_acls": true,
"ignore_public_acls": true,
"block_public_policy": true,
"restrict_public_buckets": true
}
-1
View File
@@ -1 +0,0 @@
+14 -83
View File
@@ -10,10 +10,6 @@ use std::time::Duration;
use tokio::sync::{Notify, Semaphore};
use tokio::time::Instant;
pub mod permit_set;
const MAX_PERMIT_REFERENCES: usize = usize::MAX / 2;
/// A resource owner publishes a capacity; running units retain their permits
/// after a reduction. Lease slots are reserved once, with no per-unit Arc
/// allocation. Tokio wait queues may lazily initialize platform mutex storage
@@ -40,7 +36,6 @@ pub struct AdaptiveAdmission {
maximum: u32,
slots: Box<[PermitSlots]>,
next_chunk: AtomicUsize,
completed: AtomicU64,
}
struct PermitSlots {
@@ -66,12 +61,10 @@ pub struct AdaptivePermit {
impl Clone for AdaptivePermit {
fn clone(&self) -> Self {
// Subset carriers can retain slot references through a shared owner,
// so the slot count needs its own bound in addition to Arc's guard.
// Every slot reference owns a strong owner reference. Clone the Arc
// first, so its overflow protection also bounds the slot refcount.
let owner = Arc::clone(&self.owner);
if owner.slots[self.slot / 64].references[self.slot % 64].fetch_add(1, Ordering::Relaxed) >= MAX_PERMIT_REFERENCES {
std::process::abort();
}
owner.slots[self.slot / 64].references[self.slot % 64].fetch_add(1, Ordering::Relaxed);
Self { owner, slot: self.slot }
}
}
@@ -121,22 +114,11 @@ impl NodeAdmission {
/// Publish feedback with this owner's actual completed-work counter.
pub fn update(&self, controller: &mut FeedbackController, observation: Option<FeedbackObservation>) -> FeedbackDecision {
self.update_with_probe(controller, observation, true)
}
/// Coordinate growth with other resource controllers in the same window.
/// Protective reductions never depend on permission to probe.
pub fn update_with_probe(
&self,
controller: &mut FeedbackController,
observation: Option<FeedbackObservation>,
allow_probe: bool,
) -> FeedbackDecision {
let observation = observation.map(|mut observation| {
observation.completed_units = self.completed();
observation
});
controller.update_with_probe(&self.capacity, observation, allow_probe)
controller.update(&self.capacity, observation)
}
/// Current leased scheduling cap and held async units.
@@ -267,31 +249,16 @@ impl FeedbackController {
/// Consume one sample and publish a three-second lease to the same gate.
/// If this function stops being called, admission expires independently.
pub fn update(&mut self, gate: &AdaptiveAdmission, observation: Option<FeedbackObservation>) -> FeedbackDecision {
self.update_with_probe(gate, observation, true)
}
/// Allow at most the sampling owner's selected controller to probe upward.
pub fn update_with_probe(
&mut self,
gate: &AdaptiveAdmission,
observation: Option<FeedbackObservation>,
allow_probe: bool,
) -> FeedbackDecision {
let observation = observation.map(|mut observation| {
observation.max_concurrency = observation.max_concurrency.min(gate.maximum);
observation
});
let decision = self.decide_with_probe(Instant::now(), observation, allow_probe);
let decision = self.decide(Instant::now(), observation);
gate.set_sampled_limit(decision.limit, Duration::from_secs(3));
decision
}
fn decide_with_probe(
&mut self,
now: Instant,
observation: Option<FeedbackObservation>,
allow_probe: bool,
) -> FeedbackDecision {
fn decide(&mut self, now: Instant, observation: Option<FeedbackObservation>) -> FeedbackDecision {
let Some(observation) = observation else {
self.previous = None;
self.stable_since = None;
@@ -336,8 +303,7 @@ impl FeedbackController {
FeedbackReason::NewBaseline
} else {
let since = *self.stable_since.get_or_insert(now);
if allow_probe
&& advanced
if advanced
&& now.saturating_duration_since(since) >= Duration::from_secs(8)
&& now.saturating_duration_since(self.last_change) >= Duration::from_secs(8)
&& self.limit < observation.max_concurrency
@@ -359,13 +325,6 @@ impl FeedbackController {
}
impl AdaptiveAdmission {
/// Successful async operations using this physical owner. Native clones
/// retain capacity but never count another completion.
pub fn completed(&self) -> Option<u64> {
let value = self.completed.load(Ordering::Acquire);
(value != u64::MAX).then_some(value)
}
/// Construct a bounded owner, with zero initial capacity allowed for unknown
/// input. The immutable maximum reserves at most 65,535 lease slots once;
/// acquiring or cloning a permit does not allocate a per-unit Arc.
@@ -392,7 +351,6 @@ impl AdaptiveAdmission {
maximum: maximum_u32,
slots,
next_chunk: AtomicUsize::new(0),
completed: AtomicU64::new(0),
})
}
@@ -508,21 +466,6 @@ impl AdaptiveAdmission {
self.slots[slot / 64].free.fetch_or(1_u64 << (slot % 64), Ordering::Release);
}
fn release_reference(&self, slot: usize) {
self.release_reference_with(slot, || {});
}
fn release_reference_with(&self, slot: usize, before_release: impl FnOnce()) {
if self.slots[slot / 64].references[slot % 64].fetch_sub(1, Ordering::AcqRel) != 1 {
return;
}
before_release();
// Publish the free slot before exposing another admission capacity.
self.release_slot(slot);
self.state.fetch_sub(1, Ordering::AcqRel);
self.changed.notify_one();
}
/// Wait for capacity. The future owns no permit until it returns; dropping
/// a pending future leaves capacity untouched. Wrap in the caller's existing
/// cancellation/deadline select before acquiring any storage commit locks.
@@ -550,7 +493,13 @@ impl AdaptiveAdmission {
impl Drop for AdaptivePermit {
fn drop(&mut self) {
self.owner.release_reference(self.slot);
if self.owner.slots[self.slot / 64].references[self.slot % 64].fetch_sub(1, Ordering::AcqRel) != 1 {
return;
}
// Publish the free slot before exposing another admission capacity.
self.owner.release_slot(self.slot);
self.owner.state.fetch_sub(1, Ordering::AcqRel);
self.owner.changed.notify_one();
}
}
@@ -912,24 +861,6 @@ mod tests {
}
}
#[tokio::test(start_paused = true)]
async fn shared_probe_permission_blocks_growth_but_never_protective_cuts() {
let gate = AdaptiveAdmission::new(NonZeroU16::new(8).expect("bound"), 0);
let mut controller = FeedbackController::new();
controller.update_with_probe(&gate, Some(healthy(0)), false);
for completed in 1..=20 {
tokio::time::advance(Duration::from_secs(1)).await;
assert_eq!(controller.update_with_probe(&gate, Some(healthy(completed)), false).limit, 1);
}
tokio::time::advance(Duration::from_secs(1)).await;
assert_eq!(controller.update_with_probe(&gate, Some(healthy(21)), true).reason, FeedbackReason::Probe);
assert_eq!(gate.snapshot().limit, 2);
let mut busy = healthy(22);
busy.pressure = ResourcePressure::Busy;
assert_eq!(controller.update_with_probe(&gate, Some(busy), false).limit, 1);
assert_eq!(controller.update_with_probe(&gate, None, false).limit, 0);
}
#[tokio::test(start_paused = true)]
async fn controller_grows_one_unit_per_cooldown_and_cuts_pressure_immediately() {
let gate = AdaptiveAdmission::new(NonZeroU16::new(8).expect("bounded test owner"), 0);
File diff suppressed because it is too large Load Diff
-12
View File
@@ -30,16 +30,6 @@ pub const COMMENT_KEY: &str = "comment";
pub const ENV_AUDIT_ENABLE: &str = "RUSTFS_AUDIT_ENABLE";
/// Global switch for enabling the notify module.
pub const ENV_NOTIFY_ENABLE: &str = "RUSTFS_NOTIFY_ENABLE";
/// Maximum audit entries per second for authorization denials of authenticated callers.
/// `0` disables denial auditing for that class of caller.
pub const ENV_AUDIT_DENIAL_RATE: &str = "RUSTFS_AUDIT_DENIAL_RATE";
/// Maximum audit entries per second for authorization denials of anonymous callers.
/// `0` disables denial auditing for that class of caller.
pub const ENV_AUDIT_DENIAL_ANONYMOUS_RATE: &str = "RUSTFS_AUDIT_DENIAL_ANONYMOUS_RATE";
/// Default denial audit rate for authenticated callers (entries per second).
pub const DEFAULT_AUDIT_DENIAL_RATE: u64 = 50;
/// Default denial audit rate for anonymous callers (entries per second).
pub const DEFAULT_AUDIT_DENIAL_ANONYMOUS_RATE: u64 = 5;
/// Default global audit switch (disabled by default).
pub const DEFAULT_AUDIT_ENABLE: bool = false;
/// Default global notify switch (disabled by default).
@@ -338,7 +328,5 @@ mod tests {
fn test_global_audit_notify_switch_constants() {
assert_eq!(ENV_AUDIT_ENABLE, "RUSTFS_AUDIT_ENABLE");
assert_eq!(ENV_NOTIFY_ENABLE, "RUSTFS_NOTIFY_ENABLE");
assert_eq!(ENV_AUDIT_DENIAL_RATE, "RUSTFS_AUDIT_DENIAL_RATE");
assert_eq!(ENV_AUDIT_DENIAL_ANONYMOUS_RATE, "RUSTFS_AUDIT_DENIAL_ANONYMOUS_RATE");
}
}
+3 -3
View File
@@ -30,10 +30,10 @@ pub const ENV_TRUSTED_PROXY_VALIDATION_MODE: &str = "RUSTFS_TRUSTED_PROXY_VALIDA
/// Default validation mode is "hop_by_hop".
pub const DEFAULT_TRUSTED_PROXY_VALIDATION_MODE: &str = "hop_by_hop";
/// Select RFC 7239 "Forwarded" instead of "X-Forwarded-For" in legacy mode.
/// Environment variable to enable RFC 7239 "Forwarded" header support.
pub const ENV_TRUSTED_PROXY_ENABLE_RFC7239: &str = "RUSTFS_TRUSTED_PROXY_ENABLE_RFC7239";
/// Legacy mode uses X-Forwarded-For unless RFC 7239 is explicitly selected.
pub const DEFAULT_TRUSTED_PROXY_ENABLE_RFC7239: bool = false;
/// RFC 7239 support is enabled by default.
pub const DEFAULT_TRUSTED_PROXY_ENABLE_RFC7239: bool = true;
/// Environment variable for the maximum allowed proxy hops.
pub const ENV_TRUSTED_PROXY_MAX_HOPS: &str = "RUSTFS_TRUSTED_PROXY_MAX_HOPS";
-6
View File
@@ -146,9 +146,3 @@ pub const ENV_TARGET_STORE_COMPRESS: &str = "RUSTFS_TARGET_STORE_COMPRESS";
/// Queue-store compression is enabled by default to reduce disk footprint.
pub const DEFAULT_TARGET_STORE_COMPRESS: bool = true;
/// Maximum on-disk bytes in each target queue, including failed entries and pending writes.
pub const ENV_TARGET_STORE_MAX_BYTES: &str = "RUSTFS_TARGET_STORE_MAX_BYTES";
/// Queue writes fail at one GiB unless the operator sets another positive byte budget.
pub const DEFAULT_TARGET_STORE_MAX_BYTES: u64 = 1024 * 1024 * 1024;
+1 -6
View File
@@ -175,17 +175,12 @@ pub const ENV_HTTP_SOCKET_RECV_BUFFER_BYTES: &str = "RUSTFS_HTTP_SOCKET_RECV_BUF
pub const DEFAULT_HTTP_SOCKET_RECV_BUFFER_BYTES: usize = 0;
/// Environment variable for the S3 request-body inter-chunk read timeout
/// (seconds). Default: 300. Set to 0 to disable the inactivity timeout.
/// (seconds). Default: 300. Set to 0 to disable.
///
/// This bounds how long a `PutObject`/upload may wait for the *next* body chunk
/// while more bytes are still expected. It resets on every chunk, so it does not
/// penalize slow-but-progressing uploads — it only fires when a peer sends a
/// partial body and then goes silent *without* closing the connection (no EOF).
/// Independently of this knob, `PutObject` and `UploadPart` require at least
/// 64 KiB of body progress per 60 seconds of server wait (the minimum-progress
/// floor, logged as `upload_body_progress_stalled`); it bounds trickle uploads
/// that would otherwise pin upload admission indefinitely, and it stays in
/// force when this timeout is set to 0.
/// A reverse proxy or CDN that forwards a truncated body this way would
/// otherwise make RustFS wait forever for bytes that never arrive; with this
/// timeout the stalled read is aborted and logged with the received/expected
+53 -102
View File
@@ -24,14 +24,6 @@ use std::{
time::{Duration, SystemTime},
};
/// Copy an existing cache-model value without including surrounding work.
#[inline]
pub fn clone_cache_value<T: ToOwned + ?Sized>(source: &T) -> T::Owned {
#[cfg(feature = "hotpath-alloc")]
let _copy_scope = hotpath::MeasurementGuardSync::new("cache_model.owned_copy", false, false);
source.to_owned()
}
/// Maximum amount a persisted `last_update` may lead the local wall clock before the
/// persisted timestamp is treated as untrustworthy.
///
@@ -350,7 +342,7 @@ impl UnknownTierStats {
self.diagnostics_dropped = checked_saturating_add(1, self.diagnostics_dropped, &mut self.counter_overflowed);
continue;
}
self.diagnostics.push(clone_cache_value(diagnostic));
self.diagnostics.push(diagnostic.clone());
}
}
if !self.diagnostics.is_empty() {
@@ -375,7 +367,7 @@ impl UnknownTierStats {
if !self.fits_add(other) {
return None;
}
let mut merged = clone_cache_value(self);
let mut merged = self.clone();
merged.merge(other);
Some(merged)
}
@@ -415,7 +407,7 @@ impl AllTierStats {
if st.is_empty() {
continue;
}
let entry = self.tiers.entry(clone_cache_value(tier)).or_default();
let entry = self.tiers.entry(tier.clone()).or_default();
*entry = entry.add(st);
}
}
@@ -445,7 +437,7 @@ impl AllTierStats {
.tiers
.keys()
.filter(|tier| tier.as_str() != "STANDARD" && tier.as_str() != "REDUCED_REDUNDANCY" && !known.contains(tier.as_str()))
.map(clone_cache_value)
.cloned()
.collect();
for tier in retired_tiers {
if let Some(stats) = self.tiers.remove(&tier) {
@@ -1245,10 +1237,10 @@ impl DataUsageEntry {
}
pub fn add_child(&mut self, hash: &DataUsageHash) {
if self.children.contains(&clone_cache_value(&hash.0)) {
if self.children.contains(&hash.key()) {
return;
}
self.children.insert(clone_cache_value(&hash.0));
self.children.insert(hash.key());
}
pub fn merge(&mut self, other: &DataUsageEntry) {
@@ -1265,7 +1257,7 @@ impl DataUsageEntry {
s_rep.replica_size += o_rep.replica_size;
s_rep.replica_count += o_rep.replica_count;
for (arn, stat) in o_rep.targets.iter() {
let st = s_rep.targets.entry(clone_cache_value(arn)).or_default();
let st = s_rep.targets.entry(arn.clone()).or_default();
st.pending_size += stat.pending_size;
st.replicated_size += stat.replicated_size;
st.failed_size += stat.failed_size;
@@ -1459,7 +1451,7 @@ pub fn prefix_usage_in_cache(
} else {
format!("{bucket}/{prefix}")
};
let entry = clone_cache_value(cache.get(&clone_cache_value(&hash_path(&root).0))?);
let entry = cache.get(&hash_path(&root).key())?.clone();
let usage = PrefixUsageSummary::from_entry(&flatten_entry(cache, &entry, 0)?);
@@ -1602,38 +1594,38 @@ pub struct DataUsageCache {
impl DataUsageCache {
pub fn replace(&mut self, path: &str, parent: &str, e: DataUsageEntry) {
let hash = hash_path(path);
self.cache.insert(clone_cache_value(&hash.0), e);
self.cache.insert(hash.key(), e);
if !parent.is_empty() {
let phash = hash_path(parent);
let p = {
let p = self.cache.entry(clone_cache_value(&phash.0)).or_default();
let p = self.cache.entry(phash.key()).or_default();
p.add_child(&hash);
clone_cache_value(p)
p.clone()
};
self.cache.insert(clone_cache_value(&phash.0), p);
self.cache.insert(phash.key(), p);
}
}
pub fn replace_hashed(&mut self, hash: &DataUsageHash, parent: &Option<DataUsageHash>, e: &DataUsageEntry) {
self.cache.insert(clone_cache_value(&hash.0), clone_cache_value(e));
self.cache.insert(hash.key(), e.clone());
if let Some(parent) = parent {
self.cache.entry(clone_cache_value(&parent.0)).or_default().add_child(hash);
self.cache.entry(parent.key()).or_default().add_child(hash);
}
}
pub fn find(&self, path: &str) -> Option<DataUsageEntry> {
self.cache.get(&clone_cache_value(&hash_path(path).0)).map(clone_cache_value)
self.cache.get(&hash_path(path).key()).cloned()
}
pub fn find_children_copy(&mut self, h: DataUsageHash) -> DataUsageHashMap {
clone_cache_value(&self.cache.entry(clone_cache_value(&h.0)).or_default().children)
self.cache.entry(h.string()).or_default().children.clone()
}
pub fn flatten(&self, root: &DataUsageEntry) -> DataUsageEntry {
let mut root = clone_cache_value(root);
for id in clone_cache_value(&root.children).iter() {
let mut root = root.clone();
for id in root.children.clone().iter() {
if let Some(e) = self.cache.get(id) {
let mut e = clone_cache_value(e);
let mut e = e.clone();
if !e.children.is_empty() {
e = self.flatten(&e);
}
@@ -1645,16 +1637,16 @@ impl DataUsageCache {
}
pub fn copy_with_children(&mut self, src: &DataUsageCache, hash: &DataUsageHash, parent: &Option<DataUsageHash>) {
if let Some(e) = src.cache.get(&clone_cache_value(&hash.0)) {
self.cache.insert(clone_cache_value(&hash.0), clone_cache_value(e));
if let Some(e) = src.cache.get(&hash.string()) {
self.cache.insert(hash.key(), e.clone());
for ch in e.children.iter() {
if *ch == clone_cache_value(&hash.0) {
if *ch == hash.key() {
return;
}
self.copy_with_children(src, &DataUsageHash(clone_cache_value(ch)), &Some(clone_cache_value(hash)));
self.copy_with_children(src, &DataUsageHash(ch.to_string()), &Some(hash.clone()));
}
if let Some(parent) = parent {
let p = self.cache.entry(clone_cache_value(&parent.0)).or_default();
let p = self.cache.entry(parent.key()).or_default();
p.add_child(hash);
}
}
@@ -1662,14 +1654,14 @@ impl DataUsageCache {
pub fn delete_recursive(&mut self, hash: &DataUsageHash) {
let mut need_remove = Vec::new();
if let Some(v) = self.cache.get(&clone_cache_value(&hash.0)) {
if let Some(v) = self.cache.get(&hash.string()) {
for child in v.children.iter() {
need_remove.push(clone_cache_value(child));
need_remove.push(child.clone());
}
}
self.cache.remove(&clone_cache_value(&hash.0));
self.cache.remove(&hash.string());
need_remove.iter().for_each(|child| {
self.delete_recursive(&DataUsageHash(clone_cache_value(child)));
self.delete_recursive(&DataUsageHash(child.to_string()));
});
}
@@ -1690,7 +1682,7 @@ impl DataUsageCache {
}
pub fn search_parent(&self, hash: &DataUsageHash) -> Option<DataUsageHash> {
let want = clone_cache_value(&hash.0);
let want = hash.key();
if let Some(last_index) = want.rfind('/')
&& let Some(v) = self.find(&want[0..last_index])
&& v.children.contains(&want)
@@ -1701,7 +1693,7 @@ impl DataUsageCache {
for (k, v) in self.cache.iter() {
if v.children.contains(&want) {
let found = DataUsageHash(clone_cache_value(k));
let found = DataUsageHash(k.clone());
return Some(found);
}
}
@@ -1709,7 +1701,7 @@ impl DataUsageCache {
}
pub fn is_compacted(&self, hash: &DataUsageHash) -> bool {
match self.cache.get(&clone_cache_value(&hash.0)) {
match self.cache.get(&hash.key()) {
Some(due) => due.compacted,
None => false,
}
@@ -1734,7 +1726,7 @@ impl DataUsageCache {
if self.cache.len() < limit {
return;
}
let top = clone_cache_value(&hash_path(&self.info.name).0);
let top = hash_path(&self.info.name).key();
let top_e = match self.find(&top) {
Some(e) => e,
None => return,
@@ -1761,7 +1753,7 @@ impl DataUsageCache {
}
pub fn reduce_children_of(&mut self, path: &DataUsageHash, limit: usize, compact_self: bool) {
let e = match self.cache.get(&clone_cache_value(&path.0)) {
let e = match self.cache.get(&path.key()) {
Some(e) => e,
None => return,
};
@@ -1771,13 +1763,13 @@ impl DataUsageCache {
}
if e.children.len() > limit && compact_self {
let mut flat = self.size_recursive(&clone_cache_value(&path.0)).unwrap_or_default();
let mut flat = self.size_recursive(&path.key()).unwrap_or_default();
flat.compacted = true;
self.delete_recursive(path);
self.replace_hashed(path, &None, &flat);
return;
}
let total = self.total_children_rec(&clone_cache_value(&path.0));
let total = self.total_children_rec(&path.key());
if total < limit {
return;
}
@@ -1790,12 +1782,12 @@ impl DataUsageCache {
let mut candidate_index = 0;
while remove > 0 && candidate_index < candidates.len() {
let e = &candidates[candidate_index];
let candidate = clone_cache_value(&e.path);
let candidate = e.path.clone();
if candidate == *path && !compact_self {
break;
}
let removing = self.total_children_rec(&clone_cache_value(&candidate.0));
let mut flat = match self.size_recursive(&clone_cache_value(&candidate.0)) {
let removing = self.total_children_rec(&candidate.key());
let mut flat = match self.size_recursive(&candidate.key()) {
Some(flat) => flat,
None => {
candidate_index += 1;
@@ -1832,7 +1824,7 @@ impl DataUsageCache {
if o.root().is_none() {
return;
}
*self = clone_cache_value(o);
*self = o.clone();
return;
};
@@ -1845,8 +1837,7 @@ impl DataUsageCache {
}
existing_root.merge(&other_root);
self.cache
.insert(clone_cache_value(&hash_path(&self.info.name).0), existing_root);
self.cache.insert(hash_path(&self.info.name).key(), existing_root);
let root_hash = self.root_hash();
for key in other_root.children.iter() {
@@ -1857,7 +1848,7 @@ impl DataUsageCache {
if let Some(existing) = self.cache.get_mut(key) {
existing.merge(&flat);
} else {
self.replace_hashed(&DataUsageHash(clone_cache_value(key)), &Some(clone_cache_value(&root_hash)), &flat);
self.replace_hashed(&DataUsageHash(key.clone()), &Some(root_hash.clone()), &flat);
}
}
}
@@ -1901,7 +1892,7 @@ impl DataUsageCache {
for (arn, stat) in rs.targets.iter() {
bui.replication_info.insert(
clone_cache_value(arn),
arn.clone(),
BucketTargetUsageInfo {
replication_pending_size: stat.pending_size,
replicated_size: stat.replicated_size,
@@ -1914,7 +1905,7 @@ impl DataUsageCache {
);
}
}
buckets_usage.insert(clone_cache_value(bucket_name), bui);
buckets_usage.insert(bucket_name.clone(), bui);
}
DataUsageInfo {
@@ -1946,20 +1937,20 @@ struct Inner {
}
fn add(data_usage_cache: &DataUsageCache, path: &DataUsageHash, candidates: &mut Vec<Inner>) -> usize {
let e = match data_usage_cache.cache.get(&clone_cache_value(&path.0)) {
let e = match data_usage_cache.cache.get(&path.key()) {
Some(e) => e,
None => return 0,
};
let mut objects = e.objects;
for ch in e.children.iter() {
objects += add(data_usage_cache, &DataUsageHash(clone_cache_value(ch)), candidates);
objects += add(data_usage_cache, &DataUsageHash(ch.clone()), candidates);
}
// Collect internal nodes (with children) as compaction candidates.
// Leaf nodes have no children to remove, so compacting them is a no-op.
if !e.children.is_empty() {
candidates.push(Inner {
objects,
path: clone_cache_value(path),
path: path.clone(),
});
}
objects
@@ -1967,7 +1958,7 @@ fn add(data_usage_cache: &DataUsageCache, path: &DataUsageHash, candidates: &mut
fn mark(duc: &DataUsageCache, entry: &DataUsageEntry, found: &mut HashSet<String>) {
for k in entry.children.iter() {
found.insert(clone_cache_value(k));
found.insert(k.to_string());
if let Some(ch) = duc.cache.get(k) {
mark(duc, ch, found);
}
@@ -2417,11 +2408,11 @@ impl SizeSummary {
}
for entry in &other.size_reconciliation {
self.record_size_reconciliation(clone_cache_value(entry));
self.record_size_reconciliation(entry.clone());
}
self.size_reconciliation_truncated |= other.size_reconciliation_truncated;
for scope in &other.reconciliation_scopes {
self.record_reconciliation_scope_with(&scope.bucket, &scope.object, || clone_cache_value(scope));
self.record_reconciliation_scope(&scope.bucket, &scope.object);
}
}
@@ -2446,13 +2437,6 @@ impl SizeSummary {
/// Mark one object scope as refreshed. Duplicate scopes are suppressed so
/// merging summaries remains bounded and deterministic.
pub fn record_reconciliation_scope(&mut self, bucket: &str, object: &str) {
self.record_reconciliation_scope_with(bucket, object, || SizeReconciliationScope {
bucket: bucket.to_string(),
object: object.to_string(),
});
}
fn record_reconciliation_scope_with(&mut self, bucket: &str, object: &str, build: impl FnOnce() -> SizeReconciliationScope) {
if !self
.reconciliation_scopes
.iter()
@@ -2462,7 +2446,10 @@ impl SizeSummary {
self.size_reconciliation_truncated = true;
return;
}
self.reconciliation_scopes.push(build());
self.reconciliation_scopes.push(SizeReconciliationScope {
bucket: bucket.to_string(),
object: object.to_string(),
});
}
}
}
@@ -3417,42 +3404,6 @@ mod tests {
assert_eq!(summary1.versions, 15);
}
#[test]
fn test_reconciliation_scope_builds_only_after_admission() {
let mut summary = SizeSummary::new();
let builds = std::cell::Cell::new(0);
let build = || {
builds.set(builds.get() + 1);
SizeReconciliationScope {
bucket: "bucket".to_string(),
object: "object".to_string(),
}
};
summary.record_reconciliation_scope_with("bucket", "object", build);
assert_eq!(builds.get(), 1, "an admitted scope is copied once");
assert_eq!(summary.reconciliation_scopes.len(), 1);
assert_eq!(summary.reconciliation_scopes[0].bucket, "bucket");
assert_eq!(summary.reconciliation_scopes[0].object, "object");
summary.record_reconciliation_scope_with("bucket", "object", build);
assert_eq!(builds.get(), 1, "duplicate scopes must not be copied");
assert_eq!(summary.reconciliation_scopes.len(), 1);
assert!(!summary.size_reconciliation_truncated);
for index in 1..1024 {
summary.record_reconciliation_scope("bucket", &index.to_string());
}
summary.record_reconciliation_scope_with("bucket", "object", build);
assert_eq!(builds.get(), 1, "a duplicate at capacity must not be copied");
assert!(!summary.size_reconciliation_truncated);
summary.record_reconciliation_scope_with("bucket", "overflow", build);
assert_eq!(builds.get(), 1, "a rejected scope must not be copied");
assert_eq!(summary.reconciliation_scopes.len(), 1024);
assert!(summary.size_reconciliation_truncated);
}
#[test]
fn test_size_histogram_compat_rollup_sums_all_sub_buckets() {
let mut hist = SizeHistogram::default();
-1
View File
@@ -25,7 +25,6 @@ workspace = true
[features]
default = []
scanner-peer-mixed-version = []
hotpath = [
"hotpath/hotpath",
"hotpath/tokio",
+3 -22
View File
@@ -26,7 +26,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
| **protocols** | [`src/protocols/`](src/protocols) | FTPS, WebDAV, SFTP compliance. Fixed ports, own guide: [`src/protocols/README.md`](src/protocols/README.md) |
| **reliant** | [`src/reliant/`](src/reliant) | Tests that reuse an **externally started** server (SQL/select, conditional writes, lifecycle, deleted-object reads, node-interact). Run via [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh); see [`src/reliant/README.md`](src/reliant/README.md) |
| **cluster** | `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `namespace_lock_quorum_test`, `admin_timeout_regression_test`, `object_lambda_test`, `replication_extension_test`, `tier_stats_cluster_test` | Multi-node scenarios via `RustFSTestClusterEnvironment` |
| **distributed 4×4** | [`src/distributed/`](src/distributed) | Nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
| **distributed 4×4** | [`src/distributed/`](src/distributed) | Storage-sensitive PR and nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
| **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
| **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
@@ -64,25 +64,6 @@ Each build/run holds an exclusive `rustfs.e2e.lock` marker beside the binary; co
The protocols suite has its own fixed-port and single-worker contract in [`src/protocols/README.md`](src/protocols/README.md). Use its command under [Troubleshooting](#troubleshooting).
### Scanner mixed-version gate
The Linux N/O scanner gate needs a separately qualified `33482bb` server, which supports scoped ACKs. Set `RUSTFS_SCANNER_COMPAT_OLD_BINARY` to that executable and `RUSTFS_SCANNER_COMPAT_OLD_SHA256` to its independently verified digest. The release binary used by ordinary upgrade CI is a different baseline. Missing or mismatched inputs fail the test.
Enable the harness-only `scanner-peer-mixed-version` feature explicitly; keep the server wrapper features identical between build and run:
```bash
python3 scripts/e2e_binary.py build --features e2e-test-hooks
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- \
cargo nextest run --profile e2e-distributed -p e2e_test \
--features scanner-peer-mixed-version --ignore-default-filter \
-E 'test(=distributed::scanner_peer_bucket_test::scanner_peer_bucket_new_coordinator_certifies_old_peers_and_scopes_next_cycle)' \
--test-threads 1 --retries 0 --no-tests=fail
```
The separate `scanner_peer_bucket_new_coordinator_certifies_old_worker_peers_and_scopes_next_cycle` case uses the same feature and qualified old binary inputs. Select its exact name in the command above. It keeps the original N process enabled and starts the three official O replacements with `RUSTFS_SCANNER_ENABLED=false`: their autonomous scanner loops are disabled, while real S3 dirty tracking, authenticated snapshots, publication leases, scoped ACKs, and N-dispatched RemoteScanner workers remain active. It records each node's actual scanner status and retains the same identity, durability, root, ACK, data-conservation, and physical-walk requirements. This fixed-role case does not validate autonomous O scanning or all-enabled election behavior. The original all-enabled N/O test remains separate and has not passed acceptance; a fixed-role result cannot satisfy that gate.
For this fixed-role case, the `e2e-test-hooks` server also records the N client's complete bounded worker inventory. Requests are observed before opening the normal stream and paired only after authenticated source/plan validation and terminal EOF. Closed manifests and final health receipts must bind both cycles to their actual candidate/root; the next cycle needs exactly four sets of hot-only execution and at least one positive-progress worker mapped to a measured official O process. A zero-progress Current response, missing/extra request, unhealthy recorder, or missing physical coverage fails the gate. The recorder does not add O hooks or change any protocol or timeout. Final health reads check immutable closed-cycle evidence, not current N liveness.
### `#[ignore]` semantics
Ignored tests are excluded from the default `cargo nextest run` pass because
@@ -194,9 +175,9 @@ the same profile for membership and execution with one nightly worker.
| `s3s-e2e` black-box | `e2e-tests` + `e2e-tests-rio-v2` jobs | **Active** (external conformance tool) |
| ILM / lifecycle (ignored) | `test-ilm-integration-serial` lane, `-j1` | **Active** (backlog#1148 ilm-1) |
| KMS suite | `e2e-full` job, merge queue + main | **Active** |
| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, release tags + weekly schedule + manual dispatch | **Active** |
| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
| Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (nightly / dispatch) |
| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (storage-sensitive PR / nightly / dispatch) |
| Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
| Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
| Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) |
@@ -502,163 +502,10 @@ mod tests {
assert!(response.contains(if expected == reqwest::StatusCode::FORBIDDEN {
"<Code>AccessDenied</Code>"
} else {
"<Code>BucketAlreadyOwnedByYou</Code>"
"<Code>BucketAlreadyExists</Code>"
}));
}
}
Ok(())
}
/// A deployment owns a single bucket namespace, so an IAM user that recreates
/// an existing bucket is the owner, not another account. In the default region
/// every such request returns the legacy idempotent `200 OK` (AWS CreateBucket
/// semantics), including buckets created by root and by another IAM user.
#[tokio::test]
async fn test_create_bucket_duplicate_in_default_region_is_idempotent_for_iam_user()
-> Result<(), Box<dyn Error + Send + Sync>> {
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec!["--region", "us-east-1"]).await?;
let root = env.create_s3_client();
admin_create_user(&env, "duplicate-iam", "duplicate-iam-secret").await?;
admin_attach_user_policy_via(
AdminTransport::Signed,
&env.url,
&env.access_key,
&env.secret_key,
"readwrite",
"duplicate-iam",
)
.await?;
let iam = Client::from_conf(build_test_s3_config(
&env.url,
"duplicate-iam",
"duplicate-iam-secret",
None,
"duplicate-create",
));
root.create_bucket()
.bucket("duplicate-root-owned")
.send()
.await
.expect("root creates the first bucket");
iam.create_bucket()
.bucket("duplicate-root-owned")
.send()
.await
.expect("an IAM user recreating a root-owned bucket in us-east-1 gets 200 OK");
iam.create_bucket()
.bucket("duplicate-iam-owned")
.send()
.await
.expect("the IAM user creates its own bucket");
iam.create_bucket()
.bucket("duplicate-iam-owned")
.send()
.await
.expect("the same IAM user recreating its bucket gets 200 OK");
root.create_bucket()
.bucket("duplicate-iam-owned")
.send()
.await
.expect("root recreating an IAM-created bucket gets 200 OK");
Ok(())
}
/// Outside the default region the same ownership answer becomes
/// `409 BucketAlreadyOwnedByYou` for every authorized requester. Requests that
/// lack `s3:CreateBucket` still fail authorization before the duplicate check.
#[tokio::test]
async fn test_create_bucket_duplicate_outside_default_region_reports_owned_by_you() -> Result<(), Box<dyn Error + Send + Sync>>
{
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec!["--region", "eu-west-1"]).await?;
let region = "eu-west-1";
let root = Client::from_conf(
env.create_s3_client()
.config()
.to_builder()
.region(Region::new(region))
.build(),
);
admin_create_user(&env, "duplicate-eu-iam", "duplicate-eu-iam-secret").await?;
admin_attach_user_policy_via(
AdminTransport::Signed,
&env.url,
&env.access_key,
&env.secret_key,
"readwrite",
"duplicate-eu-iam",
)
.await?;
let iam = Client::from_conf(
build_test_s3_config(&env.url, "duplicate-eu-iam", "duplicate-eu-iam-secret", None, "duplicate-create")
.to_builder()
.region(Region::new(region))
.build(),
);
admin_create_user(&env, "duplicate-eu-readonly", "duplicate-eu-readonly-secret").await?;
admin_attach_user_policy_via(
AdminTransport::Signed,
&env.url,
&env.access_key,
&env.secret_key,
"readonly",
"duplicate-eu-readonly",
)
.await?;
let readonly = Client::from_conf(
build_test_s3_config(
&env.url,
"duplicate-eu-readonly",
"duplicate-eu-readonly-secret",
None,
"duplicate-create",
)
.to_builder()
.region(Region::new(region))
.build(),
);
for (client, bucket) in [(&root, "duplicate-eu-root"), (&iam, "duplicate-eu-iam")] {
client
.create_bucket()
.bucket(bucket)
.send()
.await
.unwrap_or_else(|error| panic!("{bucket} must be created: {error:?}"));
// Both the creator and the other authenticated identity own the namespace.
for creator in [&root, &iam] {
let error = creator
.create_bucket()
.bucket(bucket)
.send()
.await
.expect_err("a duplicate create outside us-east-1 must report ownership");
assert_eq!(
error.as_service_error().and_then(ProvideErrorMetadata::code),
Some("BucketAlreadyOwnedByYou"),
"{bucket}"
);
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(409), "{bucket}");
}
}
let denied = readonly
.create_bucket()
.bucket("duplicate-eu-root")
.send()
.await
.expect_err("a requester without s3:CreateBucket must be denied before the duplicate check");
assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
Ok(())
}
}
@@ -16,193 +16,8 @@
mod tests {
use crate::common::{RustFSTestEnvironment, init_logging};
use aws_sdk_s3::Client;
use aws_sdk_s3::config::http::HttpResponse;
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration};
fn assert_not_found<E: ProvideErrorMetadata + std::fmt::Debug>(error: &SdkError<E, HttpResponse>) {
assert_eq!(error.raw_response().map(|r| r.status().as_u16()), Some(404), "{error:?}");
assert_eq!(
error.as_service_error().and_then(ProvideErrorMetadata::code),
Some("NoSuchKey"),
"{error:?}"
);
}
#[tokio::test]
async fn conditional_writes_cannot_replace_delete_marker_with_if_match() {
init_logging();
let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
env.start_rustfs_server(vec![]).await.expect("start RustFS");
let client = env.create_s3_client();
let bucket = "conditional-delete-marker-proof";
create_versioned_bucket(&client, bucket).await;
client
.put_object()
.bucket(bucket)
.key("source")
.body(ByteStream::from_static(b"source"))
.send()
.await
.expect("write copy source");
for operation in ["put", "copy", "multipart"] {
let old = client
.put_object()
.bucket(bucket)
.key(operation)
.body(ByteStream::from_static(b"old"))
.send()
.await
.expect("write historical version");
let old_etag = old.e_tag().expect("historical ETag");
let marker = client
.delete_object()
.bucket(bucket)
.key(operation)
.send()
.await
.expect("create delete marker");
let marker_id = marker.version_id().expect("delete marker version");
let (upload_id, completed) = if operation == "multipart" {
let upload = client
.create_multipart_upload()
.bucket(bucket)
.key(operation)
.send()
.await
.expect("initiate multipart upload");
let upload_id = upload.upload_id().expect("upload ID").to_owned();
let part = client
.upload_part()
.bucket(bucket)
.key(operation)
.upload_id(&upload_id)
.part_number(1)
.body(ByteStream::from_static(b"new"))
.send()
.await
.expect("upload part");
let completed = CompletedMultipartUpload::builder()
.parts(
CompletedPart::builder()
.part_number(1)
.set_e_tag(part.e_tag().map(str::to_owned))
.build(),
)
.build();
(Some(upload_id), Some(completed))
} else {
(None, None)
};
for etag in [old_etag, "\"00000000000000000000000000000000\""] {
match operation {
"put" => {
let err = client
.put_object()
.bucket(bucket)
.key(operation)
.if_match(etag)
.body(ByteStream::from_static(b"new"))
.send()
.await
.expect_err("If-Match must reject marker");
assert_not_found(&err);
}
"copy" => {
let err = client
.copy_object()
.bucket(bucket)
.key(operation)
.copy_source(format!("{bucket}/source"))
.if_match(etag)
.send()
.await
.expect_err("If-Match copy must reject marker");
assert_not_found(&err);
}
_ => {
let err = client
.complete_multipart_upload()
.bucket(bucket)
.key(operation)
.upload_id(upload_id.as_deref().expect("upload ID"))
.multipart_upload(completed.clone().expect("completed parts"))
.if_match(etag)
.send()
.await
.expect_err("If-Match completion must reject marker");
assert_not_found(&err);
}
}
let versions = client
.list_object_versions()
.bucket(bucket)
.prefix(operation)
.send()
.await
.expect("list versions after rejected write");
assert_eq!(versions.versions().len(), 1, "{operation}: failed write created a version");
assert_eq!(versions.delete_markers().len(), 1);
assert_eq!(versions.delete_markers()[0].version_id(), Some(marker_id));
assert_eq!(versions.delete_markers()[0].is_latest(), Some(true));
}
match operation {
"put" => {
client
.put_object()
.bucket(bucket)
.key(operation)
.if_none_match("*")
.body(ByteStream::from_static(b"new"))
.send()
.await
.expect("If-None-Match PUT over marker");
}
"copy" => {
client
.copy_object()
.bucket(bucket)
.key(operation)
.copy_source(format!("{bucket}/source"))
.if_none_match("*")
.send()
.await
.expect("If-None-Match copy over marker");
}
_ => {
client
.complete_multipart_upload()
.bucket(bucket)
.key(operation)
.upload_id(upload_id.as_deref().expect("upload ID"))
.multipart_upload(completed.expect("completed parts"))
.if_none_match("*")
.send()
.await
.expect("If-None-Match completion over marker");
}
}
let current = client
.get_object()
.bucket(bucket)
.key(operation)
.send()
.await
.expect("read successful conditional write")
.body
.collect()
.await
.expect("read object body")
.into_bytes();
let expected: &[u8] = if operation == "copy" { b"source" } else { b"new" };
assert_eq!(current.as_ref(), expected, "{operation}: unexpected current object");
}
}
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
async fn create_versioned_bucket(client: &Client, bucket: &str) {
client
@@ -12,249 +12,11 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use super::harness::{DistCluster, DistLayout, TestResult, payload_for, s3_probe_client, sha256_hex, unique_bucket};
use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, unique_bucket};
use crate::common::init_logging;
use aws_sdk_s3::{
Client,
error::{ProvideErrorMetadata, SdkError},
primitives::ByteStream,
};
use std::any::TypeId;
use std::collections::BTreeSet;
use std::fmt::Debug;
use std::future::Future;
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::Barrier;
use tokio::time::Instant;
const LOCK_ADMISSION_BUDGET: Duration = Duration::from_secs(45);
async fn after_lock_contention<T, E, F, Fut>(operation: &str, mut call: F) -> TestResult<Result<T, SdkError<E>>>
where
E: Debug + ProvideErrorMetadata + 'static,
F: FnMut() -> Fut,
Fut: Future<Output = Result<T, SdkError<E>>>,
{
let started = Instant::now();
let mut attempts = 0;
// The workload uses unique unversioned keys. Repeat only explicitly
// advertised lock backpressure; every step still needs its real result.
let outcome = tokio::time::timeout(LOCK_ADMISSION_BUDGET, async {
loop {
attempts += 1;
let result = call().await;
let retry = result.as_ref().err().is_some_and(|error| {
let Some(response) = error.raw_response() else {
return false;
};
let Some(service_error) = error.as_service_error() else {
return false;
};
// HEAD has no XML error body, so its SDK service error can
// lack a code even when the server supplies retry headers.
let namespace_code = service_error.code() == Some("ServiceUnavailable")
|| (service_error.code().is_none()
&& TypeId::of::<E>() == TypeId::of::<aws_sdk_s3::operation::head_object::HeadObjectError>());
response.status().as_u16() == 503
&& namespace_code
&& response.headers().get("x-rustfs-error-cause") == Some("namespace_lock_timeout")
&& response.headers().get("retry-after") == Some("1")
});
if !retry {
return result;
}
tokio::time::sleep(Duration::from_secs(1)).await;
}
})
.await;
if attempts > 1 || !matches!(&outcome, Ok(Ok(_))) {
eprintln!(
"mixed workload request: operation={operation}, attempts={attempts}, elapsed={:?}",
started.elapsed()
);
}
outcome.map_err(|_| {
format!("{operation} exceeded its {LOCK_ADMISSION_BUDGET:?} request budget after {attempts} attempts").into()
})
}
async fn assert_workload_bytes(client: &Client, bucket: &str, key: &str, expected: &[u8]) -> TestResult {
let output = after_lock_contention(&format!("GET {key}"), || client.get_object().bucket(bucket).key(key).send()).await??;
let actual = output.body.collect().await?.into_bytes();
if actual.as_ref() != expected {
return Err(format!(
"object {key} bytes mismatch: expected {} bytes sha256={} got {} bytes sha256={}",
expected.len(),
sha256_hex(expected),
actual.len(),
sha256_hex(&actual)
)
.into());
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use aws_smithy_http_client::test_util::infallible_client_fn;
use aws_smithy_types::body::SdkBody;
use std::sync::atomic::{AtomicUsize, Ordering};
type Reply = (u16, &'static str, Option<&'static str>, Option<&'static str>, &'static str);
fn client(respond: impl Fn(usize) -> Reply + Send + Sync + 'static) -> (Client, Arc<AtomicUsize>) {
let requests = Arc::new(AtomicUsize::new(0));
let seen = Arc::clone(&requests);
let transport = infallible_client_fn(move |_| {
let (status, code, cause, retry_after, body) = respond(seen.fetch_add(1, Ordering::SeqCst));
let body = if code.is_empty() {
body.to_string()
} else {
format!("<Error><Code>{code}</Code></Error>")
};
let mut response = http::Response::builder()
.status(status)
.header("content-type", "application/xml")
.header("content-length", body.len());
if let Some(cause) = cause {
response = response.header("x-rustfs-error-cause", cause);
}
if let Some(retry_after) = retry_after {
response = response.header("retry-after", retry_after);
}
response.body(SdkBody::from(body)).expect("scripted S3 response")
});
let config = crate::common::build_test_s3_config(
"http://contention.example.invalid",
"synthetic-access",
"synthetic-secret",
None,
"contention-test",
);
let client = Client::from_conf(config.to_builder().http_client(transport).build());
(s3_probe_client(&client), requests)
}
fn contention() -> Reply {
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("1"), "")
}
#[tokio::test(start_paused = true)]
async fn advertised_contention_recovers_after_standard_sdk_attempts_are_exhausted() -> TestResult {
let (client, requests) = client(|attempt| {
if attempt < 3 {
contention()
} else {
(200, "", None, None, "")
}
});
let standard = Client::from_conf(
client
.config()
.to_builder()
.retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(3))
.build(),
);
standard
.put_object()
.bucket("bucket")
.key("key")
.body(ByteStream::from_static(b"body"))
.send()
.await
.expect_err("three advertised lock responses exhaust the standard SDK policy");
assert_eq!(requests.load(Ordering::SeqCst), 3);
requests.store(0, Ordering::SeqCst);
let started = Instant::now();
after_lock_contention("PUT recovery", || {
client
.put_object()
.bucket("bucket")
.key("key")
.body(ByteStream::from_static(b"body"))
.send()
})
.await??;
assert_eq!(requests.load(Ordering::SeqCst), 4);
assert!(started.elapsed() >= Duration::from_secs(3), "honor each Retry-After response");
Ok(())
}
#[tokio::test(start_paused = true)]
async fn bodyless_head_contention_uses_explicit_retry_headers() -> TestResult {
let (client, requests) = client(|attempt| {
if attempt == 0 {
(503, "", Some("namespace_lock_timeout"), Some("1"), "")
} else {
(200, "", None, None, "")
}
});
after_lock_contention("HEAD recovery", || client.head_object().bucket("bucket").key("key").send()).await??;
assert_eq!(requests.load(Ordering::SeqCst), 2);
Ok(())
}
#[tokio::test(start_paused = true)]
async fn unrelated_or_malformed_responses_are_not_retried() -> TestResult {
for reply in [
(500, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
(403, "AccessDenied", None, None, ""),
(404, "NoSuchKey", None, None, ""),
(503, "ServiceUnavailable", Some("storage_write_quorum"), None, ""),
(503, "ServiceUnavailable", Some("namespace_lock_unavailable"), Some("1"), ""),
(503, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
(503, "", Some("namespace_lock_timeout"), Some("1"), ""),
(503, "ServiceUnavailable", None, Some("1"), ""),
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), None, ""),
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("2"), ""),
] {
let (client, requests) = client(move |_| reply);
let result = after_lock_contention("PUT fatal response", || {
client
.put_object()
.bucket("bucket")
.key("key")
.body(ByteStream::from_static(b"body"))
.send()
})
.await?;
assert!(result.is_err(), "fatal response cannot become success: {reply:?}");
assert_eq!(requests.load(Ordering::SeqCst), 1, "fatal response must remain visible: {reply:?}");
}
Ok(())
}
#[tokio::test(start_paused = true)]
async fn persistent_lock_contention_exhausts_the_request_budget() {
let (client, requests) = client(|_| contention());
let started = Instant::now();
let result = after_lock_contention("PUT unavailable", || {
client
.put_object()
.bucket("bucket")
.key("key")
.body(ByteStream::from_static(b"body"))
.send()
})
.await;
assert!(result.is_err(), "persistent contention cannot acknowledge an operation");
assert!(started.elapsed() >= LOCK_ADMISSION_BUDGET);
assert!(requests.load(Ordering::SeqCst) > 1);
assert!(requests.load(Ordering::SeqCst) <= LOCK_ADMISSION_BUDGET.as_secs() as usize + 1);
}
#[tokio::test(start_paused = true)]
async fn acknowledged_wrong_bytes_fail_without_retry() {
let (client, requests) = client(|_| (200, "", None, None, "wrong bytes"));
let error = assert_workload_bytes(&client, "bucket", "key", b"expected bytes")
.await
.expect_err("a successful but incorrect read is a consistency failure");
assert!(error.to_string().contains("bytes mismatch"));
assert_eq!(requests.load(Ordering::SeqCst), 1);
}
}
#[tokio::test]
async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node() -> TestResult {
@@ -262,9 +24,7 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
let dist = DistCluster::start(DistLayout::FourByFour).await?;
let bucket = unique_bucket("conc");
dist.create_bucket(&bucket).await?;
// This consistency probe owns its admission policy; unrelated responses
// must not be hidden by the SDK's general-purpose retries.
let clients = Arc::new(dist.clients()?.iter().map(s3_probe_client).collect::<Vec<_>>());
let clients = Arc::new(dist.clients()?);
let worker_count = 24;
let rounds = 4;
let barrier = Arc::new(Barrier::new(worker_count));
@@ -284,46 +44,31 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
let key = format!("source/worker-{idx:02}-round-{round}.bin");
let copy_key = format!("retained/worker-{idx:02}-round-{round}.bin");
let body = payload_for(&key, 64 * 1024);
let context = format!("worker={idx} round={round}");
after_lock_contention(&format!("{context} PUT source"), || {
writer
.put_object()
.bucket(&bucket)
.key(&key)
.body(ByteStream::from(body.clone()))
.send()
})
.await??;
put_object(writer, &bucket, &key, body.clone()).await?;
let head = after_lock_contention(&format!("{context} HEAD source"), || {
reader.head_object().bucket(&bucket).key(&key).send()
})
.await??;
let head = reader.head_object().bucket(&bucket).key(&key).send().await?;
if head.content_length() != Some(body.len() as i64) {
return Err(format!("HEAD returned the wrong size for {key}: {head:?}").into());
}
assert_workload_bytes(reader, &bucket, &key, &body).await?;
assert_object_bytes(reader, &bucket, &key, &body).await?;
after_lock_contention(&format!("{context} COPY retained"), || {
copier
.copy_object()
.bucket(&bucket)
.key(&copy_key)
.copy_source(format!("{bucket}/{key}"))
.send()
})
.await??;
assert_workload_bytes(writer, &bucket, &copy_key, &body).await?;
copier
.copy_object()
.bucket(&bucket)
.key(&copy_key)
.copy_source(format!("{bucket}/{key}"))
.send()
.await?;
assert_object_bytes(writer, &bucket, &copy_key, &body).await?;
after_lock_contention(&format!("{context} DELETE source"), || {
writer.delete_object().bucket(&bucket).key(&key).send()
})
.await??;
let missing = after_lock_contention(&format!("{context} HEAD deleted source"), || {
reader.head_object().bucket(&bucket).key(&key).send()
})
.await?
.expect_err("deleted source key must not remain visible");
writer.delete_object().bucket(&bucket).key(&key).send().await?;
let missing = reader
.head_object()
.bucket(&bucket)
.key(&key)
.send()
.await
.expect_err("deleted source key must not remain visible");
if missing.raw_response().map(|response| response.status().as_u16()) != Some(404) {
return Err(format!("deleted source {key} returned an unexpected result: {missing:?}").into());
}
@@ -337,22 +82,14 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
for handle in handles {
inventory.extend(handle.await??);
}
assert_eq!(
inventory.len(),
worker_count * rounds,
"every worker round must retain its acknowledged copy"
);
let expected_keys: BTreeSet<_> = inventory.iter().map(|(key, _)| key.as_str()).collect();
for (node_idx, client) in clients.iter().enumerate() {
let listed = after_lock_contention(&format!("node={node_idx} LIST retained"), || {
client.list_objects_v2().bucket(&bucket).prefix("retained/").send()
})
.await??;
let listed = client.list_objects_v2().bucket(&bucket).prefix("retained/").send().await?;
let listed_keys: BTreeSet<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
assert_eq!(listed_keys, expected_keys, "node {node_idx} returned a divergent retained-key listing");
for (key, body) in &inventory {
assert_workload_bytes(client, &bucket, key, body)
assert_object_bytes(client, &bucket, key, body)
.await
.map_err(|error| format!("node {node_idx} failed to read {key}: {error}"))?;
}
-2
View File
@@ -39,8 +39,6 @@ mod s3_during_data_movement_test;
mod scanner_heal_topology_fixture;
mod scanner_heal_topology_test;
mod scanner_heal_topology_tier_test;
mod scanner_peer_bucket_fixture;
mod scanner_peer_bucket_test;
mod site_replication_test;
mod table_catalog_test;
mod upgrade_test;
@@ -18,7 +18,7 @@ use super::harness::{
};
use crate::common::{ClusterTopology, RustFSTestClusterEnvironment};
use crate::rpc_fault_proxy::{RpcFaultProxy, RpcPause};
use crate::storage_api::multipart_fence_acceptance::{LOCK_DIAGNOSTICS_FILTER, RUSTFS_META_MULTIPART_BUCKET};
use crate::storage_api::multipart_fence_acceptance::RUSTFS_META_MULTIPART_BUCKET;
use aws_sdk_s3::{
Client,
config::retry::RetryConfig,
@@ -38,16 +38,14 @@ const DEADLINE: Duration = Duration::from_secs(45);
async fn fixture(
drives_per_node: usize,
storage_class: &str,
lock_diagnostics: bool,
) -> TestResult<(RustFSTestClusterEnvironment, Vec<RpcFaultProxy>, Vec<Client>, String)> {
fixture_from_binary(drives_per_node, storage_class, None, lock_diagnostics).await
fixture_from_binary(drives_per_node, storage_class, None).await
}
async fn fixture_from_binary(
drives_per_node: usize,
storage_class: &str,
binary: Option<&std::path::Path>,
lock_diagnostics: bool,
) -> TestResult<(RustFSTestClusterEnvironment, Vec<RpcFaultProxy>, Vec<Client>, String)> {
let mut cluster =
RustFSTestClusterEnvironment::with_topology(ClusterTopology::single_pool_multidrive(4, drives_per_node)).await?;
@@ -92,10 +90,6 @@ async fn fixture_from_binary(
] {
cluster.set_env(name, value);
}
if lock_diagnostics {
cluster.set_env("RUST_LOG", LOCK_DIAGNOSTICS_FILTER);
cluster.set_env("RUSTFS_OBJECT_LOCK_DIAG_ENABLE", "true");
}
let mut proxies = Vec::new();
if binary.is_none() {
for node in 0..4 {
@@ -369,120 +363,26 @@ fn upload_path_scope(bucket: &str, key: &str, upload_id: &str) -> TestResult<Str
))
}
async fn with_paused_peers<T, E>(
async fn with_paused_peers<T: Send + 'static>(
proxies: &[RpcFaultProxy],
methods: &[&'static str],
limits: &[usize],
scope: &str,
request: impl std::future::Future<Output = Result<T, SdkError<E>>> + Send + 'static,
) -> TestResult<Result<T, SdkError<E>>>
where
T: Send + 'static,
E: std::error::Error + ProvideErrorMetadata + Send + Sync + 'static,
{
let deadline = tokio::time::Instant::now() + DEADLINE;
request: impl std::future::Future<Output = T> + Send + 'static,
) -> TestResult<T> {
let pauses: Vec<_> = proxies[1..=limits.len()]
.iter()
.zip(limits)
.map(|(proxy, limit)| proxy.pause(methods, *limit, Some(scope)))
.collect::<Result<_, _>>()?;
let mut task = tokio::spawn(request);
let mut joined = None;
let mut deadline_expired = false;
let mut barrier_result = {
let faults = wait_for_faults(&pauses, methods);
tokio::pin!(faults);
match tokio::time::timeout_at(deadline, async {
tokio::select! {
result = &mut faults => result,
result = &mut task => {
joined = Some(result);
// A ready request is not evidence that every selected fault
// was reached. Poll the original barrier once for that proof.
match futures::poll!(faults.as_mut()) {
std::task::Poll::Ready(result) => result,
std::task::Poll::Pending => Err("SDK request completed before the selected RPC fault barrier".into()),
}
}
}
})
.await
{
Ok(result) => result,
Err(error) => {
deadline_expired = true;
Err(std::io::Error::new(std::io::ErrorKind::TimedOut, error).into())
}
}
};
// Tokio's timeout polls an already-ready inner future before its timer.
// A late observation cannot certify the original absolute deadline.
deadline_expired |= tokio::time::Instant::now() >= deadline;
if deadline_expired {
barrier_result = Err(std::io::Error::new(
std::io::ErrorKind::TimedOut,
"selected RPC fault proof observed after the absolute MPU deadline",
)
.into());
}
if barrier_result.is_err() {
for pause in &pauses {
pause.discard_pending();
}
}
let mut client_task_aborted = false;
let request_result = match joined {
Some(result) => result,
None => match tokio::time::timeout_at(deadline, &mut task).await {
Ok(result) => result,
Err(error) => {
deadline_expired = true;
for pause in &pauses {
pause.discard_pending();
}
barrier_result = Err(std::io::Error::new(std::io::ErrorKind::TimedOut, error).into());
// This owns only the test's SDK request, not server/native work.
// Retire the client task without granting another operation budget.
client_task_aborted = true;
task.abort();
task.await
}
},
};
deadline_expired |= tokio::time::Instant::now() >= deadline;
if deadline_expired {
barrier_result = Err(std::io::Error::new(
std::io::ErrorKind::TimedOut,
"SDK request result observed after the absolute MPU deadline",
)
.into());
}
let hits: Vec<_> = pauses.iter().map(RpcPause::hits).collect();
let task = tokio::spawn(request);
wait_for_faults(&pauses, methods).await?;
let result = tokio::time::timeout(DEADLINE, task).await??;
for pause in &pauses {
pause.discard_pending();
}
drop(pauses);
if barrier_result.is_err() {
let sdk_error = request_result.as_ref().ok().and_then(|result| result.as_ref().err());
let response = sdk_error.and_then(SdkError::raw_response);
eprintln!(
"MPU fault barrier failed: methods={methods:?}, hits={hits:?}, client_task_aborted={client_task_aborted}, deadline_expired={deadline_expired}, sdk_code={:?}, cause={:?}, request_id={:?}",
sdk_error
.and_then(|error| error.as_service_error())
.and_then(|error| error.meta().code()),
response.and_then(|response| response.headers().get("x-rustfs-error-cause")),
response.and_then(|response| response.headers().get("x-amz-request-id")),
);
}
match (barrier_result, request_result) {
// Preserve the actual SDK source instead of replacing an early service
// failure with an unrelated fault-wait timeout. The outer Err stays red.
(Err(_), Ok(Err(error))) => Err(Box::new(error)),
(Err(_), Err(error)) if !client_task_aborted => Err(Box::new(error)),
(Err(error), _) => Err(error),
(Ok(()), Ok(result)) => Ok(result),
(Ok(()), Err(error)) => Err(Box::new(error)),
}
Ok(result)
}
fn quorum_outcome<T: std::fmt::Debug, E: std::fmt::Debug + ProvideErrorMetadata>(
@@ -1418,7 +1318,7 @@ async fn concurrent_capped_parts_respect_limit_on_distinct_nodes(clients: &[Clie
#[tokio::test]
async fn metadata_delay_preserves_write_quorum_and_rejects_quorum_minus_one() -> TestResult {
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2", true).await?;
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
list_parts_disk_loss_after_metadata_read(&cluster, &proxies, &clients, &bucket).await?;
concurrent_capped_parts_respect_limit_on_distinct_nodes(&clients, &bucket).await?;
phase_quorum_matrix(&cluster, &proxies, &clients, &bucket, false, 1, 3).await
@@ -1426,13 +1326,13 @@ async fn metadata_delay_preserves_write_quorum_and_rejects_quorum_minus_one() ->
#[tokio::test]
async fn lock_delay_preserves_admission_quorum_and_reports_timeout() -> TestResult {
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
phase_quorum_matrix(&cluster, &proxies, &clients, &bucket, true, 1, 3).await
}
#[tokio::test]
async fn sixteen_shard_multipart_metadata_requires_twelve_votes() -> TestResult {
let (cluster, proxies, clients, bucket) = fixture(4, "EC:4", false).await?;
let (cluster, proxies, clients, bucket) = fixture(4, "EC:4").await?;
let storage = cluster_admin_ok(&cluster, http::Method::GET, "/rustfs/admin/v3/storageinfo", None).await?;
let storage: serde_json::Value = serde_json::from_str(&storage)?;
let storage: rustfs_madmin::StorageInfo = serde_json::from_value(storage["info"].clone())?;
@@ -1446,7 +1346,7 @@ async fn sixteen_shard_multipart_metadata_requires_twelve_votes() -> TestResult
#[tokio::test]
async fn mixed_version_failed_complete_requires_current_coordinator_for_abort() -> TestResult {
let previous = super::upgrade_test::source_binary()?;
let (mut cluster, mut proxies, clients, bucket) = fixture_from_binary(4, "EC:4", Some(&previous), false).await?;
let (mut cluster, mut proxies, clients, bucket) = fixture_from_binary(4, "EC:4", Some(&previous)).await?;
assert!(
proxies.is_empty(),
"the genuine previous fleet boots and seeds through its normal direct transport"
@@ -1649,7 +1549,7 @@ async fn native_certified_partial_abort_retry_reclaims_survivors_below_metadata_
capability_sha256: Option<[u8; 32]>,
abort: Option<AbortRef>,
}
let (mut cluster, proxies, clients, bucket) = fixture(4, "EC:4", false).await?;
let (mut cluster, proxies, clients, bucket) = fixture(4, "EC:4").await?;
let key = "partial-abort";
let committed = payload_for("committed-before-abort", 4096);
clients[0]
@@ -1865,7 +1765,7 @@ async fn native_certified_partial_abort_retry_reclaims_survivors_below_metadata_
#[tokio::test]
async fn late_complete_packets_cannot_publish_after_abort_and_bucket_recreation() -> TestResult {
for size in [0, 8 * 1024 * 1024] {
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
let key = "late-complete";
let upload = clients[0].create_multipart_upload().bucket(&bucket).key(key).send().await?;
let upload_id = upload.upload_id().ok_or("missing upload ID")?;
@@ -1992,7 +1892,7 @@ async fn late_complete_packets_cannot_publish_after_abort_and_bucket_recreation(
async fn late_part_packets_preserve_retired_and_fresh_uploads() -> TestResult {
for (method, settles) in [("PreparePartTransaction", false), ("SettlePartTransaction", true)] {
for recreate_bucket in [false, true] {
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
let key = "late-part-transaction";
let previous = payload_for("committed-before-late-part", 4096);
clients[0]
@@ -2175,7 +2075,7 @@ async fn late_part_packets_preserve_retired_and_fresh_uploads() -> TestResult {
#[tokio::test]
async fn cancelled_metadata_wait_releases_fence_for_bucket_recreation() -> TestResult {
let (_cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
let (_cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
let pauses: Vec<_> = proxies[1..]
.iter()
.map(|proxy| proxy.pause(&["WriteMetadata"], usize::MAX, Some(&bucket)))
@@ -2198,7 +2098,7 @@ async fn cancelled_metadata_wait_releases_fence_for_bucket_recreation() -> TestR
async fn part_resend_owner_serializes_complete_and_abort_after_client_cancellation() -> TestResult {
use md5::{Digest as _, Md5};
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
for cancel_waiter in [false, true] {
for complete in [false, true] {
let key = format!("resend-{cancel_waiter}-{complete}");
@@ -2686,7 +2586,7 @@ async fn resume_at_rpc<T: Send + 'static>(
#[tokio::test]
async fn each_multipart_phase_resumes_after_metadata_and_lock_delays() -> TestResult {
let (_cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
let (_cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
for locks in [false, true] {
let key = format!("phase-{locks}");
let client = clients[0].clone();
@@ -25,8 +25,7 @@
use super::harness::{
DistCluster, DistLayout, TestResult, cluster_admin, cluster_admin_ok, decommission_status_json, get_object_bytes,
payload_for, put_object, rebalance_complete, rebalance_status_json, s3_probe_client, s3_probe_error_code, sha256_hex,
unique_bucket, wait_until,
payload_for, put_object, s3_probe_client, s3_probe_error_code, sha256_hex, unique_bucket, wait_until,
};
use crate::chaos::{VersionShardCensus, census_object_version_on_disk};
use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, RustFSTestClusterEnvironment};
@@ -48,7 +47,6 @@ use tokio::time::{Instant, sleep};
use uuid::Uuid;
pub(crate) const MOVEMENT_BARRIER_DIR_ENV: &str = "RUSTFS_E2E_DATA_MOVEMENT_BARRIER_DIR";
pub(crate) const STAGE_AFTER_ENTRY_ADMISSION: &str = "after_entry_admission";
pub(crate) const STAGE_BEFORE_TARGET_COMMIT: &str = "before_target_commit";
pub(crate) const STAGE_BEFORE_SOURCE_CLEANUP: &str = "before_source_cleanup";
pub(crate) const STAGE_AFTER_RESPONSIBILITY_PERSISTED: &str = "after_responsibility_persisted";
@@ -1168,21 +1166,8 @@ pub(crate) async fn clear_decommission(cluster: &RustFSTestClusterEnvironment, p
.await
}
pub(crate) async fn stop_rebalance(cluster: &RustFSTestClusterEnvironment, expected_id: &str) -> TestResult<String> {
let path = "/rustfs/admin/v3/rebalance/stop";
let (status, response) = cluster_admin(cluster, Method::POST, path, None).await?;
if status.is_success() {
return Ok(response);
}
// The run can finish between the active observation and the stop request.
// Only completion of that same run makes a not-started response acceptable.
if status == StatusCode::NOT_FOUND
&& response.contains("<Code>NoSuchResource</Code>")
&& rebalance_complete(&rebalance_status_json(cluster).await?, expected_id)?
{
return Ok(response);
}
Err(format!("POST {path} failed: {status} {response}").into())
pub(crate) async fn stop_rebalance(cluster: &RustFSTestClusterEnvironment) -> TestResult<String> {
cluster_admin_ok(cluster, Method::POST, "/rustfs/admin/v3/rebalance/stop", None).await
}
pub(crate) fn decommission_pool_entry(status: &Value, pool_id: usize) -> TestResult<Value> {
@@ -1600,129 +1585,6 @@ pub(crate) async fn assert_every_node_serves(dist: &DistCluster, bucket: &str, l
mod tests {
use super::*;
const REBALANCE_NOT_STARTED: &str =
"<Error><Code>NoSuchResource</Code><Message>pool rebalance is not started</Message></Error>";
async fn mock_rebalance_stop(
stop_status: StatusCode,
stop_body: &str,
run_status: Value,
) -> TestResult<(TestResult<String>, Vec<String>)> {
use bytes::Bytes;
use http_body_util::Full;
use hyper::service::service_fn;
use hyper_util::rt::TokioIo;
use tokio::net::TcpListener;
use tokio::sync::mpsc;
use tokio::task::JoinSet;
let listener = TcpListener::bind("127.0.0.1:0").await?;
let mut cluster = RustFSTestClusterEnvironment::new(1).await?;
cluster.nodes[0].url = format!("http://{}", listener.local_addr()?);
let stop_body = stop_body.to_string();
let (requests_tx, mut requests_rx) = mpsc::unbounded_channel();
let mut server = JoinSet::new();
server.spawn(async move {
loop {
let (stream, _) = match listener.accept().await {
Ok(connection) => connection,
Err(error) => return Err::<(), Box<dyn std::error::Error + Send + Sync>>(error.into()),
};
let stop_body = stop_body.clone();
let run_status = run_status.clone();
let requests_tx = requests_tx.clone();
let handler = service_fn(move |request: hyper::Request<hyper::body::Incoming>| {
let route = format!("{} {}", request.method(), request.uri().path());
requests_tx.send(route.clone()).expect("record mock admin request");
let (status, body) = match route.as_str() {
"POST /rustfs/admin/v3/rebalance/stop" => (stop_status, stop_body.clone()),
"GET /rustfs/admin/v3/rebalance/status" => (StatusCode::OK, run_status.to_string()),
_ => (StatusCode::BAD_REQUEST, "unexpected admin request".to_string()),
};
async move { http::Response::builder().status(status).body(Full::new(Bytes::from(body))) }
});
hyper::server::conn::http1::Builder::new()
.keep_alive(false)
.serve_connection(TokioIo::new(stream), handler)
.await?;
}
});
let outcome = tokio::time::timeout(Duration::from_secs(5), stop_rebalance(&cluster, "run-1")).await?;
server.abort_all();
let requests = std::iter::from_fn(|| requests_rx.try_recv().ok()).collect();
Ok((outcome, requests))
}
fn completed_rebalance() -> Value {
serde_json::json!({
"id": "run-1",
"pools": [
{ "id": 0, "status": "Completed", "progress": { "versions": 113 } },
{ "id": 1, "status": "None" }
]
})
}
#[tokio::test]
async fn rebalance_stop_success_does_not_query_completion() -> TestResult {
let (outcome, requests) = mock_rebalance_stop(StatusCode::OK, "", Value::Null).await?;
assert_eq!(outcome?, "");
assert_eq!(requests, ["POST /rustfs/admin/v3/rebalance/stop"]);
Ok(())
}
#[tokio::test]
async fn rebalance_stop_accepts_completed_run_after_not_started_response() -> TestResult {
let (outcome, requests) =
mock_rebalance_stop(StatusCode::NOT_FOUND, REBALANCE_NOT_STARTED, completed_rebalance()).await?;
outcome?;
assert_eq!(
requests,
[
"POST /rustfs/admin/v3/rebalance/stop",
"GET /rustfs/admin/v3/rebalance/status"
]
);
Ok(())
}
#[tokio::test]
async fn rebalance_stop_rejects_unproven_completion_and_other_errors() -> TestResult {
let complete = completed_rebalance();
let mut invalid = vec![Value::Null];
for (pointer, value) in [
("/id", serde_json::json!("run-2")),
("/pools/0/status", serde_json::json!("Started")),
("/pools/0/status", serde_json::json!("Failed")),
("/pools/0/status", serde_json::json!("Stopped")),
("/pools/0/progress/versions", serde_json::json!(0)),
] {
let mut status = complete.clone();
*status.pointer_mut(pointer).expect("completion fixture field") = value;
invalid.push(status);
}
let mut pending = complete.clone();
pending["stopPropagation"] = serde_json::json!({ "pendingTerminalReload": true });
invalid.push(pending);
let mut failed = complete.clone();
failed["pools"][0]["lastError"] = serde_json::json!("migration failed");
invalid.push(failed);
for status in invalid {
let (outcome, _) = mock_rebalance_stop(StatusCode::NOT_FOUND, REBALANCE_NOT_STARTED, status.clone()).await?;
assert!(outcome.is_err(), "unproven completion accepted: {status}");
}
for (code, body) in [
(StatusCode::FORBIDDEN, REBALANCE_NOT_STARTED),
(StatusCode::INTERNAL_SERVER_ERROR, REBALANCE_NOT_STARTED),
(StatusCode::NOT_FOUND, "<Error><Code>NoSuchBucket</Code></Error>"),
] {
let (outcome, requests) = mock_rebalance_stop(code, body, complete.clone()).await?;
assert!(outcome.is_err(), "unrelated error accepted: {code} {body}");
assert_eq!(requests, ["POST /rustfs/admin/v3/rebalance/stop"]);
}
Ok(())
}
#[test]
fn version_ids_normalize_null_spellings() {
for spelling in [None, Some(""), Some("null"), Some("00000000-0000-0000-0000-000000000000")] {
@@ -28,7 +28,7 @@ use super::harness::{
wait_for_rebalance_active, wait_for_rebalance_complete,
};
use super::scanner_heal_topology_fixture::{
LiveWorkload, MOVEMENT_ACTIVE_TIMEOUT, MOVEMENT_COMPLETE_TIMEOUT, MovementOperation, SEED_POOL, STAGE_AFTER_ENTRY_ADMISSION,
LiveWorkload, MOVEMENT_ACTIVE_TIMEOUT, MOVEMENT_COMPLETE_TIMEOUT, MovementOperation, SEED_POOL,
STAGE_AFTER_RESPONSIBILITY_PERSISTED, STAGE_BEFORE_SOURCE_CLEANUP, STAGE_BEFORE_TARGET_COMMIT, TopologyManifest, TopologyRun,
assert_bucket_durability_strict, assert_every_node_serves, assert_probe_objects, assert_process_durability_strict,
assert_seed_pool_census_unchanged, assert_single_pool_placement, cancel_decommission, capture_seed_pool_census,
@@ -274,7 +274,6 @@ async fn wait_for_rebalance_stopped_or_completed(run: &TopologyRun, rebalance_id
#[tokio::test]
async fn scanner_heal_topology_rebalance_survives_live_writes_stop_and_restart() -> TestResult {
init_logging();
require_test_hooks();
let mut run = TopologyRun::start("rebalance").await?;
let manifest = seed_manifest(&run).await?;
assert_bucket_durability_strict(&run.dist.cluster, &manifest.buckets()).await?;
@@ -289,52 +288,23 @@ async fn scanner_heal_topology_rebalance_survives_live_writes_stop_and_restart()
.map(|entry| (entry.key.clone(), entry.sha256.clone()))
.collect();
// Keep an admitted entry on the source until stop cancels admission. Start
// propagation may await that entry, so observe and stop before awaiting ACK.
let first_key = manifest.plain_key("external", 0)?;
run.movement_barrier
.arm_movement("rebalance", &manifest.plain_bucket, &first_key, STAGE_AFTER_ENTRY_ADMISSION)?;
let observe_first = async {
let (observed_id, reached) = tokio::time::timeout(MOVEMENT_ACTIVE_TIMEOUT, async {
let reached = run.movement_barrier.wait_reached(MOVEMENT_ACTIVE_TIMEOUT).await?;
let status = rebalance_status_json(&run.dist.cluster).await?;
let observed_id = status
.get("id")
.and_then(Value::as_str)
.ok_or("rebalance status omitted id")?;
wait_for_rebalance_active(&run.dist.cluster, observed_id, MOVEMENT_ACTIVE_TIMEOUT).await?;
Ok::<_, Box<dyn std::error::Error + Send + Sync>>((observed_id.to_string(), reached))
})
.await??;
let probes = put_probe_objects(&run.dist, &scratch, "during-rebalance", 4).await?;
let workload = run_live_workload(LiveWorkload {
clients: run.dist.clients()?,
bucket: scratch.clone(),
prefix: "live".to_string(),
writers: LIVE_WRITERS,
rounds: LIVE_ROUNDS,
baseline_bucket: manifest.plain_bucket.clone(),
baseline: baseline.clone(),
pause: Duration::from_millis(100),
});
let stop = async {
sleep(Duration::from_secs(2)).await;
stop_rebalance(&run.dist.cluster, &observed_id).await?;
let (outcome, status) = wait_for_rebalance_stopped_or_completed(&run, &observed_id).await?;
if outcome != "stopped" {
return Err("rebalance completed before stop; mid-run stop was not exercised".into());
}
Ok::<_, Box<dyn std::error::Error + Send + Sync>>((outcome, status))
};
let (ops, (outcome, status)) = tokio::try_join!(workload, stop)?;
Ok::<_, Box<dyn std::error::Error + Send + Sync>>((observed_id, reached, probes, ops, outcome, status))
};
let (first_id, (observed_id, first_reached, during_rebalance, ops, first_outcome, first_status)) =
tokio::try_join!(start_rebalance(&run.dist.cluster), observe_first)?;
if first_id != observed_id {
return Err(format!("rebalance start acknowledged {first_id}, but the stopped run was {observed_id}").into());
}
run.movement_barrier.disarm()?;
let first_id = start_rebalance(&run.dist.cluster).await?;
wait_for_rebalance_active(&run.dist.cluster, &first_id, MOVEMENT_ACTIVE_TIMEOUT).await?;
let during_rebalance = put_probe_objects(&run.dist, &scratch, "during-rebalance", 4).await?;
let workload = tokio::spawn(run_live_workload(LiveWorkload {
clients: run.dist.clients()?,
bucket: scratch.clone(),
prefix: "live".to_string(),
writers: LIVE_WRITERS,
rounds: LIVE_ROUNDS,
baseline_bucket: manifest.plain_bucket.clone(),
baseline: baseline.clone(),
pause: Duration::from_millis(100),
}));
sleep(Duration::from_secs(2)).await;
stop_rebalance(&run.dist.cluster).await?;
let (first_outcome, first_status) = wait_for_rebalance_stopped_or_completed(&run, &first_id).await?;
let ops = workload.await??;
let live_state = expected_live_state(&ops);
for node_idx in 0..run.dist.cluster.nodes.len() {
@@ -363,40 +333,8 @@ async fn scanner_heal_topology_rebalance_survives_live_writes_stop_and_restart()
verify_from_every_node(&run.dist, &manifest, "after restart following the stop").await?;
verify_live_state(&run.dist.client(0)?, &scratch, &live_state, "after restart following the stop").await?;
// A short second run can finish before the start response reaches us.
// Hold one remaining source entry until its active state is observed.
let mut second_key = None;
for entry in manifest.plain_entries() {
if pools_holding(&run.dist, &manifest.plain_bucket, &entry.key, None)?.contains(&SEED_POOL) {
second_key = Some(entry.key.clone());
break;
}
}
let second_key = second_key.ok_or("no seed-pool object remains for the second rebalance")?;
let second_barrier = run.movement_barrier.next_phase();
second_barrier.arm_movement("rebalance", &manifest.plain_bucket, &second_key, STAGE_BEFORE_SOURCE_CLEANUP)?;
let observe_second = async {
let reached = second_barrier.wait_reached(MOVEMENT_ACTIVE_TIMEOUT).await?;
let status = rebalance_status_json(&run.dist.cluster).await?;
let observed_id = status
.get("id")
.and_then(Value::as_str)
.ok_or("rebalance status omitted id")?;
if observed_id == first_id {
return Err("the second rebalance reused the stopped run id".into());
}
wait_for_rebalance_active(&run.dist.cluster, observed_id, MOVEMENT_ACTIVE_TIMEOUT).await?;
second_barrier.disarm()?;
second_barrier.release_movement()?;
Ok::<_, Box<dyn std::error::Error + Send + Sync>>((observed_id.to_string(), reached))
};
// Worker propagation can hold the start response until movement drains.
let (second_id, (observed_id, second_reached)) = tokio::try_join!(start_rebalance(&run.dist.cluster), async {
tokio::time::timeout(MOVEMENT_ACTIVE_TIMEOUT, observe_second).await?
})?;
if second_id != observed_id {
return Err(format!("rebalance start acknowledged {second_id}, but the active run was {observed_id}").into());
}
let second_id = start_rebalance(&run.dist.cluster).await?;
wait_for_rebalance_active(&run.dist.cluster, &second_id, MOVEMENT_ACTIVE_TIMEOUT).await?;
wait_for_rebalance_complete(&run.dist.cluster, &second_id, MOVEMENT_COMPLETE_TIMEOUT).await?;
verify_from_every_node(&run.dist, &manifest, "after the second rebalance completed").await?;
verify_live_state(&run.dist.client(3)?, &scratch, &live_state, "after the second rebalance completed").await?;
@@ -413,8 +351,8 @@ async fn scanner_heal_topology_rebalance_survives_live_writes_stop_and_restart()
&json!({
"gate": "G-REB",
"manifest": manifest_summary(&manifest),
"first_run": { "id": first_id, "outcome": first_outcome, "status": first_status, "versions_held_by_two_pools_after_stop": retained_after_stop, "barrier_object": first_key, "barrier": first_reached },
"second_run": { "id": second_id, "versions_off_seed_pool": moved, "barrier_object": second_key, "barrier": second_reached },
"first_run": { "id": first_id, "outcome": first_outcome, "status": first_status, "versions_held_by_two_pools_after_stop": retained_after_stop },
"second_run": { "id": second_id, "versions_off_seed_pool": moved },
"live_ops": ops,
"probe_placement_during_rebalance": during_placement,
"process_durability_strict_nodes": strict_nodes,
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -33,7 +33,7 @@ mod tests {
use sha2::{Digest, Sha256};
use std::collections::HashSet;
use std::error::Error;
use std::io::{Read, Seek, SeekFrom, Write};
use std::io::{Read, Write};
use std::net::SocketAddr;
use std::path::{Path, PathBuf};
use std::process::Command;
@@ -1654,89 +1654,6 @@ mod tests {
TargetEndpointBlackhole,
}
async fn release_heal_commit_barrier_after_shutdown(
log: &Path,
log_offset: u64,
barrier: &Path,
) -> Result<serde_json::Value, Box<dyn Error + Send + Sync>> {
// Observe within the existing graceful process-stop budget.
timeout(Duration::from_secs(45), async {
loop {
let mut file = std::fs::File::open(log)?;
file.seek(SeekFrom::Start(log_offset))?;
let mut tail = String::new();
file.read_to_string(&mut tail)?;
if let Some(event) = tail
.lines()
.filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
.find(|event| {
event["event"] == "server_shutdown_state"
&& event["component"] == "main"
&& event["subsystem"] == "startup"
&& event["state"] == "stopping"
})
{
// Local heal has stopped and HTTP shutdown has been signalled.
// Let admitted peer RPCs finish; keeping this synthetic barrier
// until process exit would make the fixture block its own drain.
std::fs::remove_file(barrier)?;
return Ok(event);
}
sleep(Duration::from_millis(10)).await;
}
})
.await?
}
#[tokio::test(start_paused = true)]
async fn heal_commit_barrier_waits_for_new_shutdown_event() -> Result<(), Box<dyn Error + Send + Sync>> {
let directory = tempfile::tempdir()?;
let log = directory.path().join("server.log");
let barrier = directory.path().join("barrier");
let event = serde_json::json!({
"event": "server_shutdown_state", "component": "main", "subsystem": "startup", "state": "stopping"
});
std::fs::write(&log, format!("{event}\n"))?;
std::fs::write(&barrier, "bucket/prefix/")?;
let log_offset = log.metadata()?.len();
let mut writer = std::fs::OpenOptions::new().append(true).open(&log)?;
writeln!(writer, "{}", serde_json::json!({"event": "shutdown_signal_received"}))?;
writeln!(writer, "{}", serde_json::json!({"event": "server_shutdown_state", "state": "stopping"}))?;
let release = release_heal_commit_barrier_after_shutdown(&log, log_offset, &barrier);
tokio::pin!(release);
assert!(
futures::poll!(release.as_mut()).is_pending(),
"old or unrelated events must not release the barrier"
);
assert!(barrier.is_file(), "the partial-rebuild barrier must remain armed before shutdown");
tokio::time::advance(Duration::from_secs(6)).await;
assert!(
futures::poll!(release.as_mut()).is_pending(),
"background shutdown may take longer than five seconds within the process-stop budget"
);
writeln!(writer, "{event}")?;
tokio::time::advance(Duration::from_millis(10)).await;
assert_eq!(release.await?, event, "retain the actual new shutdown witness");
assert!(!barrier.exists(), "shutdown must release the synthetic hold");
Ok(())
}
#[tokio::test(start_paused = true)]
async fn heal_commit_barrier_missing_shutdown_fails_without_releasing() -> Result<(), Box<dyn Error + Send + Sync>> {
let directory = tempfile::tempdir()?;
let log = directory.path().join("server.log");
let barrier = directory.path().join("barrier");
std::fs::write(&log, "")?;
std::fs::write(&barrier, "bucket/prefix/")?;
let error = release_heal_commit_barrier_after_shutdown(&log, 0, &barrier)
.await
.expect_err("missing shutdown evidence must fail the fixture");
assert!(error.is::<tokio::time::error::Elapsed>());
assert!(barrier.is_file(), "a missing shutdown witness must not release repairs");
Ok(())
}
async fn run_cluster_root_heal_interruption(scenario: InterruptionScenario) -> Result<(), Box<dyn Error + Send + Sync>> {
let server_binary = rustfs_binary_path();
let evidence_case = match scenario {
@@ -1874,7 +1791,6 @@ mod tests {
let mut created_online_objects = Vec::with_capacity(online_object_count);
let mut expected_manifests = Vec::with_capacity(online_object_count);
let mut unclean_shutdown_marker_observed = None;
let mut shutdown_barrier_release = None;
let mut attempt_count = 0usize;
let max_online_attempts = online_object_count.saturating_mul(topology.total_drives().max(1));
while expected_manifests.len() < online_object_count {
@@ -2457,21 +2373,7 @@ mod tests {
| InterruptionScenario::BackgroundTargetRestartEc84MultiSet
| InterruptionScenario::BackgroundCoordinatorRestart
) {
if interruption_node == 1
&& let Some(barrier) = &commit_barrier
{
let target_log = PathBuf::from(format!("{log_dir}/node1.log"));
let log_offset = target_log.metadata()?.len();
let (stopped, released) = tokio::join!(
cluster.stop_node_gracefully(interruption_node),
release_heal_commit_barrier_after_shutdown(&target_log, log_offset, barrier),
);
// Await both branches even on error so the child is reaped.
stopped?;
shutdown_barrier_release = Some(released?);
} else {
cluster.stop_node_gracefully(interruption_node).await?;
}
cluster.stop_node_gracefully(interruption_node).await?;
} else {
cluster.stop_node(interruption_node)?;
}
@@ -2513,9 +2415,7 @@ mod tests {
}
}
}
if shutdown_barrier_release.is_none()
&& let Some(barrier) = &commit_barrier
{
if let Some(barrier) = &commit_barrier {
std::fs::remove_file(barrier)?;
}
cluster.start_node_from_binary(interruption_node, &server_binary).await?;
@@ -2846,7 +2746,6 @@ mod tests {
"all_peers_bound_to_generation_window": true,
"pid_before": target_pid, "pid_after": restarted_pid,
"unclean_shutdown_marker": unclean_shutdown_marker_observed.unwrap_or(false),
"graceful_barrier_release": shutdown_barrier_release,
"objects": evidence_objects, "node_listings": node_listings,
});
if let Some(outage_write) = outage_write_diagnostic {
-3
View File
@@ -283,9 +283,6 @@ mod ssec_copy_test;
#[cfg(test)]
mod multipart_storage_class_test;
#[cfg(test)]
mod multipart_upload_id_test;
#[cfg(test)]
mod storage_class_capability_test;
@@ -1,175 +0,0 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Multipart upload ID regression tests.
//!
//! Every operation that addresses an existing multipart upload must report an
//! upload ID that does not name a live upload as `404 NoSuchUpload`, whether
//! the ID is malformed or is a well-formed ID of an aborted upload.
#[cfg(test)]
mod tests {
use crate::common::{RustFSTestEnvironment, init_logging};
use aws_sdk_s3::Client;
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
const BUCKET: &str = "multipart-upload-id";
const KEY: &str = "target.bin";
const COPY_SOURCE_KEY: &str = "copy-source.bin";
// RustFS upload IDs are URL-safe base64; '.' is outside that alphabet, so
// the object layer rejects this ID as malformed instead of looking it up.
const MALFORMED_UPLOAD_ID: &str = "malformed.upload.id";
const PART_NUMBER: i32 = 1;
const PART_BODY: &[u8] = b"multipart upload id regression part";
const PART_ETAG: &str = "\"00000000000000000000000000000000\"";
const NO_SUCH_UPLOAD_STATUS: u16 = 404;
const NO_SUCH_UPLOAD_CODE: &str = "NoSuchUpload";
const OBJECT_NOT_FOUND_STATUS: u16 = 404;
/// Describes how `result` differs from `404 NoSuchUpload`, or returns `None` when it matches.
fn no_such_upload_mismatch<T, E>(operation: &str, result: Result<T, SdkError<E>>) -> Option<String>
where
E: ProvideErrorMetadata + std::fmt::Debug,
{
let Err(error) = result else {
return Some(format!("{operation}: succeeded"));
};
let status = error.raw_response().map(|response| response.status().as_u16());
let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
(status != Some(NO_SUCH_UPLOAD_STATUS) || code != Some(NO_SUCH_UPLOAD_CODE))
.then(|| format!("{operation}: status {status:?}, code {code:?}: {error:?}"))
}
async fn start_server_with_copy_source() -> Result<(RustFSTestEnvironment, Client), Box<dyn std::error::Error + Send + Sync>>
{
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(Vec::new()).await?;
env.create_test_bucket(BUCKET).await?;
let client = env.create_s3_client();
client
.put_object()
.bucket(BUCKET)
.key(COPY_SOURCE_KEY)
.body(ByteStream::from_static(PART_BODY))
.send()
.await?;
Ok((env, client))
}
/// Sends every operation that addresses an existing upload and asserts that all of them report `404 NoSuchUpload`.
async fn assert_every_upload_operation_reports_no_such_upload(client: &Client, upload_id: &str) {
let upload_part = client
.upload_part()
.bucket(BUCKET)
.key(KEY)
.upload_id(upload_id)
.part_number(PART_NUMBER)
.body(ByteStream::from_static(PART_BODY))
.send()
.await;
let upload_part_copy = client
.upload_part_copy()
.bucket(BUCKET)
.key(KEY)
.upload_id(upload_id)
.part_number(PART_NUMBER)
.copy_source(format!("/{BUCKET}/{COPY_SOURCE_KEY}"))
.send()
.await;
let list_parts = client.list_parts().bucket(BUCKET).key(KEY).upload_id(upload_id).send().await;
let complete = client
.complete_multipart_upload()
.bucket(BUCKET)
.key(KEY)
.upload_id(upload_id)
.multipart_upload(
CompletedMultipartUpload::builder()
.parts(CompletedPart::builder().part_number(PART_NUMBER).e_tag(PART_ETAG).build())
.build(),
)
.send()
.await;
let abort = client
.abort_multipart_upload()
.bucket(BUCKET)
.key(KEY)
.upload_id(upload_id)
.send()
.await;
let mismatches: Vec<String> = [
no_such_upload_mismatch("UploadPart", upload_part),
no_such_upload_mismatch("UploadPartCopy", upload_part_copy),
no_such_upload_mismatch("ListParts", list_parts),
no_such_upload_mismatch("CompleteMultipartUpload", complete),
no_such_upload_mismatch("AbortMultipartUpload", abort),
]
.into_iter()
.flatten()
.collect();
assert!(
mismatches.is_empty(),
"upload ID {upload_id:?}: operations not reporting {NO_SUCH_UPLOAD_STATUS} {NO_SUCH_UPLOAD_CODE}:\n{}",
mismatches.join("\n")
);
}
/// Asserts that the rejected operations left neither a multipart upload nor an object under `KEY`.
async fn assert_no_upload_or_object_published(client: &Client) -> TestResult {
let uploads = client.list_multipart_uploads().bucket(BUCKET).prefix(KEY).send().await?;
assert!(uploads.uploads().is_empty(), "rejected operations must not leave an upload: {uploads:?}");
let head = client.head_object().bucket(BUCKET).key(KEY).send().await;
let status = head
.as_ref()
.err()
.and_then(|error| error.raw_response())
.map(|response| response.status().as_u16());
assert_eq!(
status,
Some(OBJECT_NOT_FOUND_STATUS),
"rejected operations must not publish {KEY}: {head:?}"
);
Ok(())
}
#[tokio::test]
async fn malformed_upload_id_reports_no_such_upload_on_every_upload_operation() -> TestResult {
let (_env, client) = start_server_with_copy_source().await?;
assert_every_upload_operation_reports_no_such_upload(&client, MALFORMED_UPLOAD_ID).await;
assert_no_upload_or_object_published(&client).await
}
#[tokio::test]
async fn aborted_upload_id_reports_no_such_upload_on_every_upload_operation() -> TestResult {
let (_env, client) = start_server_with_copy_source().await?;
let created = client.create_multipart_upload().bucket(BUCKET).key(KEY).send().await?;
let upload_id = created.upload_id().ok_or("CreateMultipartUpload returned no upload ID")?;
client
.abort_multipart_upload()
.bucket(BUCKET)
.key(KEY)
.upload_id(upload_id)
.send()
.await?;
assert_every_upload_operation_reports_no_such_upload(&client, upload_id).await;
assert_no_upload_or_object_published(&client).await
}
}
@@ -31,8 +31,8 @@ use aws_sdk_s3::error::ProvideErrorMetadata;
use aws_sdk_s3::operation::list_object_versions::ListObjectVersionsOutput;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Delete, DeleteMarkerEntry, ObjectIdentifier, ObjectVersion,
ServerSideEncryption, VersioningConfiguration,
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, DeleteMarkerEntry, ObjectVersion, ServerSideEncryption,
VersioningConfiguration,
};
use base64_simd::STANDARD as BASE64_STANDARD;
use bytes::Bytes;
@@ -3729,8 +3729,6 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
init_logging();
let mut source_env = RustFSTestEnvironment::new().await?;
let source_log = format!("{}/source.log", source_env.temp_dir);
source_env.capture_log_path = Some(source_log.clone());
let mut source_env_vars = replication_fast_env();
source_env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
source_env.start_rustfs_server_with_env(vec![], &source_env_vars).await?;
@@ -3793,66 +3791,19 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
"target did not preserve the source delete-marker version ID"
);
let wait_for_source_writeback = || async {
let metrics_url = format!("{}/{source_bucket}?replication-metrics", source_env.url);
timeout(Duration::from_secs(30), async {
loop {
let response = signed_request(
http::Method::GET,
&metrics_url,
&source_env.access_key,
&source_env.secret_key,
None,
None,
)
.await?;
let metrics: serde_json::Value = response.error_for_status()?.json().await?;
let queued = metrics["queued"]["curr"]["count"]
.as_f64()
.ok_or("missing replication queue count")?;
if metrics["provider_available"] == true && queued == 0.0 {
return Ok::<(), Box<dyn Error + Send + Sync>>(());
}
sleep(Duration::from_millis(50)).await;
}
})
.await?
};
wait_for_source_writeback().await?;
let purged = source_client
.delete_objects()
source_client
.delete_object()
.bucket(source_bucket)
.delete(
Delete::builder()
.objects(
ObjectIdentifier::builder()
.key(object_key)
.version_id(&purged_version_id)
.build()?,
)
.objects(
ObjectIdentifier::builder()
.key(object_key)
.version_id(delete_marker_version_id)
.build()?,
)
.build()?,
)
.key(object_key)
.version_id(&purged_version_id)
.send()
.await?;
assert!(purged.errors().is_empty(), "batch version purge failed: {:?}", purged.errors());
assert_eq!(purged.deleted().len(), 2, "both the data version and marker must be purged");
assert_replication_converged(&source_client, source_bucket, &target_client, target_bucket).await?;
let target_state = list_replication_state(&target_client, target_bucket).await?;
assert!(
target_state.iter().all(|entry| entry.version_id != purged_version_id),
"target retained the explicitly purged object version"
);
assert!(
target_state.iter().all(|entry| entry.version_id != delete_marker_version_id),
"target retained the explicitly purged delete marker"
);
assert!(
target_state
.iter()
@@ -3868,19 +3819,6 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
.await?;
assert_eq!(retained.body.collect().await?.into_bytes().as_ref(), b"versioned replication payload v2");
// Wait for the source workers to finish their final writeback, which runs
// after the remote purge has already become visible (rustfs#9121).
wait_for_source_writeback().await?;
let logs = fs::read_to_string(source_log).await?;
let writeback_failures: Vec<_> = logs
.lines()
.filter(|line| line.contains("apply_replication_delete_state"))
.collect();
assert!(
writeback_failures.is_empty(),
"a completed batch purge must not report a source writeback failure: {writeback_failures:?}"
);
Ok(())
}
+28 -456
View File
@@ -28,8 +28,7 @@ use hyper_util::{
use prost::Message as _;
use rustfs_protos::proto_gen::node_service::{
BatchGenerallyLockRequest, DeleteRequest, GenerallyLockRequest, PreparePartTransactionResponse, ReadVersionRequest,
RecoverMultipartSourceRequest, RenameDataResponse, ScannerScopedDirtyUsageAckRequest, SettlePartTransactionRequest,
SettlePartTransactionResponse,
RecoverMultipartSourceRequest, RenameDataResponse, SettlePartTransactionRequest, SettlePartTransactionResponse,
};
use std::{
collections::HashSet,
@@ -50,9 +49,6 @@ use tokio::{
type ProxyBody = BoxBody<Bytes, io::Error>;
type ProxyClient = Client<HttpConnector, ProxyBody>;
mod scoped_ack;
pub(crate) use scoped_ack::{ScopedAckFault, ScopedAckQueue, ScopedAckScope};
#[derive(Clone, Copy, PartialEq, Eq)]
enum Disposition {
Hold,
@@ -125,8 +121,7 @@ impl RpcPause {
self.state.blocked.load(Ordering::SeqCst)
}
/// Selected RPCs with successful HTTP/gRPC response headers, excluding
/// discarded packets. This does not prove a successful gRPC trailer or ACK apply.
/// Selected RPCs with successful HTTP/gRPC headers, excluding discarded packets.
pub(crate) fn forwarded(&self) -> usize {
self.state.forwarded.load(Ordering::SeqCst)
}
@@ -238,7 +233,6 @@ pub(crate) struct RpcFaultProxy {
address: SocketAddr,
slot: PauseSlot,
lock_slot: PauseSlot,
scoped_ack: scoped_ack::QueueSlot,
task: JoinHandle<()>,
late_requests: LateRequests,
}
@@ -251,8 +245,6 @@ impl RpcFaultProxy {
let active = slot.clone();
let lock_slot: PauseSlot = Arc::new(Mutex::new(None));
let active_lock = lock_slot.clone();
let scoped_ack = scoped_ack::empty_slot();
let active_ack = scoped_ack.clone();
let late_requests: LateRequests = Arc::new(Mutex::new(Some(Vec::new())));
let late = late_requests.clone();
let task = tokio::spawn(async move {
@@ -267,7 +259,6 @@ impl RpcFaultProxy {
let h2 = h2.clone();
let active = active.clone();
let active_lock = active_lock.clone();
let active_ack = active_ack.clone();
let late = late.clone();
connections.spawn(async move {
let service = service_fn(move |request: Request<Incoming>| {
@@ -281,7 +272,6 @@ impl RpcFaultProxy {
None
};
let pause = owner_pause.or_else(|| active.lock().expect("RPC pause slot").clone());
let ack = active_ack.lock().expect("scoped ACK queue slot").clone();
let retain = pause.as_ref().is_some_and(|pause| {
pause.retain_after_disconnect
&& request.uri().path().strip_prefix("/node_service.NodeService/")
@@ -290,11 +280,11 @@ impl RpcFaultProxy {
let late = late.clone();
async move {
if !retain {
return forward_request(request, client, target, pause, ack).await;
return forward_request(request, client, target, pause).await;
}
// This opt-in fault owns the packet after client cancellation.
// Proxy teardown still aborts every retained request.
let task = tokio::spawn(forward_request(request, client, target, pause, ack));
let task = tokio::spawn(forward_request(request, client, target, pause));
{
let mut requests = late.lock().expect("late RPC ownership");
if let Some(requests) = requests.as_mut() {
@@ -319,7 +309,6 @@ impl RpcFaultProxy {
address,
slot,
lock_slot,
scoped_ack,
task,
late_requests,
})
@@ -329,11 +318,6 @@ impl RpcFaultProxy {
self.address
}
/// Retain only genuine scoped ACKs for one receiver/session and bucket set.
pub(crate) fn scoped_ack_queue(&self, scope: ScopedAckScope, capacity: usize) -> io::Result<ScopedAckQueue> {
ScopedAckQueue::arm(self.scoped_ack.clone(), scope, capacity)
}
/// Pause at most limit requests to the named unary methods; other streams
/// on the same HTTP/2 connection continue forwarding.
pub(crate) fn pause(&self, methods: &[&'static str], limit: usize, scope: Option<&str>) -> io::Result<RpcPause> {
@@ -485,9 +469,6 @@ fn unary_payload(body: &[u8]) -> io::Result<&[u8]> {
}
fn scope_matches(method: &str, body: &[u8], scope: &[u8]) -> io::Result<bool> {
if method == "ScannerScopedDirtyUsageAck" {
return scoped_ack_matches(body, Some(scope));
}
if method == "RecoverMultipartSource" {
let request = RecoverMultipartSourceRequest::decode(unary_payload(body)?).map_err(io::Error::other)?;
if request.protocol_version != u32::from(MULTIPART_SOURCE_PROTOCOL_VERSION) || request.path.as_bytes() != scope {
@@ -544,9 +525,7 @@ fn scoped_lock_owners(method: &str, body: &[u8], scope: &[u8]) -> io::Result<Vec
let args = match method {
"Lock" => vec![GenerallyLockRequest::decode(payload).map_err(io::Error::other)?.args],
"LockBatch" => BatchGenerallyLockRequest::decode(payload).map_err(io::Error::other)?.args,
_ => {
return Err(io::Error::new(io::ErrorKind::InvalidInput, "lock observer requires a lock method"));
}
_ => return Err(io::Error::new(io::ErrorKind::InvalidInput, "lock observer requires a lock method")),
};
args.into_iter()
.filter(|args| args.as_bytes().windows(scope.len()).any(|window| window == scope))
@@ -561,39 +540,18 @@ fn scoped_lock_owners(method: &str, body: &[u8], scope: &[u8]) -> io::Result<Vec
.collect()
}
fn scoped_ack_matches(body: &[u8], scope: Option<&[u8]>) -> io::Result<bool> {
let request = ScannerScopedDirtyUsageAckRequest::decode(unary_payload(body)?).map_err(io::Error::other)?;
Ok(!request.probe_only && scope.is_none_or(|scope| request.entries.iter().any(|entry| entry.bucket.as_bytes() == scope)))
}
async fn forward_request(
request: Request<Incoming>,
client: ProxyClient,
target: SocketAddr,
pause: Option<Arc<PauseState>>,
ack: Option<Arc<scoped_ack::QueueState>>,
) -> io::Result<hyper::Response<ProxyBody>> {
let request = request.map(|body| body.map_err(io::Error::other).boxed());
let request = if let Some(ack) = ack
&& request.uri().path() == "/node_service.ScannerControlService/ScannerScopedDirtyUsageAck"
{
match scoped_ack::intercept(request, client.clone(), target, ack).await? {
scoped_ack::Interception::Pass(request) => request,
scoped_ack::Interception::Handled(response) => return Ok(response),
}
} else {
request
};
let (parts, body) = request.into_parts();
let mut body = body;
let (mut parts, body) = request.into_parts();
let mut body = body.map_err(io::Error::other).boxed();
let mut observed = None;
let mut inspection = None;
if let Some(pause) = pause {
let method = parts
.uri
.path()
.strip_prefix("/node_service.NodeService/")
.or_else(|| parts.uri.path().strip_prefix("/node_service.ScannerControlService/"));
let method = parts.uri.path().strip_prefix("/node_service.NodeService/");
if let Some(method) = method.filter(|name| pause.methods.contains(name)) {
let mut route = None;
let mut lock_owners = Vec::new();
@@ -641,16 +599,6 @@ async fn forward_request(
.with_trailers(async move { trailers.map(Ok::<_, io::Error>) })
.boxed();
matches
} else if method == "ScannerScopedDirtyUsageAck" {
let collected = Limited::new(body, 1024 * 1024).collect().await.map_err(io::Error::other)?;
let trailers = collected.trailers().cloned();
let bytes = collected.to_bytes();
let matches = scoped_ack_matches(&bytes, None)?;
body = Full::new(bytes)
.map_err(|never| match never {})
.with_trailers(async move { trailers.map(Ok::<_, io::Error>) })
.boxed();
matches
} else {
true
};
@@ -694,7 +642,24 @@ async fn forward_request(
}
}
}
let response = forward_to_backend(Request::from_parts(parts, body), client, target).await?;
if !parts.headers.contains_key(http::header::HOST)
&& let Some(authority) = parts.uri.authority()
{
parts.headers.insert(
http::header::HOST,
http::HeaderValue::from_str(authority.as_str()).map_err(io::Error::other)?,
);
}
parts.uri = http::Uri::builder()
.scheme("http")
.authority(target.to_string())
.path_and_query(parts.uri.path_and_query().map_or("/", |value| value.as_str()))
.build()
.map_err(io::Error::other)?;
let response = client
.request(Request::from_parts(parts, body))
.await
.map_err(io::Error::other)?;
if let Some((pause, disk)) = inspection
&& response.status().is_success()
&& response.headers().get("grpc-status").is_some_and(|status| status == "12")
@@ -747,32 +712,6 @@ async fn forward_request(
Ok(response.map(|body| body.map_err(io::Error::other).boxed()))
}
async fn forward_to_backend(
request: Request<ProxyBody>,
client: ProxyClient,
target: SocketAddr,
) -> io::Result<hyper::Response<Incoming>> {
let (mut parts, body) = request.into_parts();
if !parts.headers.contains_key(http::header::HOST)
&& let Some(authority) = parts.uri.authority()
{
parts.headers.insert(
http::header::HOST,
http::HeaderValue::from_str(authority.as_str()).map_err(io::Error::other)?,
);
}
parts.uri = http::Uri::builder()
.scheme("http")
.authority(target.to_string())
.path_and_query(parts.uri.path_and_query().map_or("/", |value| value.as_str()))
.build()
.map_err(io::Error::other)?;
client
.request(Request::from_parts(parts, body))
.await
.map_err(io::Error::other)
}
impl Drop for RpcFaultProxy {
fn drop(&mut self) {
if let Some(requests) = self.late_requests.lock().expect("late RPC ownership").take() {
@@ -792,7 +731,6 @@ mod tests {
struct Backend {
received: Arc<AtomicUsize>,
auth_headers: Arc<Mutex<Vec<(String, String)>>>,
address: SocketAddr,
task: JoinHandle<()>,
}
@@ -816,8 +754,6 @@ mod tests {
let address = listener.local_addr()?;
let received = Arc::new(AtomicUsize::new(0));
let received_task = received.clone();
let auth_headers = Arc::new(Mutex::new(Vec::new()));
let backend_auth = auth_headers.clone();
let task = tokio::spawn(async move {
let mut children = JoinSet::new();
loop {
@@ -825,12 +761,10 @@ mod tests {
accepted = listener.accept() => {
let Ok((socket, _)) = accepted else { break };
let received = received_task.clone();
let backend_auth = backend_auth.clone();
let reply = reply.clone();
children.spawn(async move {
let service = service_fn(move |request: Request<Incoming>| {
let received = received.clone();
let backend_auth = backend_auth.clone();
let reply = reply.clone();
async move {
received.fetch_add(1, Ordering::SeqCst);
@@ -839,10 +773,6 @@ mod tests {
http::HeaderValue::from_str(authority.as_str()).expect("valid HTTP authority")
})
});
if let (Some(nonce), Some(signature)) = (request.headers().get("x-rustfs-rpc-nonce"), request.headers().get("x-rustfs-rpc-signature-v2")) {
backend_auth.lock().expect("backend auth header observations").push((
nonce.to_str().expect("test nonce").to_string(), signature.to_str().expect("test signature").to_string()));
}
let body = request.into_body().collect().await?.to_bytes();
let body = reply.unwrap_or(body);
let mut trailers = http::HeaderMap::new();
@@ -865,12 +795,7 @@ mod tests {
}
}
});
Ok(Backend {
received,
auth_headers,
address,
task,
})
Ok(Backend { received, address, task })
}
async fn call(client: &ProxyClient, address: SocketAddr, method: &str) -> io::Result<hyper::Response<Incoming>> {
@@ -884,14 +809,9 @@ mod tests {
bytes: Bytes,
) -> io::Result<hyper::Response<Incoming>> {
let body: ProxyBody = Full::new(bytes).map_err(|never| match never {}).boxed();
let service = if method == "ScannerScopedDirtyUsageAck" {
"ScannerControlService"
} else {
"NodeService"
};
let request = Request::builder()
.method("POST")
.uri(format!("http://{address}/node_service.{service}/{method}"))
.uri(format!("http://{address}/node_service.NodeService/{method}"))
.header(http::header::TE, "trailers")
.body(body)
.map_err(io::Error::other)?;
@@ -906,353 +826,6 @@ mod tests {
frame.into()
}
const ACK_OWNER: &str = "11111111-1111-1111-1111-111111111111";
const ACK_INSTANCE: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
fn ack_scope() -> ScopedAckScope {
ScopedAckScope {
nonce: uuid::Uuid::new_v4().to_string(),
owner_id: ACK_OWNER.to_string(),
instance_id: ACK_INSTANCE.to_string(),
buckets: std::collections::BTreeSet::from(["cold-bucket".to_string()]),
}
}
fn ack_request(bucket: &str, generation: u64) -> ScannerScopedDirtyUsageAckRequest {
ScannerScopedDirtyUsageAckRequest {
challenge: vec![7; 16].into(),
protocol_version: 1,
owner_id: ACK_OWNER.to_string(),
instance_id: ACK_INSTANCE.to_string(),
scope: 1,
probe_only: false,
entries: vec![rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageEntry {
bucket: bucket.to_string(),
bucket_incarnation: vec![4; 16].into(),
generation,
}],
}
}
fn ack_frame(request: &ScannerScopedDirtyUsageAckRequest) -> Bytes {
let payload = request.encode_to_vec();
let mut frame = vec![0];
frame.extend_from_slice(&u32::try_from(payload.len()).expect("bounded test frame").to_be_bytes());
frame.extend_from_slice(&payload);
Bytes::from(frame)
}
fn ack_hash(bytes: &[u8]) -> String {
use sha2::Digest as _;
hex_simd::encode_to_string(&sha2::Sha256::digest(bytes)[..], hex_simd::AsciiCase::Lower)
}
// Transport-only fixtures: the distributed tests supply production signatures.
async fn call_ack_frame(
client: &ProxyClient,
address: SocketAddr,
frame: Bytes,
nonce: &str,
) -> io::Result<hyper::Response<Incoming>> {
let request = Request::builder()
.method("POST")
.uri(format!("http://{address}/node_service.ScannerControlService/ScannerScopedDirtyUsageAck"))
.header(http::header::TE, "trailers")
.header("x-rustfs-rpc-nonce", nonce)
.header("x-rustfs-rpc-signature-v2", format!("opaque-signature-{nonce}"))
.body(Full::new(frame).map_err(|never| match never {}).boxed())
.map_err(io::Error::other)?;
client.request(request).await.map_err(io::Error::other)
}
#[tokio::test]
async fn scoped_ack_queue_loss_preserves_backend_boundary() {
tokio::time::timeout(Duration::from_secs(10), async {
for fault in [ScopedAckFault::RequestLoss, ScopedAckFault::ResponseLoss] {
let backend = backend().await.expect("backend");
let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
.await
.expect("proxy");
let queue = proxy.scoped_ack_queue(ack_scope(), 1).expect("ACK queue");
let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
let frame = ack_frame(&ack_request("cold-bucket", 7));
let address = proxy.local_addr();
let body = frame.clone();
let caller = tokio::spawn(async move { call_ack_frame(&client, address, body, "loss").await });
queue.wait_for_packets(1).await.expect("captured ACK");
assert_eq!(backend.received.load(Ordering::SeqCst), 0, "retention must precede backend dispatch");
queue.settle(0, fault).expect("settle actual retained packet");
assert!(
caller.await.expect("caller task").is_err(),
"the caller must observe the injected transport failure"
);
let receipt = queue.wait_for_completion(0).await.expect("complete fault receipt");
let response_loss = fault == ScopedAckFault::ResponseLoss;
assert_eq!(receipt["forwarded"], response_loss);
assert_eq!(receipt["backend_complete"], response_loss);
assert_eq!(backend.received.load(Ordering::SeqCst), usize::from(response_loss));
if response_loss {
assert_eq!(receipt["outcome"], "response_lost");
assert_eq!(receipt["response"]["grpc_status"], "0", "status exists only in real backend trailers");
assert_eq!(receipt["response"]["bytes"], frame.len());
assert_eq!(receipt["response"]["sha256"], ack_hash(&frame));
assert!(
receipt["response"]["trailers_sha256"]
.as_str()
.is_some_and(|value| value.len() == 64)
);
} else {
assert_eq!(receipt["outcome"], "request_lost");
}
}
})
.await
.expect("bounded ACK loss checks");
}
#[tokio::test]
async fn scoped_ack_queue_keeps_cancelled_packets_and_releases_out_of_order() {
tokio::time::timeout(Duration::from_secs(10), async {
let backend = backend().await.expect("backend");
let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
.await
.expect("proxy");
let queue = proxy.scoped_ack_queue(ack_scope(), 2).expect("two-packet ACK queue");
let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
let first = ack_frame(&ack_request("cold-bucket", 7));
let second = ack_frame(&ack_request("cold-bucket", 9));
let address = proxy.local_addr();
let old_client = client.clone();
let old_body = first.clone();
let old_caller = tokio::spawn(async move { call_ack_frame(&old_client, address, old_body, "first").await });
queue.wait_for_packets(1).await.expect("first retained packet");
old_caller.abort();
assert!(old_caller.await.expect_err("cancel original caller").is_cancelled());
let new_body = second.clone();
let new_caller = tokio::spawn(async move { call_ack_frame(&client, address, new_body, "second").await });
queue.wait_for_packets(2).await.expect("second retained packet");
assert_eq!(backend.received.load(Ordering::SeqCst), 0);
queue.settle(1, ScopedAckFault::Forward).expect("release newer packet first");
let response = new_caller
.await
.expect("new caller task")
.expect("new response")
.into_body()
.collect()
.await
.expect("complete body");
assert_eq!(response.trailers().expect("response trailers")["x-proof"], "unchanged-proof");
assert_eq!(response.to_bytes(), second);
queue.wait_for_completion(1).await.expect("new packet complete");
queue
.settle(0, ScopedAckFault::Forward)
.expect("release old packet after its caller disappeared");
let old = queue.wait_for_completion(0).await.expect("detached old packet complete");
assert_eq!(old["backend_complete"], true);
assert_eq!(old["request_sha256"], ack_hash(&first));
assert_eq!(old["response"]["sha256"], ack_hash(&first));
assert_eq!(backend.received.load(Ordering::SeqCst), 2);
assert_eq!(
*backend.auth_headers.lock().expect("backend header observations"),
vec![
("second".to_string(), "opaque-signature-second".to_string()),
("first".to_string(), "opaque-signature-first".to_string())
]
);
assert!(
queue.settle(0, ScopedAckFault::Forward).is_err(),
"one original packet may only be forwarded once"
);
})
.await
.expect("bounded detached retention check");
}
#[tokio::test]
async fn scoped_ack_queue_bounds_capacity_and_excludes_other_scopes() {
tokio::time::timeout(Duration::from_secs(10), async {
let backend = backend().await.expect("backend");
let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
.await
.expect("proxy");
assert!(proxy.scoped_ack_queue(ack_scope(), 0).is_err());
assert!(proxy.scoped_ack_queue(ack_scope(), 5).is_err());
let mut invalid = ack_scope();
invalid.nonce.clear();
assert!(proxy.scoped_ack_queue(invalid, 1).is_err());
let mut scope = ack_scope();
scope.buckets.insert("cold-bucket-z".to_string());
let queue = proxy.scoped_ack_queue(scope, 1).expect("bounded ACK queue");
let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
let mut probe = ack_request("cold-bucket", 7);
probe.probe_only = true;
let mut other_peer = ack_request("cold-bucket", 7);
other_peer.owner_id = uuid::Uuid::new_v4().to_string();
let mut other_instance = ack_request("cold-bucket", 7);
other_instance.instance_id = "b".repeat(32);
let mut invalid_challenge = ack_request("cold-bucket", 7);
invalid_challenge.challenge = vec![7; 15].into();
let mut noncanonical_entries = ack_request("cold-bucket-z", 7);
noncanonical_entries.entries.extend(ack_request("cold-bucket", 7).entries);
let unselected = [
probe,
other_peer,
other_instance,
ack_request("cold-bucket-other", 7),
ack_request("cold-bucket", u64::MAX),
invalid_challenge,
noncanonical_entries,
];
let unselected_count = unselected.len();
for (index, request) in unselected.into_iter().enumerate() {
let body = ack_frame(&request);
let response = call_ack_frame(&client, proxy.local_addr(), body.clone(), "unselected")
.await
.expect("unselected ACK progresses");
assert_eq!(response.into_body().collect().await.expect("unselected body").to_bytes(), body);
assert!(queue.receipts().is_empty(), "unselected or protocol-invalid ACK must not consume a slot");
assert_eq!(backend.received.load(Ordering::SeqCst), index + 1);
}
assert!(queue.receipts().is_empty());
let address = proxy.local_addr();
let held_client = client.clone();
let valid_frame = ack_frame(&ack_request("cold-bucket", 7));
let held_body = valid_frame.clone();
let held = tokio::spawn(async move { call_ack_frame(&held_client, address, held_body, "held").await });
queue.wait_for_packets(1).await.expect("one matching packet");
assert!(!held.is_finished(), "the subsequent valid ACK must be retained");
assert_eq!(queue.receipts()[0]["request_sha256"], ack_hash(&valid_frame));
assert_eq!(backend.received.load(Ordering::SeqCst), unselected_count);
let extra = call_ack_frame(&client, address, ack_frame(&ack_request("cold-bucket", 8)), "beyond-capacity")
.await
.expect("capacity overflow passes without retention");
extra.into_body().collect().await.expect("overflow response");
assert_eq!(queue.receipts().len(), 1);
assert_eq!(backend.received.load(Ordering::SeqCst), unselected_count + 1);
drop(queue);
assert!(
held.await.expect("held caller").is_err(),
"dropping the session must cancel its retained packet"
);
assert_eq!(
backend.received.load(Ordering::SeqCst),
unselected_count + 1,
"session drop must never forward the held packet"
);
})
.await
.expect("bounded scope and lifecycle checks");
}
#[tokio::test]
async fn scoped_ack_queue_expiry_never_forwards_retained_packet() {
let (backend, _proxy, queue, caller) = tokio::time::timeout(Duration::from_secs(10), async {
let backend = backend().await.expect("backend");
let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
.await
.expect("proxy");
let queue = proxy.scoped_ack_queue(ack_scope(), 1).expect("ACK queue");
let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
let address = proxy.local_addr();
let caller = tokio::spawn(async move {
call_ack_frame(&client, address, ack_frame(&ack_request("cold-bucket", 7)), "expires").await
});
queue.wait_for_packets(1).await.expect("retained packet");
(backend, proxy, queue, caller)
})
.await
.expect("bounded queue setup");
tokio::time::pause();
tokio::time::advance(Duration::from_secs(120)).await;
tokio::time::resume();
assert!(
tokio::time::timeout(Duration::from_secs(10), caller)
.await
.expect("expiry completes caller")
.expect("caller task")
.is_err()
);
let receipt = queue.receipts().into_iter().next().expect("expired receipt");
assert_eq!(receipt["completed"], true);
assert_eq!(receipt["forwarded"], false);
assert_eq!(receipt["backend_complete"], false);
assert_eq!(backend.received.load(Ordering::SeqCst), 0);
assert_eq!(
queue
.settle(0, ScopedAckFault::Forward)
.expect_err("expired packet cannot be released")
.kind(),
io::ErrorKind::TimedOut
);
}
#[tokio::test]
async fn scanner_ack_fault_matches_exact_bucket_and_excludes_probes() {
use rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageEntry;
tokio::time::timeout(Duration::from_secs(10), async {
let backend = backend().await.expect("backend listener");
let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
.await
.expect("proxy listener");
let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
let pause = proxy
.pause(&["ScannerScopedDirtyUsageAck"], 1, Some("cold-bucket"))
.expect("scoped ACK fault");
let frame = |bucket: &str, probe_only: bool| {
let payload = ScannerScopedDirtyUsageAckRequest {
challenge: vec![3; 16].into(),
protocol_version: 1,
owner_id: "11111111-1111-1111-1111-111111111111".to_string(),
instance_id: "a".repeat(32),
scope: 1,
probe_only,
entries: vec![ScannerScopedDirtyUsageEntry {
bucket: bucket.to_string(),
bucket_incarnation: vec![4; 16].into(),
generation: 17,
}],
}
.encode_to_vec();
let mut frame = vec![0];
frame.extend_from_slice(&u32::try_from(payload.len()).expect("small ACK frame").to_be_bytes());
frame.extend_from_slice(&payload);
Bytes::from(frame)
};
for bytes in [frame("cold-bucket", true), frame("cold-bucket-other", false)] {
let response = call_body(&client, proxy.local_addr(), "ScannerScopedDirtyUsageAck", bytes.clone())
.await
.expect("probe and other bucket must progress");
assert_eq!(response.into_body().collect().await.expect("unselected response").to_bytes(), bytes);
}
assert_eq!(pause.hits(), 0);
let signed_bytes = frame("cold-bucket", false);
let task_bytes = signed_bytes.clone();
let address = proxy.local_addr();
let blocked =
tokio::spawn(async move { call_body(&client, address, "ScannerScopedDirtyUsageAck", task_bytes).await });
pause
.wait_for_hits(1)
.await
.expect("real ScannerControlService method must reach the fault");
assert!(!blocked.is_finished());
assert_eq!(backend.received.load(Ordering::SeqCst), 2, "held ACK must not apply at the backend");
pause.release();
let collected = blocked
.await
.expect("ACK task")
.expect("released ACK")
.into_body()
.collect()
.await
.expect("ACK body and trailers");
assert_eq!(collected.trailers().expect("ACK trailers")["x-proof"], "unchanged-proof");
assert_eq!(collected.to_bytes(), signed_bytes, "proxy must preserve the complete signed ACK frame");
})
.await
.expect("scanner ACK fault deadline");
}
fn native_abort_request(path: &str) -> RecoverMultipartSourceRequest {
use crate::storage_api::multipart_source_rpc::{
MultipartAbortCertificate, MultipartAbortProof, MultipartAbortProofKind, MultipartAbortRound, MultipartSourcePhase,
@@ -2146,4 +1719,3 @@ mod tests {
);
}
}
// event-drill: isolated harness-only change
@@ -1,404 +0,0 @@
// Copyright 2026 RustFS Team
// SPDX-License-Identifier: Apache-2.0
//! Bounded retention of original scoped ACK frames, independent of caller cancellation.
use super::{ProxyBody, ProxyClient, forward_to_backend, unary_payload};
use bytes::Bytes;
use http_body_util::{BodyExt as _, Full, Limited};
use hyper::{Request, Response};
use prost::Message as _;
use rustfs_protos::proto_gen::node_service::{ScannerScopedDirtyUsageAckRequest, ScannerScopedDirtyUsageAckResponse};
use rustfs_protos::scoped_dirty_usage::validate_scoped_dirty_usage_request;
use serde_json::{Value, json};
use sha2::{Digest as _, Sha256};
use std::{
collections::BTreeSet,
io,
net::SocketAddr,
sync::{
Arc, Mutex,
atomic::{AtomicBool, Ordering},
},
time::Duration,
};
use tokio::{
sync::{Notify, oneshot, watch},
task::AbortHandle,
time::Instant,
};
const MAX_RETAINED: usize = 4;
const MAX_BYTES: usize = 1024 * 1024;
const RETENTION: Duration = Duration::from_secs(120);
pub(crate) struct ScopedAckScope {
pub nonce: String,
pub owner_id: String,
pub instance_id: String,
pub buckets: BTreeSet<String>,
}
impl ScopedAckScope {
fn validate(&self) -> io::Result<()> {
let valid_uuid = |value: &str| uuid::Uuid::parse_str(value).is_ok_and(|value| !value.is_nil());
if !valid_uuid(&self.nonce)
|| !valid_uuid(&self.owner_id)
|| self.instance_id.is_empty()
|| self.instance_id.len() > 128
|| self.buckets.is_empty()
|| self.buckets.len() > 16
|| self.buckets.iter().any(|bucket| bucket.is_empty() || bucket.len() > 63)
{
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
"scoped ACK queue needs a bounded session, receiver and bucket scope",
));
}
Ok(())
}
fn matches(&self, request: &ScannerScopedDirtyUsageAckRequest) -> bool {
!request.probe_only
&& validate_scoped_dirty_usage_request(request).is_ok()
&& request.owner_id == self.owner_id
&& request.instance_id == self.instance_id
&& request.entries.iter().all(|entry| self.buckets.contains(&entry.bucket))
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum ScopedAckFault {
Forward,
RequestLoss,
ResponseLoss,
}
struct Packet {
action: watch::Sender<Option<ScopedAckFault>>,
receipt: Mutex<Value>,
task: Mutex<Option<AbortHandle>>,
}
pub(super) struct QueueState {
scope: ScopedAckScope,
capacity: usize,
started: Instant,
deadline: Instant,
closed: AtomicBool,
// The packet inventory lock is released before any packet lock or await.
packets: Mutex<Vec<Arc<Packet>>>,
changed: Notify,
}
pub(super) type QueueSlot = Arc<Mutex<Option<Arc<QueueState>>>>;
pub(super) fn empty_slot() -> QueueSlot {
Arc::new(Mutex::new(None))
}
pub(crate) struct ScopedAckQueue {
state: Arc<QueueState>,
slot: QueueSlot,
}
impl ScopedAckQueue {
pub(super) fn arm(slot: QueueSlot, scope: ScopedAckScope, capacity: usize) -> io::Result<Self> {
scope.validate()?;
if !(1..=MAX_RETAINED).contains(&capacity) {
return Err(io::Error::new(io::ErrorKind::InvalidInput, "scoped ACK capacity must be 1..=4"));
}
let mut active = slot.lock().expect("scoped ACK queue slot");
if active.is_some() {
return Err(io::Error::new(io::ErrorKind::AlreadyExists, "scoped ACK queue is already armed"));
}
let started = Instant::now();
let state = Arc::new(QueueState {
scope,
capacity,
started,
deadline: started + RETENTION,
closed: AtomicBool::new(false),
packets: Mutex::new(Vec::new()),
changed: Notify::new(),
});
*active = Some(state.clone());
drop(active);
Ok(Self { state, slot })
}
pub(crate) fn receipts(&self) -> Vec<Value> {
let packets = self.state.packets.lock().expect("scoped ACK packets").clone();
packets
.iter()
.map(|packet| packet.receipt.lock().expect("scoped ACK receipt").clone())
.collect()
}
pub(crate) async fn wait_for_packets(&self, count: usize) -> io::Result<()> {
self.wait_until(|| self.receipts().len() >= count).await
}
pub(crate) async fn wait_for_completion(&self, id: usize) -> io::Result<Value> {
self.wait_until(|| {
self.receipts()
.get(id)
.is_some_and(|receipt| receipt["completed"].as_bool() == Some(true))
})
.await?;
self.receipts()
.get(id)
.cloned()
.ok_or_else(|| io::Error::other("scoped ACK receipt disappeared"))
}
async fn wait_until(&self, ready: impl Fn() -> bool) -> io::Result<()> {
tokio::time::timeout_at(self.state.deadline, async {
loop {
let changed = self.state.changed.notified();
if ready() {
return;
}
changed.await;
}
})
.await
.map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "scoped ACK queue lifetime elapsed"))
}
pub(crate) fn settle(&self, id: usize, fault: ScopedAckFault) -> io::Result<()> {
if Instant::now() >= self.state.deadline {
return Err(io::Error::new(io::ErrorKind::TimedOut, "scoped ACK retention expired"));
}
let packet = self
.state
.packets
.lock()
.expect("scoped ACK packets")
.get(id)
.cloned()
.ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "scoped ACK packet is not retained"))?;
if !packet.action.send_if_modified(|current| {
if current.is_none() {
*current = Some(fault);
true
} else {
false
}
}) {
return Err(io::Error::new(io::ErrorKind::AlreadyExists, "scoped ACK packet was already settled"));
}
Ok(())
}
}
impl Drop for ScopedAckQueue {
fn drop(&mut self) {
self.state.closed.store(true, Ordering::SeqCst);
let mut slot = self.slot.lock().expect("scoped ACK queue slot");
if slot.as_ref().is_some_and(|current| Arc::ptr_eq(current, &self.state)) {
*slot = None;
}
drop(slot);
let packets = self.state.packets.lock().expect("scoped ACK packets").clone();
for packet in packets {
if let Some(task) = packet.task.lock().expect("scoped ACK task").take() {
task.abort();
}
}
}
}
pub(super) enum Interception {
Pass(Request<ProxyBody>),
Handled(Response<ProxyBody>),
}
fn restored_body(bytes: Bytes, trailers: Option<http::HeaderMap>) -> ProxyBody {
Full::new(bytes)
.map_err(|never| match never {})
.with_trailers(async move { trailers.map(Ok::<_, io::Error>) })
.boxed()
}
fn hash(bytes: &[u8]) -> String {
hex(Sha256::digest(bytes).as_ref())
}
fn hex(bytes: &[u8]) -> String {
hex_simd::encode_to_string(bytes, hex_simd::AsciiCase::Lower)
}
fn trailer_receipt(trailers: Option<&http::HeaderMap>) -> Value {
trailers.map_or(Value::Null, |headers| {
json!(
headers
.iter()
.map(|(name, value)| { json!({"name": name.as_str(), "value_hex": hex(value.as_bytes())}) })
.collect::<Vec<_>>()
)
})
}
fn header_hash(headers: &http::HeaderMap) -> String {
let mut hasher = Sha256::new();
for (name, value) in headers {
hasher.update(name.as_str().as_bytes());
hasher.update([0]);
hasher.update(value.as_bytes());
hasher.update([0]);
}
hex(hasher.finalize().as_ref())
}
fn millis(state: &QueueState) -> u64 {
u64::try_from(state.started.elapsed().as_millis()).unwrap_or(u64::MAX)
}
pub(super) async fn intercept(
request: Request<ProxyBody>,
client: ProxyClient,
target: SocketAddr,
state: Arc<QueueState>,
) -> io::Result<Interception> {
if Instant::now() >= state.deadline || state.closed.load(Ordering::SeqCst) {
return Ok(Interception::Pass(request));
}
let (parts, body) = request.into_parts();
let collected = tokio::time::timeout_at(state.deadline, Limited::new(body, MAX_BYTES).collect())
.await
.map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "scoped ACK frame collection expired"))?
.map_err(io::Error::other)?;
let trailers = collected.trailers().cloned();
let bytes = collected.to_bytes();
let decoded = ScannerScopedDirtyUsageAckRequest::decode(unary_payload(&bytes)?).map_err(io::Error::other)?;
let request = Request::from_parts(parts, restored_body(bytes.clone(), trailers.clone()));
if Instant::now() >= state.deadline || !state.scope.matches(&decoded) {
return Ok(Interception::Pass(request));
}
let header = |name| request.headers().get(name).and_then(|value| value.to_str().ok());
let (action, mut selected) = watch::channel(None);
let packet = {
let mut packets = state.packets.lock().expect("scoped ACK packets");
if packets.len() >= state.capacity || state.closed.load(Ordering::SeqCst) {
return Ok(Interception::Pass(request));
}
let id = packets.len();
let packet = Arc::new(Packet {
action,
task: Mutex::new(None),
receipt: Mutex::new(json!({
"id": id, "session_nonce": state.scope.nonce, "receiver": target.to_string(),
"authority": request.uri().authority().map(|value| value.as_str()),
"captured_at_ms": millis(&state), "completed": false, "forwarded": false,
"backend_complete": false, "outcome": "held", "request_bytes": bytes.len(),
"request_sha256": hash(&bytes), "request_frame_hex": hex(&bytes), "request_headers_sha256": header_hash(request.headers()),
"request_trailers_sha256": trailers.as_ref().map(header_hash),
"request_trailers": trailer_receipt(trailers.as_ref()),
"auth_nonce": header("x-rustfs-rpc-nonce"), "replay_nonce": header("x-rustfs-rpc-replay-nonce"),
"signature_sha256": header("x-rustfs-rpc-signature-v2").map(|value| hash(value.as_bytes())),
"replay_signature_sha256": header("x-rustfs-rpc-signature-v3").map(|value| hash(value.as_bytes())),
"request": {"owner_id": decoded.owner_id, "instance_id": decoded.instance_id,
"probe_only": decoded.probe_only, "entries": decoded.entries.iter().map(|entry| json!({
"bucket": entry.bucket, "bucket_incarnation": uuid::Uuid::from_slice(&entry.bucket_incarnation).ok().map(|value| value.to_string()),
"generation": entry.generation })).collect::<Vec<_>>() }
})),
});
packets.push(packet.clone());
packet
};
let (reply, response) = oneshot::channel();
let worker_packet = packet.clone();
let worker_state = state.clone();
// This owner survives an HTTP/2 stream reset from the original 5 s caller.
// Every request is settled once and is discarded at the session deadline.
let task = tokio::spawn(async move {
let result = tokio::time::timeout_at(worker_state.deadline, async {
let fault = loop {
if let Some(fault) = *selected.borrow_and_update() {
break fault;
}
selected
.changed()
.await
.map_err(|_| io::Error::other("scoped ACK queue closed"))?;
};
if fault == ScopedAckFault::RequestLoss {
worker_packet.receipt.lock().expect("scoped ACK receipt")["outcome"] = json!("request_lost");
return Err(io::Error::new(io::ErrorKind::ConnectionAborted, "injected scoped ACK request loss"));
}
if worker_state.closed.load(Ordering::SeqCst) || Instant::now() >= worker_state.deadline {
return Err(io::Error::new(
io::ErrorKind::ConnectionAborted,
"scoped ACK session ended or expired before forwarding",
));
}
{
let mut receipt = worker_packet.receipt.lock().expect("scoped ACK receipt");
receipt["forwarded"] = json!(true);
receipt["forwarded_at_ms"] = json!(millis(&worker_state));
}
let backend = forward_to_backend(request, client, target).await?;
let (parts, body) = backend.into_parts();
let collected = Limited::new(body, MAX_BYTES).collect().await.map_err(io::Error::other)?;
let trailers = collected.trailers().cloned();
let bytes = collected.to_bytes();
let grpc_status = trailers
.as_ref()
.and_then(|values| values.get("grpc-status"))
.or_else(|| parts.headers.get("grpc-status"))
.and_then(|value| value.to_str().ok());
let response = unary_payload(&bytes)
.ok()
.and_then(|payload| ScannerScopedDirtyUsageAckResponse::decode(payload).ok())
.map(|response| {
json!({"owner_id": response.owner_id, "instance_id": response.instance_id,
"supported": response.supported, "cleared": response.cleared,
"response_proof_bytes": response.response_proof.len()})
});
{
let mut receipt = worker_packet.receipt.lock().expect("scoped ACK receipt");
receipt["backend_complete"] = json!(true);
receipt["backend_completed_at_ms"] = json!(millis(&worker_state));
receipt["response"] = json!({"status": parts.status.as_u16(), "grpc_status": grpc_status,
"bytes": bytes.len(), "sha256": hash(&bytes), "frame_hex": hex(&bytes), "headers_sha256": header_hash(&parts.headers),
"trailers": trailer_receipt(trailers.as_ref()),
"trailers_sha256": trailers.as_ref().map(header_hash), "ack": response});
receipt["outcome"] = json!(if fault == ScopedAckFault::ResponseLoss {
"response_lost"
} else {
"forwarded"
});
}
if fault == ScopedAckFault::ResponseLoss {
return Err(io::Error::new(
io::ErrorKind::ConnectionAborted,
"injected scoped ACK response loss after backend trailers",
));
}
Ok(Response::from_parts(parts, restored_body(bytes, trailers)))
})
.await
.unwrap_or_else(|_| Err(io::Error::new(io::ErrorKind::TimedOut, "scoped ACK retention expired")));
{
let mut receipt = worker_packet.receipt.lock().expect("scoped ACK receipt");
if result.is_err() && receipt["outcome"] == "held" {
receipt["outcome"] = json!("transport_failed_or_expired");
}
receipt["completed"] = json!(true);
receipt["completed_at_ms"] = json!(millis(&worker_state));
}
let _ = reply.send(result);
worker_state.changed.notify_one();
});
*packet.task.lock().expect("scoped ACK task") = Some(task.abort_handle());
if state.closed.load(Ordering::SeqCst) {
task.abort();
}
drop(task);
state.changed.notify_one();
response
.await
.map_err(|_| io::Error::new(io::ErrorKind::ConnectionAborted, "scoped ACK queue ended"))?
.map(Interception::Handled)
}
-23
View File
@@ -70,9 +70,6 @@ pub(crate) mod multipart_rpc_delay {
#[cfg(test)]
pub(crate) mod multipart_fence_acceptance {
pub(crate) use super::RUSTFS_META_MULTIPART_BUCKET;
/// Targeted lock diagnostics for the metadata-delay quorum case.
pub(crate) const LOCK_DIAGNOSTICS_FILTER: &str = "warn,rustfs_lock::distributed_lock=debug,rustfs_ecstore::object_lock_diag=trace,rustfs_ecstore::cluster::rpc::remote_locker=info";
}
#[cfg(test)]
@@ -83,23 +80,3 @@ pub(crate) mod multipart_source_rpc {
PublicationIdentity, SourceGeneration, SourceGeometry, SourceLineage,
};
}
#[cfg(test)]
pub(crate) mod scanner_peer_bucket {
use rustfs_ecstore::api::disk as ecstore_disk;
use rustfs_ecstore::api::erasure as ecstore_erasure;
use rustfs_ecstore::api::rpc as ecstore_rpc;
pub(crate) use ecstore_disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
pub(crate) use ecstore_erasure::{ReedSolomonEncoder, calc_shard_size};
pub(crate) use ecstore_rpc::PeerRestClient;
}
/// Storage handles used to seed a pre-validation bucket record while the fixture server is stopped.
#[cfg(test)]
pub(crate) mod bucket_tag_fixture {
pub(crate) use rustfs_ecstore::api::bucket::metadata::load_bucket_metadata;
pub(crate) use rustfs_ecstore::api::layout::{DisksLayout, EndpointServerPools};
pub(crate) use rustfs_ecstore::api::runtime::InstanceContext;
pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks_with_instance_ctx};
}
+29 -147
View File
@@ -360,26 +360,12 @@ impl OpaMock {
pub(crate) async fn expect_bucket_tags(&mut self, action: &str, bucket: &str, department: Option<&str>) -> TestResult {
let payload = self.next_request().await?;
Self::assert_bucket_tags(&payload, action, bucket, department);
Ok(())
}
async fn expect_versioned_tag_permission(&mut self, bucket: &str, object: &str, version: &str) -> TestResult {
let payload = self.next_request().await?;
Self::assert_bucket_tags(&payload, "s3:GetObjectVersionTagging", bucket, Some("finance"));
assert_eq!(payload["input"]["resource"]["object"], object);
assert_eq!(payload["input"]["context"]["conditions"]["versionid"], serde_json::json!([version]));
Ok(())
}
fn assert_bucket_tags(payload: &Value, action: &str, bucket: &str, department: Option<&str>) {
let input = &payload["input"];
assert_eq!(input["identity"]["account"], "opabuckettags");
assert_eq!(input["identity"]["is_owner"], false);
assert_eq!(input["action"], action);
assert_eq!(input["resource"]["bucket"], bucket);
let expected = department.map(|value| serde_json::json!([value]));
assert_eq!(input["context"]["conditions"].get("ExistingBucketTag/department"), expected.as_ref());
Ok(())
}
async fn wait_for_validation(&mut self) -> TestResult {
@@ -692,61 +678,22 @@ async fn test_list_buckets_opa_contract() -> TestResult {
Ok(())
}
/// Stop and reap the directory's owning server before opening the fixture store.
async fn seed_legacy_duplicate_bucket_tags(env: &mut RustFSTestEnvironment, bucket: &str) -> TestResult {
use crate::storage_api::bucket_tag_fixture::{
DisksLayout, ECStore, EndpointServerPools, InstanceContext, init_local_disks_with_instance_ctx, load_bucket_metadata,
};
let server = env
.process
.as_mut()
.ok_or("missing server process before offline fixture write")?;
server.kill()?;
server.wait()?;
env.process = None;
let layout = DisksLayout::from_volumes(std::slice::from_ref(&env.temp_dir))?;
let (pools, _) = EndpointServerPools::create_server_endpoints(&env.address, &layout).await?;
let context = std::sync::Arc::new(InstanceContext::new());
init_local_disks_with_instance_ctx(&context, pools.clone()).await?;
let store = ECStore::new_with_instance_ctx(env.address.parse()?, pools, Default::default(), context.clone()).await?;
let _cancel = context
.background_cancel_token()
.expect("store binds its shutdown token")
.drop_guard();
let mut metadata = load_bucket_metadata(store.clone(), bucket).await?;
let incarnation = metadata.bucket_incarnation_id;
let policy = metadata.policy_config_json.clone();
assert!(!incarnation.is_nil());
assert!(!policy.is_empty(), "the bucket-policy Allow must survive legacy seeding");
// Encode the duplicate tag set with the same bucket record and erasure writer used by the store.
let xml = b"<Tagging><TagSet><Tag><Key>department</Key><Value>finance</Value></Tag><Tag><Key>department</Key><Value>engineering</Value></Tag></TagSet></Tagging>";
metadata.tagging_config_xml = xml.to_vec();
metadata.save_with_store(store.clone()).await?;
let persisted = load_bucket_metadata(store, bucket).await?;
assert_eq!(persisted.bucket_incarnation_id, incarnation);
assert_eq!(persisted.policy_config_json, policy);
assert_eq!(persisted.tagging_config_xml.as_slice(), xml);
Ok(())
}
#[tokio::test]
async fn test_bucket_tags_opa_contract() -> TestResult {
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Tag, Tagging, TaggingDirective, VersioningConfiguration,
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Tag, Tagging, VersioningConfiguration,
};
init_logging();
let mut opa = OpaMock::start().await?;
let mut env = RustFSTestEnvironment::new().await?;
let opa_url = opa.url.clone();
let server_env = [
("RUSTFS_POLICY_PLUGIN_URL", opa_url.as_str()),
env.start_rustfs_server_without_cleanup_with_env(&[
("RUSTFS_POLICY_PLUGIN_URL", opa.url.as_str()),
("RUSTFS_POLICY_PLUGIN_AUTH_TOKEN", OPA_AUTH_TOKEN),
("NO_PROXY", "127.0.0.1,localhost"),
];
env.start_rustfs_server_without_cleanup_with_env(&server_env).await?;
])
.await?;
let secret = uuid::Uuid::new_v4().to_string();
create_user(&env, "opabuckettags", &secret).await?;
let client = aws_sdk_s3::Client::from_conf(build_test_s3_config(&env.url, "opabuckettags", &secret, None, "e2e-bucket-tags"));
@@ -786,7 +733,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
.await?;
assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
let head = client
client
.head_object()
.bucket(source)
.key("report")
@@ -794,8 +741,6 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
.send()
.await?;
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
assert_eq!(head.tag_count(), Some(1));
opa.expect_versioned_tag_permission(source, "report", version).await?;
client.list_objects_v2().bucket(source).send().await?;
opa.expect_bucket_tags("s3:ListBucket", source, Some("finance")).await?;
client.list_object_versions().bucket(source).send().await?;
@@ -805,65 +750,28 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
admin.create_bucket().bucket(destination).send().await?;
set_department(&admin, destination, "engineering").await?;
let copy_source = format!("{source}/report?versionId={version}");
for directive in [TaggingDirective::Copy, TaggingDirective::Replace] {
let error = client
.copy_object()
.bucket(destination)
.key("copied")
.copy_source(&copy_source)
.set_tagging_directive((directive == TaggingDirective::Replace).then_some(TaggingDirective::Replace))
.set_tagging((directive == TaggingDirective::Replace).then(|| "department=legal".to_owned()))
.send()
.await
.expect_err("source tags must not authorize a different destination");
assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
if directive == TaggingDirective::Copy {
opa.expect_versioned_tag_permission(source, "report", version).await?;
}
opa.expect_bucket_tags("s3:PutObject", destination, Some("engineering"))
.await?;
let missing = admin
.head_object()
.bucket(destination)
.key("copied")
.send()
.await
.expect_err("denied COPY and REPLACE must leave no destination object");
assert_eq!(missing.raw_response().map(|response| response.status().as_u16()), Some(404));
}
let error = client
.copy_object()
.bucket(destination)
.key("copied")
.copy_source(&copy_source)
.send()
.await
.expect_err("source tags must not authorize a different destination");
assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
opa.expect_bucket_tags("s3:PutObject", destination, Some("engineering"))
.await?;
set_department(&admin, destination, "finance").await?;
for directive in [TaggingDirective::Copy, TaggingDirective::Replace] {
client
.copy_object()
.bucket(destination)
.key("copied")
.copy_source(&copy_source)
.set_tagging_directive((directive == TaggingDirective::Replace).then_some(TaggingDirective::Replace))
.set_tagging((directive == TaggingDirective::Replace).then(|| "department=legal".to_owned()))
.send()
.await?;
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
// The access hook checks preliminary tags, then the copy use case checks
// the exact source reader's tags before publishing the destination.
for _ in 0..2 {
if directive == TaggingDirective::Copy {
opa.expect_versioned_tag_permission(source, "report", version).await?;
}
opa.expect_bucket_tags("s3:PutObject", destination, Some("finance")).await?;
}
let tags = admin.get_object_tagging().bucket(destination).key("copied").send().await?;
assert_eq!(tags.tag_set().len(), 1);
assert_eq!(tags.tag_set()[0].key(), "department");
assert_eq!(
tags.tag_set()[0].value(),
if directive == TaggingDirective::Copy {
"engineering"
} else {
"legal"
}
);
}
client
.copy_object()
.bucket(destination)
.key("copied")
.copy_source(&copy_source)
.send()
.await?;
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
opa.expect_bucket_tags("s3:PutObject", destination, Some("finance")).await?;
let upload = client
.create_multipart_upload()
@@ -939,14 +847,6 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("engineering"))
.await?;
let missing = admin
.head_object()
.bucket(destination)
.key("denied-source")
.send()
.await
.expect_err("a denied source must leave no destination object");
assert_eq!(missing.raw_response().map(|response| response.status().as_u16()), Some(404));
set_department(&client, source, "finance")
.await
.expect_err("requested tags must not authorize their own mutation");
@@ -985,7 +885,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
opa.expect_bucket_tags("s3:GetObject", destination, Some("engineering"))
.await?;
let invalid_write = admin
admin
.put_bucket_tagging()
.bucket(destination)
.tagging(
@@ -995,25 +895,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
.build()?,
)
.send()
.await
.expect_err("new duplicate tags must be rejected before publication");
assert_eq!(invalid_write.as_service_error().and_then(ProvideErrorMetadata::code), Some("InvalidTag"));
assert_eq!(invalid_write.raw_response().map(|response| response.status().as_u16()), Some(400));
set_department(&admin, source, "finance").await?;
seed_legacy_duplicate_bucket_tags(&mut env, destination).await?;
env.start_rustfs_server_without_cleanup_with_env(&server_env).await?;
// Prove this process can authorize the same caller and read a healthy bucket's persisted tags.
let read = client
.get_object()
.bucket(source)
.key("report")
.version_id(version)
.send()
.await?;
assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
let error = client
.get_object()
.bucket(destination)
@@ -48,7 +48,6 @@ const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY";
const G09_EVIDENCE_DIR_ENV: &str = "RUSTFS_SCANNER_HEAL_G09_EVIDENCE_DIR";
const RC5_COMMIT: &str = "40a2470feb567201165a5b809b7598bb4b1f68f5";
const STABLE_1_0_1_COMMIT: &str = "6de965ae3c965a78ff819fbcd7acd4aa44177d92";
const STABLE_1_0_4_COMMIT: &str = "46337942d55f3a8e6a97b147ca98d9ea8115555d";
const SSE_MASTER_KEY_ENV: &str = "RUSTFS_SSE_S3_MASTER_KEY";
const SSE_MASTER_KEY: &str = "QkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkI=";
const PLAIN_BUCKET: &str = "upgrade-plain-data";
@@ -955,8 +954,7 @@ async fn exercise_mixed_cluster(
phase: &str,
current_node: usize,
previous_node: usize,
previous_has_delete_recovery: bool,
) -> Result<Option<String>, BoxError> {
) -> TestResult {
let clients = cluster.create_all_clients()?;
wait_for_upgrade_write_readiness(&clients, MIXED_BUCKET, phase, LISTING_CONVERGENCE_TIMEOUT).await?;
wait_for_drive_readmission(&clients, MIXED_BUCKET, phase, DRIVE_READMISSION_TIMEOUT).await?;
@@ -1011,30 +1009,6 @@ async fn exercise_mixed_cluster(
vec![u8::try_from(MULTIPART_WORKERS - 1)?; 64 * 1024]
);
// 1.0.4 already implements the delete decision protocol. Use a separate
// old-writer key so the original seed still spans the entire rollout.
if previous_has_delete_recovery {
let key = format!(".upgrade-delete/{phase}/previous-writer");
previous_client
.put_object()
.bucket(MIXED_BUCKET)
.key(&key)
.body(ByteStream::from_static(UPGRADE_READINESS_BODY))
.send()
.await?;
for client in &clients {
assert_eq!(read_object(client, MIXED_BUCKET, &key, None).await?.1, UPGRADE_READINESS_BODY);
}
upgrade_probe_client(current_client)
.delete_object()
.bucket(MIXED_BUCKET)
.key(&key)
.send()
.await?;
assert_mixed_deletes_visible(&clients, std::slice::from_ref(&key)).await?;
return Ok(Some(key));
}
// Every online participant must understand the delete decision protocol.
// An upgrade phase can serve reads/writes while intentionally refusing a
// modern DELETE; rejecting it must leave the old writer's seed intact.
@@ -1055,31 +1029,6 @@ async fn exercise_mixed_cluster(
);
}
Ok(None)
}
async fn assert_mixed_deletes_visible(clients: &[Client], keys: &[String]) -> TestResult {
for (node, client) in clients.iter().enumerate() {
for key in keys {
let error = upgrade_probe_client(client)
.get_object()
.bucket(MIXED_BUCKET)
.key(key)
.send()
.await
.expect_err("a successful mixed-version DELETE must remain visible through every reader");
assert_eq!(
error.raw_response().map(|response| response.status().as_u16()),
Some(404),
"node {node}: {key}"
);
assert_eq!(
error.as_service_error().and_then(ProvideErrorMetadata::code),
Some("NoSuchKey"),
"node {node}: {key}"
);
}
}
Ok(())
}
@@ -1163,9 +1112,8 @@ async fn wait_for_upgrade_writes(
match response {
Ok(Ok(_)) => break,
Ok(Err(error)) => {
let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
if error.raw_response().map(|response| response.status().as_u16()) != Some(503)
|| !matches!(code, Some("ServiceUnavailable" | "SlowDown" | "SlowDownRead"))
|| error.as_service_error().and_then(ProvideErrorMetadata::code) != Some("ServiceUnavailable")
{
return Err(format!("{phase}: node {node} {probe} failed: {error:?}").into());
}
@@ -2291,9 +2239,6 @@ async fn direct_upgrade_from_previous_release_preserves_object_contracts() -> Te
async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts() -> TestResult {
init_logging();
let previous_binary = source_binary()?;
// Bind the expected protocol to the verified release source, never to the
// DELETE result itself. The older CI source pins retain their 503 contract.
let previous_has_delete_recovery = binary_source_revision(&previous_binary).await? == STABLE_1_0_4_COMMIT;
let evidence_context = g09_evidence_context(&previous_binary).await?;
let current_binary = rustfs_binary_path();
let mut cluster = RustFSTestClusterEnvironment::new(MIXED_NODE_COUNT).await?;
@@ -2314,10 +2259,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
cluster.stop_node(0)?;
cluster.start_node_from_binary(0, &current_binary).await?;
let mut mixed_deleted_keys = Vec::new();
if let Some(key) = exercise_mixed_cluster(&cluster, "one-current-node", 0, 1, previous_has_delete_recovery).await? {
mixed_deleted_keys.push(key);
}
exercise_mixed_cluster(&cluster, "one-current-node", 0, 1).await?;
// One controlled out-of-policy attempt proves fail-closed behavior below
// native write quorum; positive MPU writers remain paused while mixed.
assert_mixed_pending_refusal(&cluster, &pending, &sources).await?;
@@ -2325,9 +2267,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
for (node_idx, phase) in [(1, "two-current-nodes"), (2, "one-previous-node")] {
cluster.stop_node(node_idx)?;
cluster.start_node_from_binary(node_idx, &current_binary).await?;
if let Some(key) = exercise_mixed_cluster(&cluster, phase, 0, 3, previous_has_delete_recovery).await? {
mixed_deleted_keys.push(key);
}
exercise_mixed_cluster(&cluster, phase, 0, 3).await?;
}
cluster.stop_node(3)?;
@@ -2337,7 +2277,6 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
let completed_pending = finish_pending_multipart(&clients, &pending, pending_parts.concat()).await?;
exercise_homogeneous_multipart(&cluster, "homogeneous-current").await?;
verify_load_bodies(&clients, "homogeneous-previous", "multipart").await?;
assert_mixed_deletes_visible(&clients, &mixed_deleted_keys).await?;
for (node_idx, client) in clients.iter().enumerate() {
assert_eq!(
@@ -2386,7 +2325,6 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
cluster.start_with_binary(&previous_binary).await?;
prepare_release_baseline(&mut cluster, &previous_binary, "rollback").await?;
let old_clients = cluster.create_all_clients()?;
assert_mixed_deletes_visible(&old_clients, &mixed_deleted_keys).await?;
assert_resumed_multipart(&old_clients, &completed_pending).await?;
for phase in ["homogeneous-previous", "homogeneous-current"] {
verify_load_bodies(&old_clients, phase, "multipart").await?;
@@ -2451,8 +2389,6 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
"phases": phases,
"objects_per_phase": objects_per_phase,
"mixed_load": "plain-put",
"mixed_delete_protocol_supported": previous_has_delete_recovery,
"mixed_deleted_keys": mixed_deleted_keys,
"homogeneous_multipart_phases": ["homogeneous-previous", "homogeneous-current", "homogeneous-rollback"],
"multipart_objects_per_homogeneous_phase": MULTIPART_WORKERS * MULTIPART_UPLOADS_PER_WORKER,
"assertions": [
@@ -2485,11 +2421,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
"16x16 multipart load runs in homogeneous previous, current and rollback phases",
"one-current-node Complete returns typed 503 and preserves actual source authority and parts",
"the same legacy upload completes and replays once fully upgraded without adding a version",
if previous_has_delete_recovery {
"mixed-version DELETE succeeds and stays absent through every reader after upgrade and rollback"
} else {
"modern DELETE fails closed and preserves every byte while old peers remain"
}
"modern DELETE fails closed and preserves every byte while old peers remain"
],
}),
)?;
+3 -6
View File
@@ -31,7 +31,6 @@ workspace = true
[features]
default = []
fips = ["rustfs-s3-client/fips", "rustfs-utils/fips"]
diagnostic-read-version-caller = ["rustfs-io-metrics/diagnostic-read-version-caller"]
# Sampled wall/CPU diagnostics only; does not change the read backend.
diagnostic-local-batch = ["rustix/time"]
@@ -226,11 +225,9 @@ metrics = { workspace = true }
# The guard scripts/check_no_tokio_io_uring.sh allows an explicit
# io-uring integration; only the tokio "io-uring" runtime feature is banned.
[target.'cfg(target_os = "linux")'.dependencies]
# Native completion owners keep physical I/O leases alive after cancellation.
rustfs-uring = { git = "https://github.com/rustfs/uring.git", rev = "eb8fc160e71d804427de1e905f5c226d580f71bf" }
[target.'cfg(target_os = "linux")'.dev-dependencies]
rustfs-uring = { git = "https://github.com/rustfs/uring.git", rev = "eb8fc160e71d804427de1e905f5c226d580f71bf", features = ["fault-injection"] }
# SharedReadBudget and the bounded read APIs are available in the 0.3.0
# registry release; keep the application on the published contract.
rustfs-uring = "0.3.0"
[target.'cfg(windows)'.dependencies]
winapi-util.workspace = true
+12 -32
View File
@@ -192,7 +192,6 @@ pub mod bucket {
table_catalog_path_hash, unreadable_config_refusal,
};
pub use crate::bucket::metadata::{BUCKET_DURABILITY_CONFIG, BUCKET_ON_DEMAND_MIGRATION_CONFIG};
pub use crate::bucket::metadata::{MAX_BUCKET_TAGS, validate_tag_set_entries};
}
pub mod config_parse_mode {
@@ -332,8 +331,8 @@ pub mod bucket {
pub mod utils {
pub use crate::bucket::utils::{
check_bucket_and_object_names, check_list_objs_args, check_object_name_for_length_and_slash,
check_valid_bucket_name_strict, deserialize, has_bad_path_component, has_xml_invalid_control_characters,
is_meta_bucketname, is_valid_object_prefix, serialize,
check_valid_bucket_name_strict, deserialize, has_bad_path_component, is_meta_bucketname, is_valid_object_prefix,
serialize,
};
}
@@ -448,19 +447,10 @@ pub mod data_usage {
}
pub mod disk {
pub use crate::cluster::rpc::background_pipe::{BackgroundPipeReader, BackgroundPipeWriter, background_pipe};
pub use crate::cluster::rpc::background_reader_stream::BackgroundReaderStream;
pub use crate::disk::background_memory::BackgroundEncodeBuffer;
pub use crate::disk::disk_store::get_object_disk_read_timeout;
#[cfg(feature = "test-util")]
pub use crate::disk::disk_store::wait_for_multipart_index_bootstrap;
pub use crate::disk::local::{ReplacementExecutionLease, ScanGuard};
#[cfg(feature = "test-util")]
pub use crate::disk::local::{local_durability_modes, set_local_rename_data_fail_after_metadata_commit};
pub use crate::disk::multipart_index::{
MULTIPART_INDEX_RPC_MAX_BYTES, MULTIPART_INDEX_RPC_MAX_REQUEST_BYTES, MultipartIndexPage, MultipartIndexRpcReply,
MultipartIndexRpcRequest,
};
pub use crate::disk::multipart_source::{
MULTIPART_SOURCE_PROTOCOL_VERSION, MultipartAbortCertificate, MultipartAbortProof, MultipartAbortProofKind,
MultipartAbortRound, MultipartSourceOperation, MultipartSourcePhase, MultipartSourceRequest, MultipartSourceResponse,
@@ -473,24 +463,18 @@ pub mod disk {
pub use crate::disk::os::{LocalPublicationPause, LocalPublicationStage};
pub use crate::disk::{
BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp,
BoundedDeleteRecoveryDiscovery, BoundedDeleteRecoveryInspection, BoundedDeleteRecoverySettlement,
BoundedDeleteRecoverySettlementReply, CheckPartsResp, ConditionalFileUpdate, CurrentDeleteGeneration,
CurrentDeleteTransition, DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES, DELETE_RECOVERY_MAX_INSPECTION_ENTRIES,
CheckPartsResp, ConditionalFileUpdate, CurrentDeleteGeneration, CurrentDeleteTransition,
DELETE_RECOVERY_PROTOCOL_VERSION, DeleteOptions, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest,
DeleteRecoveryInspection, DeleteRecoveryPage, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, DeleteRecoveryState,
Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore, FileInfoVersions, FileReader, FileWriter,
GuardedDeleteVersionsPayload, GuardedFileInfoVersions, HEAL_REPLACEMENT_RECOVERY_DIR, HEALING_MARKER_PATH,
NsScannerOpenRequest, OldCurrentSize, POOL_META_NATIVE_CAS_METADATA_SUFFIX, PartTransactionAction, PoolMetaNativeBackup,
PoolMetaNativeInspection, PoolMetaNativeObject, PoolMetaNativeRecoveryRequest, PoolMetaNativeRecoveryResponse,
GuardedDeleteVersionsPayload, GuardedFileInfoVersions, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize,
POOL_META_NATIVE_CAS_METADATA_SUFFIX, PartTransactionAction, PoolMetaNativeBackup, PoolMetaNativeInspection,
PoolMetaNativeObject, PoolMetaNativeRecoveryRequest, PoolMetaNativeRecoveryResponse,
QUOTA_MUTATION_FENCE_METADATA_SUFFIX, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_TMP_BUCKET,
ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, SnapshotLeaseToken,
UpdateMetadataOpts, VolumeInfo, WalkDirOptions, is_quota_mutation_fence_path, new_disk,
validate_batch_read_version_item_count,
};
pub use crate::disk::{
BackgroundMemoryBudget, PhysicalDiskBinding, PhysicalDiskPlan, PhysicalDiskSample, PhysicalDiskSampler,
PhysicalResourcePlan, RemoteBackgroundIo, with_background_resource_unit, with_remote_background_io,
};
pub use bytes::Bytes;
pub use endpoint::Endpoint;
pub use error::DiskError;
@@ -626,12 +610,11 @@ pub mod rebalance {
pub mod rio {
pub use crate::io_support::rio::{
DecryptReader, DynReader, EncryptReader, HardLimitReader, HashReader, ReadStream, Reader, WriteEncryption, WritePlan,
boxed_reader, compression_metadata_value, validate_compression_scheme, wrap_reader,
boxed_reader, compression_metadata_value, wrap_reader,
};
}
pub mod rpc {
pub use crate::cluster::rpc::http_auth::with_rpc_background_request;
pub use crate::cluster::rpc::{
AuthenticatedChannel, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, MAX_NETWORK_PROBE_BYTES, MAX_NETWORK_PROBE_DURATION,
NetworkPeerProbeClient, NetworkPeerProbeError, NetworkPeerProbeMeasurement, NetworkPeerTarget, PEER_RESTDRY_RUN,
@@ -641,8 +624,8 @@ pub mod rpc {
TONIC_RPC_PREFIX, TonicInterceptor, build_put_file_auth_trailer, check_and_record_signed_rpc_nonce,
decode_heal_bucket_rpc_options, encode_heal_bucket_rpc_options, gen_signature_headers, gen_tonic_replay_scope_headers,
gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client,
node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, rpc_background_admission,
set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_ns_scanner_capability_with_checkpoint_coverage,
node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
sign_ns_scanner_capability, sign_ns_scanner_capability_with_checkpoint_coverage,
sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, sign_tonic_rpc_response_proof,
tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
verify_ns_scanner_capability, verify_ns_scanner_capability_with_checkpoint_coverage,
@@ -663,9 +646,7 @@ pub mod set_disk {
#[cfg(feature = "test-util")]
pub mod test_util {
pub use crate::bucket::quota::reservation::{
fail_next_quota_ledger_save_for_test, initialize_quota_ledger_for_test, retry_quota_reconciliation_for_test,
};
pub use crate::bucket::quota::reservation::fail_next_quota_ledger_save_for_test;
pub use crate::set_disk::ConditionalReadBenchmarkMetadataGuard;
pub use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, PutObjectCommitPause};
@@ -694,9 +675,8 @@ pub mod storage {
pub use crate::store::HealWalkVersion;
pub use crate::store::{
BootstrapLocalTarget, ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk,
all_local_disk_path, find_local_disk_by_ref, find_local_disk_by_ref_for_rpc, init_local_disks,
init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map,
prewarm_local_disk_id_map_with_instance_ctx,
all_local_disk_path, find_local_disk_by_ref, init_local_disks, init_local_disks_with_instance_ctx, init_lock_clients,
prewarm_local_disk_id_map, prewarm_local_disk_id_map_with_instance_ctx,
};
pub use crate::store::{HealObjectAbsenceProof, HealObjectStorageResult};
}
+5 -325
View File
@@ -16,8 +16,7 @@ use crate::bucket::metadata::BucketMetadata;
use crate::bucket::metadata_sys::get_bucket_targets_config;
use crate::bucket::metadata_sys::get_replication_config;
use crate::bucket::remote_s3_client::{
PathStyle, REPLICATION_TARGET_RETRY_POLICY, RemoteCredentials, RemoteS3EndpointSpec, RemoteS3RetryPolicy,
UploadResponseDeadline, build_remote_s3_client,
PathStyle, REPLICATION_TARGET_RETRY_POLICY, RemoteCredentials, RemoteS3EndpointSpec, build_remote_s3_client,
};
use crate::bucket::replication::{ObjectLockIntegrity, object_lock_put_integrity, replication_etags_match};
use crate::bucket::replication::{ReplicationStatusType, ReplicationTargetConfigBridge};
@@ -120,7 +119,6 @@ impl From<&BucketTarget> for RemoteS3EndpointSpec {
ca_cert_pem: (!target.ca_cert_pem.trim().is_empty()).then(|| target.ca_cert_pem.clone()),
connect_timeout: None,
read_timeout: None,
operation_timeout: Some(TargetRequestTimeouts::from_env().request),
// Replication has no retry budget of its own on the request path,
// so it keeps the SDK's standard three attempts; stating it here
// pins the behaviour to this line instead of an SDK default.
@@ -1193,7 +1191,6 @@ impl BucketTargetSys {
health_check_duration: target.health_check_duration,
replicate_sync: target.replication_sync,
client: Arc::new(client),
timeouts: TargetRequestTimeouts::from_env(),
})
}
@@ -1925,83 +1922,6 @@ impl std::fmt::Display for S3ClientError {
}
}
/// Replication request budgets, snapshotted when a target client is built.
/// Streamed GET bodies are not subject to the upload-response budget.
#[derive(Debug, Clone, Copy)]
pub struct TargetRequestTimeouts {
request: Duration,
complete: Duration,
minimum_transfer_rate: u64,
}
impl Default for TargetRequestTimeouts {
fn default() -> Self {
Self {
request: Duration::from_secs(60),
complete: Duration::from_secs(15 * 60),
minimum_transfer_rate: 1024 * 1024,
}
}
}
impl TargetRequestTimeouts {
fn from_env() -> Self {
let defaults = Self::default();
let seconds = |name: &str, fallback: Duration| {
let value = rustfs_utils::get_env_u64(name, fallback.as_secs());
Duration::from_secs(if (1..=86400).contains(&value) {
value
} else {
fallback.as_secs()
})
};
let minimum_transfer_rate =
rustfs_utils::get_env_u64("RUSTFS_REPLICATION_MIN_TRANSFER_RATE_BYTES_PER_SEC", defaults.minimum_transfer_rate);
Self {
request: seconds("RUSTFS_REPLICATION_REQUEST_TIMEOUT_SECS", defaults.request),
complete: seconds("RUSTFS_REPLICATION_COMPLETE_TIMEOUT_SECS", defaults.complete),
minimum_transfer_rate: if minimum_transfer_rate == 0 {
defaults.minimum_transfer_rate
} else {
minimum_transfer_rate
},
}
}
fn upload_timeout(self, size: i64) -> Result<Duration, S3ClientError> {
let size = u64::try_from(size).map_err(|_| S3ClientError::new("replication upload size must be non-negative"))?;
// The SDK can make three attempts. Leave transmission headroom for
// all three; the separate response deadline still bounds a trickling
// target after each upload finishes.
let attempts = match REPLICATION_TARGET_RETRY_POLICY {
RemoteS3RetryPolicy::Disabled => 1,
RemoteS3RetryPolicy::Standard { max_attempts } => u64::from(max_attempts.max(1)),
};
let seconds = size
.div_ceil(self.minimum_transfer_rate)
.checked_mul(attempts)
.ok_or_else(|| S3ClientError::new("replication upload timeout exceeds supported duration"))?;
let timeout = self
.request
.checked_add(Duration::from_secs(seconds))
.ok_or_else(|| S3ClientError::new("replication upload timeout exceeds supported duration"))?;
if Instant::now().checked_add(timeout).is_none() {
return Err(S3ClientError::new("replication upload timeout exceeds supported duration"));
}
Ok(timeout)
}
fn upload_override(self, size: i64) -> Result<aws_sdk_s3::config::Builder, S3ClientError> {
Ok(aws_sdk_s3::config::Builder::new()
.timeout_config(
aws_smithy_types::timeout::TimeoutConfig::builder()
.operation_timeout(self.upload_timeout(size)?)
.build(),
)
.interceptor(UploadResponseDeadline(self.request)))
}
}
#[derive(Debug)]
pub struct TargetClient {
pub endpoint: String,
@@ -2015,7 +1935,6 @@ pub struct TargetClient {
pub health_check_duration: Duration,
pub replicate_sync: bool,
pub client: Arc<S3Client>,
pub timeouts: TargetRequestTimeouts,
}
impl TargetClient {
@@ -2441,11 +2360,12 @@ impl TargetClient {
Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
});
let mut config = self.timeouts.upload_override(size)?;
if forwards_source_checksum {
config = config.request_checksum_calculation(aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired);
operation = operation.config_override(
aws_sdk_s3::config::Builder::new()
.request_checksum_calculation(aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired),
);
}
operation = operation.config_override(config);
match operation.send().await {
Ok(output) => {
// Under SSE-KMS/DSSE or SSE-C the target's ETag is not the MD5
@@ -2581,7 +2501,6 @@ impl TargetClient {
}
Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
})
.config_override(self.timeouts.upload_override(size)?)
.send()
.await
{
@@ -2619,13 +2538,6 @@ impl TargetClient {
}
Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
})
.config_override(
aws_sdk_s3::config::Builder::new().timeout_config(
aws_smithy_types::timeout::TimeoutConfig::builder()
.operation_timeout(self.timeouts.complete)
.build(),
),
)
.send()
.await
{
@@ -3054,241 +2966,11 @@ mod tests {
health_check_duration: Duration::from_secs(5),
replicate_sync: false,
client: Arc::new(client),
timeouts: TargetRequestTimeouts::default(),
},
request_headers,
)
}
#[derive(Debug, Clone)]
struct TimedResponseConnector {
upload_delay: Duration,
response_delay: Duration,
prefix_bytes: usize,
response: bytes::Bytes,
frames: Arc<std::sync::atomic::AtomicUsize>,
dropped: Arc<std::sync::atomic::AtomicUsize>,
}
struct ResponseDrop(Arc<std::sync::atomic::AtomicUsize>);
impl Drop for ResponseDrop {
fn drop(&mut self) {
self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
}
}
impl SmithyHttpConnector for TimedResponseConnector {
fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
let settings = self.clone();
HttpConnectorFuture::new(async move {
http_body_util::BodyExt::collect(request.into_body())
.await
.map_err(aws_smithy_runtime_api::client::result::ConnectorError::io)?;
tokio::time::sleep(settings.upload_delay).await;
let guard = ResponseDrop(Arc::clone(&settings.dropped));
let stream = futures::stream::unfold(
(settings.prefix_bytes, Some(settings.response), true, guard),
move |(remaining, mut body, first, guard)| {
let frames = Arc::clone(&settings.frames);
async move {
if remaining == 0 && body.is_none() {
return None;
}
if !first {
tokio::time::sleep(settings.response_delay).await;
}
let bytes = if remaining > 0 {
bytes::Bytes::from_static(b" ")
} else {
body.take()?
};
frames.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
Some((
Ok::<_, std::io::Error>(http_body::Frame::data(bytes)),
(remaining.saturating_sub(1), body, false, guard),
))
}
},
)
.fuse();
let body = SdkBody::from_body_1_x(http_body_util::StreamBody::new(stream));
let mut response =
HttpResponse::new(aws_smithy_runtime_api::http::StatusCode::try_from(200u16).expect("status"), body);
response.headers_mut().insert("etag", "\"test-etag\"");
Ok(response)
})
}
}
fn timed_response_target(
upload_delay: Duration,
response_delay: Duration,
prefix_bytes: usize,
response: &'static [u8],
) -> (TargetClient, Arc<std::sync::atomic::AtomicUsize>, Arc<std::sync::atomic::AtomicUsize>) {
let frames = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let dropped = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let connector = SharedHttpConnector::new(TimedResponseConnector {
upload_delay,
response_delay,
prefix_bytes,
response: bytes::Bytes::from_static(response),
frames: Arc::clone(&frames),
dropped: Arc::clone(&dropped),
});
let http_client = http_client_fn(move |_settings, _components| connector.clone());
let (mut target, _) = header_recording_target_client(Vec::new());
target.timeouts = TargetRequestTimeouts {
request: Duration::from_secs(2),
complete: Duration::from_secs(10),
minimum_transfer_rate: 1024 * 1024,
};
target.client = Arc::new(S3Client::from_conf(
target
.client
.config()
.to_builder()
.http_client(http_client)
.retry_config(aws_sdk_s3::config::retry::RetryConfig::disabled())
.timeout_config(
aws_smithy_types::timeout::TimeoutConfig::builder()
.operation_timeout(target.timeouts.request)
.build(),
)
.build(),
));
(target, frames, dropped)
}
#[test]
fn replication_upload_budgets_preserve_large_transfers_and_reject_invalid_sizes() {
let policy = TargetRequestTimeouts::default();
assert_eq!(policy.upload_timeout(0).expect("empty upload"), Duration::from_secs(60));
assert_eq!(
policy.upload_timeout(5 * 1024 * 1024 * 1024).expect("maximum single PUT"),
Duration::from_secs(15420)
);
assert!(policy.upload_timeout(-1).is_err());
let slow = TargetRequestTimeouts {
minimum_transfer_rate: 1,
..policy
};
assert!(slow.upload_timeout(i64::MAX).is_err());
}
#[tokio::test(start_paused = true)]
async fn replication_upload_response_deadline_is_independent_of_size_and_checksum_override() {
for operation in ["put", "forwarded-checksum", "part"] {
let (target, frames, dropped) = timed_response_target(Duration::from_secs(4), Duration::from_millis(500), 100, b"");
let body = vec![b'x'; 2 * 1024 * 1024];
let size = i64::try_from(body.len()).expect("size");
let started = tokio::time::Instant::now();
let error = if operation == "part" {
target
.put_object_part(
"bucket",
"key",
"upload",
1,
size,
ByteStream::from(body),
&PutObjectPartOptions::default(),
)
.await
.expect_err("response deadline")
} else {
let mut opts = PutObjectOptions::default();
if operation == "forwarded-checksum" {
opts.user_metadata.insert(
"x-amz-checksum-sha256".to_owned(),
"OoJ3yNhRwv3wwtZoGqEIPrPX9xwTnfLl+ka0wStN1g0=".to_owned(),
);
}
target
.put_object("bucket", "key", size, ByteStream::from(body), &opts)
.await
.expect_err("response deadline")
};
assert!(
started.elapsed() >= Duration::from_secs(6),
"upload gets its full independent window: {error}"
);
assert!(
started.elapsed() < target.timeouts.upload_timeout(size).expect("budget"),
"response must end before the larger total deadline"
);
assert!(
frames.load(std::sync::atomic::Ordering::SeqCst) >= 2,
"trickle delivered progress before failure"
);
assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1, "transport body released");
}
}
#[tokio::test(start_paused = true)]
async fn replication_slow_upload_preserves_transfer_window_and_streaming_get_body() {
let (target, _, _) = timed_response_target(Duration::from_secs(4), Duration::from_millis(100), 5, b"");
let body = vec![b'x'; 2 * 1024 * 1024];
let size = i64::try_from(body.len()).expect("size");
target
.put_object("bucket", "key", size, ByteStream::from(body), &PutObjectOptions::default())
.await
.expect("slow upload, prompt response");
let (target, _, dropped) = timed_response_target(Duration::ZERO, Duration::from_secs(1), 5, b"payload");
let response = target
.get_object("bucket", "key", None, None, None, HeaderMap::new())
.await
.expect("GET headers");
let started = tokio::time::Instant::now();
let body = response
.body
.collect()
.await
.expect("streamed GET remains outside upload deadline")
.into_bytes();
assert!(started.elapsed() > target.timeouts.request);
assert_eq!(&body[..], b" payload");
assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
}
#[tokio::test(start_paused = true)]
async fn replication_complete_processing_whitespace_has_a_separate_bounded_window() {
const RESULT: &[u8] = b"<CompleteMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><ETag>\"etag\"</ETag></CompleteMultipartUploadResult>";
for prefix in [5, 100] {
let (target, frames, dropped) = timed_response_target(Duration::ZERO, Duration::from_secs(1), prefix, RESULT);
let started = tokio::time::Instant::now();
let result = target
.complete_multipart_upload(
"bucket",
"key",
"upload",
vec![CompletedPart::builder().part_number(1).e_tag("etag").build()],
&PutObjectOptions::default(),
)
.await;
if prefix == 5 {
assert!(result.is_ok(), "legal processing whitespace: {result:?}");
} else {
assert!(result.is_err());
assert!(started.elapsed() <= Duration::from_secs(11));
}
assert!(started.elapsed() > target.timeouts.request);
assert!(frames.load(std::sync::atomic::Ordering::SeqCst) >= 2);
assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
}
}
#[tokio::test(start_paused = true)]
async fn replication_control_operation_deadline_covers_response_trickle() {
let (target, frames, dropped) =
timed_response_target(Duration::ZERO, Duration::from_millis(500), 100, b"<VersioningConfiguration/>");
let started = tokio::time::Instant::now();
assert!(target.get_bucket_versioning("bucket").await.is_err());
assert!(started.elapsed() <= Duration::from_secs(3));
assert!(frames.load(std::sync::atomic::Ordering::SeqCst) >= 2);
assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
}
fn streaming_test_body(payload: &'static [u8]) -> ByteStream {
let stream = tokio_util::io::ReaderStream::new(std::io::Cursor::new(payload));
let body = http_body_util::StreamBody::new(futures::StreamExt::map(stream, |r| r.map(http_body::Frame::data)));
@@ -3597,7 +3279,6 @@ mod tests {
health_check_duration: Duration::from_secs(5),
replicate_sync: false,
client: Arc::new(client),
timeouts: TargetRequestTimeouts::default(),
},
request_uris,
)
@@ -3910,7 +3591,6 @@ mod tests {
health_check_duration: Duration::from_secs(5),
replicate_sync: false,
client: Arc::new(client),
timeouts: TargetRequestTimeouts::default(),
})
}
@@ -5487,14 +5487,6 @@ pub async fn put_restore_opts(
if !oi.user_tags.is_empty() {
meta.insert(AMZ_OBJECT_TAGGING.to_string(), (*oi.user_tags).clone());
}
if let Some(expires) = oi.expires {
meta.insert(
"expires".to_string(),
expires
.format(&time::format_description::well_known::Rfc3339)
.map_err(std::io::Error::other)?,
);
}
let restore_expiry = lifecycle::expected_expiry_time(OffsetDateTime::now_utc(), rreq.days.unwrap_or(1));
meta.insert(
metadata_keys::RESTORE.to_string(),
@@ -5514,6 +5506,7 @@ pub async fn put_restore_opts(
// computed MD5 is not the object's public plaintext ETag.
preserve_etag: oi.etag.clone(),
shard_integrity_write_mode: Some(oi.shard_integrity_write_mode()),
//expires: oi.expires,
..Default::default()
})
}
@@ -7151,25 +7144,6 @@ mod tests {
0,
"tier generation lease should release after EOF"
);
let unread = get_transitioned_object_reader_with_tier_manager(
&object_info.bucket,
&object_info.name,
&None,
&HeaderMap::new(),
&object_info,
&ObjectOptions::default(),
&manager,
None,
)
.await
.expect("open an unread transitioned response");
assert_eq!(TierConfigMgr::active_operation_lease_count(&manager, &tier).await, 1);
drop(unread);
assert_eq!(
TierConfigMgr::active_operation_lease_count(&manager, &tier).await,
0,
"canceling an unread response must release the tier lease"
);
}
#[cfg(feature = "test-util")]
+2 -77
View File
@@ -256,38 +256,6 @@ pub const BUCKET_POLICY_CONFIG: &str = "policy.json";
pub const BUCKET_NOTIFICATION_CONFIG: &str = "notification.xml";
pub const BUCKET_LIFECYCLE_CONFIG: &str = "lifecycle.xml";
pub const BUCKET_SSECONFIG: &str = "bucket-encryption.xml";
/// Shared tag-entry validation for object requests and persisted bucket tags.
pub fn validate_tag_set_entries(tag_set: &[s3s::dto::Tag]) -> std::result::Result<(), &'static str> {
fn is_valid_tag_text(value: &str) -> bool {
static VALID_TAG_TEXT: std::sync::OnceLock<std::result::Result<regex::Regex, regex::Error>> = std::sync::OnceLock::new();
VALID_TAG_TEXT
.get_or_init(|| regex::Regex::new(r"^[\p{L}\p{N}\p{Z}+\-=._:/@]*$"))
.as_ref()
.is_ok_and(|pattern| pattern.is_match(value))
}
let mut keys = std::collections::HashSet::with_capacity(tag_set.len());
for tag in tag_set {
let key = tag
.key
.as_deref()
.filter(|key| !key.is_empty())
.ok_or("Tag key cannot be empty")?;
if key.encode_utf16().count() > 128 || !is_valid_tag_text(key) {
return Err("The TagKey you have provided is invalid");
}
let value = tag.value.as_deref().ok_or("Tag value cannot be null")?;
if value.encode_utf16().count() > 256 || !is_valid_tag_text(value) {
return Err("The TagValue you have provided is invalid");
}
if !keys.insert(key) {
return Err("Cannot provide multiple Tags with the same key");
}
}
Ok(())
}
pub const MAX_BUCKET_TAGS: usize = 50;
pub const BUCKET_TAGGING_CONFIG: &str = "tagging.xml";
pub const BUCKET_QUOTA_CONFIG_FILE: &str = "quota.json";
pub const OBJECT_LOCK_CONFIG: &str = "object-lock.xml";
@@ -1022,20 +990,6 @@ impl BucketMetadata {
self.encryption_config_updated_at = updated;
}
BUCKET_TAGGING_CONFIG => {
if !data.is_empty() {
let tagging = deserialize::<Tagging>(&data).map_err(|_| {
Error::InvalidArgument(self.name.clone(), String::new(), "Invalid bucket tagging XML".to_owned())
})?;
if tagging.tag_set.len() > MAX_BUCKET_TAGS {
return Err(Error::InvalidArgument(
self.name.clone(),
String::new(),
"Cannot have more than 50 tags per bucket".to_owned(),
));
}
validate_tag_set_entries(&tagging.tag_set)
.map_err(|message| Error::InvalidArgument(self.name.clone(), String::new(), message.to_owned()))?;
}
self.tagging_config_xml = data;
// Drop the parsed form (like lifecycle above) so clearing the
// payload can't leave stale parsed tags to be cached.
@@ -1549,11 +1503,7 @@ pub(crate) async fn load_bucket_metadata_parse_with_presence(
bucket: &str,
parse: bool,
) -> Result<(BucketMetadata, bool)> {
let (metadata_res, incarnation_res) = tokio::join!(
Box::pin(read_bucket_metadata(api.clone(), bucket)),
Box::pin(load_bucket_incarnation(api.clone(), bucket)),
);
let (mut bm, persisted) = match metadata_res {
let (mut bm, persisted) = match read_bucket_metadata(api.clone(), bucket).await {
Ok(res) => (res, true),
Err(err) => {
if err != Error::ConfigNotFound {
@@ -1564,7 +1514,7 @@ pub(crate) async fn load_bucket_metadata_parse_with_presence(
}
};
let incarnation = incarnation_res?;
let incarnation = load_bucket_incarnation(api.clone(), bucket).await?;
if persisted {
if bm.name.is_empty() {
bm.name = bucket.to_string();
@@ -1830,31 +1780,6 @@ mod test {
/// backlog#2292: a replicated config is stamped with the source
/// `updated_at` it was given, not the local clock, while the plain
/// `update_config` entry keeps stamping the local clock.
#[test]
fn bucket_tag_updates_reject_unreadable_shapes_without_replacing_metadata() {
let mut metadata = BucketMetadata::new("validated-tags");
let valid = br#"<Tagging><TagSet><Tag><Key>team</Key><Value></Value></Tag></TagSet></Tagging>"#.to_vec();
metadata
.update_config(BUCKET_TAGGING_CONFIG, valid.clone())
.expect("empty value is valid");
let updated = metadata.tagging_config_updated_at;
for invalid in [
"<Tagging><TagSet><Tag><Key>team</Key></Tag></TagSet></Tagging>".to_owned(),
"<Tagging><TagSet><Tag><Key></Key><Value>x</Value></Tag></TagSet></Tagging>".to_owned(),
"<Tagging><TagSet><Tag><Key>team</Key><Value>x</Value></Tag><Tag><Key>team</Key><Value>y</Value></Tag></TagSet></Tagging>".to_owned(),
format!("<Tagging><TagSet>{}</TagSet></Tagging>", (0..51).map(|i| format!("<Tag><Key>k{i}</Key><Value>v</Value></Tag>")).collect::<String>()),
] {
assert!(metadata.update_config_at(BUCKET_TAGGING_CONFIG, invalid.into_bytes(), updated + time::Duration::hours(1)).is_err());
assert_eq!(metadata.tagging_config_xml, valid);
assert_eq!(metadata.tagging_config_updated_at, updated);
}
metadata.tagging_config_xml = b"legacy corrupt tags".to_vec();
metadata
.update_config(BUCKET_TAGGING_CONFIG, Vec::new())
.expect("deletion repairs legacy corruption");
assert!(metadata.tagging_config_xml.is_empty());
}
#[test]
fn update_config_at_stamps_the_given_time_and_update_config_stamps_now() {
let source_time = OffsetDateTime::now_utc() - time::Duration::hours(3);
+27 -219
View File
@@ -683,7 +683,6 @@ pub(crate) async fn update_validated_in<F>(
bucket: &str,
config_file: &str,
data: Vec<u8>,
expected_incarnation_id: Option<Uuid>,
validate: F,
) -> Result<OffsetDateTime>
where
@@ -691,7 +690,7 @@ where
{
Box::pin(async {
let sys = require_bucket_metadata_sys_in(ctx)?;
let guard = acquire_config_write_guard_for_incarnation(sys.clone(), bucket, expected_incarnation_id).await?;
let guard = acquire_config_write_guard(sys.clone(), bucket).await?;
validate()?;
update_under_config_write_guard(sys, &guard, config_file, data, None).await
})
@@ -1436,7 +1435,6 @@ pub(crate) async fn get_object_lock_config_state_in(
/// Re-read the Object Lock state and bucket incarnation from the same
/// authoritative metadata blob while the caller holds the transaction lock.
#[cfg(all(test, feature = "test-util"))]
pub(crate) async fn get_object_lock_config_and_incarnation_from_disk_in(
ctx: &crate::runtime::instance::InstanceContext,
bucket: &str,
@@ -1455,27 +1453,6 @@ pub(crate) async fn get_object_lock_config_and_incarnation_from_disk_with_guard_
get_delete_config_from_disk_in(ctx, bucket, guard, None).await
}
#[allow(clippy::type_complexity)]
pub(crate) async fn get_object_lock_and_quota_config_from_disk_with_guard_in(
ctx: &crate::runtime::instance::InstanceContext,
bucket: &str,
guard: Option<&rustfs_lock::NamespaceLockGuard>,
) -> Result<(
ObjectLockConfigState,
Uuid,
OffsetDateTime,
std::result::Result<(Option<BucketQuota>, Uuid, OffsetDateTime), String>,
)> {
let (state, incarnation_id, updated_at, quota_res) =
get_delete_and_quota_config_from_disk_in(ctx, bucket, guard, None, true).await?;
Ok((
state,
incarnation_id,
updated_at,
quota_res.unwrap_or(Ok((None, incarnation_id, updated_at))),
))
}
/// Validate queued expiration against the same fresh metadata used for Object
/// Lock admission. The caller holds the transaction read lock through commit,
/// so a successful lifecycle update cannot be followed by an old-policy delete.
@@ -1485,24 +1462,6 @@ pub(crate) async fn get_delete_config_from_disk_in(
guard: Option<&rustfs_lock::NamespaceLockGuard>,
expiry_policy: Option<&rustfs_lifecycle::ExpiryPolicySnapshot>,
) -> Result<(ObjectLockConfigState, Uuid, OffsetDateTime)> {
let (state, incarnation_id, updated_at, _) =
get_delete_and_quota_config_from_disk_in(ctx, bucket, guard, expiry_policy, false).await?;
Ok((state, incarnation_id, updated_at))
}
#[allow(clippy::type_complexity)]
async fn get_delete_and_quota_config_from_disk_in(
ctx: &crate::runtime::instance::InstanceContext,
bucket: &str,
guard: Option<&rustfs_lock::NamespaceLockGuard>,
expiry_policy: Option<&rustfs_lifecycle::ExpiryPolicySnapshot>,
include_quota: bool,
) -> Result<(
ObjectLockConfigState,
Uuid,
OffsetDateTime,
Option<std::result::Result<(Option<BucketQuota>, Uuid, OffsetDateTime), String>>,
)> {
let bucket_meta_sys_lock = bucket_metadata_sys_of(ctx)?;
let bucket_meta_sys = bucket_meta_sys_lock.read().await.clone();
@@ -1518,7 +1477,7 @@ async fn get_delete_and_quota_config_from_disk_in(
.await?
}
};
match &authority {
match authority {
BucketMetadataAuthority::Authoritative(metadata)
if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() =>
{
@@ -1530,11 +1489,11 @@ async fn get_delete_and_quota_config_from_disk_in(
}
}
let object_lock_config_updated_at = metadata.object_lock_config_updated_at;
let state = object_lock_config_state_from_authoritative_metadata(metadata)?;
let incarnation_id = metadata.bucket_incarnation_id;
let quota_res = include_quota
.then(|| quota_config_and_incarnation_from_authority(bucket, authority).map_err(|err| err.to_string()));
Ok((state, incarnation_id, object_lock_config_updated_at, quota_res))
Ok((
object_lock_config_state_from_authoritative_metadata(&metadata)?,
metadata.bucket_incarnation_id,
object_lock_config_updated_at,
))
}
BucketMetadataAuthority::Authoritative(_) => {
Err(Error::other(format!("bucket incarnation metadata is not authoritative: {bucket}")))
@@ -1605,23 +1564,6 @@ pub(crate) async fn get_quota_config_and_incarnation_from_disk_for_options_in(
return Err(Error::other("quota snapshot metadata transaction fence was released"));
}
#[cfg(test)]
let has_injected_error = sys.object_lock_disk_read_errors.read().await.contains(bucket);
#[cfg(not(test))]
let has_injected_error = false;
if !ctx.lock_manager().is_disabled()
&& !has_injected_error
&& let Some(fresh_quota) = snapshot.take_fresh_put_quota()
{
if snapshot_guard.is_lock_lost()
|| snapshot_guard.is_released()
|| !snapshot.is_valid_for_destructive_put(store.id, bucket, opts.expected_bucket_incarnation_id.unwrap_or_default())
{
return Err(Error::other("quota snapshot fence or bucket incarnation changed during the disk read"));
}
return fresh_quota.map_err(Error::other);
}
// The snapshot carries the caller's lifecycle fence, excluding deletion.
// Its transaction guard excludes config writes, so a same-name object
// owner need not recursively acquire the bucket namespace here.
@@ -1808,22 +1750,9 @@ struct MetadataPublishGuard {
_guard: tokio::sync::OwnedMutexGuard<MetadataPublishLockState>,
}
static LOCAL_META_MUTATION_GEN: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);
pub(crate) fn bump_local_meta_mutation_gen() {
LOCAL_META_MUTATION_GEN.fetch_add(1, std::sync::atomic::Ordering::Release);
}
pub(crate) fn current_local_meta_mutation_gen() -> u64 {
LOCAL_META_MUTATION_GEN.load(std::sync::atomic::Ordering::Acquire)
}
type LocalAuthorityCache = Arc<StdMutex<HashMap<String, (u64, Arc<BucketMetadata>)>>>;
#[derive(Debug, Clone)]
pub struct BucketMetadataSys {
metadata_map: Arc<RwLock<HashMap<String, Arc<BucketMetadata>>>>,
local_authority_cache: LocalAuthorityCache,
/// Serializes metadata-map commits and their derived cache updates for one
/// bucket. Namespace locks, when present, are acquired before this lock.
metadata_publish_locks: Arc<MetadataPublishLockRegistry>,
@@ -1867,7 +1796,6 @@ impl BucketMetadataSys {
pub fn new(api: Arc<ECStore>) -> Self {
Self {
metadata_map: Arc::new(RwLock::new(HashMap::new())),
local_authority_cache: Arc::new(StdMutex::new(HashMap::new())),
metadata_publish_locks: Arc::new(MetadataPublishLockRegistry {
locks: StdMutex::new(HashMap::new()),
}),
@@ -1902,15 +1830,6 @@ impl BucketMetadataSys {
self.api.upgrade()
}
/// Whether the backing `ECStore` has been dropped.
///
/// Used by `InstanceContext::clear_bucket_metadata_sys_if_stale` during
/// `ECStore::drop` so a shared bootstrap context does not retain a
/// `BucketMetadataSys` whose weak handle is dangling.
pub(crate) fn is_store_stale(&self) -> bool {
self.api.upgrade().is_none()
}
fn metadata_publish_lock(&self, bucket: &str) -> Arc<Mutex<MetadataPublishLockState>> {
Self::bucket_lock_in(&self.metadata_publish_locks, bucket)
}
@@ -2241,7 +2160,6 @@ impl BucketMetadataSys {
}
pub async fn set(&self, bucket: String, bm: Arc<BucketMetadata>) {
bump_local_meta_mutation_gen();
if !is_meta_bucketname(&bucket) {
let publish_lock = self.metadata_publish_lock(&bucket);
let _publish_guard = publish_lock.lock().await;
@@ -2260,7 +2178,6 @@ impl BucketMetadataSys {
///
/// Returns `true` if an entry was present. Reserved meta buckets are ignored.
pub async fn remove(&self, bucket: &str) -> bool {
bump_local_meta_mutation_gen();
if is_meta_bucketname(bucket) {
return false;
}
@@ -2281,7 +2198,6 @@ impl BucketMetadataSys {
}
async fn _reset(&mut self) {
bump_local_meta_mutation_gen();
let mut map = self.metadata_map.write().await;
map.clear();
drop(map);
@@ -3085,77 +3001,30 @@ impl BucketMetadataSys {
return Err(Error::other(format!("injected Object Lock metadata disk read failure: {bucket}")));
}
let store = self.object_store();
let can_use_local_authority_cache = !store.ctx.lock_manager().is_disabled() && store.all_disks_local_and_online();
let gen_before = current_local_meta_mutation_gen();
let cached_authority = can_use_local_authority_cache
.then(|| {
self.local_authority_cache
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.get(bucket)
.filter(|(cached_gen, _)| *cached_gen == gen_before)
.map(|(_, bm)| Arc::clone(bm))
})
.flatten();
if let Some(cached) = cached_authority {
await_bucket_namespace_operation(
Some(guard),
bucket,
"bucket metadata authoritative snapshot",
std::future::ready(Ok::<_, Error>(())),
)
.await?;
if current_local_meta_mutation_gen() == gen_before && store.all_disks_local_and_online() {
return Ok(BucketMetadataAuthority::Authoritative(cached));
}
}
let (bucket_info_res, metadata_res) = tokio::join!(
Box::pin(await_bucket_namespace_operation(
Some(guard),
bucket,
"bucket metadata snapshot existence check",
async {
store
.get_bucket_info_from_sets_at_read_quorum(
bucket,
&crate::storage_api_contracts::bucket::BucketOptions::default(),
)
.await
},
)),
Box::pin(await_bucket_namespace_operation(
Some(guard),
bucket,
"bucket metadata authoritative snapshot",
load_bucket_metadata_parse_with_presence(Arc::clone(&store), bucket, true),
)),
);
match bucket_info_res {
match await_bucket_namespace_operation(Some(guard), bucket, "bucket metadata snapshot existence check", async {
self.object_store()
.get_bucket_info_from_sets_at_read_quorum(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
.await
})
.await
{
Ok(_) => {}
Err(Error::VolumeNotFound) => return Ok(BucketMetadataAuthority::MissingBucket),
Err(err) => return Err(err),
}
let (metadata, persisted) = metadata_res?;
let (metadata, persisted) = await_bucket_namespace_operation(
Some(guard),
bucket,
"bucket metadata authoritative snapshot",
load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true),
)
.await?;
if persisted && metadata.lock_enabled && !metadata.bucket_creation_committed {
return Err(Error::ErasureWriteQuorum);
}
if persisted {
let metadata = Arc::new(metadata);
if can_use_local_authority_cache
&& metadata.bucket_incarnation_sidecar
&& !metadata.bucket_incarnation_id.is_nil()
&& current_local_meta_mutation_gen() == gen_before
&& store.all_disks_local_and_online()
{
self.local_authority_cache
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.insert(bucket.to_string(), (gen_before, Arc::clone(&metadata)));
}
Ok(BucketMetadataAuthority::Authoritative(metadata))
Ok(BucketMetadataAuthority::Authoritative(Arc::new(metadata)))
} else {
Ok(BucketMetadataAuthority::Fabricated)
}
@@ -3428,7 +3297,7 @@ mod tests {
),
(
BUCKET_TAGGING_CONFIG,
r#"<Tagging><TagSet><Tag><Key>environment</Key><Value>测试-rust</Value></Tag></TagSet></Tagging>"#.as_bytes(),
r#"<Tagging><TagSet><Tag><Key>environment</Key><Value>测试-🦀</Value></Tag></TagSet></Tagging>"#.as_bytes(),
),
(BUCKET_REPLICATION_CONFIG, NEW_WRITER_REPLICATION_XML),
(
@@ -4556,13 +4425,8 @@ mod tests {
std::fs::create_dir_all(dir.path().join(bucket)).expect("bucket volume should be created");
}
let mut bm = BucketMetadata::new(bucket);
if config_file == crate::bucket::metadata::BUCKET_TAGGING_CONFIG {
// Model legacy persisted corruption without passing the write-time validator.
bm.tagging_config_xml = raw.to_vec();
} else {
bm.update_config(config_file, raw.to_vec())
.expect("raw config should be stored");
}
bm.update_config(config_file, raw.to_vec())
.expect("raw config should be stored");
sys.persist_new_and_set(bm).await.expect("initial metadata should persist");
}
@@ -5215,61 +5079,6 @@ mod tests {
);
}
#[tokio::test]
#[serial]
async fn validated_config_update_rejects_a_different_bucket_incarnation() {
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = "catalog-cleanup-incarnation";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
let incarnation = store
.bucket_incarnation_id_from_disk(bucket)
.await
.expect("read bucket incarnation");
let called = std::sync::atomic::AtomicBool::new(false);
let result = store
.update_bucket_metadata_config_for_incarnation_validated(
bucket,
"quota.json",
br#"{"quota":1}"#.to_vec(),
Uuid::new_v4(),
|| {
called.store(true, std::sync::atomic::Ordering::SeqCst);
Ok(())
},
)
.await;
assert!(matches!(result, Err(Error::BucketNotFound(name)) if name == bucket));
assert!(
!called.load(std::sync::atomic::Ordering::SeqCst),
"incarnation must be checked before the external fence"
);
let (metadata, _) = get_config_from_disk_with_presence_in(&store.ctx, bucket)
.await
.expect("read unchanged bucket metadata");
assert!(
metadata.quota_config_json.is_empty(),
"a stale cleanup must not modify the replacement bucket"
);
store
.update_bucket_metadata_config_for_incarnation_validated(
bucket,
"quota.json",
br#"{"quota":1}"#.to_vec(),
incarnation,
|| Ok(()),
)
.await
.expect("the current incarnation permits the validated update");
let (metadata, _) = get_config_from_disk_with_presence_in(&store.ctx, bucket)
.await
.expect("read updated bucket metadata");
assert_eq!(metadata.quota_config_json, br#"{"quota":1}"#);
}
#[tokio::test]
#[serial]
async fn quota_admission_rejects_stale_cached_no_quota() {
@@ -5743,8 +5552,7 @@ mod tests {
// update only needs one interleaving to show up.
const ROUNDS: usize = 8;
for round in 0..ROUNDS {
let tagging =
format!("<Tagging><TagSet><Tag><Key>round</Key><Value>{round}</Value></Tag></TagSet></Tagging>").into_bytes();
let tagging = format!("<Tagging><Round>{round}</Round></Tagging>").into_bytes();
let policy = format!(r#"{{"Version":"2012-10-17","Round":{round}}}"#).into_bytes();
let start = Arc::new(tokio::sync::Barrier::new(2));
@@ -5817,7 +5625,7 @@ mod tests {
let blocked = tokio::spawn({
let node_b = node_b.clone();
async move { update_with_sys(node_b, bucket, BUCKET_TAGGING_CONFIG, b"<Tagging><TagSet/></Tagging>".to_vec()).await }
async move { update_with_sys(node_b, bucket, BUCKET_TAGGING_CONFIG, b"<Tagging/>".to_vec()).await }
});
// Long enough for the write to have finished had it not waited: the
@@ -119,21 +119,6 @@ async fn durable_quota_reporting_retires_degraded_floors_without_changing_ledger
// Exercise the real storage entry point before testing the new
// reporting semantics: the same ledger must admit and refuse
// these writes even when a scanner floor claims a million bytes.
if protocol == 1 {
let mut bootstrap = PutObjReader::from_vec(vec![0x5a; 512]);
assert!(
matches!(
store
.put_object(&bucket, "bootstrap", &mut bootstrap, &ObjectOptions::default())
.await,
Err(StorageError::SlowDown)
),
"a missing authoritative ledger must not admit a write from scanner reporting"
);
crate::bucket::quota::reservation::retry_quota_reconciliation_for_test(store.clone(), &bucket)
.await
.expect("bootstrap authoritative usage through the production worker");
}
let mut too_large = PutObjReader::from_vec(vec![0x5a; 8192]);
assert!(
matches!(
File diff suppressed because it is too large Load Diff
+2 -87
View File
@@ -40,9 +40,7 @@ use aws_smithy_runtime_api::client::http::{
HttpConnector as SmithyHttpConnector, HttpConnectorFuture, SharedHttpConnector, http_client_fn,
};
use aws_smithy_runtime_api::client::interceptors::Intercept;
use aws_smithy_runtime_api::client::interceptors::context::{
BeforeDeserializationInterceptorContextMut, BeforeTransmitInterceptorContextMut,
};
use aws_smithy_runtime_api::client::interceptors::context::BeforeTransmitInterceptorContextMut;
use aws_smithy_runtime_api::client::orchestrator::{HttpRequest, HttpResponse};
use aws_smithy_runtime_api::client::result::ConnectorError;
use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents;
@@ -157,8 +155,6 @@ pub struct RemoteS3EndpointSpec {
pub ca_cert_pem: Option<String>,
pub connect_timeout: Option<Duration>,
pub read_timeout: Option<Duration>,
/// Entire SDK call, including non-streaming response bodies and retries.
pub operation_timeout: Option<Duration>,
/// How many wire requests one logical call may cost. Every consumer
/// declares it; see [`RemoteS3RetryPolicy`].
pub retry: RemoteS3RetryPolicy,
@@ -256,78 +252,6 @@ pub(crate) fn remote_sdk_credentials(credentials: &RemoteCredentials, now: Syste
Ok(builder.build())
}
/// Only installed on non-streaming upload operations. The budget begins at
/// response headers, independently of the potentially long request upload.
#[derive(Debug)]
pub(crate) struct UploadResponseDeadline(pub Duration);
impl Intercept for UploadResponseDeadline {
fn name(&self) -> &'static str {
"RustfsUploadResponseDeadline"
}
fn modify_before_deserialization(
&self,
context: &mut BeforeDeserializationInterceptorContextMut<'_>,
_runtime_components: &RuntimeComponents,
_cfg: &mut ConfigBag,
) -> Result<(), BoxError> {
let response = context.response_mut();
let inner = std::mem::replace(response.body_mut(), SdkBody::taken());
*response.body_mut() = SdkBody::from_body_1_x(UploadResponseBody {
inner: Some(inner),
deadline: Box::pin(tokio::time::sleep(self.0)),
});
Ok(())
}
}
struct UploadResponseBody {
inner: Option<SdkBody>,
deadline: std::pin::Pin<Box<tokio::time::Sleep>>,
}
impl http_body::Body for UploadResponseBody {
type Data = bytes::Bytes;
type Error = BoxError;
fn poll_frame(
mut self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
) -> std::task::Poll<Option<Result<http_body::Frame<Self::Data>, Self::Error>>> {
use std::future::Future as _;
use std::task::Poll;
if self.inner.is_none() {
return Poll::Ready(None);
}
// The absolute response budget never renews on data, empty frames,
// or trailers. A retry gets its own bounded response window.
if self.deadline.as_mut().poll(cx).is_ready() {
self.inner.take();
return Poll::Ready(Some(Err(std::io::Error::new(
std::io::ErrorKind::TimedOut,
"replication response body deadline exceeded",
)
.into())));
}
let Some(inner) = self.inner.as_mut() else {
return Poll::Ready(None);
};
let result = std::pin::Pin::new(inner).poll_frame(cx);
if matches!(result, Poll::Ready(None) | Poll::Ready(Some(Err(_)))) {
self.inner.take();
}
result
}
fn is_end_stream(&self) -> bool {
self.inner.as_ref().is_none_or(http_body::Body::is_end_stream)
}
fn size_hint(&self) -> http_body::SizeHint {
self.inner.as_ref().map(http_body::Body::size_hint).unwrap_or_default()
}
}
/// Appends a caller-identifying token to the SDK `User-Agent`. Runs after
/// signing: SigV4 excludes `user-agent` from the canonical request, so the
/// signature stays valid.
@@ -385,7 +309,7 @@ pub async fn build_remote_s3_config(spec: &RemoteS3EndpointSpec) -> Result<aws_s
config_builder = config_builder.http_client(http_client);
}
if spec.connect_timeout.is_some() || spec.read_timeout.is_some() || spec.operation_timeout.is_some() {
if spec.connect_timeout.is_some() || spec.read_timeout.is_some() {
let mut timeouts = TimeoutConfig::builder();
if let Some(connect_timeout) = spec.connect_timeout {
timeouts = timeouts.connect_timeout(connect_timeout);
@@ -393,9 +317,6 @@ pub async fn build_remote_s3_config(spec: &RemoteS3EndpointSpec) -> Result<aws_s
if let Some(read_timeout) = spec.read_timeout {
timeouts = timeouts.read_timeout(read_timeout);
}
if let Some(operation_timeout) = spec.operation_timeout {
timeouts = timeouts.operation_timeout(operation_timeout);
}
config_builder = config_builder.timeout_config(timeouts.build());
}
@@ -753,7 +674,6 @@ mod tests {
ca_cert_pem: None,
connect_timeout: None,
read_timeout: None,
operation_timeout: None,
retry: RemoteS3RetryPolicy::Disabled,
user_agent_suffix: "",
}
@@ -1107,16 +1027,11 @@ mod tests {
spec.user_agent_suffix = "RustFS-Test/0.0";
spec.connect_timeout = Some(Duration::from_secs(5));
spec.read_timeout = Some(Duration::from_secs(5));
spec.operation_timeout = Some(Duration::from_secs(12));
let config = build_remote_s3_config(&spec)
.await
.expect("spec should build")
.http_client(http_client)
.build();
assert_eq!(
config.timeout_config().and_then(TimeoutConfig::operation_timeout),
Some(Duration::from_secs(12))
);
S3Client::from_conf(config)
.head_bucket()
.bucket("bucket")
@@ -33,7 +33,6 @@ pub(crate) struct ReplicationMetadataStore;
impl ReplicationMetadataStore {
pub(crate) const MRF_REPLICATION_FILE: &'static str = "config/replication/mrf.bin";
pub(crate) const MRF_REPLICATION_RECOVERY_LOCK: &'static str = "config/replication/mrf.bin.recovery";
pub(crate) const MRF_REPLICATION_REPLAY_STAMP: &'static str = "config/replication/mrf.bin.replay";
pub(crate) const FORCE_DELETE_REPLICATION_FILE: &'static str = "config/replication/force-delete.bin";
pub(crate) const FORCE_DELETE_REPLICATION_TRANSACTION_LOCK: &'static str = "config/replication/force-delete.bin.transaction";
@@ -117,10 +116,6 @@ mod tests {
ReplicationMetadataStore::MRF_REPLICATION_RECOVERY_LOCK,
"config/replication/mrf.bin.recovery"
);
assert_eq!(
ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
"config/replication/mrf.bin.replay"
);
assert_eq!(
ReplicationMetadataStore::FORCE_DELETE_REPLICATION_FILE,
"config/replication/force-delete.bin"
@@ -16,8 +16,8 @@ use super::replication_config_store::ReplicationConfigStore;
use super::replication_error_boundary::{Error as EcstoreError, is_err_object_not_found, is_err_version_not_found};
use super::replication_filemeta_boundary::{
MrfOpKind, MrfReplicateEntry, REPLICATE_HEAL_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicatedTargetInfo,
ReplicationGenerationSnapshot, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision,
replicate_decision_for_admitted_targets, replication_statuses_map, version_purge_statuses_map,
ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision, replicate_decision_for_admitted_targets,
replication_statuses_map, version_purge_statuses_map,
};
use super::replication_lock_boundary::ReplicationLockTiming;
use super::replication_logging::{EVENT_REPLICATION_CONFIG_LOOKUP_SKIPPED, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REPLICATION};
@@ -86,10 +86,6 @@ const MRF_CORRUPT_FILE_PREFIX: &str = "config/replication/mrf.corrupt";
const MRF_PENDING_CAP: usize = 200_000;
const MRF_RETRY_INITIAL_DELAY: Duration = Duration::from_millis(100);
const MRF_RETRY_MAX_DELAY: Duration = Duration::from_secs(5);
/// Interval between runtime replays of the MRF journal, cluster-wide. A pass
/// re-attempts every retained entry, so a target that stays down costs one
/// attempt per entry per interval.
const MRF_REPLAY_INTERVAL: Duration = Duration::from_secs(300);
#[derive(Debug, Default)]
pub struct DurableMrfBacklog {
@@ -619,7 +615,7 @@ async fn read_mrf_entries<S: ReplicationObjectIO>(storage: Arc<S>) -> Result<Vec
}
/// Acknowledge only the generation read by the recovery leader. Entries appended
/// after that generation are retained as a suffix and replayed on the next pass.
/// after that generation are retained as a suffix and replayed on the next startup.
/// Lock order: recovery leader lock -> MRF journal object lock.
async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
storage: Arc<S>,
@@ -671,7 +667,7 @@ async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
Err(EcstoreError::PreconditionFailed)
}
/// Acquires the MRF recovery leader lock for one journal replay pass.
/// Acquires the MRF recovery leader lock for the startup replay.
/// Returns `None` (after logging) when the lock cannot be created or another
/// node is already processing the backlog.
async fn acquire_mrf_recovery_guard<S: ReplicationStorage>(storage: &Arc<S>) -> Option<rustfs_lock::NamespaceLockGuard> {
@@ -1022,7 +1018,7 @@ fn admitted_mrf_replicate_object(oi: ObjectInfo, entry: &MrfReplicateEntry, op_t
}
/// Acknowledges the replayed MRF prefix and returns the retained backlog.
/// On acknowledgement failure the backlog is preserved for the next replay pass and
/// On acknowledgement failure the backlog is preserved for the next startup and
/// re-read (falling back to the replayed snapshot) so the published summary stays accurate.
async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
storage: &Arc<S>,
@@ -1037,7 +1033,7 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
error = %error,
"Failed to acknowledge the MRF recovery prefix; preserving it for the next replay pass"
"Failed to acknowledge the MRF recovery prefix; preserving it for the next startup"
);
match read_mrf_entries(storage.clone()).await {
Ok(current) => current,
@@ -1055,120 +1051,6 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
}
}
fn unix_millis_now() -> i64 {
i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000_000).unwrap_or(i64::MAX)
}
/// Whether a periodic pass may run, given the start time of the last pass on
/// any node. A stamp more than one interval ahead of this node's clock is skew
/// or a bad value, so it does not suppress replay.
fn mrf_replay_due(last_started_ms: Option<i64>, now_ms: i64, interval: Duration) -> bool {
let Some(last_started_ms) = last_started_ms else {
return true;
};
let interval_ms = i64::try_from(interval.as_millis()).unwrap_or(i64::MAX);
now_ms.saturating_sub(last_started_ms) >= interval_ms || last_started_ms.saturating_sub(now_ms) >= interval_ms
}
async fn read_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>) -> Option<i64> {
let data = ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP)
.await
.ok()?;
std::str::from_utf8(&data).ok()?.trim().parse().ok()
}
async fn write_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>, started_ms: i64) {
if let Err(error) = ReplicationConfigStore::save(
storage.clone(),
ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
started_ms.to_string().into_bytes(),
)
.await
{
warn!(
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
error = %error,
"Failed to record the MRF replay start time"
);
}
}
/// One MRF journal replay pass. The recovery leader reads the journal,
/// re-drives every entry, and acknowledges the replayed prefix; entries that
/// could not be admitted, and entries appended meanwhile, stay for the next pass.
///
/// Every node runs the periodic loop, and the leader lock only serializes
/// passes. With `min_interval`, a pass is skipped when a pass on any node
/// started within that interval, read from the shared replay stamp, so a
/// cluster replays once per interval rather than once per node. The startup
/// pass passes `None` and always runs. A pass that finds entries records its
/// start time. Returns whether the journal was read.
/// Lock order: recovery leader lock -> replay stamp object lock.
async fn replay_mrf_journal<S: ReplicationStorage>(storage: &Arc<S>, min_interval: Option<Duration>) -> bool {
let Some(recovery_guard) = acquire_mrf_recovery_guard(storage).await else {
return false;
};
let started_ms = unix_millis_now();
if let Some(interval) = min_interval
&& !mrf_replay_due(read_mrf_replay_stamp(storage).await, started_ms, interval)
{
return false;
}
let Some(entries) = load_mrf_recovery_entries(storage).await else {
return true;
};
if !entries.is_empty() {
write_mrf_replay_stamp(storage, started_ms).await;
}
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
let total = entries.len();
let mut queued_count = 0usize;
let mut retry_entries = Vec::new();
for entry in entries.iter() {
let Some(admission) = replay_mrf_entry(entry, storage, &mut retry_entries).await else {
continue;
};
if admission == ReplicationQueueAdmission::Missed {
retry_entries.push(entry.clone());
} else if admission == ReplicationQueueAdmission::Queued {
queued_count += 1;
}
}
let retained = resolve_retained_mrf_entries(storage, &recovery_guard, &entries, &retry_entries).await;
let retained_count = retained.len();
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
if queued_count > 0 {
info!(
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
recovered = queued_count,
total,
retained = retained_count,
"Replayed MRF entries admitted for retry"
);
}
true
}
/// Re-runs [`replay_mrf_journal`] every `period`. The recovery leader lock keeps
/// one pass at a time across the cluster, including the startup pass, and the
/// shared replay stamp spaces passes from different nodes by `period`.
fn spawn_mrf_replay_loop<S: ReplicationStorage>(storage: Arc<S>, period: Duration) -> JoinHandle<()> {
tokio::spawn(async move {
loop {
tokio::time::sleep(period).await;
replay_mrf_journal(&storage, Some(period)).await;
}
})
}
#[derive(Debug, thiserror::Error)]
#[error("replication resync {active_resync_id} is already active for {bucket}/{arn}")]
struct ResyncActiveConflictError {
@@ -1289,7 +1171,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
// Start background tasks
pool.start_mrf_persister().await;
pool.start_mrf_processor().await;
pool.start_mrf_replayer().await;
pool.start_force_delete_processor().await;
pool
@@ -1581,7 +1462,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
/// Queues a replica task
pub async fn queue_replica_task(&self, ri: ReplicateObjectInfo) -> ReplicationQueueAdmission {
// A source generation already queued or being uploaded is not driven a
// A version that is already queued or being uploaded is not driven a
// second time: the scanner heal pass sees it as PENDING until the
// first upload lands and would otherwise re-queue it every cycle
// (backlog#2362). The key is released when the worker finishes, or
@@ -1758,23 +1639,52 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
/// Starts the MRF processor — one-shot at startup.
///
/// Replays the on-disk MRF journal once; [`Self::start_mrf_replayer`] repeats
/// the pass while the server runs.
/// Reads the on-disk MRF file, re-injects admitted entries as Heal operations, and
/// rewrites any entries that could not be admitted for a later startup retry.
async fn start_mrf_processor(&self) {
let storage = self.storage.clone();
let handle = tokio::spawn(async move {
replay_mrf_journal(&storage, None).await;
});
self.task_handles.lock().await.push(handle);
}
/// Starts the runtime MRF replayer — ongoing background task.
///
/// Entries reach the journal while the server runs (worker-queue overflow,
/// delete-marker purge intents whose watch window drained with targets still
/// dirty). Replaying only at startup left them waiting for a restart.
async fn start_mrf_replayer(&self) {
let handle = spawn_mrf_replay_loop(self.storage.clone(), MRF_REPLAY_INTERVAL);
let handle = tokio::spawn(async move {
let Some(recovery_guard) = acquire_mrf_recovery_guard(&storage).await else {
return;
};
let Some(entries) = load_mrf_recovery_entries(&storage).await else {
return;
};
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
let total = entries.len();
let mut queued_count = 0usize;
let mut retry_entries = Vec::new();
for entry in entries.iter() {
let Some(admission) = replay_mrf_entry(entry, &storage, &mut retry_entries).await else {
continue;
};
if admission == ReplicationQueueAdmission::Missed {
retry_entries.push(entry.clone());
} else if admission == ReplicationQueueAdmission::Queued {
queued_count += 1;
}
}
let retained = resolve_retained_mrf_entries(&storage, &recovery_guard, &entries, &retry_entries).await;
let retained_count = retained.len();
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
if queued_count > 0 {
info!(
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
recovered = queued_count,
total,
retained = retained_count,
"Replayed MRF entries admitted for retry"
);
}
});
self.task_handles.lock().await.push(handle);
}
@@ -1845,7 +1755,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
/// writes them to the on-disk MRF file every flush interval (default 10s,
/// overridable via `RUSTFS_REPL_MRF_FLUSH_INTERVAL_MS`) or when 1 000 new
/// entries accumulate. Runtime writers append under the journal lock; only the
/// recovery leader of a replay pass may remove an acknowledged prefix.
/// startup recovery leader may remove an acknowledged prefix.
async fn start_mrf_persister(&self) {
let Some(mut rx) = self.mrf_save_rx.lock().await.take() else {
return;
@@ -2404,47 +2314,28 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
}
/// Object versions currently queued or being uploaded, keyed by bucket,
/// object name, version and source generation. `queue_replica_task` admits a generation only once
/// object name and version. `queue_replica_task` admits a version only once
/// while it is in flight; the scanner heal pass and MRF replays that arrive
/// in the meantime are `Skipped` instead of driving a second complete upload
/// (backlog#2362). Entries are removed when the worker finishes the task or
/// when no worker accepted it.
#[derive(Debug, Default)]
pub(crate) struct ReplicationInFlight {
keys: std::sync::Mutex<std::collections::HashSet<ReplicationInFlightKey>>,
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
struct ReplicationInFlightKey {
bucket: String,
object: String,
version_id: Option<uuid::Uuid>,
generation: ReplicationGenerationSnapshot,
}
impl From<&ReplicateObjectInfo> for ReplicationInFlightKey {
fn from(info: &ReplicateObjectInfo) -> Self {
Self {
bucket: info.bucket.clone(),
object: info.name.clone(),
version_id: info.version_id,
generation: info.replication_generation.clone(),
}
}
keys: std::sync::Mutex<std::collections::HashSet<(String, String, Option<uuid::Uuid>)>>,
}
impl ReplicationInFlight {
fn lock(&self) -> std::sync::MutexGuard<'_, std::collections::HashSet<ReplicationInFlightKey>> {
fn lock(&self) -> std::sync::MutexGuard<'_, std::collections::HashSet<(String, String, Option<uuid::Uuid>)>> {
self.keys.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
}
/// Claim `ri`; a metadata mutation is independent of its earlier generation.
/// Claim `ri`; `false` when the same version is already in flight.
fn try_begin(&self, ri: &ReplicateObjectInfo) -> bool {
self.lock().insert(ReplicationInFlightKey::from(ri))
self.lock().insert((ri.bucket.clone(), ri.name.clone(), ri.version_id))
}
fn finish(&self, ri: &ReplicateObjectInfo) {
self.lock().remove(&ReplicationInFlightKey::from(ri));
self.lock().remove(&(ri.bucket.clone(), ri.name.clone(), ri.version_id));
}
#[cfg(test)]
@@ -2457,21 +2348,26 @@ impl ReplicationInFlight {
/// including when replication panics.
struct ReplicationInFlightGuard {
in_flight: Arc<ReplicationInFlight>,
key: ReplicationInFlightKey,
key: ReplicateObjectInfo,
}
impl ReplicationInFlightGuard {
fn new(in_flight: Arc<ReplicationInFlight>, ri: &ReplicateObjectInfo) -> Self {
Self {
in_flight,
key: ReplicationInFlightKey::from(ri),
key: ReplicateObjectInfo {
bucket: ri.bucket.clone(),
name: ri.name.clone(),
version_id: ri.version_id,
..Default::default()
},
}
}
}
impl Drop for ReplicationInFlightGuard {
fn drop(&mut self) {
self.in_flight.lock().remove(&self.key);
self.in_flight.finish(&self.key);
}
}
@@ -3525,8 +3421,6 @@ mod tests {
allow_first_read: Notify,
read_count: AtomicUsize,
delete_storage_calls: AtomicUsize,
delete_error: StdMutex<EcstoreError>,
replay_stamp: StdMutex<Option<Vec<u8>>>,
reported_size: StdMutex<Option<i64>>,
stream_read_bytes: Arc<AtomicUsize>,
write_count: AtomicUsize,
@@ -3594,24 +3488,6 @@ mod tests {
_h: Self::HeaderMap,
_opts: &Self::ObjectOptions,
) -> Result<Self::GetObjectReader, Self::Error> {
if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
let stamp = self
.shared
.replay_stamp
.lock()
.expect("test replay stamp lock should not be poisoned")
.clone()
.ok_or(EcstoreError::FileNotFound)?;
return Ok(Self::GetObjectReader {
object_info: ObjectInfo {
size: i64::try_from(stamp.len()).expect("test stamp length should fit i64"),
..Default::default()
},
stream: Box::new(Cursor::new(stamp)),
buffered_body: None,
body_source: Default::default(),
});
}
if !object.ends_with("/.replication/resync.bin")
&& !object.ends_with("config/replication/mrf.bin")
&& !object.ends_with("config/replication/force-delete.bin")
@@ -3675,16 +3551,6 @@ mod tests {
let lock = self.new_ns_lock(bucket, object).await?;
Some(lock.get_write_lock(Duration::from_secs(10)).await?)
};
if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
let mut stamp = Vec::new();
data.stream.read_to_end(&mut stamp).await.map_err(EcstoreError::from)?;
*self
.shared
.replay_stamp
.lock()
.expect("test replay stamp lock should not be poisoned") = Some(stamp);
return Ok(ObjectInfo::default());
}
if opts.http_preconditions.is_some()
&& let Some(replacement) = self
.shared
@@ -3806,12 +3672,7 @@ mod tests {
_opts: Self::ObjectOptions,
) -> Result<Self::ObjectInfo, Self::Error> {
self.shared.delete_storage_calls.fetch_add(1, Ordering::SeqCst);
Err(self
.shared
.delete_error
.lock()
.expect("delete error lock should not be poisoned")
.clone())
Err(EcstoreError::NotImplemented)
}
async fn delete_objects(
@@ -4068,48 +3929,6 @@ mod tests {
assert_eq!(current_queue(&pool, "in-flight-bucket").await, (3, 3 * 4096));
}
#[tokio::test]
async fn queue_replica_task_admits_metadata_changes_during_an_initial_upload() {
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
let (tx, mut rx) = mpsc::channel(4);
pool.workers.write().await.push(tx);
let initial = ReplicateObjectInfo {
bucket: "pending-tags-bucket".to_string(),
name: "object".to_string(),
version_id: Some(uuid::Uuid::new_v4()),
size: 4096,
replication_generation: ReplicationGenerationSnapshot {
mutation_id: Some(uuid::Uuid::new_v4().to_string()),
..Default::default()
},
..Default::default()
};
let metadata = ReplicateObjectInfo {
op_type: ReplicationType::Metadata,
user_tags: String::new(),
replication_generation: ReplicationGenerationSnapshot {
mutation_id: Some(uuid::Uuid::new_v4().to_string()),
..Default::default()
},
..initial.clone()
};
assert_eq!(pool.queue_replica_task(initial.clone()).await, ReplicationQueueAdmission::Queued);
assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Queued);
assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Skipped);
assert_eq!(pool.in_flight.len(), 2);
drop(ReplicationInFlightGuard::new(pool.in_flight.clone(), &initial));
assert_eq!(pool.in_flight.len(), 1);
assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Skipped);
for expected in [&initial, &metadata] {
let ReplicationOperation::Object(queued) = rx.recv().await.expect("queued operation") else {
panic!("expected object replication");
};
assert_eq!(queued.replication_generation, expected.replication_generation);
}
drop(ReplicationInFlightGuard::new(pool.in_flight.clone(), &metadata));
assert_eq!(pool.in_flight.len(), 0);
}
#[tokio::test]
async fn queue_replica_task_releases_the_version_when_no_worker_accepts_it() {
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
@@ -4352,8 +4171,6 @@ mod tests {
allow_first_read: Notify::new(),
read_count: AtomicUsize::new(0),
delete_storage_calls: AtomicUsize::new(0),
delete_error: StdMutex::new(EcstoreError::NotImplemented),
replay_stamp: StdMutex::new(None),
reported_size: StdMutex::new(None),
stream_read_bytes: Arc::new(AtomicUsize::new(0)),
write_count: AtomicUsize::new(0),
@@ -4404,103 +4221,6 @@ mod tests {
}
}
#[tokio::test]
async fn delete_worker_already_purged_source_completes_without_retry() {
let arn = "arn:test";
for delete_marker_version in [false, true] {
for version_id in [Uuid::new_v4(), Uuid::nil()] {
for error in [
EcstoreError::VersionNotFound("source".to_string(), "purged".to_string(), version_id.to_string()),
EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
EcstoreError::FileVersionNotFound,
EcstoreError::FileNotFound,
] {
let shared = empty_resync_shared_state();
*shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
let storage = Arc::new(LoadResyncNodeStore::new("purged-source", shared.clone()));
let delete = DeletedObjectReplicationInfo {
bucket: "source".to_string(),
delete_object: ReplicationDeletedObject {
object_name: "purged".to_string(),
version_id: (!delete_marker_version).then_some(version_id),
delete_marker_version_id: delete_marker_version.then_some(version_id),
replication_state: Some(ReplicationState {
replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
purge_targets: HashMap::from([(arn.to_string(), VersionPurgeStatusType::Complete)]),
..Default::default()
}),
..Default::default()
},
op_type: ReplicationType::Delete,
..Default::default()
};
assert!(
replicate_delete_with_outcome(delete, storage).await,
"a confirmed target purge must complete when the source is already absent (marker={delete_marker_version}, version={version_id})"
);
assert_eq!(
shared.delete_storage_calls.load(Ordering::SeqCst),
2 + usize::from(!delete_marker_version && !version_id.is_nil()),
"the worker must lock and attempt writeback, reading a non-null data version first"
);
}
}
}
}
#[tokio::test]
async fn delete_worker_missing_source_does_not_hide_unfinished_replication() {
let arn = "arn:test";
for (version_purge, target_status, error) in [
(
true,
VersionPurgeStatusType::Pending,
EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
),
(true, VersionPurgeStatusType::Complete, EcstoreError::ErasureWriteQuorum),
(
false,
VersionPurgeStatusType::Empty,
EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
),
] {
let shared = empty_resync_shared_state();
*shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
let storage = Arc::new(LoadResyncNodeStore::new("unfinished-delete", shared));
let delete = DeletedObjectReplicationInfo {
bucket: "source".to_string(),
delete_object: ReplicationDeletedObject {
object_name: "purged".to_string(),
version_id: version_purge.then(Uuid::new_v4),
delete_marker: !version_purge,
replication_state: Some(ReplicationState {
replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
targets: if version_purge {
HashMap::new()
} else {
HashMap::from([(arn.to_string(), ReplicationStatusType::Completed)])
},
purge_targets: if version_purge {
HashMap::from([(arn.to_string(), target_status)])
} else {
HashMap::new()
},
..Default::default()
}),
..Default::default()
},
op_type: ReplicationType::Delete,
..Default::default()
};
assert!(
!replicate_delete_with_outcome(delete, storage).await,
"unfinished target work, source I/O failures, and marker creation must remain retryable"
);
}
}
#[tokio::test]
async fn delete_worker_preserves_persisted_admission_without_live_config() {
let arn = "arn:test";
@@ -5584,8 +5304,6 @@ mod tests {
allow_first_read: Notify::new(),
read_count: AtomicUsize::new(0),
delete_storage_calls: AtomicUsize::new(0),
delete_error: StdMutex::new(EcstoreError::NotImplemented),
replay_stamp: StdMutex::new(None),
reported_size: StdMutex::new(None),
stream_read_bytes: Arc::new(AtomicUsize::new(0)),
write_count: AtomicUsize::new(0),
@@ -6550,108 +6268,6 @@ mod tests {
assert_eq!(retained.target_arns, entry.target_arns);
}
#[tokio::test]
async fn mrf_replay_loop_drains_entries_persisted_after_startup() {
let shared = empty_resync_shared_state();
let storage = Arc::new(LoadResyncNodeStore::new("mrf-runtime-replay", shared.clone()));
// A force-delete intent from a superseded generation replays as
// Skipped, so a pass that reaches it acknowledges it off the journal.
let stale_intent = |object: &str| MrfReplicateEntry {
bucket: "mrf-runtime-replay".to_string(),
object: object.to_string(),
op: MrfOpKind::Delete,
force_delete_id: Some(uuid::Uuid::new_v4()),
force_delete_local_commit: false,
target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
..Default::default()
};
let journal_drained = || async {
tokio::time::timeout(Duration::from_secs(5), async {
while !shared.data.lock().expect("test data lock should not be poisoned").is_empty() {
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.is_ok()
};
let handle = spawn_mrf_replay_loop(storage, Duration::from_millis(20));
for object in ["first", "second"] {
*shared.data.lock().expect("test data lock should not be poisoned") =
encode_mrf_file(&[stale_intent(object)]).expect("MRF entry should encode");
assert!(
journal_drained().await,
"an MRF entry persisted while the server runs must be replayed without a restart ({object})"
);
}
handle.abort();
}
#[tokio::test]
async fn mrf_periodic_replay_runs_once_per_interval_across_nodes() {
let shared = empty_resync_shared_state();
let node_a = Arc::new(LoadResyncNodeStore::new("mrf-node-a", shared.clone()));
let node_b = Arc::new(LoadResyncNodeStore::new("mrf-node-b", shared.clone()));
let interval = Duration::from_secs(1);
let persist_backlog = |object: &str| {
let entry = MrfReplicateEntry {
bucket: "mrf-node-replay".to_string(),
object: object.to_string(),
op: MrfOpKind::Delete,
force_delete_id: Some(uuid::Uuid::new_v4()),
force_delete_local_commit: false,
target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
..Default::default()
};
*shared.data.lock().expect("test data lock should not be poisoned") =
encode_mrf_file(&[entry]).expect("MRF entry should encode");
};
let journal_is_empty = || shared.data.lock().expect("test data lock should not be poisoned").is_empty();
persist_backlog("first");
assert!(replay_mrf_journal(&node_a, Some(interval)).await, "node A's periodic pass should replay");
assert!(journal_is_empty(), "node A's pass should acknowledge the replayed entry");
// Node B's timer fires within the same interval with a backlog present.
persist_backlog("second");
assert!(
!replay_mrf_journal(&node_b, Some(interval)).await,
"a node must not replay within one interval of another node's pass"
);
assert!(!journal_is_empty(), "the backlog should wait for the next interval");
assert!(
replay_mrf_journal(&node_b, None).await,
"the startup pass should replay regardless of the shared stamp"
);
assert!(journal_is_empty(), "the startup pass should acknowledge the replayed entry");
persist_backlog("third");
tokio::time::sleep(interval).await;
assert!(
replay_mrf_journal(&node_a, Some(interval)).await,
"a periodic pass should replay once the interval has elapsed"
);
assert!(journal_is_empty(), "the due pass should acknowledge the replayed entry");
}
#[test]
fn mrf_replay_due_tolerates_missing_and_skewed_stamps() {
let interval = Duration::from_secs(300);
let now = 1_000_000_000;
assert!(mrf_replay_due(None, now, interval));
assert!(!mrf_replay_due(Some(now - 299_999), now, interval));
assert!(mrf_replay_due(Some(now - 300_000), now, interval));
assert!(
!mrf_replay_due(Some(now + 1_000), now, interval),
"small clock skew should still space passes"
);
assert!(
mrf_replay_due(Some(now + 300_000), now, interval),
"a stamp an interval ahead must not suppress replay"
);
}
#[test]
fn mrf_entry_delete_marker_roundtrip() {
let dm_vid = Uuid::new_v4();
@@ -2809,22 +2809,17 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
true
}
Err(e) => {
// A batch purge or another worker may have removed this version
// already. Remote failures still control the final outcome below.
let source_already_purged = is_version_purge && (is_err_object_not_found(&e) || is_err_version_not_found(&e));
if !source_already_purged {
error!(
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %dobj.target_arn,
object = %dobj.delete_object.object_name,
operation = "apply_replication_delete_state",
error = %e,
"Replication target operation failed"
);
}
error!(
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %dobj.target_arn,
object = %dobj.delete_object.object_name,
operation = "apply_replication_delete_state",
error = %e,
"Replication target operation failed"
);
send_event(EventArgs {
event_name,
bucket_name: bucket.clone(),
@@ -2837,7 +2832,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
},
..Default::default()
});
source_already_purged
false
}
};
@@ -3015,7 +3010,7 @@ const DELETE_MARKER_PURGE_WATCH_INTERVAL: TokioDuration = TokioDuration::from_se
/// If the marker disappears (deleted before or while the replica landed),
/// purge the replicated marker from the targets, retrying failed targets on
/// later rounds. When the window drains with targets still dirty, persist the
/// purge intent as a durable MRF entry so the next MRF replay pass re-drives it through
/// purge intent as a durable MRF entry so the next startup replays it through
/// `purge_stale_delete_marker_targets`.
async fn watch_and_purge_source_delete_marker<S: ReplicationStorage>(
bucket: String,
@@ -3106,7 +3101,7 @@ async fn enqueue_delete_marker_purge_mrf(dobj: &DeletedObjectReplicationInfo, fa
/// Heal callers retry for the full watch window because the startup MRF
/// processor runs before bucket metadata (and thus target clients) finishes
/// initializing — the first pass can see `target_client_missing` and a later
/// round resolves the client; replay passes are serial background work, so
/// round resolves the client; the replay loop is serial and startup-only, so
/// blocking it for up to the window per dirty entry is acceptable. Live
/// callers run on replication workers where a down target would pin a worker
/// for the whole window, so they attempt once and lean on the durable intent
@@ -3287,18 +3282,10 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
let mut join_set = JoinSet::new();
let mut all_succeeded = true;
let mut target_infos = Vec::with_capacity(tgt_arns.len());
for arn in tgt_arns {
let Some(tgt_client) = ReplicationTargetStore::remote_target_client(bucket, &arn).await else {
all_succeeded = false;
target_infos.push(force_delete_target_info(
dobj.op_type,
&arn,
"",
false,
Err(TARGET_CLIENT_UNAVAILABLE_ERROR.to_string()),
));
debug!(
event = EVENT_REPLICATION_FORCE_DELETE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
@@ -3324,7 +3311,6 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
let bucket = bucket.clone();
let object_name = object_name.clone();
let op_type = dobj.op_type;
join_set.spawn(async move {
if ReplicationTargetStore::target_is_offline(&tgt_client).await {
@@ -3349,16 +3335,9 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
user_agent: "Internal: [Replication]".to_string(),
..Default::default()
});
return force_delete_target_info(
op_type,
&tgt_client.arn,
&tgt_client.endpoint,
tgt_client.secure,
Err(FORCE_DELETE_TARGET_OFFLINE_ERROR.to_string()),
);
return false;
}
let started = Instant::now();
if let Err(e) = tgt_client
.remove_object(&tgt_client.bucket, &object_name, None, replication_force_delete_remove_options())
.await
@@ -3385,25 +3364,16 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
user_agent: "Internal: [Replication]".to_string(),
..Default::default()
});
return force_delete_target_info(
op_type,
&tgt_client.arn,
&tgt_client.endpoint,
tgt_client.secure,
Err(e.to_string()),
);
return false;
}
force_delete_target_info(op_type, &tgt_client.arn, &tgt_client.endpoint, tgt_client.secure, Ok(started.elapsed()))
true
});
}
while let Some(result) = join_set.join_next().await {
match result {
Ok(info) => {
all_succeeded &= info.replication_status == ReplicationStatusType::Completed;
target_infos.push(info);
}
Ok(success) => all_succeeded &= success,
Err(error) => {
all_succeeded = false;
error!(
@@ -3420,8 +3390,6 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
}
}
record_force_delete_target_stats(bucket, &target_infos).await;
if all_succeeded
&& let Some(operation_id) = dobj.delete_object.force_delete_id
&& let Err(error) = super::replication_pool::complete_force_delete_intent(storage, operation_id).await
@@ -3441,53 +3409,6 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
all_succeeded
}
const FORCE_DELETE_TARGET_OFFLINE_ERROR: &str = "replication target is offline";
/// One target's outcome of a replicated force-delete in the shape the
/// replication statistics consume. A force-delete carries no per-version
/// replication state, so the previous status is always `Pending`: that is
/// the transition `ReplicationStats::update` counts as a failure, whereas an
/// `Empty` previous status would drop the outcome and leave `replication
/// status` reporting zero failures while the peer still holds the objects.
fn force_delete_target_info(
op_type: ReplicationType,
arn: &str,
endpoint: &str,
secure: bool,
outcome: std::result::Result<std::time::Duration, String>,
) -> ReplicatedTargetInfo {
let mut rinfo = ReplicatedTargetInfo {
arn: arn.to_string(),
op_type,
endpoint: endpoint.to_string(),
secure,
prev_replication_status: ReplicationStatusType::Pending,
..Default::default()
};
match outcome {
Ok(duration) => {
rinfo.replication_status = ReplicationStatusType::Completed;
rinfo.duration = duration;
}
Err(error) => {
rinfo.replication_status = ReplicationStatusType::Failed;
rinfo.error = Some(error);
}
}
rinfo
}
async fn record_force_delete_target_stats(bucket: &str, targets: &[ReplicatedTargetInfo]) {
let Some(stats) = runtime_sources::replication_stats() else {
return;
};
for target in targets {
stats
.update(bucket, target, target.replication_status.clone(), target.prev_replication_status.clone())
.await;
}
}
fn unavailable_delete_target_info(dobj: &DeletedObjectReplicationInfo, arn: &str) -> ReplicatedTargetInfo {
let mut rinfo = dobj
.delete_object
@@ -6609,7 +6530,6 @@ mod tests {
health_check_duration: std::time::Duration::from_secs(5),
replicate_sync: false,
client: Arc::new(aws_sdk_s3::Client::from_conf(config)),
timeouts: Default::default(),
})
}
@@ -7524,51 +7444,6 @@ mod tests {
server.join().expect("test HTTP server should finish");
}
/// A force-delete the peer rejects must reach the replication failure
/// counters. A replicated force-delete used to only log the rejection, so
/// `replication status` reported zero failures while the sites diverged.
#[tokio::test]
async fn force_delete_target_outcomes_reach_replication_stats() {
let stats = super::super::replication_state::ReplicationStats::new();
let denied = force_delete_target_info(
ReplicationType::Delete,
"arn:rustfs:replication::denied:bucket",
"http://peer-a:9000",
false,
Err("remove_object failed: AccessDenied".to_string()),
);
assert_eq!(denied.replication_status, ReplicationStatusType::Failed);
assert_eq!(denied.prev_replication_status, ReplicationStatusType::Pending);
assert_eq!(denied.error.as_deref(), Some("remove_object failed: AccessDenied"));
let accepted = force_delete_target_info(
ReplicationType::Heal,
"arn:rustfs:replication::accepted:bucket",
"https://peer-b:9000",
true,
Ok(std::time::Duration::from_millis(3)),
);
assert_eq!(accepted.replication_status, ReplicationStatusType::Completed);
assert!(accepted.error.is_none());
for target in [&denied, &accepted] {
stats
.update(
"bucket",
target,
target.replication_status.clone(),
target.prev_replication_status.clone(),
)
.await;
}
let bucket_stats = stats.get("bucket").await;
let denied_stat = bucket_stats.stats.get(&denied.arn).expect("denied target: stats entry");
assert_eq!((denied_stat.failed.count, denied_stat.replicated_count), (1, 0));
let accepted_stat = bucket_stats.stats.get(&accepted.arn).expect("accepted target: stats entry");
assert_eq!((accepted_stat.failed.count, accepted_stat.replicated_count), (0, 1));
}
/// AWS answers `HEAD ?versionId=<source marker id>` with a bodiless 400.
/// The marker is verified through the latest version instead: a
/// versionless 404 with `x-amz-delete-marker: true` proves it propagated.
-12
View File
@@ -259,10 +259,6 @@ pub fn check_object_name_for_length_and_slash(bucket: &str, object: &str) -> Res
pub fn check_new_object_args(bucket: &str, object: &str, data_movement: bool) -> Result<()> {
check_put_object_args(bucket, object)?;
if !data_movement && !is_meta_bucketname(bucket) && has_xml_invalid_control_characters(object) {
return Err(StorageError::ObjectNameInvalid(bucket.to_owned(), object.to_owned()));
}
if cfg!(target_os = "windows") && !data_movement && !is_meta_bucketname(bucket) && object.contains('\\') {
return Err(StorageError::InvalidArgument(
bucket.to_owned(),
@@ -274,14 +270,6 @@ pub fn check_new_object_args(bucket: &str, object: &str, data_movement: bool) ->
Ok(())
}
/// New public keys must remain representable in an unencoded XML listing.
/// Existing keys still use the read/delete validators so they can be removed.
pub fn has_xml_invalid_control_characters(object: &str) -> bool {
object
.chars()
.any(|character| character < ' ' && !matches!(character, '\t' | '\n' | '\r'))
}
pub fn check_copy_obj_args(bucket: &str, object: &str) -> Result<()> {
check_bucket_and_object_names(bucket, object)
}
+36 -253
View File
@@ -34,7 +34,7 @@ use tokio::sync::Mutex as TokioMutex;
use tokio::task::JoinSet;
use tokio::time::timeout;
use tokio_util::sync::CancellationToken;
use tracing::{Instrument, debug, error, warn};
use tracing::{debug, error, warn};
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_METACACHE: &str = "metacache";
@@ -224,8 +224,6 @@ pub struct ListPathRawOptions {
pub recursive: bool,
pub incl_deleted: bool,
pub skip_hidden_prefix_check: bool,
pub hidden_entry_budget: usize,
pub pending_prefix: Option<String>,
pub filter_prefix: Option<String>,
pub forward_to: Option<String>,
pub min_disks: usize,
@@ -262,8 +260,6 @@ impl Clone for ListPathRawOptions {
recursive: self.recursive,
incl_deleted: self.incl_deleted,
skip_hidden_prefix_check: self.skip_hidden_prefix_check,
hidden_entry_budget: self.hidden_entry_budget,
pending_prefix: self.pending_prefix.clone(),
filter_prefix: self.filter_prefix.clone(),
forward_to: self.forward_to.clone(),
min_disks: self.min_disks,
@@ -293,8 +289,6 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
recursive: opts.recursive,
incl_deleted: opts.incl_deleted,
skip_hidden_prefix_check: opts.skip_hidden_prefix_check,
hidden_entry_budget: opts.hidden_entry_budget,
pending_prefix: opts.pending_prefix.clone(),
report_notfound: opts.report_not_found,
filter_prefix: opts.filter_prefix.clone(),
forward_to: opts.forward_to.clone(),
@@ -307,15 +301,13 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
}
/// How a merged raw listing stream ended.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ListPathRawOutcome {
/// At least one producer stopped at its per-disk limit instead of at the
/// end of its namespace, so the closed stream is not proof of exhaustion.
/// Carried in-band by the metacache stream trailer, so remote producers
/// report it as well.
pub limit_reached: bool,
/// Inclusive sorted boundary proved by every reader before a bounded stop.
pub scan_cut: Option<String>,
}
pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> disk::error::Result<()> {
@@ -371,15 +363,10 @@ async fn list_path_raw_inner(
let producer_errs: Arc<[OnceLock<DiskError>]> = (0..opts.disks.len()).map(|_| OnceLock::new()).collect::<Vec<_>>().into();
let cancel_rx = CancellationToken::new();
let walk_options = Arc::new(walk_dir_options(&opts));
for (disk_idx, disk) in opts.disks.iter().enumerate() {
let opdisk = disk.clone();
let walk_options = walk_options.clone();
#[cfg(test)]
let test_reader_behavior = opts.test_reader_behaviors.get(disk_idx).cloned();
#[cfg(test)]
let test_disk_io_context_observed = opts.test_disk_io_context_observed.clone();
let opts_clone = opts.clone();
let fallback_claim_tracker = fallback_claim_tracker.clone();
let fds_clone = fds.clone();
#[cfg(test)]
@@ -389,17 +376,17 @@ async fn list_path_raw_inner(
let disk_io_context_clone = disk_io_context.clone();
let (rd, wr) = tokio::io::duplex(64 * 1024);
readers.push(MetacacheReader::new(rd));
let producer = with_optional_disk_io_context(disk_io_context_clone, async move {
jobs.spawn(with_optional_disk_io_context(disk_io_context_clone, async move {
let mut wr = PublishedBytesWriter::new(wr);
#[cfg(test)]
if let Some(observer) = &test_disk_io_context_observed {
if let Some(observer) = &opts_clone.test_disk_io_context_observed {
observer.store(
crate::disk::current_disk_io_context().is_some_and(|context| context.class == DiskIoClass::Background),
std::sync::atomic::Ordering::Release,
);
}
#[cfg(test)]
let test_primary_error = if let Some(behavior) = test_reader_behavior {
let test_primary_error = if let Some(behavior) = opts_clone.test_reader_behaviors.get(disk_idx).cloned() {
match behavior {
TestReaderBehavior::Eof => return Ok(()),
TestReaderBehavior::Entries(entries) => {
@@ -456,6 +443,8 @@ async fn list_path_raw_inner(
None
};
let wakl_opts = walk_dir_options(&opts_clone);
let mut need_fallback = false;
let mut last_err = None;
#[cfg(test)]
@@ -470,7 +459,7 @@ async fn list_path_raw_inner(
if !need_fallback && let Some(disk) = opdisk {
let primary_walk_started = std::time::Instant::now();
match disk.walk_dir((*walk_options).clone(), &mut wr).await {
match disk.walk_dir(wakl_opts, &mut wr).await {
Ok(_res) => {
rustfs_io_metrics::record_stage_duration(
"metacache_walk_dir_primary",
@@ -494,8 +483,8 @@ async fn list_path_raw_inner(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
bucket = %walk_options.bucket,
path = %walk_options.base_dir,
bucket = %opts_clone.bucket,
path = %opts_clone.path,
disk_index = disk_idx,
state = "walk_dir_missing_path",
error = ?err,
@@ -506,8 +495,8 @@ async fn list_path_raw_inner(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
bucket = %walk_options.bucket,
path = %walk_options.base_dir,
bucket = %opts_clone.bucket,
path = %opts_clone.path,
disk_index = disk_idx,
state = "walk_dir_failed",
error = ?err,
@@ -609,8 +598,8 @@ async fn list_path_raw_inner(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
bucket = %walk_options.bucket,
path = %walk_options.base_dir,
bucket = %opts_clone.bucket,
path = %opts_clone.path,
disk_index = disk_idx,
state = "fallback_disk_missing_for_path",
error = ?err,
@@ -621,8 +610,8 @@ async fn list_path_raw_inner(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
bucket = %walk_options.bucket,
path = %walk_options.base_dir,
bucket = %opts_clone.bucket,
path = %opts_clone.path,
disk_index = disk_idx,
state = "fallback_disk_missing",
error = ?err,
@@ -637,7 +626,7 @@ async fn list_path_raw_inner(
}
let fallback_walk_started = std::time::Instant::now();
match disk.as_ref().walk_dir((*walk_options).clone(), &mut wr).await {
match disk.as_ref().walk_dir(walk_dir_options(&opts_clone), &mut wr).await {
Ok(_r) => {
rustfs_io_metrics::record_stage_duration(
"metacache_walk_dir_fallback",
@@ -663,8 +652,8 @@ async fn list_path_raw_inner(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
bucket = %walk_options.bucket,
path = %walk_options.base_dir,
bucket = %opts_clone.bucket,
path = %opts_clone.path,
disk_index = disk_idx,
state = "fallback_walk_dir_missing_path",
error = ?err,
@@ -675,8 +664,8 @@ async fn list_path_raw_inner(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
bucket = %walk_options.bucket,
path = %walk_options.base_dir,
bucket = %opts_clone.bucket,
path = %opts_clone.path,
disk_index = disk_idx,
state = "fallback_walk_dir_failed",
error = ?err,
@@ -698,13 +687,12 @@ async fn list_path_raw_inner(
// warn!("list_path_raw: while need_fallback done");
Ok(())
});
jobs.spawn(producer.in_current_span());
}));
}
let revjob_rx = rx.clone();
let mut revjobs = JoinSet::new();
let merge = async move {
revjobs.spawn(async move {
// Consumer-side peek timeout, and a caveat worth understanding
// (rustfs/backlog#1217).
//
@@ -734,8 +722,6 @@ async fn list_path_raw_inner(
// Sticky across rounds: once any reader's trailer says its producer
// stopped at the per-disk limit, the merged stream may hide more keys.
let mut limit_reached = false;
let mut scan_cut = None;
let mut last_scanned_key: Option<String> = None;
loop {
let mut current = MetaCacheEntry::default();
@@ -747,7 +733,7 @@ async fn list_path_raw_inner(
// );
if revjob_rx.is_cancelled() {
return Ok(ListPathRawOutcome { limit_reached, scan_cut });
return Ok(ListPathRawOutcome { limit_reached });
}
let mut top_entries: Vec<Option<MetaCacheEntry>> = vec![None; readers.len()];
@@ -761,16 +747,6 @@ async fn list_path_raw_inner(
// Start every missing head read in the same round so one stalled
// disk cannot multiply the wait budget by the erasure-set width.
// Outcomes are still consumed below in stable disk-index order.
if !revjob_rx.is_cancelled() {
for (i, reader) in readers.iter_mut().enumerate() {
if errs[i].is_none()
&& pending_entries[i].is_none()
&& let Some(Ok(Some(entry))) = reader.try_peek_buffered()
{
pending_entries[i] = Some(entry);
}
}
}
let concurrent_peeks = readers.iter_mut().enumerate().filter_map(|(i, reader)| {
if errs[i].is_some() || pending_entries[i].is_some() {
return None;
@@ -807,12 +783,7 @@ async fn list_path_raw_inner(
continue;
}
// eof
if r.limit_reached() == Some(true) {
limit_reached = true;
scan_cut = Some(last_scanned_key.clone().ok_or_else(|| {
DiskError::other("bounded listing reader ended without a scan boundary")
})?);
}
limit_reached |= r.limit_reached() == Some(true);
at_eof += 1;
// warn!("list_path_raw: peek eof, disk: {}", i);
continue;
@@ -886,7 +857,7 @@ async fn list_path_raw_inner(
*r = MetacacheReader::new(detached_rd);
continue;
}
PeekOutcome::Cancelled => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
PeekOutcome::Cancelled => return Ok(ListPathRawOutcome { limit_reached }),
}
};
@@ -907,7 +878,7 @@ async fn list_path_raw_inner(
}
if entry.name == current.name {
// If exact match, we agree.
if current.matches_strict_borrowed(&entry) {
if current.matches_strict(&entry) {
top_entries[i] = Some(entry);
agree += 1;
} else {
@@ -945,7 +916,7 @@ async fn list_path_raw_inner(
if let Some(finished_fn) = opts.finished.as_ref() {
tokio::select! {
biased;
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
_ = finished_fn(&errs) => {}
}
}
@@ -995,14 +966,6 @@ async fn list_path_raw_inner(
return Err(classify_listing_quorum_failure(&failures));
}
// The next head from a bounded reader is unknown. The previous
// globally consumed group is a safe cut; consuming another reader's
// later head would turn this absence of evidence into a skipped key.
// Check quorum failures above before accepting the bounded result.
if limit_reached {
break;
}
// Break if all at EOF or error.
if at_eof + has_err == readers.len() {
if has_err > 0
@@ -1010,7 +973,7 @@ async fn list_path_raw_inner(
{
tokio::select! {
biased;
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
_ = finished_fn(&errs) => {}
}
}
@@ -1021,19 +984,16 @@ async fn list_path_raw_inner(
break;
}
last_scanned_key = Some(current.name.clone());
if agree == readers.len() && !opts.preserve_replica_metadata {
for r in readers.iter_mut() {
if !r.skip_current() {
let _ = r.skip(1).await;
}
let _ = r.skip(1).await;
}
if let Some(agreed_fn) = opts.agreed.as_ref() {
// warn!("list_path_raw: agreed_fn start, current: {:?}", &current.name);
tokio::select! {
biased;
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
result = agreed_fn(current) => result?
}
// warn!("list_path_raw: agreed_fn done");
@@ -1049,7 +1009,7 @@ async fn list_path_raw_inner(
}
for (i, r) in readers.iter_mut().enumerate() {
if top_entries[i].is_some() && !r.skip_current() {
if top_entries[i].is_some() {
let _ = r.skip(1).await;
}
}
@@ -1057,14 +1017,13 @@ async fn list_path_raw_inner(
if let Some(partial_fn) = opts.partial.as_ref() {
tokio::select! {
biased;
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
_ = partial_fn(MetaCacheEntries(top_entries), &errs) => {}
}
}
}
Ok(ListPathRawOutcome { limit_reached, scan_cut })
};
revjobs.spawn(merge.in_current_span());
Ok(ListPathRawOutcome { limit_reached })
});
let merge_started = std::time::Instant::now();
let Some(revjob) = revjobs.join_next().await else {
@@ -1228,24 +1187,6 @@ mod tests {
assert!(!options.skip_total_timeout);
}
#[test]
fn cloned_reader_options_preserve_hidden_scan_budget_and_pending_prefix() {
let request = ListPathRawOptions {
bucket: "versioned".to_owned(),
hidden_entry_budget: 100,
pending_prefix: Some("pending/".to_owned()),
forward_to: Some("pending/0099".to_owned()),
..Default::default()
};
// Both walks are built once from the shared options; the per-job clone
// must carry every field that the walk derives from the request.
let job = request.clone();
let walk = walk_dir_options(&job);
assert_eq!(walk.hidden_entry_budget, 100);
assert_eq!(walk.pending_prefix.as_deref(), request.pending_prefix.as_deref());
assert_eq!(walk.forward_to.as_deref(), Some("pending/0099"));
}
#[tokio::test]
async fn list_path_raw_empty_disks_returns_read_quorum() {
let err = list_path_raw(CancellationToken::new(), ListPathRawOptions::default())
@@ -1296,163 +1237,6 @@ mod tests {
);
}
#[tokio::test]
async fn list_path_raw_fanout_retains_only_linear_disk_references() {
let root = tempfile::tempdir().expect("create disk root");
let endpoint = crate::disk::endpoint::Endpoint::try_from(root.path().to_str().expect("UTF-8 disk path"))
.expect("parse disk endpoint");
let local = Arc::new(
crate::disk::local::LocalDisk::new(&endpoint, false)
.await
.expect("open local disk"),
);
let wrapper = crate::disk::disk_store::LocalDiskWrapper::new(local, false);
let disk = Arc::new(crate::disk::Disk::Local(Box::new(wrapper)));
let baseline = Arc::strong_count(&disk);
for primary_count in [1, 4, 16] {
for fallback_count in [0, 2] {
let entered: Vec<_> = (0..primary_count).map(|_| Arc::new(tokio::sync::Notify::new())).collect();
let stopped: Vec<_> = (0..primary_count).map(|_| Arc::new(tokio::sync::Notify::new())).collect();
let behaviors = entered
.iter()
.zip(&stopped)
.map(|(entered, stopped)| TestReaderBehavior::StallWithSignals {
entered: entered.clone(),
stopped: stopped.clone(),
})
.collect();
let cancel = CancellationToken::new();
let listing = tokio::spawn(list_path_raw(
cancel.clone(),
ListPathRawOptions {
disks: vec![Some(disk.clone()); primary_count],
fallback_disks: vec![Some(disk.clone()); fallback_count],
min_disks: primary_count,
test_reader_behaviors: behaviors,
peek_timeout: Some(Duration::from_secs(60)),
..Default::default()
},
));
for signal in &entered {
timeout(Duration::from_secs(2), signal.notified())
.await
.expect("every producer must enter the real fanout before counting references");
}
// The merge owns the configured primary/fallback slots, each producer
// owns its primary, and the shared fallback queue owns each fallback.
let expected = baseline + 2 * (primary_count + fallback_count);
let retained = Arc::strong_count(&disk);
cancel.cancel();
timeout(Duration::from_secs(2), listing)
.await
.expect("cancellation must stop the listing")
.expect("listing task must join")
.expect("external cancellation must finish cleanly");
for signal in &stopped {
timeout(Duration::from_secs(2), signal.notified())
.await
.expect("cancellation must drop every producer");
}
assert_eq!(Arc::strong_count(&disk), baseline, "cancellation must release every fanout reference");
assert_eq!(
retained, expected,
"fanout must retain linear references for {primary_count} primary and {fallback_count} fallback slots"
);
}
}
}
#[tokio::test(flavor = "current_thread")]
async fn list_path_raw_producer_and_merge_preserve_request_span() {
use tracing_subscriber::{Layer, layer::SubscriberExt};
struct ProducerSpan(Arc<AtomicBool>);
#[derive(Default)]
struct StateField(bool);
impl tracing::field::Visit for StateField {
fn record_debug(&mut self, _: &tracing::field::Field, _: &dyn std::fmt::Debug) {}
fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
self.0 |= field.name() == "state" && value == "walk_dir_missing_path";
}
}
impl<S> Layer<S> for ProducerSpan
where
S: tracing::Subscriber + for<'a> tracing_subscriber::registry::LookupSpan<'a>,
{
fn on_event(&self, event: &tracing::Event<'_>, context: tracing_subscriber::layer::Context<'_, S>) {
let mut state = StateField::default();
event.record(&mut state);
if state.0
&& context
.lookup_current()
.is_some_and(|span| span.name() == "flink_list_request")
{
self.0.store(true, std::sync::atomic::Ordering::Release);
}
}
}
// The subscriber is thread-local; keep the spawned tasks on this runtime thread.
let producer_observed = Arc::new(AtomicBool::new(false));
let subscriber = tracing_subscriber::registry().with(ProducerSpan(producer_observed.clone()));
let _subscriber = tracing::subscriber::set_default(subscriber);
let request = tracing::info_span!("flink_list_request", request_id = "list-span-regression");
let root = tempfile::tempdir().expect("create disk root");
let endpoint = crate::disk::endpoint::Endpoint::try_from(root.path().to_str().expect("UTF-8 disk path"))
.expect("parse disk endpoint");
let local = Arc::new(
crate::disk::local::LocalDisk::new(&endpoint, false)
.await
.expect("open local disk"),
);
let wrapper = crate::disk::disk_store::LocalDiskWrapper::new(local, false);
let disk = Arc::new(crate::disk::Disk::Local(Box::new(wrapper)));
let error = list_path_raw(
CancellationToken::new(),
ListPathRawOptions {
disks: vec![Some(disk)],
bucket: "missing-bucket".to_owned(),
min_disks: 1,
..Default::default()
},
)
.instrument(request.clone())
.await
.expect_err("missing bucket must fail the real disk producer");
assert_eq!(error, DiskError::VolumeNotFound);
assert!(
producer_observed.load(std::sync::atomic::Ordering::Acquire),
"disk error must retain the request span"
);
let merge_observed = Arc::new(AtomicBool::new(false));
let merge_sink = merge_observed.clone();
let request_id = request.id();
list_path_raw(
CancellationToken::new(),
ListPathRawOptions {
disks: vec![None],
min_disks: 1,
test_reader_behaviors: vec![TestReaderBehavior::Entries(vec![fallback_test_entry()])],
agreed: Some(Box::new(move |_| {
merge_sink.store(tracing::Span::current().id() == request_id, std::sync::atomic::Ordering::Release);
Box::pin(async { Ok(()) })
})),
..Default::default()
},
)
.instrument(request)
.await
.expect("merge should deliver the test entry");
assert!(
merge_observed.load(std::sync::atomic::Ordering::Acquire),
"merge callback must retain the request span"
);
}
#[tokio::test]
async fn scanner_joinset_walk_uses_the_local_disk_background_gate() {
let root = tempfile::tempdir().expect("local disk root should be created");
@@ -1520,7 +1304,6 @@ mod tests {
outcome.limit_reached,
"a reader whose trailer reports a limit stop must mark the merged stream as possibly incomplete"
);
assert_eq!(outcome.scan_cut.as_deref(), Some("bucket/object"));
}
#[tokio::test]
@@ -1,682 +0,0 @@
// Copyright 2026 RustFS Team
// Licensed under the Apache License, Version 2.0.
use crate::disk::background_memory::BackgroundBuffer;
use std::io::{self, IoSlice};
use std::pin::Pin;
use std::sync::{Arc, Mutex};
use std::task::{Context, Poll, Waker, ready};
use tokio::io::{AsyncRead, AsyncWrite, DuplexStream, ReadBuf};
pub struct BackgroundPipeReader {
inner: PipeEnd,
}
pub struct BackgroundPipeWriter {
inner: PipeEnd,
}
impl BackgroundPipeWriter {
/// Capture only the payload owner, without retaining node or physical units.
pub fn memory_budget(&self) -> Option<Arc<crate::disk::BackgroundMemoryBudget>> {
match &self.inner {
PipeEnd::Foreground(_) => None,
PipeEnd::Background(state) => state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.buffer
.memory_budget(),
}
}
}
enum PipeEnd {
Foreground(DuplexStream),
Background(Arc<Mutex<PipeState>>),
}
// The pipe lock may reserve memory. The memory budget never locks a pipe.
struct PipeState {
buffer: BackgroundBuffer,
read_offset: usize,
readable: usize,
capacity: usize,
reader_closed: bool,
writer_closed: bool,
admission_failure: Option<io::ErrorKind>,
read_waker: Option<Waker>,
write_waker: Option<Waker>,
}
/// Capture only memory ownership while the request is admitted. Payload storage
/// is reserved and initialized once, before the first nonempty background write.
pub fn background_pipe(capacity: usize) -> (BackgroundPipeReader, BackgroundPipeWriter) {
let capacity = capacity.max(1);
let memory = crate::disk::current_resource_unit().and_then(|resources| resources.memory_budget());
match memory {
Some(memory) => {
let state = Arc::new(Mutex::new(PipeState {
buffer: BackgroundBuffer::new(Some(memory)),
read_offset: 0,
readable: 0,
capacity,
reader_closed: false,
writer_closed: false,
admission_failure: None,
read_waker: None,
write_waker: None,
}));
(
BackgroundPipeReader {
inner: PipeEnd::Background(state.clone()),
},
BackgroundPipeWriter {
inner: PipeEnd::Background(state),
},
)
}
None => {
let (reader, writer) = tokio::io::duplex(capacity);
(
BackgroundPipeReader {
inner: PipeEnd::Foreground(reader),
},
BackgroundPipeWriter {
inner: PipeEnd::Foreground(writer),
},
)
}
}
}
fn advance_ring(offset: usize, count: usize, capacity: usize) -> usize {
let to_end = capacity - offset;
if count < to_end { offset + count } else { count - to_end }
}
fn poll_background_read(state: &Mutex<PipeState>, cx: &mut Context<'_>, output: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
if output.remaining() == 0 {
return Poll::Ready(Ok(()));
}
let mut state = state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
if state.readable == 0 {
if let Some(kind) = state.admission_failure.take() {
drop(state);
return Poll::Ready(Err(io::Error::new(kind, "background pipe storage admission failed")));
}
if state.writer_closed {
return Poll::Ready(Ok(()));
}
let replaced = state.read_waker.replace(cx.waker().clone());
drop(state);
drop(replaced);
return Poll::Pending;
}
let length = state.readable.min(output.remaining());
let first = length.min(state.capacity - state.read_offset);
output.put_slice(&state.buffer.as_slice()[state.read_offset..state.read_offset + first]);
output.put_slice(&state.buffer.as_slice()[..length - first]);
state.read_offset = advance_ring(state.read_offset, length, state.capacity);
state.readable -= length;
let wake = state.write_waker.take();
drop(state);
if let Some(wake) = wake {
wake.wake();
}
Poll::Ready(Ok(()))
}
fn poll_background_write(state: &Mutex<PipeState>, cx: &mut Context<'_>, input: &[IoSlice<'_>]) -> Poll<io::Result<usize>> {
let mut state = state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
if state.reader_closed || state.writer_closed {
return Poll::Ready(Err(io::ErrorKind::BrokenPipe.into()));
}
if input.iter().all(|part| part.is_empty()) {
return Poll::Ready(Ok(0));
}
let available = state.capacity - state.readable;
if available == 0 {
let replaced = state.write_waker.replace(cx.waker().clone());
drop(state);
drop(replaced);
return Poll::Pending;
}
let capacity = state.capacity;
if let Err(error) = state.buffer.ensure_len(capacity) {
// Legacy walk completion may suppress ordinary producer errors. A new
// pipe admission failure must also reach the reader instead of EOF.
state.admission_failure = Some(error.kind());
state.writer_closed = true;
let wake = state.read_waker.take();
drop(state);
if let Some(wake) = wake {
wake.wake();
}
return Poll::Ready(Err(error));
}
let mut offset = advance_ring(state.read_offset, state.readable, capacity);
let mut written = 0;
for part in input {
let length = part.len().min(available - written);
let first = length.min(capacity - offset);
state.buffer.as_mut_slice()[offset..offset + first].copy_from_slice(&part[..first]);
state.buffer.as_mut_slice()[..length - first].copy_from_slice(&part[first..length]);
offset = advance_ring(offset, length, capacity);
written += length;
if written == available {
break;
}
}
state.readable += written;
let wake = state.read_waker.take();
drop(state);
if let Some(wake) = wake {
wake.wake();
}
Poll::Ready(Ok(written))
}
impl AsyncRead for BackgroundPipeReader {
fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, output: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
match &mut self.get_mut().inner {
PipeEnd::Foreground(reader) => Pin::new(reader).poll_read(cx, output),
PipeEnd::Background(state) => {
let coop = ready!(tokio::task::coop::poll_proceed(cx));
let result = poll_background_read(state, cx, output);
if result.is_ready() {
coop.made_progress();
}
result
}
}
}
}
impl AsyncWrite for BackgroundPipeWriter {
fn poll_write(self: Pin<&mut Self>, cx: &mut Context<'_>, input: &[u8]) -> Poll<io::Result<usize>> {
match &mut self.get_mut().inner {
PipeEnd::Foreground(writer) => Pin::new(writer).poll_write(cx, input),
PipeEnd::Background(state) => {
let coop = ready!(tokio::task::coop::poll_proceed(cx));
let result = poll_background_write(state, cx, &[IoSlice::new(input)]);
if result.is_ready() {
coop.made_progress();
}
result
}
}
}
fn poll_write_vectored(self: Pin<&mut Self>, cx: &mut Context<'_>, input: &[IoSlice<'_>]) -> Poll<io::Result<usize>> {
match &mut self.get_mut().inner {
PipeEnd::Foreground(writer) => Pin::new(writer).poll_write_vectored(cx, input),
PipeEnd::Background(state) => {
let coop = ready!(tokio::task::coop::poll_proceed(cx));
let result = poll_background_write(state, cx, input);
if result.is_ready() {
coop.made_progress();
}
result
}
}
}
fn is_write_vectored(&self) -> bool {
match &self.inner {
PipeEnd::Foreground(writer) => writer.is_write_vectored(),
PipeEnd::Background(_) => true,
}
}
fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
match &mut self.get_mut().inner {
PipeEnd::Foreground(writer) => Pin::new(writer).poll_flush(cx),
PipeEnd::Background(_) => Poll::Ready(Ok(())),
}
}
fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
match &mut self.get_mut().inner {
PipeEnd::Foreground(writer) => Pin::new(writer).poll_shutdown(cx),
PipeEnd::Background(state) => {
let mut state = state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
state.writer_closed = true;
let wake = state.read_waker.take();
drop(state);
if let Some(wake) = wake {
wake.wake();
}
Poll::Ready(Ok(()))
}
}
}
}
impl Drop for BackgroundPipeReader {
fn drop(&mut self) {
if let PipeEnd::Background(state) = &self.inner {
let mut state = state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
state.reader_closed = true;
let wake = state.write_waker.take();
let cancelled = state.read_waker.take();
drop(state);
drop(cancelled);
if let Some(wake) = wake {
wake.wake();
}
}
}
}
impl Drop for BackgroundPipeWriter {
fn drop(&mut self) {
if let PipeEnd::Background(state) = &self.inner {
let mut state = state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
state.writer_closed = true;
let wake = state.read_waker.take();
let cancelled = state.write_waker.take();
drop(state);
drop(cancelled);
if let Some(wake) = wake {
wake.wake();
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::disk::BackgroundMemoryBudget;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
pub(super) fn budget(capacity: usize) -> Arc<BackgroundMemoryBudget> {
let budget = BackgroundMemoryBudget::new();
budget.set_sampled_limit(u64::try_from(capacity).expect("test capacity fits u64"), Duration::from_secs(30));
budget
}
pub(super) fn local_pipe(
memory: &Arc<BackgroundMemoryBudget>,
capacity: usize,
) -> (BackgroundPipeReader, BackgroundPipeWriter) {
let state = Arc::new(Mutex::new(PipeState {
buffer: BackgroundBuffer::new(Some(memory.clone())),
read_offset: 0,
readable: 0,
capacity,
reader_closed: false,
writer_closed: false,
admission_failure: None,
read_waker: None,
write_waker: None,
}));
(
BackgroundPipeReader {
inner: PipeEnd::Background(state.clone()),
},
BackgroundPipeWriter {
inner: PipeEnd::Background(state),
},
)
}
pub(super) fn write_now(writer: &mut (impl AsyncWrite + Unpin), bytes: &[u8]) -> Poll<io::Result<usize>> {
Pin::new(writer).poll_write(&mut Context::from_waker(Waker::noop()), bytes)
}
pub(super) fn read_now(reader: &mut (impl AsyncRead + Unpin), bytes: &mut [u8]) -> Poll<io::Result<usize>> {
let mut output = ReadBuf::new(bytes);
match Pin::new(reader).poll_read(&mut Context::from_waker(Waker::noop()), &mut output) {
Poll::Ready(Ok(())) => Poll::Ready(Ok(output.filled().len())),
Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
Poll::Pending => Poll::Pending,
}
}
#[test]
fn fixed_ring_allocates_lazily_once_and_keeps_last_owner_charged() {
for capacity in [1, 7, 64 * 1024, 1024 * 1024] {
let memory = budget(capacity);
let (mut reader, mut writer) = local_pipe(&memory, capacity);
assert_eq!(memory.snapshot().used, 0);
assert!(matches!(write_now(&mut writer, &[]), Poll::Ready(Ok(0))));
assert_eq!(memory.snapshot().used, 0);
assert!(matches!(write_now(&mut writer, b"x"), Poll::Ready(Ok(1))));
assert_eq!(memory.snapshot().used, capacity as u64);
memory.set_sampled_limit(0, Duration::ZERO);
let mut output = [0];
assert!(matches!(read_now(&mut reader, &mut output), Poll::Ready(Ok(1))));
assert_eq!(output, *b"x");
assert!(matches!(write_now(&mut writer, b"y"), Poll::Ready(Ok(1))));
drop(writer);
assert_eq!(memory.snapshot().used, capacity as u64, "reader still owns the real ring");
assert!(matches!(read_now(&mut reader, &mut output), Poll::Ready(Ok(1))));
assert_eq!(output, *b"y");
assert!(matches!(read_now(&mut reader, &mut output), Poll::Ready(Ok(0))));
assert_eq!(memory.snapshot().used, capacity as u64, "draining does not destroy allocation ownership");
drop(reader);
assert_eq!(memory.snapshot().used, 0);
}
}
#[test]
fn admission_error_reaches_writer_and_reader_once_without_payload() {
for sample in [None, Some((6, Duration::from_secs(30))), Some((7, Duration::ZERO))] {
let memory = BackgroundMemoryBudget::new();
if let Some((limit, validity)) = sample {
memory.set_sampled_limit(limit, validity);
}
let (mut reader, mut writer) = local_pipe(&memory, 7);
let error = match write_now(&mut writer, b"x") {
Poll::Ready(Err(e)) => e,
_ => panic!("reject before copying first payload"),
};
assert!(
crate::disk::background_memory::is_memory_pressure(&error),
"writer retains the original budget marker"
);
assert_eq!(memory.snapshot().used, 0);
let error = match read_now(&mut reader, &mut [0]) {
Poll::Ready(Err(e)) => e,
_ => panic!("reader must see a pipe admission failure"),
};
assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
assert!(matches!(read_now(&mut reader, &mut [0]), Poll::Ready(Ok(0))), "failure is delivered once");
assert!(matches!(write_now(&mut writer,b"x"),Poll::Ready(Err(e)) if e.kind()==io::ErrorKind::BrokenPipe));
drop((reader, writer));
assert_eq!(memory.snapshot().used, 0);
}
}
#[test]
fn allocation_overflow_is_a_reader_failure_without_a_leaked_reservation() {
let memory = BackgroundMemoryBudget::new();
memory.set_sampled_limit(u64::MAX, Duration::from_secs(30));
let (mut reader, mut writer) = local_pipe(&memory, usize::MAX);
assert!(matches!(write_now(&mut writer,b"x"),Poll::Ready(Err(e)) if e.kind()==io::ErrorKind::OutOfMemory));
assert_eq!(memory.snapshot().used, 0);
assert!(matches!(read_now(&mut reader,&mut[0]),Poll::Ready(Err(e)) if e.kind()==io::ErrorKind::OutOfMemory));
}
#[test]
fn scalar_and_vectored_wrap_match_a_reference_queue() {
use std::collections::VecDeque;
for capacity in [1, 3, 7, 31] {
let memory = budget(capacity);
let (mut reader, mut writer) = local_pipe(&memory, capacity);
let mut expected = VecDeque::new();
let mut next = 0u8;
for turn in 0..300 {
let input: Vec<_> = (0..(turn % 11 + 1))
.map(|_| {
let byte = next;
next = next.wrapping_add(1);
byte
})
.collect();
let available = capacity - expected.len();
let written = if turn % 2 == 0 {
write_now(&mut writer, &input)
} else {
let split = input.len() / 2;
Pin::new(&mut writer).poll_write_vectored(
&mut Context::from_waker(Waker::noop()),
&[
io::IoSlice::new(&[]),
io::IoSlice::new(&input[..split]),
io::IoSlice::new(&input[split..]),
],
)
};
if available == 0 {
assert!(written.is_pending());
} else {
let accepted = input.len().min(available);
assert!(matches!(written,Poll::Ready(Ok(n)) if n==accepted));
expected.extend(&input[..accepted]);
}
let mut output = vec![0; turn % 5 + 1];
let count = output.len().min(expected.len());
assert!(matches!(read_now(&mut reader,&mut output),Poll::Ready(Ok(n)) if n==count));
for byte in &output[..count] {
assert_eq!(Some(*byte), expected.pop_front());
}
assert_eq!(memory.snapshot().used, capacity as u64);
}
drop(writer);
let mut output = vec![0; capacity];
let count = expected.len();
assert!(matches!(read_now(&mut reader,&mut output),Poll::Ready(Ok(n)) if n==count));
assert_eq!(&output[..count], expected.into_iter().collect::<Vec<_>>().as_slice());
assert!(matches!(read_now(&mut reader, &mut output), Poll::Ready(Ok(0))));
drop(reader);
assert_eq!(memory.snapshot().used, 0);
}
}
#[tokio::test]
async fn reader_drop_wakes_blocked_writer_and_writer_drop_wakes_empty_reader() {
let memory = budget(7);
let (reader, mut writer) = local_pipe(&memory, 7);
writer.write_all(b"1234567").await.expect("fill the ring");
let task = tokio::spawn(async move { writer.write_all(b"blocked").await });
tokio::task::yield_now().await;
assert!(!task.is_finished());
drop(reader);
let error = tokio::time::timeout(Duration::from_secs(1), task)
.await
.expect("reader drop wakes writer")
.expect("writer task joins")
.expect_err("closed reader is BrokenPipe");
assert_eq!(error.kind(), io::ErrorKind::BrokenPipe);
assert_eq!(memory.snapshot().used, 0);
let (mut reader, writer) = local_pipe(&memory, 7);
let task = tokio::spawn(async move { reader.read(&mut [0]).await });
tokio::task::yield_now().await;
assert!(!task.is_finished());
drop(writer);
assert_eq!(
tokio::time::timeout(Duration::from_secs(1), task)
.await
.expect("writer drop wakes reader")
.expect("reader task joins")
.expect("clean EOF"),
0
);
}
#[tokio::test]
async fn read_wakes_a_full_writer_without_changing_byte_limit() {
let memory = budget(7);
let (mut reader, mut writer) = local_pipe(&memory, 7);
writer.write_all(b"1234567").await.expect("fill initial byte limit");
let task = tokio::spawn(async move {
writer.write_all(b"x").await.expect("resume after one byte is read");
writer
});
tokio::task::yield_now().await;
assert!(!task.is_finished());
let mut first = [0];
reader.read_exact(&mut first).await.expect("read one byte");
let writer = tokio::time::timeout(Duration::from_secs(1), task)
.await
.expect("read wakes writer")
.expect("writer joins");
assert_eq!(&first, b"1");
drop(writer);
let mut all = Vec::new();
reader.read_to_end(&mut all).await.expect("drain wrapped bytes");
assert_eq!(all, b"234567x");
drop(reader);
assert_eq!(memory.snapshot().used, 0);
}
#[tokio::test]
async fn shutdown_drains_then_eof_and_cancelled_writes_accept_nothing() {
let memory = budget(7);
let (mut reader, mut writer) = local_pipe(&memory, 7);
writer
.write_all(b"1234567")
.await
.expect("fill before cancelling a blocked write");
{
let blocked = writer.write(b"lost");
tokio::pin!(blocked);
assert!(futures::poll!(&mut blocked).is_pending());
}
writer.shutdown().await.expect("shutdown does not discard buffered bytes");
assert!(matches!(write_now(&mut writer,b"x"),Poll::Ready(Err(e)) if e.kind()==io::ErrorKind::BrokenPipe));
let mut out = Vec::new();
reader.read_to_end(&mut out).await.expect("drain after shutdown");
assert_eq!(out, b"1234567");
drop(reader);
assert_eq!(memory.snapshot().used, 7, "writer still holds the allocation");
drop(writer);
assert_eq!(memory.snapshot().used, 0);
}
#[test]
fn foreground_keeps_native_duplex_semantics() {
let (reader, mut writer) = background_pipe(7);
assert!(matches!(write_now(&mut writer, b"12345678"), Poll::Ready(Ok(7))));
assert!(write_now(&mut writer, b"x").is_pending());
drop(reader);
assert!(matches!(write_now(&mut writer,b"x"),Poll::Ready(Err(e)) if e.kind()==io::ErrorKind::BrokenPipe));
let (mut reader, writer) = background_pipe(7);
drop(writer);
assert!(matches!(read_now(&mut reader, &mut [0]), Poll::Ready(Ok(0))));
}
#[test]
fn cancelled_reader_leaves_the_real_ring_charged_until_writer_drop() {
let memory = budget(7);
let (reader, mut writer) = local_pipe(&memory, 7);
assert!(matches!(write_now(&mut writer, b"x"), Poll::Ready(Ok(1))));
drop(reader);
assert_eq!(memory.snapshot().used, 7, "the surviving writer still owns the ring allocation");
assert!(matches!(write_now(&mut writer,b"y"),Poll::Ready(Err(e)) if e.kind()==io::ErrorKind::BrokenPipe));
drop(writer);
assert_eq!(memory.snapshot().used, 0);
}
#[test]
fn runtime_shutdown_releases_queued_writer_but_not_the_surviving_reader() {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("build isolated runtime");
let memory = budget(7);
let (reader, mut writer) = local_pipe(&memory, 7);
runtime.block_on(async {
writer.write_all(b"x").await.expect("allocate ring before runtime shutdown");
tokio::spawn(async move {
let _writer = writer;
std::future::pending::<()>().await;
});
tokio::task::yield_now().await;
});
drop(runtime);
assert_eq!(memory.snapshot().used, 7, "reader holds the actual allocation after runtime cancellation");
drop(reader);
assert_eq!(memory.snapshot().used, 0);
}
#[tokio::test]
async fn ready_operations_keep_tokio_cooperative_yields() {
let memory = budget(7);
let (_reader, mut writer) = local_pipe(&memory, 7);
let observed = Arc::new(AtomicUsize::new(0));
let flag = observed.clone();
let task = tokio::spawn(async move {
flag.store(1, Ordering::SeqCst);
});
for _ in 0..1000 {
assert_eq!(writer.write(&[]).await.expect("empty operation remains successful"), 0);
}
assert_eq!(
observed.load(Ordering::SeqCst),
1,
"ready pipe operations must not starve other ready tasks"
);
task.await.expect("observer task joins");
}
}
#[cfg(test)]
mod waker_tests {
use super::*;
use std::sync::{
Weak,
atomic::{AtomicUsize, Ordering},
};
use std::task::Wake;
struct LockCheck {
state: Weak<Mutex<PipeState>>,
wakes: Arc<AtomicUsize>,
drops: Arc<AtomicUsize>,
}
impl LockCheck {
fn unlocked(&self) {
if let Some(state) = self.state.upgrade() {
assert!(state.try_lock().is_ok(), "waker code must run outside the pipe lock");
}
}
}
impl Wake for LockCheck {
fn wake(self: Arc<Self>) {
self.unlocked();
self.wakes.fetch_add(1, Ordering::SeqCst);
}
}
impl Drop for LockCheck {
fn drop(&mut self) {
self.unlocked();
self.drops.fetch_add(1, Ordering::SeqCst);
}
}
#[test]
fn replaced_and_woken_reader_and_writer_wakers_run_outside_the_lock() {
for waiting_reader in [true, false] {
let memory = super::tests::budget(7);
let (mut reader, mut writer) = super::tests::local_pipe(&memory, 7);
let PipeEnd::Background(state) = &reader.inner else { panic!("background fixture") };
let weak = Arc::downgrade(state);
let wakes = Arc::new(AtomicUsize::new(0));
let drops = Arc::new(AtomicUsize::new(0));
if !waiting_reader {
assert!(matches!(super::tests::write_now(&mut writer, b"1234567"), Poll::Ready(Ok(7))));
}
for _ in 0..2 {
let waker = Waker::from(Arc::new(LockCheck {
state: weak.clone(),
wakes: wakes.clone(),
drops: drops.clone(),
}));
let mut cx = Context::from_waker(&waker);
if waiting_reader {
let mut bytes = [0];
assert!(
Pin::new(&mut reader)
.poll_read(&mut cx, &mut ReadBuf::new(&mut bytes))
.is_pending()
);
} else {
assert!(Pin::new(&mut writer).poll_write(&mut cx, b"x").is_pending());
}
drop(waker);
}
assert_eq!(drops.load(Ordering::SeqCst), 1, "the replaced waker has been dropped");
if waiting_reader {
assert!(matches!(super::tests::write_now(&mut writer, b"x"), Poll::Ready(Ok(1))));
} else {
assert!(matches!(super::tests::read_now(&mut reader, &mut [0]), Poll::Ready(Ok(1))));
}
assert_eq!(wakes.load(Ordering::SeqCst), 1);
assert_eq!(drops.load(Ordering::SeqCst), 2);
}
}
}
@@ -1,586 +0,0 @@
// Copyright 2026 RustFS Team
// Licensed under the Apache License, Version 2.0.
use crate::disk::background_memory::{BackgroundMemoryBudget, Reservation};
use bytes::{Bytes, BytesMut};
use futures::Stream;
use std::io;
use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll};
use tokio::io::{AsyncRead, ReadBuf};
use tokio_util::io::ReaderStream;
struct ReservedReadBytes {
// Free each payload view before releasing its shared allocation receipt.
bytes: Bytes,
_reservation: Arc<Reservation>,
}
impl AsRef<[u8]> for ReservedReadBytes {
fn as_ref(&self) -> &[u8] {
&self.bytes
}
}
struct ReaderBuffer {
// These initialized bytes are spare storage, not unconsumed source data.
bytes: BytesMut,
reservation: Option<Arc<Reservation>>,
budget: Arc<BackgroundMemoryBudget>,
capacity: usize,
}
impl ReaderBuffer {
fn new(budget: Arc<BackgroundMemoryBudget>, capacity: usize) -> Self {
Self {
bytes: BytesMut::new(),
reservation: None,
budget,
capacity: capacity.max(1),
}
}
fn clear(&mut self) {
self.bytes = BytesMut::new();
self.reservation = None;
}
fn ensure_readable(&mut self) -> io::Result<()> {
if !self.bytes.is_empty() {
return Ok(());
}
if self.reservation.is_some() {
if self.bytes.try_reclaim(self.capacity) {
self.bytes.resize(self.capacity, 0);
return Ok(());
}
// A live output owns this allocation. Drop our empty view before
// its receipt, then reserve independent replacement storage.
self.clear();
}
let reservation = self.budget.reserve(self.capacity)?;
let mut bytes = Vec::new();
bytes
.try_reserve_exact(self.capacity)
.map_err(|_| io::ErrorKind::OutOfMemory)?;
bytes.resize(self.capacity, 0);
// Fresh ordinary Bytes is unique and spans the complete allocation.
// Unlike from_owner output, this conversion cannot require a copy.
let bytes = Bytes::from(bytes.into_boxed_slice())
.try_into_mut()
.map_err(|_| io::Error::other("fresh reader storage was not unique"))?;
self.bytes = bytes;
self.reservation = Some(Arc::new(reservation));
Ok(())
}
fn take_bytes(&mut self, length: usize) -> io::Result<Bytes> {
let reservation = self
.reservation
.as_ref()
.cloned()
.ok_or_else(|| io::Error::other("reader storage has no memory reservation"))?;
Ok(Bytes::from_owner(ReservedReadBytes {
bytes: self.bytes.split_to(length).freeze(),
_reservation: reservation,
}))
}
}
/// RPC response chunks retain their background memory reservation until the
/// last clone or slice is dropped. Foreground readers use the existing stream.
pub struct BackgroundReaderStream<R> {
inner: ReaderState<R>,
}
enum ReaderState<R> {
Foreground(ReaderStream<R>),
Background { reader: Option<R>, buffer: ReaderBuffer },
}
impl<R: AsyncRead> BackgroundReaderStream<R> {
/// Capture memory ownership inside the admitted scope, before moving the
/// response body out of it. This stream retains no node or device capacity.
pub fn with_capacity(reader: R, capacity: usize) -> Self {
let memory = crate::disk::current_resource_unit().and_then(|resources| resources.memory_budget());
let inner = match memory {
Some(memory) => ReaderState::Background {
reader: Some(reader),
buffer: ReaderBuffer::new(memory, capacity),
},
None => ReaderState::Foreground(ReaderStream::with_capacity(reader, capacity)),
};
Self { inner }
}
}
impl<R: AsyncRead + Unpin> Stream for BackgroundReaderStream<R> {
type Item = io::Result<Bytes>;
fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
let this = self.get_mut();
match &mut this.inner {
ReaderState::Foreground(reader) => Pin::new(reader).poll_next(cx),
ReaderState::Background { reader, buffer } => {
let Some(inner) = reader.as_mut() else {
return Poll::Ready(None);
};
if let Err(error) = buffer.ensure_readable() {
*reader = None;
return Poll::Ready(Some(Err(error)));
}
let mut output = ReadBuf::new(&mut buffer.bytes);
match std::task::ready!(Pin::new(inner).poll_read(cx, &mut output)) {
Ok(()) => {
let length = output.filled().len();
if length == 0 {
buffer.clear();
*reader = None;
Poll::Ready(None)
} else {
let bytes = buffer.take_bytes(length);
if bytes.is_err() {
buffer.clear();
*reader = None;
}
Poll::Ready(Some(bytes))
}
}
Err(error) => {
buffer.clear();
*reader = None;
Poll::Ready(Some(Err(error)))
}
}
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::disk::{BackgroundMemoryBudget, test_resource_units, with_optional_resource_unit};
use futures::StreamExt;
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
use tokio::io::AsyncWriteExt;
struct ObservedReader {
polls: Arc<AtomicUsize>,
fail: bool,
}
impl AsyncRead for ObservedReader {
fn poll_read(self: Pin<&mut Self>, _cx: &mut Context<'_>, output: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
self.polls.fetch_add(1, Ordering::SeqCst);
if self.fail {
Poll::Ready(Err(io::Error::from(io::ErrorKind::PermissionDenied)))
} else {
output.put_slice(b"data");
Poll::Ready(Ok(()))
}
}
}
#[tokio::test]
async fn background_frames_keep_capacity_until_the_last_slice() {
let memory = BackgroundMemoryBudget::new();
memory.set_sampled_limit(16, Duration::from_secs(30));
let (resources, gates) = test_resource_units(2);
let mut stream = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
BackgroundReaderStream::with_capacity(io::Cursor::new(b"data"), 16)
})
.await;
assert!(
gates.iter().all(|gate| gate.snapshot().active == 0),
"the stream captures only memory ownership"
);
assert_eq!(memory.snapshot().used, 0, "construction does not allocate payload storage");
let bytes = stream.next().await.expect("first frame").expect("read frame");
assert_eq!(bytes.as_ref(), b"data");
assert_eq!(memory.snapshot().used, 16, "charge full capacity rather than visible length");
let slice = bytes.slice(1..3);
memory.set_sampled_limit(0, Duration::ZERO);
drop(bytes);
drop(stream);
assert_eq!(slice.as_ref(), b"at");
assert_eq!(memory.snapshot().used, 16);
drop(slice);
assert_eq!(memory.snapshot().used, 0);
}
#[tokio::test]
async fn background_denial_happens_before_reading_and_never_falls_back() {
for sample in [None, Some((7, Duration::from_secs(30))), Some((8, Duration::ZERO))] {
let memory = BackgroundMemoryBudget::new();
if let Some((limit, validity)) = sample {
memory.set_sampled_limit(limit, validity);
}
let polls = Arc::new(AtomicUsize::new(0));
let (resources, _) = test_resource_units(1);
let mut stream = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
BackgroundReaderStream::with_capacity(
ObservedReader {
polls: polls.clone(),
fail: false,
},
8,
)
})
.await;
let error = stream
.next()
.await
.expect("allocation failure")
.expect_err("background allocation denied");
assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
assert_eq!(polls.load(Ordering::SeqCst), 0, "denial cannot consume source bytes");
assert!(stream.next().await.is_none());
assert_eq!(memory.snapshot().used, 0);
}
}
#[tokio::test]
async fn background_pending_read_keeps_storage_through_shrink_or_cancellation() {
for cancelled in [false, true] {
let memory = BackgroundMemoryBudget::new();
memory.set_sampled_limit(16, Duration::from_secs(30));
let (resources, _) = test_resource_units(1);
let (mut sender, reader) = tokio::io::duplex(4);
let mut stream = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
BackgroundReaderStream::with_capacity(reader, 16)
})
.await;
assert!(futures::poll!(stream.next()).is_pending());
memory.set_sampled_limit(0, Duration::ZERO);
assert_eq!(memory.snapshot().used, 16);
let bytes = if cancelled {
None
} else {
sender.write_all(b"data").await.expect("release pending read");
Some(
stream
.next()
.await
.expect("pending frame")
.expect("existing allocation survives shrink"),
)
};
drop(stream);
if let Some(bytes) = bytes {
assert_eq!(bytes.as_ref(), b"data");
assert_eq!(memory.snapshot().used, 16);
drop(bytes);
}
assert_eq!(memory.snapshot().used, 0);
}
}
#[tokio::test]
async fn background_read_error_and_eof_release_storage() {
for fail in [false, true] {
let memory = BackgroundMemoryBudget::new();
memory.set_sampled_limit(8, Duration::from_secs(30));
let (resources, _) = test_resource_units(1);
let mut stream = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
let reader: Box<dyn AsyncRead + Unpin> = if fail {
Box::new(ObservedReader {
polls: Arc::new(AtomicUsize::new(0)),
fail: true,
})
} else {
Box::new(tokio::io::empty())
};
BackgroundReaderStream::with_capacity(reader, 8)
})
.await;
if fail {
let error = stream.next().await.expect("read failure").expect_err("preserve reader error");
assert_eq!(error.kind(), io::ErrorKind::PermissionDenied);
}
assert!(stream.next().await.is_none());
assert_eq!(memory.snapshot().used, 0);
}
}
#[tokio::test]
async fn retained_frame_blocks_new_storage_until_released() {
let memory = BackgroundMemoryBudget::new();
memory.set_sampled_limit(8, Duration::from_secs(30));
let (resources, _) = test_resource_units(1);
let mut stream = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
BackgroundReaderStream::with_capacity(io::Cursor::new(b"firstonefollowup"), 8)
})
.await;
let first = stream.next().await.expect("first frame").expect("fits exact budget");
assert_eq!(first.as_ref(), b"firstone");
let error = stream
.next()
.await
.expect("next allocation")
.expect_err("held frame exhausts budget");
assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
assert_eq!(memory.snapshot().used, 8);
drop(stream);
drop(first);
assert_eq!(memory.snapshot().used, 0);
}
#[tokio::test]
async fn consumed_frames_keep_reusable_capacity_until_eof() {
let memory = BackgroundMemoryBudget::new();
memory.set_sampled_limit(8, Duration::from_secs(30));
let (resources, _) = test_resource_units(1);
let mut stream = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
BackgroundReaderStream::with_capacity(io::Cursor::new(b"firstonefollowup"), 8)
})
.await;
for expected in [b"firstone", b"followup"] {
let bytes = stream.next().await.expect("next frame").expect("released previous capacity");
assert_eq!(bytes.as_ref(), expected);
assert_eq!(memory.snapshot().used, 8);
drop(bytes);
assert_eq!(memory.snapshot().used, 8, "the reusable allocation remains owned by the stream");
}
assert!(stream.next().await.is_none());
assert_eq!(memory.snapshot().used, 0);
}
#[tokio::test]
async fn foreground_stream_and_unpolled_cancellation_preserve_existing_behavior() {
let mut stream = BackgroundReaderStream::with_capacity(io::Cursor::new(b"foreground"), 4);
assert!(matches!(stream.inner, ReaderState::Foreground(_)));
let mut bytes = Vec::new();
while let Some(chunk) = stream.next().await {
bytes.extend_from_slice(&chunk.expect("foreground chunk"));
}
assert_eq!(bytes, b"foreground");
let memory = BackgroundMemoryBudget::new();
let (resources, _) = test_resource_units(1);
let stream = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
BackgroundReaderStream::with_capacity(tokio::io::empty(), 8)
})
.await;
drop(stream);
assert_eq!(memory.snapshot().used, 0);
}
}
#[cfg(test)]
mod reuse_tests {
use super::*;
use bytes::Bytes;
use futures::StreamExt;
use std::sync::Arc;
use std::time::Duration;
use std::{
io,
pin::Pin,
task::{Context, Poll},
};
use tokio::io::{AsyncRead, ReadBuf};
static DATA: [u8; 1024] = [0x37; 1024];
struct ShortReader {
offset: usize,
length: usize,
chunk: usize,
fail_at: Option<usize>,
}
impl AsyncRead for ShortReader {
fn poll_read(mut self: Pin<&mut Self>, _: &mut Context<'_>, out: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
if self.fail_at == Some(self.offset) {
return Poll::Ready(Err(io::ErrorKind::PermissionDenied.into()));
}
let n = (self.length - self.offset).min(self.chunk).min(out.remaining());
out.put_slice(&DATA[self.offset..self.offset + n]);
self.offset += n;
Poll::Ready(Ok(()))
}
}
fn budget(limit: u64) -> Arc<crate::disk::BackgroundMemoryBudget> {
let b = crate::disk::BackgroundMemoryBudget::new();
b.set_sampled_limit(limit, Duration::from_secs(30));
b
}
async fn stream<R: AsyncRead>(
budget: &Arc<crate::disk::BackgroundMemoryBudget>,
reader: R,
capacity: usize,
) -> BackgroundReaderStream<R> {
let (resources, _) = crate::disk::test_resource_units(1);
crate::disk::with_optional_resource_unit(Some(resources.with_memory_budget(budget.clone())), async {
BackgroundReaderStream::with_capacity(reader, capacity)
})
.await
}
#[tokio::test]
async fn complete_consumed_frames_reclaim_the_same_storage_after_expiry() {
let b = budget(64);
let mut s = stream(&b, io::Cursor::new(&DATA[..]), 64).await;
let mut pointer = None;
for _ in 0..16 {
let frame = s.next().await.expect("next full frame").expect("reclaimed backing");
assert_eq!(frame.len(), 64);
assert_eq!(frame.as_ref(), &DATA[..64]);
if let Some(p) = pointer {
assert_eq!(frame.as_ptr(), p);
} else {
pointer = Some(frame.as_ptr());
}
b.set_sampled_limit(0, Duration::ZERO);
drop(frame);
assert_eq!(b.snapshot().used, 64);
}
assert!(s.next().await.is_none());
assert_eq!(b.snapshot().used, 0);
}
#[tokio::test]
async fn short_live_frames_use_disjoint_tail_then_reserve_separate_storage() {
let b = budget(32);
let mut s = stream(
&b,
ShortReader {
offset: 0,
length: 32,
chunk: 4,
fail_at: None,
},
16,
)
.await;
let mut frames: Vec<Bytes> = Vec::new();
for i in 0..4 {
let frame = s.next().await.expect("short frame").expect("remaining initialized tail");
if i != 0 {
assert_eq!(frame.as_ptr(), frames[0].as_ptr().wrapping_add(i * 4));
}
assert_eq!(frame.as_ref(), &DATA[..4]);
frames.push(frame);
assert_eq!(b.snapshot().used, 16);
}
let next = s.next().await.expect("independent frame").expect("second backing fits");
assert_ne!(next.as_ptr(), frames[0].as_ptr());
assert_eq!(b.snapshot().used, 32);
drop(frames);
assert_eq!(b.snapshot().used, 16);
drop(s);
assert_eq!(b.snapshot().used, 16);
drop(next);
assert_eq!(b.snapshot().used, 0);
}
#[tokio::test]
async fn final_slice_prevents_reclaim_and_keeps_old_plus_new_charged() {
let b = budget(32);
let mut s = stream(&b, io::Cursor::new(&DATA[..32]), 16).await;
let first = s.next().await.expect("old frame").expect("old backing");
let last = first.slice(15..);
drop(first);
let next = s.next().await.expect("new frame").expect("second backing fits");
assert_eq!(last.as_ref(), &DATA[..1]);
assert_ne!(next.as_ptr(), last.as_ptr());
assert_eq!(b.snapshot().used, 32);
drop(s);
assert_eq!(b.snapshot().used, 32);
drop(next);
assert_eq!(b.snapshot().used, 16);
drop(last);
assert_eq!(b.snapshot().used, 0);
}
#[tokio::test]
async fn expiry_uses_tail_but_rejects_new_storage_before_consuming_source() {
let b = budget(16);
let mut s = stream(
&b,
ShortReader {
offset: 0,
length: 32,
chunk: 4,
fail_at: None,
},
16,
)
.await;
let first = s.next().await.expect("first frame").expect("initial backing");
b.set_sampled_limit(0, Duration::ZERO);
for _ in 0..3 {
drop(s.next().await.expect("tail frame").expect("reuse charged storage"));
}
let error = s
.next()
.await
.expect("new backing denied")
.expect_err("live frame prevents reclaim");
assert!(crate::disk::background_memory::is_memory_pressure(&error));
assert!(s.next().await.is_none());
assert_eq!(b.snapshot().used, 16);
drop(first);
assert_eq!(b.snapshot().used, 0);
}
#[tokio::test]
async fn eof_and_error_release_spare_but_keep_live_output_charged() {
for fail_at in [None, Some(4)] {
let b = budget(16);
let mut s = stream(
&b,
ShortReader {
offset: 0,
length: 4,
chunk: 4,
fail_at,
},
16,
)
.await;
let frame = s.next().await.expect("first short frame").expect("payload");
match fail_at {
None => assert!(s.next().await.is_none()),
Some(_) => assert_eq!(
s.next()
.await
.expect("read error")
.expect_err("preserve source failure")
.kind(),
io::ErrorKind::PermissionDenied
),
}
assert!(s.next().await.is_none());
assert_eq!(b.snapshot().used, 16);
assert_eq!(frame.as_ref(), &DATA[..4]);
drop(frame);
assert_eq!(b.snapshot().used, 0);
}
}
#[test]
fn runtime_shutdown_drops_a_pending_stream_without_refunding_early() {
for abort in [false, true] {
let b = budget(16);
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("runtime");
let (_tx, reader) = tokio::io::duplex(4);
let (began, wait) = tokio::sync::oneshot::channel();
let memory = b.clone();
let handle = rt.spawn(async move {
let mut s = stream(&memory, reader, 16).await;
assert!(futures::poll!(s.next()).is_pending());
assert_eq!(memory.snapshot().used, 16);
began.send(()).expect("notify pending allocation");
let _ = s.next().await;
});
rt.block_on(wait).expect("stream pending");
assert_eq!(b.snapshot().used, 16);
if abort {
handle.abort();
let _ = rt.block_on(handle);
}
drop(rt);
assert_eq!(b.snapshot().used, 0);
}
}
}
+4 -13
View File
@@ -17,10 +17,10 @@ use crate::cluster::rpc::http_auth::RPC_REPLAY_SCOPE_VERSION_HEADER;
use crate::cluster::rpc::http_auth::{
AuthenticatedPeerReplayCapabilities, RPC_AUTH_VERSION_HEADER, RPC_AUTH_VERSION_V2, RPC_BOOT_EPOCH_CHALLENGE_HEADER,
RPC_CONTENT_SHA256_HEADER, RPC_REPLAY_CACHE_CAPABILITY_HEADER, RPC_REPLAY_CACHE_CAPABILITY_PROOF_HEADER,
RollingMutationBodyDigest, TIMESTAMP_HEADER, gen_tonic_replay_scope_headers_with_admission, internode_rpc_body_digest_strict,
rpc_background_admission, verify_tonic_peer_replay_capabilities_response,
RollingMutationBodyDigest, TIMESTAMP_HEADER, internode_rpc_body_digest_strict,
verify_tonic_peer_replay_capabilities_response,
};
use crate::cluster::rpc::{gen_tonic_signature_headers, normalize_tonic_rpc_audience};
use crate::cluster::rpc::{gen_tonic_replay_scope_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience};
#[cfg(test)]
use crate::cluster::rpc::{tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers};
use crate::disk::error::{DiskError, Error as DiskErrorType, RpcStatusError};
@@ -415,16 +415,7 @@ where
.get(RPC_CONTENT_SHA256_HEADER)
.and_then(|value| value.to_str().ok()),
) {
match rpc_background_admission(request.headers()).and_then(|background| {
gen_tonic_replay_scope_headers_with_admission(
audience,
request.uri().path(),
timestamp,
content_sha256,
boot_epoch,
background,
)
}) {
match gen_tonic_replay_scope_headers(audience, request.uri().path(), timestamp, content_sha256, boot_epoch) {
Ok(headers) => request.headers_mut().extend(headers),
Err(error) => debug!(error = %error, "could not attach replay-scoped RPC signature"),
}
+28 -240
View File
@@ -32,6 +32,7 @@ use crate::storage_api_contracts::internode::{
PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_CAPABILITY_VERSION,
};
use hmac::{Hmac, KeyInit, Mac};
use http::uri::Authority;
use http::{HeaderMap, HeaderValue, Method, Uri};
#[cfg(test)]
@@ -46,7 +47,6 @@ use rustfs_io_metrics::internode_metrics::{
global_internode_metrics,
};
use rustfs_object_data_cache::{MemoryBasis, resolve_effective_memory};
use rustfs_utils::crypto::{HmacSha256, KeyInit, Mac};
use rustfs_utils::get_env_bool;
use sha2::Digest as _;
use sha2::Sha256;
@@ -58,10 +58,9 @@ use time::OffsetDateTime;
use tracing::{error, info, warn};
use uuid::Uuid;
type HmacSha256 = Hmac<Sha256>;
const SIGNATURE_HEADER: &str = "x-rustfs-signature";
pub const RPC_BACKGROUND_ADMISSION_HEADER: &str = "x-rustfs-rpc-background-admission";
const RPC_BACKGROUND_ADMISSION_V1: &str = "v1";
const RPC_BACKGROUND_ADMISSION_DOMAIN: &[u8] = b"rustfs-rpc-background-admission-v1\0";
pub(crate) const TIMESTAMP_HEADER: &str = "x-rustfs-timestamp";
pub(crate) const RPC_AUTH_VERSION_HEADER: &str = "x-rustfs-rpc-auth-version";
const RPC_SIGNATURE_V2_HEADER: &str = "x-rustfs-rpc-signature-v2";
@@ -78,46 +77,6 @@ pub(crate) const RPC_REPLAY_CACHE_CAPABILITY_HEADER: &str = "x-rustfs-rpc-replay
pub(crate) const RPC_REPLAY_CACHE_CAPABILITY_PROOF_HEADER: &str = "x-rustfs-rpc-replay-cache-capability-proof";
const RPC_REPLAY_SCOPE_VERSION_V3: &str = "3";
const RPC_RESPONSE_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-response-proof-v1\0";
tokio::task_local! {
static RPC_BACKGROUND_REQUEST: bool;
}
pub(crate) fn current_rpc_background_request() -> bool {
RPC_BACKGROUND_REQUEST
.try_with(|background| *background)
.unwrap_or_else(|_| crate::disk::has_resource_io_scope())
}
/// Preserve the authenticated wire class while polling locally admitted work.
/// This scope carries only the class, never node, disk, or memory ownership.
pub async fn with_rpc_background_request<F: std::future::Future>(background: bool, operation: F) -> F::Output {
RPC_BACKGROUND_REQUEST.scope(background, operation).await
}
/// Parse the request class only; callers must verify the signature before using it.
pub fn rpc_background_admission(headers: &HeaderMap) -> std::io::Result<bool> {
let mut values = headers.get_all(RPC_BACKGROUND_ADMISSION_HEADER).iter();
match (values.next(), values.next()) {
(None, None) => Ok(false),
(Some(value), None) if value == RPC_BACKGROUND_ADMISSION_V1 => Ok(true),
_ => Err(std::io::Error::other("Invalid RPC background admission version")),
}
}
fn bind_background_admission(mac: &mut HmacSha256, background: bool) {
if background {
// Every auth lane binds the class, so stripping the marker or falling
// back to an older signature cannot turn admitted work into foreground I/O.
mac.update(RPC_BACKGROUND_ADMISSION_DOMAIN);
}
}
fn insert_background_admission(headers: &mut HeaderMap, background: bool) {
if background {
headers.insert(RPC_BACKGROUND_ADMISSION_HEADER, HeaderValue::from_static(RPC_BACKGROUND_ADMISSION_V1));
}
}
const RPC_REPLAY_SCOPE_DOMAIN: &[u8] = b"rustfs-rpc-replay-scope-v3\0";
const RPC_BOOT_EPOCH_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-boot-epoch-proof-v1\0";
const RPC_REPLAY_CACHE_CAPABILITY_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-replay-cache-capability-proof-v1\0";
@@ -560,23 +519,21 @@ fn redacted_rpc_path(url: &str) -> String {
}
/// Generate HMAC-SHA256 signature for the given data
fn generate_signature(secret: &str, url: &str, method: &Method, timestamp: i64, background: bool) -> String {
fn generate_signature(secret: &str, url: &str, method: &Method, timestamp: i64) -> String {
let data = signature_payload(url, method, timestamp);
let mut mac = <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).expect("HMAC can take key of any size");
bind_background_admission(&mut mac, background);
mac.update(data.as_bytes());
let result = mac.finalize();
base64_simd::STANDARD.encode_to_string(result.into_bytes())
}
fn verify_signature(secret: &str, url: &str, method: &Method, timestamp: i64, signature: &str, background: bool) -> bool {
fn verify_signature(secret: &str, url: &str, method: &Method, timestamp: i64, signature: &str) -> bool {
let Ok(signature) = base64_simd::STANDARD.decode_to_vec(signature) else {
return false;
};
let data = signature_payload(url, method, timestamp);
let mut mac = <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).expect("HMAC can take key of any size");
bind_background_admission(&mut mac, background);
mac.update(data.as_bytes());
mac.verify_slice(&signature).is_ok()
}
@@ -814,7 +771,6 @@ pub fn verify_ns_scanner_capability_with_checkpoint_coverage(
#[derive(Clone, Copy)]
struct SignatureV2Scope<'a> {
background: bool,
audience: &'a str,
service: &'a str,
rpc_method: &'a str,
@@ -824,7 +780,6 @@ struct SignatureV2Scope<'a> {
}
fn update_signature_v2(mac: &mut HmacSha256, scope: SignatureV2Scope<'_>) {
bind_background_admission(mac, scope.background);
for part in [
b"rustfs-rpc-auth-v2|".as_slice(),
scope.audience.as_bytes(),
@@ -863,7 +818,6 @@ fn verify_signature_v2(secret: &str, scope: SignatureV2Scope<'_>, signature: &st
#[derive(Clone, Copy)]
struct ReplayScope<'a> {
background: bool,
audience: &'a str,
path: &'a str,
timestamp: &'a str,
@@ -873,7 +827,6 @@ struct ReplayScope<'a> {
}
fn update_replay_scope(mac: &mut HmacSha256, scope: ReplayScope<'_>) {
bind_background_admission(mac, scope.background);
mac.update(RPC_REPLAY_SCOPE_DOMAIN);
for part in [
scope.audience.as_bytes(),
@@ -1017,24 +970,6 @@ pub fn gen_tonic_replay_scope_headers(
timestamp: &str,
content_sha256: &str,
boot_epoch: Uuid,
) -> std::io::Result<HeaderMap> {
gen_tonic_replay_scope_headers_with_admission(
audience,
path,
timestamp,
content_sha256,
boot_epoch,
current_rpc_background_request(),
)
}
pub(crate) fn gen_tonic_replay_scope_headers_with_admission(
audience: &str,
path: &str,
timestamp: &str,
content_sha256: &str,
boot_epoch: Uuid,
background: bool,
) -> std::io::Result<HeaderMap> {
if audience.is_empty() || !path.starts_with('/') || !valid_content_sha256(content_sha256) || boot_epoch.is_nil() {
return Err(std::io::Error::other("Invalid replay-scoped RPC signing scope"));
@@ -1048,7 +983,6 @@ pub(crate) fn gen_tonic_replay_scope_headers_with_admission(
let signature = generate_replay_scope_signature(
&get_shared_secret()?,
ReplayScope {
background,
audience,
path,
timestamp,
@@ -1058,7 +992,6 @@ pub(crate) fn gen_tonic_replay_scope_headers_with_admission(
},
)?;
let mut headers = HeaderMap::new();
insert_background_admission(&mut headers, background);
headers.insert(RPC_REPLAY_SCOPE_VERSION_HEADER, HeaderValue::from_static(RPC_REPLAY_SCOPE_VERSION_V3));
headers.insert(
RPC_REPLAY_SCOPE_SIGNATURE_HEADER,
@@ -1293,12 +1226,10 @@ pub fn build_auth_headers(url: &str, method: &Method, headers: &mut HeaderMap) -
pub fn gen_signature_headers(url: &str, method: &Method) -> std::io::Result<HeaderMap> {
let secret = get_shared_secret()?;
let timestamp = OffsetDateTime::now_utc().unix_timestamp();
let background = current_rpc_background_request();
let signature = generate_signature(&secret, url, method, timestamp, background);
let signature = generate_signature(&secret, url, method, timestamp);
let mut headers = HeaderMap::new();
insert_background_admission(&mut headers, background);
headers.insert(SIGNATURE_HEADER, HeaderValue::from_str(&signature).expect("Invalid header value"));
headers.insert(
TIMESTAMP_HEADER,
@@ -1333,12 +1264,10 @@ pub fn gen_tonic_signature_headers(
let timestamp_header = timestamp.to_string();
let body_nonce = (content_sha256 != UNSIGNED_PAYLOAD).then(|| Uuid::new_v4().to_string());
let nonce = body_nonce.as_deref().unwrap_or(UNSIGNED_PAYLOAD_NONCE);
let background = current_rpc_background_request();
let legacy_signature = generate_signature(&secret, TONIC_RPC_PREFIX, &Method::GET, timestamp, background);
let legacy_signature = generate_signature(&secret, TONIC_RPC_PREFIX, &Method::GET, timestamp);
let signature_v2 = generate_signature_v2(
&secret,
SignatureV2Scope {
background,
audience,
service,
rpc_method,
@@ -1349,7 +1278,6 @@ pub fn gen_tonic_signature_headers(
)?;
let mut headers = HeaderMap::new();
insert_background_admission(&mut headers, background);
headers.insert(SIGNATURE_HEADER, header_value(&legacy_signature, SIGNATURE_HEADER)?);
headers.insert(TIMESTAMP_HEADER, header_value(&timestamp_header, TIMESTAMP_HEADER)?);
headers.insert(RPC_AUTH_VERSION_HEADER, HeaderValue::from_static(RPC_AUTH_VERSION_V2));
@@ -1573,7 +1501,6 @@ fn verify_tonic_replay_scope_signature(audience: &str, path: &str, headers: &Hea
if !verify_replay_scope_signature(
&secret,
ReplayScope {
background: rpc_background_admission(headers)?,
audience,
path,
timestamp,
@@ -1588,13 +1515,7 @@ fn verify_tonic_replay_scope_signature(audience: &str, path: &str, headers: &Hea
if boot_epoch != tonic_rpc_boot_epoch() {
return Err(std::io::Error::other("RPC boot epoch is stale"));
}
check_and_record_tonic_nonce(nonce, signed_at, path)?;
// A body-bound request also carries a usable v2 signature. Consume both
// nonces so stripping the replay-scope headers cannot authorize it again.
if content_sha256 != UNSIGNED_PAYLOAD {
verify_tonic_rpc_signature_with_strictness(audience, path, headers, true)?;
}
Ok(())
check_and_record_tonic_nonce(nonce, signed_at, path)
}
/// Verify gRPC authentication, preferring v2 without downgrade on malformed v2 metadata.
@@ -1704,9 +1625,6 @@ fn verify_tonic_rpc_signature_with_strictness(
strict: bool,
) -> std::io::Result<()> {
if !has_v2_auth_headers(headers) {
if rpc_background_admission(headers)? {
return Err(std::io::Error::other("RPC background admission requires v2 authentication"));
}
// RUSTFS_COMPAT_TODO(heal-rpc-auth-v2): accept old peers during rolling upgrades. Remove after the minimum
// supported RustFS peer version sends v2 authentication on every internode gRPC request.
if strict {
@@ -1778,7 +1696,6 @@ fn verify_tonic_rpc_signature_with_strictness(
if !verify_signature_v2(
&secret,
SignatureV2Scope {
background: rpc_background_admission(headers)?,
audience,
service,
rpc_method,
@@ -1819,7 +1736,7 @@ pub fn verify_rpc_signature(url: &str, method: &Method, headers: &HeaderMap) ->
// Verify signature with constant-time HMAC comparison.
let secret = get_shared_secret()?;
if !verify_signature(&secret, url, method, timestamp, signature, rpc_background_admission(headers)?) {
if !verify_signature(&secret, url, method, timestamp, signature) {
let rpc_path = redacted_rpc_path(url);
error!(
rpc_path = %rpc_path,
@@ -1911,98 +1828,6 @@ mod tests {
runtime_sources::ensure_test_rpc_secret();
}
#[tokio::test]
async fn background_admission_is_bound_in_http_and_both_tonic_signature_lanes() {
ensure_test_rpc_secret();
let secret = get_shared_secret().expect("RPC secret");
let path = "/node_service.NodeService/ReadAll";
for background in [false, true] {
let (http, tonic) = with_rpc_background_request(background, async {
(
gen_signature_headers("/rustfs/rpc/read?disk=a", &Method::GET).expect("HTTP signature"),
gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "ReadAll", None)
.expect("gRPC signature"),
)
})
.await;
assert_eq!(rpc_background_admission(&http).expect("HTTP class"), background);
assert_eq!(rpc_background_admission(&tonic).expect("gRPC class"), background);
verify_rpc_signature("/rustfs/rpc/read?disk=a", &Method::GET, &http).expect("HTTP authentication");
verify_tonic_rpc_signature_with_strictness("node-a:9000", path, &tonic, true).expect("v2 authentication");
let timestamp = tonic[TIMESTAMP_HEADER].to_str().expect("timestamp");
let content = tonic[RPC_CONTENT_SHA256_HEADER].to_str().expect("content digest");
assert_eq!(
verify_signature_v2(
&secret,
SignatureV2Scope {
background: false,
audience: "node-a:9000",
service: "node_service.NodeService",
rpc_method: "ReadAll",
timestamp,
nonce: tonic[RPC_NONCE_HEADER].to_str().expect("nonce"),
content_sha256: content,
},
tonic[RPC_SIGNATURE_V2_HEADER].to_str().expect("signature"),
),
!background,
"an old peer cannot authenticate a background request as foreground"
);
let mut replay = tonic.clone();
// The channel can run outside the caller's task-local scope.
replay.extend(
gen_tonic_replay_scope_headers_with_admission(
"node-a:9000",
path,
timestamp,
content,
tonic_rpc_boot_epoch(),
background,
)
.expect("v3 signature"),
);
for mutation in ["strip-or-add", "unknown", "duplicate"] {
let alter = |headers: &HeaderMap| {
let mut changed = headers.clone();
match mutation {
"strip-or-add" if background => {
changed.remove(RPC_BACKGROUND_ADMISSION_HEADER);
}
"strip-or-add" => insert_background_admission(&mut changed, true),
"unknown" => {
changed.insert(RPC_BACKGROUND_ADMISSION_HEADER, HeaderValue::from_static("v2"));
}
_ => {
insert_background_admission(&mut changed, true);
changed.append(RPC_BACKGROUND_ADMISSION_HEADER, HeaderValue::from_static("v1"));
}
}
changed
};
assert!(verify_rpc_signature("/rustfs/rpc/read?disk=a", &Method::GET, &alter(&http)).is_err());
assert!(verify_tonic_rpc_signature_with_strictness("node-a:9000", path, &alter(&tonic), true).is_err());
assert!(verify_tonic_replay_scope_signature("node-a:9000", path, &alter(&replay)).is_err());
}
verify_tonic_replay_scope_signature("node-a:9000", path, &replay).expect("v3 authentication");
assert!(
verify_tonic_replay_scope_signature("node-a:9000", path, &replay).is_err(),
"replay rejected"
);
if background {
let mut downgraded = HeaderMap::new();
for name in [SIGNATURE_HEADER, TIMESTAMP_HEADER, RPC_BACKGROUND_ADMISSION_HEADER] {
downgraded.insert(name, tonic[name].clone());
}
assert!(verify_tonic_rpc_signature_with_strictness("node-a:9000", path, &downgraded, false).is_err());
downgraded.remove(RPC_BACKGROUND_ADMISSION_HEADER);
assert!(verify_tonic_rpc_signature_with_strictness("node-a:9000", path, &downgraded, false).is_err());
}
}
}
#[test]
fn namespace_scanner_capability_proof_binds_challenge_and_server_epoch() {
let secret = "test-scanner-capability-secret";
@@ -2140,8 +1965,8 @@ mod tests {
let method = Method::GET;
let timestamp = 1640995200; // Fixed timestamp
let signature1 = generate_signature(secret, url, &method, timestamp, false);
let signature2 = generate_signature(secret, url, &method, timestamp, false);
let signature1 = generate_signature(secret, url, &method, timestamp);
let signature2 = generate_signature(secret, url, &method, timestamp);
assert_eq!(signature1, signature2, "Same inputs should produce same signature");
assert!(!signature1.is_empty(), "Signature should not be empty");
@@ -2154,10 +1979,10 @@ mod tests {
let method = Method::GET;
let timestamp = 1640995200;
let signature1 = generate_signature(secret, url, &method, timestamp, false);
let signature2 = generate_signature(secret, "http://different.com/api/test2", &method, timestamp, false);
let signature3 = generate_signature(secret, url, &Method::POST, timestamp, false);
let signature4 = generate_signature(secret, url, &method, timestamp + 1, false);
let signature1 = generate_signature(secret, url, &method, timestamp);
let signature2 = generate_signature(secret, "http://different.com/api/test2", &method, timestamp);
let signature3 = generate_signature(secret, url, &Method::POST, timestamp);
let signature4 = generate_signature(secret, url, &method, timestamp + 1);
assert_ne!(signature1, signature2, "Different URLs should produce different signatures");
assert_ne!(signature1, signature3, "Different methods should produce different signatures");
@@ -2263,14 +2088,14 @@ mod tests {
let url = "http://example.com/api/test";
let method = Method::GET;
let timestamp = 1640995200;
let signature = generate_signature(secret, url, &method, timestamp, false);
let signature = generate_signature(secret, url, &method, timestamp);
let mut tampered = base64_simd::STANDARD.decode_to_vec(&signature).unwrap();
tampered[0] ^= 1;
let tampered_signature = base64_simd::STANDARD.encode_to_string(tampered);
assert!(verify_signature(secret, url, &method, timestamp, &signature, false));
assert!(!verify_signature(secret, url, &method, timestamp, &tampered_signature, false));
assert!(!verify_signature(secret, url, &method, timestamp, "invalid-signature", false));
assert!(verify_signature(secret, url, &method, timestamp, &signature));
assert!(!verify_signature(secret, url, &method, timestamp, &tampered_signature));
assert!(!verify_signature(secret, url, &method, timestamp, "invalid-signature"));
}
#[test]
@@ -2287,11 +2112,11 @@ mod tests {
);
let method = Method::GET;
let timestamp = 1_640_995_200;
let signature = generate_signature(secret, signed_url, &method, timestamp, false);
let signature = generate_signature(secret, signed_url, &method, timestamp);
assert!(verify_signature(secret, signed_url, &method, timestamp, &signature, false));
assert!(!verify_signature(secret, downgraded_url, &method, timestamp, &signature, false));
assert!(!verify_signature(secret, tampered_body_digest, &method, timestamp, &signature, false));
assert!(verify_signature(secret, signed_url, &method, timestamp, &signature));
assert!(!verify_signature(secret, downgraded_url, &method, timestamp, &signature));
assert!(!verify_signature(secret, tampered_body_digest, &method, timestamp, &signature));
}
#[test]
@@ -2301,7 +2126,7 @@ mod tests {
let method = Method::GET;
let timestamp = OffsetDateTime::now_utc().unix_timestamp();
let secret = get_shared_secret().expect("test RPC secret should resolve");
let expected_signature = generate_signature(&secret, url, &method, timestamp, false);
let expected_signature = generate_signature(&secret, url, &method, timestamp);
let invalid_signature = "invalid-signature";
let logs = CapturedLogs::default();
let subscriber = tracing_subscriber::fmt()
@@ -2339,7 +2164,7 @@ mod tests {
// Set expired timestamp (older than SIGNATURE_VALID_DURATION)
let expired_timestamp = OffsetDateTime::now_utc().unix_timestamp() - SIGNATURE_VALID_DURATION - 10;
let secret = get_shared_secret().expect("test RPC secret should resolve");
let signature = generate_signature(&secret, url, &method, expired_timestamp, false);
let signature = generate_signature(&secret, url, &method, expired_timestamp);
headers.insert(SIGNATURE_HEADER, HeaderValue::from_str(&signature).unwrap());
headers.insert(TIMESTAMP_HEADER, HeaderValue::from_str(&expired_timestamp.to_string()).unwrap());
@@ -2361,7 +2186,7 @@ mod tests {
let future_timestamp = OffsetDateTime::now_utc().unix_timestamp() + SIGNATURE_VALID_DURATION + 10;
let secret = get_shared_secret().expect("test RPC secret should resolve");
let signature = generate_signature(&secret, url, &method, future_timestamp, false);
let signature = generate_signature(&secret, url, &method, future_timestamp);
headers.insert(SIGNATURE_HEADER, HeaderValue::from_str(&signature).unwrap());
headers.insert(TIMESTAMP_HEADER, HeaderValue::from_str(&future_timestamp.to_string()).unwrap());
@@ -2475,7 +2300,7 @@ mod tests {
// Test timestamp just within valid duration
let valid_timestamp = current_time - SIGNATURE_VALID_DURATION + 1;
let signature = generate_signature(&secret, url, &method, valid_timestamp, false);
let signature = generate_signature(&secret, url, &method, valid_timestamp);
headers.insert(SIGNATURE_HEADER, HeaderValue::from_str(&signature).unwrap());
headers.insert(TIMESTAMP_HEADER, HeaderValue::from_str(&valid_timestamp.to_string()).unwrap());
@@ -2486,7 +2311,7 @@ mod tests {
// Test timestamp just outside valid duration
let mut headers = HeaderMap::new();
let invalid_timestamp = current_time - SIGNATURE_VALID_DURATION - 15;
let signature = generate_signature(&secret, url, &method, invalid_timestamp, false);
let signature = generate_signature(&secret, url, &method, invalid_timestamp);
headers.insert(SIGNATURE_HEADER, HeaderValue::from_str(&signature).unwrap());
headers.insert(TIMESTAMP_HEADER, HeaderValue::from_str(&invalid_timestamp.to_string()).unwrap());
@@ -2585,43 +2410,6 @@ mod tests {
assert_eq!(path_error.to_string(), "Invalid RPC replay scope signature");
}
#[test]
fn replay_scope_consumes_the_body_bound_v2_nonce() {
ensure_test_rpc_secret();
let audience = "node-a:9000";
let path = "/node_service.NodeService/RecoverDeleteTransaction";
let digest = "a".repeat(64);
for v3_first in [true, false] {
let mut headers =
gen_tonic_signature_headers(audience, "node_service.NodeService", "RecoverDeleteTransaction", Some(&digest))
.expect("body-bound v2 headers should build");
let timestamp = headers[TIMESTAMP_HEADER].to_str().expect("timestamp").to_string();
headers.extend(
gen_tonic_replay_scope_headers(audience, path, &timestamp, &digest, tonic_rpc_boot_epoch())
.expect("replay-scope headers should build"),
);
let mut downgraded = headers.clone();
for header in [
RPC_REPLAY_SCOPE_VERSION_HEADER,
RPC_REPLAY_SCOPE_SIGNATURE_HEADER,
RPC_REPLAY_SCOPE_NONCE_HEADER,
RPC_BOOT_EPOCH_HEADER,
] {
downgraded.remove(header);
}
let (first, replay) = if v3_first {
(&headers, &downgraded)
} else {
(&downgraded, &headers)
};
verify_tonic_rpc_signature_with_policy(audience, path, first, true, false, false)
.expect("the first delivery must be accepted");
let error = verify_tonic_rpc_signature_with_policy(audience, path, replay, true, false, false)
.expect_err("a mutation cannot replay through the other signature generation");
assert_eq!(error.to_string(), "RPC request replay detected");
}
}
#[test]
fn replay_scope_rejects_partial_metadata_and_stale_epoch_without_fallback() {
ensure_test_rpc_secret();
@@ -33,9 +33,7 @@ use rustfs_config::{
DEFAULT_INTERNODE_DATA_TRANSPORT, ENV_RUSTFS_INTERNODE_DATA_TRANSPORT, INTERNODE_DATA_TRANSPORT_TCP,
KNOWN_INTERNODE_DATA_TRANSPORT_BACKENDS,
};
use rustfs_rio::{
ChunkReaderBox, HttpChunkReader, HttpReader, HttpWriter, HttpWriterMemoryReservation, HttpWriterMemoryReserver,
};
use rustfs_rio::{ChunkReaderBox, HttpChunkReader, HttpReader, HttpWriter};
use sha2::{Digest, Sha256};
use std::collections::HashMap;
use std::future::Future;
@@ -68,19 +66,6 @@ const PUT_FILE_CAPABILITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
const CONTENT_TYPE_JSON: &str = "application/json";
const CONTENT_TYPE_MSGPACK: &str = "application/msgpack";
fn http_writer_memory_reserver() -> Option<HttpWriterMemoryReserver> {
crate::disk::current_resource_unit()
.and_then(|resources| resources.memory_budget())
.map(|budget| {
// Payloads can outlive the operation; never retain its node or disk units.
Arc::new(move |capacity| {
budget
.reserve(capacity)
.map(|reservation| Box::new(reservation) as HttpWriterMemoryReservation)
}) as HttpWriterMemoryReserver
})
}
fn unsupported_transport_message(transport: &str) -> String {
format!(
"invalid {ENV_RUSTFS_INTERNODE_DATA_TRANSPORT}={transport:?}; supported values: {}",
@@ -361,17 +346,13 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
}
async fn open_write(&self, request: WriteStreamRequest) -> Result<FileWriter> {
let memory_reserver = http_writer_memory_reserver();
let server_epoch = self.put_file_auth_capability(&request.endpoint).await?;
let auth_scope = server_epoch.map(|server_epoch| (Uuid::new_v4(), server_epoch));
let url = build_put_file_stream_url(&request, auth_scope);
let endpoint = request.endpoint;
let mut headers = json_headers();
build_auth_headers(&url, &Method::PUT, &mut headers)?;
let writer = match memory_reserver {
Some(reserve) => HttpWriter::new_with_memory_reserver(url.clone(), Method::PUT, headers, reserve).await?,
None => HttpWriter::new(url.clone(), Method::PUT, headers).await?,
};
let writer = HttpWriter::new(url.clone(), Method::PUT, headers).await?;
match auth_scope {
Some((nonce, server_epoch)) => Ok(Box::new(PutFileAuthWriter::new(writer, url, nonce, endpoint, server_epoch))),
None => Ok(Box::new(writer)),
@@ -967,193 +948,6 @@ mod tests {
use tokio::io::AsyncWriteExt;
use tokio::sync::{Barrier, Notify};
fn memory_test_write_request(authenticated: bool) -> WriteStreamRequest {
crate::runtime::sources::ensure_test_rpc_secret();
let endpoint = format!("http://writer-memory-{}.invalid", Uuid::new_v4());
put_file_capability_cache_entry(&endpoint).write().cached = Some(if authenticated {
PutFileCapabilityState::V1 {
server_epoch: Uuid::new_v4(),
revalidate_after: Instant::now() + PUT_FILE_V1_CAPABILITY_TTL,
}
} else {
PutFileCapabilityState::LegacyUntil(Instant::now() + PUT_FILE_LEGACY_CAPABILITY_TTL)
});
WriteStreamRequest {
endpoint,
disk: "disk-a".to_owned(),
volume: "bucket".to_owned(),
path: "part.1".to_owned(),
append: false,
size: 4,
}
}
#[tokio::test]
async fn http_write_memory_reserver_outlives_only_the_memory_scope() {
use crate::disk::{BackgroundMemoryBudget, test_resource_units, with_optional_resource_unit};
assert!(http_writer_memory_reserver().is_none(), "foreground must use the existing writer");
let memory = BackgroundMemoryBudget::new();
memory.set_sampled_limit(2 * 1024 * 1024, Duration::from_secs(30));
let (resources, gates) = test_resource_units(2);
let request = memory_test_write_request(true);
let endpoint = request.endpoint.clone();
let writer = with_optional_resource_unit(
Some(resources.with_memory_budget(memory.clone())),
TcpHttpInternodeDataTransport.open_write(request),
)
.await;
PUT_FILE_CAPABILITY_CACHE.write().remove(&endpoint);
let mut writer = writer.expect("admitted transport writer");
assert!(
gates.iter().all(|gate| gate.snapshot().active == 0),
"the callback must not retain disk units"
);
assert_eq!(memory.snapshot().used, 1024 * 1024);
writer.write_all(b"data").await.expect("buffered bytes");
writer.flush().await.expect("transfer charged allocation to queue");
memory.set_sampled_limit(2 * 1024 * 1024, Duration::ZERO);
writer
.write_all(&vec![0x5a; 1024 * 1024 - 4])
.await
.expect("reuse the already charged tail after observation expiry");
assert_eq!(memory.snapshot().used, 1024 * 1024, "reuse must not allocate separate storage");
let error = writer
.write(b"rejected")
.await
.expect_err("expired observation must reject new storage");
assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
assert!(
crate::disk::background_memory::is_memory_pressure(&error),
"preserve the typed memory-pressure marker"
);
assert_eq!(memory.snapshot().used, 1024 * 1024, "queued storage remains charged after expiry");
drop(writer);
tokio::task::yield_now().await;
assert_eq!(memory.snapshot().used, 0);
}
#[tokio::test]
async fn http_write_memory_denial_preserves_the_budget_error() {
use crate::disk::{BackgroundMemoryBudget, test_resource_units, with_optional_resource_unit};
for (authenticated, sample) in [false, true].into_iter().flat_map(|authenticated| {
[
None,
Some((1024 * 1024 - 1, Duration::from_secs(30))),
Some((1024 * 1024, Duration::ZERO)),
]
.into_iter()
.map(move |sample| (authenticated, sample))
}) {
let memory = BackgroundMemoryBudget::new();
if let Some((limit, valid_for)) = sample {
memory.set_sampled_limit(limit, valid_for);
}
let (resources, _) = test_resource_units(1);
let request = memory_test_write_request(authenticated);
let endpoint = request.endpoint.clone();
let result = with_optional_resource_unit(
Some(resources.with_memory_budget(memory.clone())),
TcpHttpInternodeDataTransport.open_write(request),
)
.await;
PUT_FILE_CAPABILITY_CACHE.write().remove(&endpoint);
let error = match result {
Err(Error::Io(error)) => error,
Err(error) => panic!("unexpected transport error: {error}"),
Ok(_) => panic!("unavailable memory must reject construction"),
};
assert!(crate::disk::background_memory::is_memory_pressure(&error));
assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
assert_eq!(memory.snapshot().used, 0);
}
}
#[tokio::test]
async fn admitted_remote_reads_bind_receiver_admission_to_the_http_signature() {
use crate::disk::with_background_resource_unit;
use rustfs_concurrency::adaptive::AdaptiveAdmission;
use std::num::NonZeroU16;
crate::runtime::sources::ensure_test_rpc_secret();
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
.await
.expect("remote admission listener");
let address = listener.local_addr().expect("remote admission address");
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.expect("remote read connection");
let mut bytes = Vec::new();
while !bytes.windows(4).any(|window| window == b"\r\n\r\n") {
let mut chunk = [0; 1024];
let read = socket.read(&mut chunk).await.expect("remote read headers");
assert!(read > 0 && bytes.len() < 16 * 1024, "bounded complete request headers");
bytes.extend_from_slice(&chunk[..read]);
}
let request = std::str::from_utf8(&bytes).expect("HTTP request");
let mut lines = request.split("\r\n");
let target = lines
.next()
.expect("request line")
.split_whitespace()
.nth(1)
.expect("request target")
.to_owned();
let mut headers = HeaderMap::new();
for line in lines.take_while(|line| !line.is_empty()) {
let (name, value) = line.split_once(':').expect("header field");
headers.append(
http::header::HeaderName::from_bytes(name.as_bytes()).expect("header name"),
HeaderValue::from_str(value.trim()).expect("header value"),
);
}
socket
.write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 5\r\nconnection: close\r\n\r\nhello")
.await
.expect("remote read response");
(target, headers)
});
let gate = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
let resource = gate.try_acquire().expect("sender device admission");
let mut reader = tokio::time::timeout(
Duration::from_secs(5),
with_background_resource_unit(
resource,
TcpHttpInternodeDataTransport.open_read(ReadStreamRequest {
endpoint: format!("http://{address}"),
disk: "disk-a".to_owned(),
volume: "bucket".to_owned(),
path: "part.1".to_owned(),
offset: 0,
length: 5,
stall_timeout: Some(Duration::from_secs(5)),
}),
),
)
.await
.expect("bounded remote request")
.expect("remote reader");
let mut body = Vec::new();
reader.read_to_end(&mut body).await.expect("complete remote body");
assert_eq!(body, b"hello");
let (target, headers) = server.await.expect("remote request capture");
assert_eq!(gate.snapshot().active, 0, "remote body does not retain the sender's device");
assert_eq!(
headers
.get("x-rustfs-rpc-background-admission")
.and_then(|value| value.to_str().ok()),
Some("v1"),
"resource-controlled remote work must request receiver-side admission"
);
crate::cluster::rpc::verify_rpc_signature(&target, &Method::GET, &headers).expect("bound request signature");
let mut stripped = headers;
stripped.remove("x-rustfs-rpc-background-admission");
assert!(
crate::cluster::rpc::verify_rpc_signature(&target, &Method::GET, &stripped).is_err(),
"removing background admission must invalidate the signature"
);
}
async fn wait_for_capability_flight_waiters(entry: &PutFileCapabilityCacheEntry, waiters: usize) {
tokio::time::timeout(Duration::from_secs(5), async {
loop {
+10 -13
View File
@@ -13,8 +13,6 @@
// limitations under the License.
pub(crate) mod background_monitor;
pub(crate) mod background_pipe;
pub(crate) mod background_reader_stream;
pub(crate) mod client;
pub(crate) mod context_propagation;
pub(crate) mod http_auth;
@@ -35,17 +33,16 @@ pub use client::{
// Re-exported through `api::rpc`; not every item is consumed inside this crate.
#[allow(unused_imports)]
pub use http_auth::{
RPC_BACKGROUND_ADMISSION_HEADER, TONIC_RPC_PREFIX, build_auth_headers, build_put_file_auth_trailer,
check_and_record_signed_rpc_nonce, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
normalize_tonic_rpc_audience, rpc_background_admission, set_tonic_canonical_body_digest, set_tonic_mutation_body_digest,
set_tonic_rolling_canonical_body_digest, set_tonic_rolling_mutation_body_digest, sign_ns_scanner_capability,
sign_ns_scanner_capability_with_checkpoint_coverage, sign_ns_scanner_capability_with_tier_registry_generation,
sign_put_file_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
tonic_rpc_auth_failure_reason, verify_ns_scanner_capability, verify_ns_scanner_capability_with_checkpoint_coverage,
verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer, verify_put_file_capability,
verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned, verify_tonic_rpc_response_proof,
verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
TONIC_RPC_PREFIX, build_auth_headers, build_put_file_auth_trailer, check_and_record_signed_rpc_nonce, gen_signature_headers,
gen_tonic_replay_scope_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
set_tonic_mutation_body_digest, set_tonic_rolling_canonical_body_digest, set_tonic_rolling_mutation_body_digest,
sign_ns_scanner_capability, sign_ns_scanner_capability_with_checkpoint_coverage,
sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, sign_tonic_rpc_response_proof,
tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_ns_scanner_capability,
verify_ns_scanner_capability_with_checkpoint_coverage, verify_ns_scanner_capability_with_tier_registry_generation,
verify_put_file_auth_trailer, verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response,
verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned,
verify_tonic_rpc_response_proof, verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
};
#[cfg(test)]
pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;
@@ -2014,21 +2014,9 @@ impl PeerRestClient {
}
result
})
.await;
self.settle_bucket_metadata_reload(bucket, result).await
}
async fn settle_bucket_metadata_reload(
&self,
bucket: &str,
result: std::result::Result<Result<()>, tokio::time::error::Elapsed>,
) -> Result<()> {
match result {
Ok(result) => self.finalize_result(result).await,
// A local operation deadline does not establish that the peer is
// unreachable. In particular, it must not gate IAM revocations.
Err(_) => Err(Error::other(format!("load_bucket_metadata({bucket}) timed out"))),
}
.await
.unwrap_or_else(|_| Err(Error::other(format!("load_bucket_metadata({bucket}) timed out"))));
self.finalize_result(result).await
}
async fn load_bucket_metadata_once(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
@@ -4765,31 +4753,6 @@ mod tests {
assert!(client.offline.load(Ordering::Acquire));
}
#[tokio::test]
async fn bucket_metadata_deadline_does_not_gate_healthy_peer_notifications() {
let client = test_peer_client();
let elapsed = tokio::time::timeout(std::time::Duration::ZERO, std::future::pending::<Result<()>>()).await;
let error = client
.settle_bucket_metadata_reload("slow-policy", elapsed)
.await
.expect_err("local budget expires");
assert!(error.to_string().contains("timed out"));
assert!(
!client.offline.load(Ordering::Acquire),
"local work must not suppress subsequent IAM RPCs"
);
client
.settle_bucket_metadata_reload("healthy", Ok(Ok(())))
.await
.expect("peer reply");
let error = client
.settle_bucket_metadata_reload("unreachable", Ok(Err(Error::from(tonic::Status::unavailable("connection refused")))))
.await
.expect_err("real transport failure");
assert!(error.to_string().contains("connection refused"));
assert!(client.offline.load(Ordering::Acquire), "transport failures must still enter recovery");
}
#[tokio::test]
async fn peer_rest_client_finalize_result_keeps_online_for_business_errors() {
let client = test_peer_client();
+7 -538
View File
@@ -17,7 +17,6 @@ use crate::cluster::rpc::client::{
node_service_delete_recovery_client, node_service_time_out_client, node_service_time_out_client_for_class,
node_service_time_out_client_no_auth,
};
use crate::cluster::rpc::http_auth::{current_rpc_background_request, with_rpc_background_request};
use crate::cluster::rpc::internode_data_transport::{
InternodeDataTransport, NsScannerCapabilityRequest, NsScannerStreamRequest, ReadStreamRequest, WalkDirStreamRequest,
WriteStreamRequest,
@@ -571,7 +570,6 @@ struct RetryingRemoteReader {
reader: Option<FileReader>,
transport: Arc<dyn InternodeDataTransport>,
request: ReadStreamRequest,
background: bool,
emitted: usize,
retried: bool,
initial_read_timeout: Option<Duration>,
@@ -592,7 +590,6 @@ impl RetryingRemoteReader {
reader: Some(reader),
transport,
request,
background: current_rpc_background_request(),
emitted: 0,
retried: false,
initial_read_timeout,
@@ -610,10 +607,9 @@ impl RetryingRemoteReader {
let request = resumed_read_request(&self.request, self.emitted)?;
let recovery_timeout = self.recovery_timeout;
let transport = Arc::clone(&self.transport);
self.resume = Some(AbortOnDropTask::new(tokio::spawn(with_rpc_background_request(
self.background,
async move { with_remote_read_recovery_timeout(recovery_timeout, transport.open_read_fresh(request)).await },
))));
self.resume = Some(AbortOnDropTask::new(tokio::spawn(async move {
with_remote_read_recovery_timeout(recovery_timeout, transport.open_read_fresh(request)).await
})));
Ok(())
}
@@ -721,7 +717,6 @@ struct RetryingRemoteChunkReader {
reader: Option<rustfs_rio::ChunkReaderBox>,
transport: Arc<dyn InternodeDataTransport>,
request: ReadStreamRequest,
background: bool,
emitted: usize,
retried: bool,
initial_read_timeout: Option<Duration>,
@@ -742,7 +737,6 @@ impl RetryingRemoteChunkReader {
reader: Some(reader),
transport,
request,
background: current_rpc_background_request(),
emitted: 0,
retried: false,
initial_read_timeout,
@@ -760,10 +754,9 @@ impl RetryingRemoteChunkReader {
let request = resumed_read_request(&self.request, self.emitted)?;
let recovery_timeout = self.recovery_timeout;
let transport = Arc::clone(&self.transport);
self.resume = Some(AbortOnDropTask::new(tokio::spawn(with_rpc_background_request(
self.background,
async move { with_remote_read_recovery_timeout(recovery_timeout, transport.open_read_chunks_fresh(request)).await },
))));
self.resume = Some(AbortOnDropTask::new(tokio::spawn(async move {
with_remote_read_recovery_timeout(recovery_timeout, transport.open_read_chunks_fresh(request)).await
})));
Ok(())
}
@@ -1082,10 +1075,7 @@ impl RemoteDisk {
if payload.len() > operation.max_wire_payload_bytes() {
return Err(Error::other("delete recovery operation exceeds payload limit"));
}
let settle_timeout = if matches!(
&operation,
DeleteRecoveryRpcOperation::Settle { .. } | DeleteRecoveryRpcOperation::SettleBounded(_)
) {
let settle_timeout = if matches!(&operation, DeleteRecoveryRpcOperation::Settle { .. }) {
Some(crate::disk::get_delete_recovery_timeout()?)
} else {
None
@@ -1640,7 +1630,6 @@ impl RemoteDisk {
let initial_probe_ok = Self::perform_connectivity_check(&addr).await.is_ok();
if initial_probe_ok {
health.record_operation_success(&endpoint, "connectivity_probe_success");
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
} else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
warn!(
event = EVENT_REMOTE_DISK_HEALTH,
@@ -1712,7 +1701,6 @@ impl RemoteDisk {
// Perform basic connectivity check
if Self::perform_connectivity_check(&addr).await.is_ok() {
health.record_operation_success(&endpoint, "connectivity_probe_success");
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
} else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
warn!(
event = EVENT_REMOTE_DISK_HEALTH,
@@ -1783,7 +1771,6 @@ impl RemoteDisk {
}
_ = interval.tick() => {
if Self::perform_recovery_probe(&addr, &endpoint).await.is_ok() {
rustfs_io_metrics::internode_metrics::record_peer_reachable(&addr);
let became_online = health.mark_recovery_success(&endpoint, "disk_info_probe_success");
debug!(
event = EVENT_REMOTE_DISK_HEALTH,
@@ -2015,7 +2002,6 @@ impl RemoteDisk {
let operation_result = operation().await;
if operation_result.is_ok() {
self.health.record_operation_success(&self.endpoint, "operation_success");
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
}
self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
.await;
@@ -2030,7 +2016,6 @@ impl RemoteDisk {
// Log success; the waiting guard balances every exit path.
if operation_result.is_ok() {
self.health.record_operation_success(&self.endpoint, "operation_success");
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
}
self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
.await;
@@ -2719,54 +2704,6 @@ impl RemoteDisk {
#[async_trait::async_trait]
impl DiskAPI for RemoteDisk {
async fn list_multipart_index_roots(
&self,
bucket: &str,
marker: Option<&str>,
) -> Result<crate::disk::multipart_index::MultipartIndexPage> {
use crate::disk::multipart_index::{MULTIPART_INDEX_RPC_MAX_BYTES, MultipartIndexRpcReply, MultipartIndexRpcRequest};
let expected_disk = self
.id
.lock()
.await
.filter(|id| !id.is_nil())
.ok_or(DiskError::DiskNotFound)?;
let query = MultipartIndexRpcRequest::new(bucket, marker);
let endpoint = self.endpoint.to_string();
let canonical = query.canonical_request(&endpoint)?;
let opts = serde_json::to_string(&query)?;
self.execute_with_timeout(
|| async {
let mut client = self.get_client().await?;
let mut request = Request::new(DiskInfoRequest {
disk: endpoint.clone(),
opts: opts.clone(),
});
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
let response = client.disk_info(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
if response.disk_info.len() > MULTIPART_INDEX_RPC_MAX_BYTES {
return Err(DiskError::FileCorrupt);
}
// An old server ignores the additive query and returns ordinary DiskInfo.
// It cannot satisfy the dedicated reply schema or the fresh challenge.
let reply: MultipartIndexRpcReply =
serde_json::from_str(&response.disk_info).map_err(|_| DiskError::MethodNotAllowed)?;
reply.validate(&query, expected_disk)?;
crate::cluster::rpc::verify_tonic_rpc_response_proof(
&query.canonical_response(&endpoint, &reply.page)?,
&reply.proof,
)
.map_err(Error::other)?;
Ok(reply.page)
},
get_drive_disk_info_timeout(),
)
.await
}
async fn supports_guarded_delete_versions(&self) -> Result<()> {
let response = self.guarded_delete_versions_rpc("", Vec::new(), true).await?;
if !response.errors.is_empty() || !response.item_errors.is_empty() {
@@ -4350,38 +4287,6 @@ impl DiskAPI for RemoteDisk {
}
}
async fn discover_delete_transactions_bounded(
&self,
volume: &str,
object: &str,
max_entries: u32,
) -> Result<crate::disk::BoundedDeleteRecoveryDiscovery> {
// Older peers reject this variant. Do not retry through the unbounded protocol;
// they are reported as unsupported participants (rolling-upgrade contract).
match self
.recover_delete_transaction(
volume,
DeleteRecoveryRpcOperation::DiscoverBounded {
object: object.to_owned(),
max_entries,
},
)
.await
{
Ok(DeleteRecoveryRpcReply::DiscoveredBounded(page)) => Ok(page),
Ok(_) => Err(Error::FileCorrupt),
Err(error)
if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
|| matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
.is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
{
// No paginated fallback through the unbounded protocol; the peer
// is an unsupported participant until it is upgraded.
Err(Error::UnsupportedDisk)
}
Err(error) => Err(error),
}
}
async fn discover_delete_transactions(
&self,
volume: &str,
@@ -4417,62 +4322,6 @@ impl DiskAPI for RemoteDisk {
_ => Err(Error::other("unexpected delete discovery reply")),
}
}
async fn settle_delete_transaction_bounded(
&self,
volume: &str,
request: crate::disk::BoundedDeleteRecoverySettlement,
_namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> Result<crate::disk::BoundedDeleteRecoverySettlementReply> {
match self
.recover_delete_transaction(volume, DeleteRecoveryRpcOperation::SettleBounded(request))
.await
{
Ok(DeleteRecoveryRpcReply::SettledBounded(reply)) => Ok(reply),
Ok(_) => Err(Error::FileCorrupt),
Err(error)
if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
|| matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
.is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
{
// No paginated fallback through the unbounded protocol; the peer
// is an unsupported participant until it is upgraded.
Err(Error::UnsupportedDisk)
}
Err(error) => Err(error),
}
}
async fn inspect_delete_transaction_bounded(
&self,
volume: &str,
object: &str,
transaction: Uuid,
max_entries: u32,
) -> Result<crate::disk::BoundedDeleteRecoveryInspection> {
match self
.recover_delete_transaction(
volume,
DeleteRecoveryRpcOperation::InspectBounded {
object: object.to_owned(),
transaction,
max_entries,
},
)
.await
{
Ok(DeleteRecoveryRpcReply::InspectedBounded(page)) => Ok(page),
Ok(_) => Err(Error::FileCorrupt),
Err(error)
if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
|| matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
.is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
{
// No paginated fallback through the unbounded protocol; the peer
// is an unsupported participant until it is upgraded.
Err(Error::UnsupportedDisk)
}
Err(error) => Err(error),
}
}
async fn inspect_delete_transaction(
&self,
volume: &str,
@@ -4993,113 +4842,6 @@ mod tests {
}
}
#[tokio::test]
async fn bounded_delete_discovery_lost_remote_reply_leaves_no_cursor() {
runtime_sources::ensure_test_rpc_secret();
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
.await
.expect("test peer must bind");
let directory = tempfile::tempdir().expect("temporary disk");
let endpoint = Endpoint::try_from(directory.path().to_str().expect("disk path")).expect("endpoint");
let disk = Arc::new(
crate::disk::local::LocalDisk::new(&endpoint, false)
.await
.expect("local disk"),
);
disk.make_volume("bucket").await.expect("bucket");
let prefix = directory.path().join("bucket/prefix");
std::fs::create_dir(&prefix).expect("prefix");
for index in 0..64 {
std::fs::create_dir(prefix.join(format!("child-{index}"))).expect("legitimate child");
}
let fixture = Arc::new(TestBoundedDiscovery {
disk: Arc::clone(&disk),
response_gate: TestFormatReadGate::default(),
});
*peer.peer.bounded_discovery.lock().expect("bounded fixture") = Some(Arc::clone(&fixture));
let remote = remote_disk_for_addr(&peer.addr).await;
let task = tokio::spawn(async move { remote.discover_delete_transactions_bounded("bucket", "prefix", 8).await });
tokio::time::timeout(Duration::from_secs(5), fixture.response_gate.entered.notified())
.await
.expect("disk completed bounded scan");
task.abort();
assert!(task.await.expect_err("caller cancelled").is_cancelled());
assert_eq!(disk.public_delete_discovery_sessions_for_test(), 0);
fixture.response_gate.release.cancel();
peer.stop().await;
assert_eq!(disk.public_delete_discovery_sessions_for_test(), 0);
assert!(prefix.join("child-63").is_dir());
}
#[tokio::test]
async fn multipart_index_old_disk_info_handler_is_not_a_ready_empty_index() {
runtime_sources::ensure_test_rpc_secret();
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
.await
.expect("old peer must bind");
let remote = remote_disk_for_addr(&peer.addr).await;
remote
.set_disk_id(Some(Uuid::new_v4()))
.await
.expect("expected disk identity");
assert!(matches!(
remote.list_multipart_index_roots("bucket", None).await,
Err(DiskError::MethodNotAllowed)
));
assert_eq!(
peer.peer.disk_info_calls.load(Ordering::Acquire),
1,
"exercise the actual legacy JSON handler"
);
peer.stop().await;
}
#[tokio::test]
async fn bounded_delete_discovery_old_peer_rejects_without_paginated_fallback() {
runtime_sources::ensure_test_rpc_secret();
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
.await
.expect("old peer must bind");
let remote = remote_disk_for_addr(&peer.addr).await;
assert_eq!(
remote
.discover_delete_transactions_bounded("bucket", "prefix", 4096)
.await
.expect_err("old peer cannot prove a bounded inventory"),
DiskError::UnsupportedDisk
);
let transaction = Uuid::new_v4();
assert_eq!(
remote
.inspect_delete_transaction_bounded("bucket", "prefix", transaction, 65536)
.await
.expect_err("old peer cannot inspect with a budget"),
DiskError::UnsupportedDisk
);
assert_eq!(
remote
.settle_delete_transaction_bounded(
"bucket",
crate::disk::BoundedDeleteRecoverySettlement {
object: "prefix".to_owned(),
transaction,
expected_fingerprint: [0; 32],
action: DeleteRecoveryAction::PublishCommit,
max_entries: 65536,
},
None
)
.await
.expect_err("old peer cannot settle with a budget"),
DiskError::UnsupportedDisk
);
assert_eq!(
*peer.peer.unsupported_methods.lock().expect("observed methods"),
vec!["/node_service.NodeService/RecoverDeleteTransaction"; 3]
);
peer.stop().await;
}
#[tokio::test]
async fn multipart_source_recovery_old_peer_rejects_without_generic_fallback() {
runtime_sources::ensure_test_rpc_secret();
@@ -5991,18 +5733,10 @@ mod tests {
object_read_all_disks: Arc<StdMutex<Vec<String>>>,
format_data: Bytes,
read_all_data: Bytes,
fail_format_reads: Arc<AtomicBool>,
next_format_read_gate: Arc<StdMutex<Option<Arc<TestFormatReadGate>>>>,
bounded_discovery: Arc<StdMutex<Option<Arc<TestBoundedDiscovery>>>>,
unsupported_methods: Arc<StdMutex<Vec<String>>>,
}
#[derive(Debug)]
struct TestBoundedDiscovery {
disk: Arc<crate::disk::local::LocalDisk>,
response_gate: TestFormatReadGate,
}
#[derive(Debug, Default)]
struct TestFormatReadGate {
entered: tokio::sync::Notify,
@@ -6018,9 +5752,7 @@ mod tests {
object_read_all_disks: Arc::default(),
format_data,
read_all_data,
fail_format_reads: Arc::default(),
next_format_read_gate: Arc::default(),
bounded_discovery: Arc::default(),
unsupported_methods: Arc::default(),
}
}
@@ -6120,56 +5852,6 @@ mod tests {
Ok(grpc.unary(method, request).await)
})
}
"/node_service.NodeService/RecoverDeleteTransaction"
if self.peer.bounded_discovery.lock().expect("bounded fixture").is_some() =>
{
#[derive(Clone)]
struct BoundedSvc(Arc<TestBoundedDiscovery>);
impl tonic::server::UnaryService<RecoverDeleteTransactionRequest> for BoundedSvc {
type Response = rustfs_protos::proto_gen::node_service::RecoverDeleteTransactionResponse;
type Future = Pin<Box<dyn Future<Output = std::result::Result<Response<Self::Response>, Status>> + Send>>;
fn call(&mut self, request: Request<RecoverDeleteTransactionRequest>) -> Self::Future {
let fixture = Arc::clone(&self.0);
Box::pin(async move {
let request = request.into_inner();
let operation: DeleteRecoveryRpcOperation =
rustfs_protos::decode_delete_recovery_payload(&request.operation)
.map_err(|_| Status::invalid_argument("invalid bounded request"))?;
let DeleteRecoveryRpcOperation::DiscoverBounded { object, max_entries } = operation else {
return Err(Status::invalid_argument("unexpected fallback"));
};
let page = fixture
.disk
.discover_delete_transactions_bounded(&request.volume, &object, max_entries)
.await
.map_err(|_| Status::internal("bounded discovery failed"))?;
// Lose the response only after the real disk operation has completed.
fixture.response_gate.entered.notify_one();
fixture.response_gate.release.cancelled().await;
let result = rmp_serde::to_vec_named(&DeleteRecoveryRpcReply::DiscoveredBounded(page))
.map_err(|_| Status::internal("encode bounded reply"))?;
Ok(Response::new(rustfs_protos::proto_gen::node_service::RecoverDeleteTransactionResponse {
success: true,
protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION,
result: result.into(),
error: None,
}))
})
}
}
let fixture = self
.peer
.bounded_discovery
.lock()
.expect("bounded fixture")
.clone()
.expect("configured fixture");
Box::pin(async move {
let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec);
Ok(grpc.unary(BoundedSvc(fixture), request).await)
})
}
"/node_service.NodeService/ReadAll" => {
#[derive(Clone)]
struct ReadAllSvc(AuthenticatedReadPeer);
@@ -6193,9 +5875,6 @@ mod tests {
gate.entered.notify_one();
gate.release.cancelled().await;
}
if peer.fail_format_reads.load(Ordering::Acquire) {
return Err(Status::unavailable("format read temporarily unavailable"));
}
peer.format_data.clone()
} else {
peer.object_read_all_disks
@@ -8090,98 +7769,6 @@ mod tests {
.await;
}
#[tokio::test]
async fn remote_read_retries_preserve_signed_admission_after_sender_scope_ends() {
use crate::cluster::rpc::{rpc_background_admission, verify_rpc_signature};
use crate::disk::with_background_resource_unit;
use rustfs_concurrency::adaptive::AdaptiveAdmission;
use std::num::NonZeroU16;
crate::runtime::sources::ensure_test_rpc_secret();
for background in [false, true] {
for chunked in [false, true] {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("retry listener");
let address = listener.local_addr().expect("retry address");
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.expect("fresh connection");
let mut bytes = Vec::new();
while !bytes.windows(4).any(|window| window == b"\r\n\r\n") {
let mut chunk = [0; 1024];
let read = socket.read(&mut chunk).await.expect("retry request");
assert!(read > 0 && bytes.len() < 16 * 1024, "bounded complete headers");
bytes.extend_from_slice(&chunk[..read]);
}
let request = std::str::from_utf8(&bytes).expect("HTTP request");
let mut lines = request.split("\r\n");
let target = lines
.next()
.expect("request line")
.split_whitespace()
.nth(1)
.expect("target")
.to_owned();
let mut headers = http::HeaderMap::new();
for line in lines.take_while(|line| !line.is_empty()) {
let (name, value) = line.split_once(':').expect("header");
headers.append(
http::header::HeaderName::from_bytes(name.as_bytes()).expect("header name"),
http::HeaderValue::from_str(value.trim()).expect("header value"),
);
}
socket
.write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 6\r\nconnection: close\r\n\r\n456789")
.await
.expect("retry response");
(target, headers)
});
let gate = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
let request = ReadStreamRequest {
endpoint: format!("http://{address}"),
..resume_request(10)
};
let create = async {
let transport = Arc::new(TcpHttpInternodeDataTransport);
if chunked {
Box::new(RetryingRemoteChunkReader::new_with_timeouts(
resume_step_chunk_reader(ResumeReadStep::PartialThenReset(b"0123".to_vec())),
transport,
request,
None,
None,
)) as FileReader
} else {
Box::new(RetryingRemoteReader::new_with_timeouts(
resume_step_reader(ResumeReadStep::PartialThenReset(b"0123".to_vec())),
transport,
request,
None,
None,
)) as FileReader
}
};
let mut reader = if background {
with_background_resource_unit(gate.try_acquire().expect("sender admission"), create).await
} else {
create.await
};
assert_eq!(gate.snapshot().active, 0, "retry reader cannot retain sender disk capacity");
let mut output = Vec::new();
tokio::time::timeout(
Duration::from_secs(5),
with_rpc_background_request(!background, reader.read_to_end(&mut output)),
)
.await
.expect("bounded retry")
.expect("resumed body");
assert_eq!(output, b"0123456789", "chunked={chunked}, background={background}");
let (target, headers) = server.await.expect("captured retry");
assert!(target.contains("offset=11") && target.contains("length=6"), "{target}");
assert_eq!(rpc_background_admission(&headers).expect("admission marker"), background);
verify_rpc_signature(&target, &http::Method::GET, &headers).expect("signed retry class");
}
}
}
#[tokio::test]
async fn remote_reader_resumes_from_emitted_bytes_without_duplicates() {
let transport = Arc::new(ResumeTransport::with_read_steps(vec![ResumeReadStep::Data(b"456789".to_vec())]));
@@ -9161,124 +8748,6 @@ mod tests {
peer.stop().await;
}
#[tokio::test]
#[serial(remote_disk_recovery_probe)]
async fn endpoint_reconnect_failure_preserves_registered_remote_recovery() {
temp_env::async_with_vars(
[
(crate::disk::disk_store::ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, Some("true")),
(rustfs_config::ENV_DRIVE_RETURNING_SUCCESS_THRESHOLD, Some("1")),
],
async {
runtime_sources::ensure_test_rpc_secret();
let mut reference = crate::layout::format::FormatV3::new(1, 1);
reference.erasure.this = reference.erasure.sets[0][0];
let peer = TestGrpcPeer::spawn(
Bytes::from(reference.to_json().expect("format should serialize")),
Bytes::from_static(b"recovered-peer-data"),
)
.await
.expect("authenticated regression peer is required");
let set = missing_read_set(&peer, reference).await;
let disk = crate::disk::new_disk(
&set.set_endpoints[0],
&DiskOption {
cleanup: false,
health_check: true,
},
)
.await
.expect("registered remote disk should construct");
let crate::disk::Disk::Remote(remote) = disk.as_ref() else {
panic!("fixture must create a remote disk");
};
assert!(remote.health_check);
remote.force_runtime_state_for_test(RuntimeDriveHealthState::Offline);
set.disks.write().await[0] = Some(Arc::clone(&disk));
peer.peer.fail_format_reads.store(true, Ordering::Release);
set.connect_disks().await;
peer.peer.fail_format_reads.store(false, Ordering::Release);
let cancelled_after_failure = remote.cancel_token.is_cancelled();
assert!(peer.peer.read_all_calls() > 0, "reconnect must attempt the failing format RPC");
assert!(
Arc::ptr_eq(&disk, set.disks.read().await[0].as_ref().expect("existing slot must remain")),
"a failed reconnect must not publish its unvalidated candidate"
);
remote.spawn_recovery_monitor_if_needed();
let recovery = time::timeout(Duration::from_secs(5), async {
while !disk.is_online().await {
tokio::task::yield_now().await;
}
disk.read_all("bucket", "object").await
})
.await;
disk.close().await.expect("stop the registered disk's monitoring");
peer.stop().await;
assert!(!cancelled_after_failure, "failed replacement must retain the registered monitor");
assert_eq!(
recovery
.expect("failed reconnect must leave the registered disk able to recover without another sweep")
.expect("the recovered registered disk must serve real I/O"),
Bytes::from_static(b"recovered-peer-data")
);
},
)
.await;
}
#[tokio::test]
#[serial(remote_disk_recovery_probe)]
async fn endpoint_reconnect_stops_old_remote_monitor_only_after_publication() {
runtime_sources::ensure_test_rpc_secret();
let mut reference = crate::layout::format::FormatV3::new(1, 1);
reference.erasure.this = reference.erasure.sets[0][0];
let peer = TestGrpcPeer::spawn(
Bytes::from(reference.to_json().expect("format should serialize")),
Bytes::from_static(b"replacement-peer-data"),
)
.await
.expect("authenticated regression peer is required");
let set = missing_read_set(&peer, reference).await;
let disk = crate::disk::new_disk(
&set.set_endpoints[0],
&DiskOption {
cleanup: false,
health_check: true,
},
)
.await
.expect("registered remote disk should construct");
let crate::disk::Disk::Remote(remote) = disk.as_ref() else {
panic!("fixture must create a remote disk");
};
remote.force_runtime_state_for_test(RuntimeDriveHealthState::Offline);
set.disks.write().await[0] = Some(Arc::clone(&disk));
let gate = peer.peer.pause_next_format_read();
let pending_set = Arc::clone(&set);
let pending = tokio::spawn(async move { pending_set.connect_disks().await });
time::timeout(Duration::from_secs(5), gate.entered.notified())
.await
.expect("replacement must reach the format validation barrier");
let cancelled_before_publication = remote.cancel_token.is_cancelled();
gate.release.cancel();
pending.await.expect("validated replacement must finish publication");
let replacement = set.disks.read().await[0].clone().expect("replacement must occupy the slot");
let readback = replacement.read_all("bucket", "object").await;
replacement.close().await.expect("stop replacement monitoring");
peer.stop().await;
assert!(
!cancelled_before_publication,
"the registered monitor must survive pending format validation"
);
assert!(!Arc::ptr_eq(&disk, &replacement), "validated replacement must be published");
assert!(remote.cancel_token.is_cancelled(), "the displaced monitor must stop after publication");
assert_eq!(
readback.expect("replacement must serve real I/O"),
Bytes::from_static(b"replacement-peer-data")
);
}
#[tokio::test]
#[serial(remote_disk_recovery_probe)]
async fn endpoint_reconnect_admits_ready_drive_while_another_format_probe_is_blocked() {
+84 -532
View File
@@ -18,7 +18,6 @@ use crate::cluster::rpc::client::{
use crate::cluster::rpc::set_tonic_rolling_mutation_body_digest;
use async_trait::async_trait;
use bytes::Bytes;
use futures::{FutureExt, future::BoxFuture};
use rustfs_lock::{
LockClient, LockError, LockInfo, LockRequest, LockResponse, LockStats, LockStatus, LockType, Result,
types::{LockId, LockMetadata, LockPriority},
@@ -33,17 +32,16 @@ use rustfs_protos::{
};
use std::collections::HashMap;
use std::future::Future;
use std::panic::AssertUnwindSafe;
use std::pin::Pin;
use std::sync::{Mutex, OnceLock};
use std::time::Duration;
use tokio::task::JoinHandle;
use tokio::time::{Instant, timeout};
use tonic::service::interceptor::InterceptedService;
use tonic::{Request, Response};
use tracing::{debug, info, warn};
const EVENT_NAMESPACE_LOCK_RELEASE: &str = "namespace_lock_release";
const EVENT_REMOTE_LOCK_RPC_CLEANUP: &str = "remote_lock_rpc_cleanup";
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_REMOTE_LOCK: &str = "remote_lock";
@@ -51,38 +49,18 @@ fn attach_lock_mutation_body_digest<T: rustfs_protos::CanonicalMutationBody>(req
set_tonic_rolling_mutation_body_digest(request)
}
/// Work to run if an RPC completes after its caller stops waiting.
/// Work to run if an RPC that already timed out for its caller completes later.
type LateCompletion<T> = Option<Box<dyn FnOnce(T) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send>>;
type LateFailure = LateCompletion<()>;
type RpcFuture<T> = BoxFuture<'static, std::thread::Result<std::result::Result<T, tonic::Status>>>;
/// Own admission before connecting as well as while an RPC is in flight.
struct LockRequestGuard<'a> {
client: &'a RemoteClient,
op: &'static str,
struct RpcCleanup<T> {
late: LateCompletion<T>,
late_failure: LateFailure,
admission: Option<LockRequestAdmission>,
}
impl LockRequestGuard<'_> {
fn into_admission(mut self) -> Option<LockRequestAdmission> {
self.admission.take()
}
}
impl Drop for LockRequestGuard<'_> {
fn drop(&mut self) {
self.client.release_lock_request(self.op, self.admission.take());
}
}
struct RpcCleanup<'a, T> {
late: LateCompletion<T>,
late_failure: LateFailure,
admission: Option<LockRequestGuard<'a>>,
}
impl<'a, T> RpcCleanup<'a, T> {
fn new(late: LateCompletion<T>, late_failure: LateFailure, admission: Option<LockRequestGuard<'a>>) -> Self {
impl<T> RpcCleanup<T> {
fn new(late: LateCompletion<T>, late_failure: LateFailure, admission: Option<LockRequestAdmission>) -> Self {
Self {
late,
late_failure,
@@ -91,25 +69,6 @@ impl<'a, T> RpcCleanup<'a, T> {
}
}
/// A dropped caller hands its RPC and cleanup to the same bounded owner as a timeout.
struct InFlightRpc<'a, T: Send + 'static> {
client: &'a RemoteClient,
op: &'static str,
resource_summary: &'a str,
future: Option<RpcFuture<T>>,
cleanup: Option<RpcCleanup<'a, T>>,
}
impl<T: Send + 'static> Drop for InFlightRpc<'_, T> {
fn drop(&mut self) {
if let Some(future) = self.future.take()
&& let Some(cleanup) = self.cleanup.take()
{
self.client.detach_rpc(self.op, self.resource_summary, future, cleanup);
}
}
}
/// The liveness window is this many RPC deadlines: a peer that completed a
/// lock RPC within it is slow, not gone, and keeps its channel on a timeout.
const LOCK_RPC_LIVENESS_WINDOW_DEADLINES: u32 = 2;
@@ -178,9 +137,6 @@ fn with_lock_peer_health<R>(addr: &str, update: impl FnOnce(&mut LockPeerChannel
let mut peers = lock_peer_channel_health()
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if let Some(health) = peers.get_mut(addr) {
return update(health);
}
update(peers.entry(addr.to_string()).or_default())
}
@@ -202,15 +158,6 @@ fn reset_lock_peer_health_for_test(addr: &str) {
.remove(addr);
}
#[cfg(test)]
type EvictionPause = (tokio::sync::oneshot::Sender<()>, tokio::sync::oneshot::Receiver<()>);
#[cfg(test)]
fn eviction_pauses_for_test() -> &'static Mutex<HashMap<String, EvictionPause>> {
static PAUSES: OnceLock<Mutex<HashMap<String, EvictionPause>>> = OnceLock::new();
PAUSES.get_or_init(Mutex::default)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum EvictionTrigger {
/// The caller's deadline expired while the stream was still open.
@@ -498,10 +445,6 @@ impl RemoteClient {
}
}
});
// A successful lock RPC over a cached or fresh channel proves the peer is
// reachable; reconcile the internode peer-health gauge so it does not stay
// stale when no fresh dial fires (issue #9119).
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
}
fn admit_late_release(&self) -> bool {
@@ -559,14 +502,6 @@ impl RemoteClient {
admission
}
fn admit_lock_request_guard(&self, op: &'static str) -> Option<LockRequestGuard<'_>> {
self.admit_lock_request(op).map(|admission| LockRequestGuard {
client: self,
op,
admission: Some(admission),
})
}
fn release_lock_request(&self, op: &'static str, admission: Option<LockRequestAdmission>) {
if is_request_breaker_operation(op) && admission.is_some() {
with_lock_peer_health(&self.addr, |health| {
@@ -648,17 +583,6 @@ impl RemoteClient {
trigger: EvictionTrigger,
deadline: Duration,
) {
#[cfg(test)]
let pause = eviction_pauses_for_test()
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.remove(&self.addr);
#[cfg(test)]
if let Some((entered, resume)) = pause {
let _ = entered.send(());
let _ = resume.await;
}
let now = Instant::now();
let cooldown = Self::eviction_cooldown();
let liveness_window = Self::liveness_window(deadline);
@@ -689,26 +613,22 @@ impl RemoteClient {
);
}
/// Keep an abandoned RPC running instead of cancelling its stream.
/// Keep a timed-out RPC running instead of cancelling its stream.
///
/// Dropping the future sends `RST_STREAM`; under load those resets pile up
/// in the server's pending-accept queue until it answers `GOAWAY
/// too_many_resets` and kills every stream on the connection. A detached
/// stream ends on its own within the internode RPC timeout, the number per
/// peer is bounded, and a lock granted after its caller gave up is released.
fn detach_rpc<T: Send + 'static>(
fn detach_timed_out_rpc<T: Send + 'static>(
&self,
op: &'static str,
resource_summary: &str,
future: RpcFuture<T>,
cleanup: RpcCleanup<'_, T>,
handle: JoinHandle<std::result::Result<T, tonic::Status>>,
late: LateCompletion<T>,
late_failure: LateFailure,
admission: Option<LockRequestAdmission>,
) {
let RpcCleanup {
late,
late_failure,
admission,
} = cleanup;
let admission = admission.and_then(LockRequestGuard::into_admission);
let limit = Self::detached_rpc_limit();
let admitted = with_lock_peer_health(&self.addr, |health| {
if health.detached_rpcs >= limit {
@@ -719,29 +639,25 @@ impl RemoteClient {
}
});
if !admitted {
drop(future);
handle.abort();
self.release_lock_request(op, admission);
if let Some(late_failure) = late_failure {
drop(tokio::spawn(late_failure(())));
}
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_detached(op, "aborted");
debug!(
event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
state = "detached_budget_exhausted",
addr = %self.addr,
op,
resource_summary,
limit,
"Abandoned remote lock RPC cancelled"
"Cancelled timed-out remote lock RPC because the detached stream budget is exhausted"
);
return;
}
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_detached(op, "detached");
let addr = self.addr.clone();
tokio::spawn(async move {
let outcome = future.await;
let outcome = handle.await;
with_lock_peer_health(&addr, |health| {
health.detached_rpcs = health.detached_rpcs.saturating_sub(1);
if is_request_breaker_operation(op) && admission.is_some() {
@@ -756,7 +672,6 @@ impl RemoteClient {
with_lock_peer_health(&addr, |health| {
health.last_success = Some(Instant::now());
});
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "success");
if let Some(late) = late {
late(response).await;
@@ -768,31 +683,19 @@ impl RemoteClient {
}
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "error");
debug!(
event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
state = "late_rpc_failed",
addr = %addr,
op,
tonic_code = ?status.code(),
tonic_message = status.message(),
"Abandoned remote lock RPC failed"
"Detached remote lock RPC failed after its caller timed out"
);
}
Err(_) => {
Err(join_error) => {
if let Some(late_failure) = late_failure {
late_failure(()).await;
}
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "join_error");
debug!(
event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
state = "late_rpc_panicked",
addr = %addr,
op,
"Abandoned remote lock RPC panicked"
);
debug!(addr = %addr, op, error = %join_error, "Detached remote lock RPC task ended abnormally");
}
}
});
@@ -880,28 +783,16 @@ impl RemoteClient {
self.release_late_release();
rustfs_io_metrics::lock_metrics::record_remote_lock_late_release(outcome);
if outcome == "released" {
debug!(
event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
state = "late_grants_released",
addr = %self.addr,
count = original_count,
"Released remote locks granted after their caller stopped waiting"
);
debug!(addr = %self.addr, count = original_count, "Released remote locks granted after their caller timed out");
} else {
let (log_warning, suppressed_timeouts) = self.timeout_log_decision("late_release", Self::rpc_timeout());
if log_warning {
warn!(
event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
state = "late_grants_unreleased",
addr = %self.addr,
count = original_count,
outcome,
suppressed_timeouts,
"Could not release every remote lock granted after its caller stopped waiting; the server lease will expire it"
"Could not release every remote lock granted after its caller timed out; the server lease will expire it"
);
} else {
debug!(addr = %self.addr, count = original_count, outcome, suppressed_timeouts, "Suppressed repeated late remote lock release warning");
@@ -915,47 +806,25 @@ impl RemoteClient {
resource_summary: &str,
deadline: Duration,
future: Fut,
cleanup: RpcCleanup<'_, T>,
cleanup: RpcCleanup<T>,
) -> std::result::Result<T, LockError>
where
Fut: Future<Output = std::result::Result<T, tonic::Status>> + Send + 'static,
T: Send + 'static,
{
let mut rpc = InFlightRpc {
client: self,
op,
resource_summary,
future: Some(AssertUnwindSafe(future).catch_unwind().boxed()),
cleanup: Some(cleanup),
};
let outcome = match rpc.future.as_mut() {
Some(future) => tokio::select! {
// A response becoming ready on the deadline tick must not turn an
// expired caller budget into a transport error and channel eviction.
biased;
_ = tokio::time::sleep(deadline) => None,
outcome = future => Some(outcome),
},
None => return Err(LockError::internal("remote lock RPC has no in-flight future")),
};
if outcome.is_some() {
rpc.future.take();
}
let Some(RpcCleanup {
late: _,
let RpcCleanup {
late,
late_failure,
admission,
}) = rpc.cleanup.as_mut()
else {
return Err(LockError::internal("remote lock RPC has no cleanup owner"));
};
let token = admission.as_ref().and_then(|guard| guard.admission);
let result = match outcome {
Some(Ok(Ok(response))) => {
self.record_rpc_success(op, token);
} = cleanup;
let mut handle = tokio::spawn(future);
match timeout(deadline, &mut handle).await {
Ok(Ok(Ok(response))) => {
self.record_rpc_success(op, admission);
self.release_lock_request(op, admission);
Ok(response)
}
Some(Ok(Err(err))) => {
Ok(Ok(Err(err))) => {
let reason = err.to_string();
if err.code() == tonic::Code::DeadlineExceeded {
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_timeout(&self.addr, op);
@@ -988,11 +857,12 @@ impl RemoteClient {
"Suppressed repeated remote lock RPC deadline"
);
}
if let Some(late_failure) = late_failure.take() {
drop(tokio::spawn(late_failure(())));
}
self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Timeout, deadline)
.await;
self.release_lock_request(op, admission);
if let Some(late_failure) = late_failure {
drop(tokio::spawn(late_failure(())));
}
return Err(LockError::timeout(format!("remote lock RPC {op} on {}", self.addr), deadline));
}
// Only evict (and re-dial) the cached channel when the failure is a genuine
@@ -1053,32 +923,30 @@ impl RemoteClient {
);
}
if transport_failure {
// Ambiguous-grant cleanup owns its work before channel eviction can suspend.
if let Some(late_failure) = late_failure.take() {
drop(tokio::spawn(late_failure(())));
}
self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Transport, deadline)
.await;
}
self.release_lock_request(op, admission);
if transport_failure && let Some(late_failure) = late_failure {
drop(tokio::spawn(late_failure(())));
}
Err(LockError::internal(format!("{op} RPC failed: {reason}")))
}
Some(Err(_)) => {
Ok(Err(join_error)) => {
warn!(
event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
state = "rpc_panicked",
addr = %self.addr,
op,
resource_summary,
"Remote lock RPC panicked"
error = %join_error,
"Remote lock RPC task ended abnormally"
);
if let Some(late_failure) = late_failure.take() {
self.release_lock_request(op, admission);
if let Some(late_failure) = late_failure {
drop(tokio::spawn(late_failure(())));
}
Err(LockError::internal(format!("{op} RPC future panicked")))
Err(LockError::internal(format!("{op} RPC task failed: {join_error}")))
}
None => {
Err(_) => {
let reason = format!("RPC timed out after {deadline:?}");
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_timeout(&self.addr, op);
self.record_request_timeout(op, deadline);
@@ -1110,17 +978,12 @@ impl RemoteClient {
"Suppressed repeated remote lock RPC timeout"
);
}
// Continue draining the abandoned stream while channel eviction waits.
drop(rpc);
self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Timeout, deadline)
.await;
return Err(LockError::timeout(format!("remote lock RPC {op} on {}", self.addr), deadline));
self.detach_timed_out_rpc(op, resource_summary, handle, late, late_failure, admission);
Err(LockError::timeout(format!("remote lock RPC {op} on {}", self.addr), deadline))
}
};
if rpc.future.is_none() {
rpc.cleanup.take();
}
result
}
fn rpc_timeout_failure_response(request: &LockRequest, err: &LockError) -> LockResponse {
@@ -1211,7 +1074,7 @@ impl RemoteClient {
#[async_trait]
impl LockClient for RemoteClient {
async fn acquire_lock(&self, request: &LockRequest) -> Result<LockResponse> {
let admission = match self.admit_lock_request_guard("lock") {
let admission = match self.admit_lock_request("lock") {
Some(admission) => admission,
None => return Ok(Self::rpc_backoff_failure_response(request)),
};
@@ -1222,6 +1085,7 @@ impl LockClient for RemoteClient {
Ok(client) => client,
Err(err) => {
self.record_request_timeout("lock", rpc_timeout);
self.release_lock_request("lock", Some(admission));
return Err(err);
}
};
@@ -1229,12 +1093,14 @@ impl LockClient for RemoteClient {
let args = match serde_json::to_string(&bounded_request) {
Ok(args) => args,
Err(err) => {
self.release_lock_request("lock", Some(admission));
return Err(LockError::internal(format!("Failed to serialize request: {err}")));
}
};
let mut req = Request::new(GenerallyLockRequest { args });
req.set_timeout(rpc_timeout);
if let Err(err) = attach_lock_mutation_body_digest(&mut req) {
self.release_lock_request("lock", Some(admission));
return Err(err.into());
}
let late = self.late_release_hook(request.lock_id.clone());
@@ -1276,7 +1142,7 @@ impl LockClient for RemoteClient {
return Ok(Vec::new());
}
let admission = match self.admit_lock_request_guard("lock_batch") {
let admission = match self.admit_lock_request("lock_batch") {
Some(admission) => admission,
None => return Ok(Self::rpc_backoff_failure_batch(requests)),
};
@@ -1290,6 +1156,7 @@ impl LockClient for RemoteClient {
Ok(client) => client,
Err(err) => {
self.record_request_timeout("lock_batch", rpc_timeout);
self.release_lock_request("lock_batch", Some(admission));
return Err(err);
}
};
@@ -1303,12 +1170,14 @@ impl LockClient for RemoteClient {
{
Ok(args) => args,
Err(err) => {
self.release_lock_request("lock_batch", Some(admission));
return Err(err);
}
};
let mut req = Request::new(BatchGenerallyLockRequest { args });
req.set_timeout(rpc_timeout);
if let Err(err) = attach_lock_mutation_body_digest(&mut req) {
self.release_lock_request("lock_batch", Some(admission));
return Err(err.into());
}
let late = self.late_release_batch_hook(requests.iter().map(|request| request.lock_id.clone()).collect());
@@ -1446,6 +1315,34 @@ impl LockClient for RemoteClient {
Ok(resp.success)
}
async fn downgrade_committed_tail(&self, lock_id: &LockId) -> Result<bool> {
let marker = rustfs_lock::client::COMMITTED_TAIL_CONVERSION_MARKER;
let mut request = Self::create_unlock_request(lock_id);
request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
let mut client = self.get_client().await?;
let resource_summary = request.resource.to_string();
let mut req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&request)
.map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
});
attach_lock_mutation_body_digest(&mut req)?;
let response = self
.execute_rpc(
"refresh",
&resource_summary,
Self::rpc_timeout(),
async move { client.refresh(req).await },
RpcCleanup::new(None, None, None),
)
.await?
.into_inner();
if let Some(error) = response.error_info {
return Err(LockError::internal(error));
}
// Older nodes refresh normally but return no conversion marker.
Ok(response.success && response.lock_info.as_deref() == Some(marker))
}
async fn force_release(&self, lock_id: &LockId) -> Result<bool> {
info!("remote force_release for {}", lock_id);
let force_request = Self::create_unlock_request(lock_id);
@@ -1836,351 +1733,6 @@ mod tests {
assert!(acquired_lock_ids(&lock_ids, &[]).is_empty());
}
#[tokio::test]
async fn test_remote_client_cancelled_connection_releases_request_admission() {
ensure_test_rpc_secret();
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind pending-handshake peer");
let addr = format!("http://{}", listener.local_addr().expect("pending-handshake peer address"));
let client = RemoteClient::new(addr.clone());
let request = test_lock_request(Duration::from_secs(30));
for batch in [false, true] {
reset_lock_peer_health_for_test(&addr);
with_lock_peer_health(&addr, |health| {
health.request_backoff_until = Some(Instant::now() - Duration::from_secs(1));
});
let mut acquisition = Box::pin(async {
if batch {
client.acquire_locks_batch(std::slice::from_ref(&request)).await.map(|_| ())
} else {
client.acquire_lock(&request).await.map(|_| ())
}
});
std::future::poll_fn(|cx| {
assert!(acquisition.as_mut().poll(cx).is_pending(), "peer must wait for its HTTP/2 handshake");
std::task::Poll::Ready(())
})
.await;
let pending = lock_peer_health_for_test(&addr);
assert_eq!(pending.request_in_flight, 1, "pending connection owns one request admission");
assert!(pending.request_probe_generation.is_some(), "the request is the half-open probe");
drop(acquisition);
let cancelled = lock_peer_health_for_test(&addr);
assert_eq!(cancelled.request_in_flight, 0, "cancellation must return request admission");
assert!(cancelled.request_probe_generation.is_none(), "cancellation must unblock the next probe");
let probe = client.admit_lock_request("lock").expect("cancelled peer can be probed again");
client.release_lock_request("lock", Some(probe));
}
reset_lock_peer_health_for_test(&addr);
}
#[tokio::test]
async fn test_remote_client_cancelled_rpc_runs_late_grant_cleanup() {
let addr = "http://cancelled-rpc-late-grant";
reset_lock_peer_health_for_test(addr);
let client = RemoteClient::new(addr.to_string());
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
let (grant_tx, grant_rx) = tokio::sync::oneshot::channel();
let (released_tx, released_rx) = tokio::sync::oneshot::channel();
let caller = tokio::spawn(async move {
let late: LateCompletion<bool> = Some(Box::new(move |granted| {
Box::pin(async move {
released_tx.send(granted).expect("late grant observer remains alive");
})
}));
client
.execute_rpc(
"lock",
"bucket/object",
Duration::from_secs(30),
async move {
started_tx.send(()).expect("caller observes RPC dispatch");
Ok(grant_rx.await.expect("test decides when peer grants lock"))
},
RpcCleanup::new(late, None, None),
)
.await
});
timeout(Duration::from_secs(1), started_rx)
.await
.expect("RPC must dispatch")
.expect("dispatch signal");
caller.abort();
assert!(caller.await.expect_err("caller was cancelled").is_cancelled());
assert_eq!(
lock_peer_health_for_test(addr).detached_rpcs,
1,
"cancelled RPC uses bounded detached ownership"
);
grant_tx.send(true).expect("cancellation must preserve the running RPC");
assert!(
timeout(Duration::from_secs(1), released_rx)
.await
.expect("late grant must be cleaned up")
.expect("cleanup result")
);
assert_eq!(lock_peer_health_for_test(addr).detached_rpcs, 0, "completed cleanup returns its slot");
reset_lock_peer_health_for_test(addr);
}
#[tokio::test]
async fn test_remote_client_cancelled_rpc_runs_ambiguous_failure_cleanup() {
let addr = "http://cancelled-rpc-late-failure";
reset_lock_peer_health_for_test(addr);
let client = RemoteClient::new(addr.to_string());
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
let (failure_tx, failure_rx) = tokio::sync::oneshot::channel();
let (released_tx, released_rx) = tokio::sync::oneshot::channel();
let caller = tokio::spawn(async move {
let late_failure: LateFailure = Some(Box::new(move |()| {
Box::pin(async move {
released_tx.send(()).expect("ambiguous cleanup observer remains alive");
})
}));
client
.execute_rpc(
"lock_batch",
"bucket/object",
Duration::from_secs(30),
async move {
started_tx.send(()).expect("caller observes RPC dispatch");
failure_rx.await.expect("test decides when transport fails");
Err::<(), _>(tonic::Status::cancelled("transport failure after caller cancellation"))
},
RpcCleanup::new(None, late_failure, None),
)
.await
});
timeout(Duration::from_secs(1), started_rx)
.await
.expect("RPC must dispatch")
.expect("dispatch signal");
caller.abort();
assert!(caller.await.expect_err("caller was cancelled").is_cancelled());
failure_tx.send(()).expect("cancellation must preserve the running RPC");
timeout(Duration::from_secs(1), released_rx)
.await
.expect("ambiguous grant must be cleaned up")
.expect("cleanup signal");
assert_eq!(lock_peer_health_for_test(addr).detached_rpcs, 0, "failed detached RPC returns its slot");
reset_lock_peer_health_for_test(addr);
}
#[tokio::test]
async fn test_remote_client_cancelled_completed_failure_runs_cleanup() {
for (label, status) in [
("transport", tonic::Status::cancelled("ambiguous transport failure")),
("deadline", tonic::Status::deadline_exceeded("ambiguous deadline failure")),
] {
let addr = format!("http://cancelled-completed-{label}");
reset_lock_peer_health_for_test(&addr);
let client = RemoteClient::new(addr.clone());
let (evicting_tx, evicting_rx) = tokio::sync::oneshot::channel();
let (_resume_tx, resume_rx) = tokio::sync::oneshot::channel();
eviction_pauses_for_test()
.lock()
.expect("eviction pause registry")
.insert(addr.clone(), (evicting_tx, resume_rx));
let (released_tx, released_rx) = tokio::sync::oneshot::channel();
let caller = tokio::spawn(async move {
let late_failure: LateFailure = Some(Box::new(move |()| {
Box::pin(async move {
released_tx
.send(())
.expect("completed-failure cleanup observer remains alive");
})
}));
client
.execute_rpc(
"lock",
"bucket/object",
Duration::from_secs(30),
async move { Err::<(), _>(status) },
RpcCleanup::new(None, late_failure, None),
)
.await
});
timeout(Duration::from_secs(1), evicting_rx)
.await
.expect("completed error must reach channel eviction")
.expect("eviction signal");
timeout(Duration::from_secs(1), released_rx)
.await
.expect("cleanup must transfer before channel eviction suspends")
.expect("cleanup signal");
caller.abort();
assert!(caller.await.expect_err("caller is cancelled during eviction").is_cancelled());
assert_eq!(
lock_peer_health_for_test(&addr).detached_rpcs,
0,
"completed RPC cannot consume a detached slot"
);
reset_lock_peer_health_for_test(&addr);
}
}
#[tokio::test(start_paused = true)]
async fn test_remote_client_deadline_wins_over_same_tick_transport_timeout() {
let addr = "http://same-tick-rpc-deadline";
reset_lock_peer_health_for_test(addr);
with_lock_peer_health(addr, |health| health.last_success = Some(Instant::now()));
let client = RemoteClient::new(addr.to_string());
let budget = Duration::from_millis(50);
let (released_tx, released_rx) = tokio::sync::oneshot::channel();
let late_failure: LateFailure = Some(Box::new(move |()| {
Box::pin(async move {
released_tx.send(()).expect("same-tick cleanup observer remains alive");
})
}));
let result = client
.execute_rpc(
"lock",
"bucket/object",
budget,
async move {
tokio::time::sleep(budget).await;
Err::<(), _>(tonic::Status::cancelled("inner transport deadline expired"))
},
RpcCleanup::new(None, late_failure, None),
)
.await;
assert!(
matches!(result, Err(LockError::Timeout { .. })),
"expired caller budget must retain timeout classification"
);
let health = lock_peer_health_for_test(addr);
assert_eq!(health.consecutive_timeouts, 1, "same-tick completion must follow timeout eviction policy");
assert!(
health.last_eviction.is_none(),
"a recently serving peer keeps its channel after a caller timeout"
);
timeout(Duration::from_secs(1), released_rx)
.await
.expect("same-tick ambiguous completion must trigger cleanup")
.expect("cleanup signal");
assert_eq!(
lock_peer_health_for_test(addr).detached_rpcs,
0,
"same-tick completion returns detached admission"
);
reset_lock_peer_health_for_test(addr);
}
#[tokio::test]
async fn test_remote_client_timeout_drains_late_grant_during_eviction() {
let addr = "http://timeout-late-grant-during-eviction";
reset_lock_peer_health_for_test(addr);
let client = RemoteClient::new(addr.to_string());
let (evicting_tx, evicting_rx) = tokio::sync::oneshot::channel();
let (resume_tx, resume_rx) = tokio::sync::oneshot::channel();
eviction_pauses_for_test()
.lock()
.expect("eviction pause registry")
.insert(addr.to_string(), (evicting_tx, resume_rx));
let (grant_tx, grant_rx) = tokio::sync::oneshot::channel();
let (released_tx, released_rx) = tokio::sync::oneshot::channel();
let caller = tokio::spawn(async move {
let late: LateCompletion<bool> = Some(Box::new(move |granted| {
Box::pin(async move {
released_tx.send(granted).expect("late grant observer remains alive");
})
}));
client
.execute_rpc(
"lock",
"bucket/object",
Duration::from_millis(25),
async move { Ok(grant_rx.await.expect("test controls the late grant")) },
RpcCleanup::new(late, None, None),
)
.await
});
timeout(Duration::from_secs(1), evicting_rx)
.await
.expect("timed-out RPC must reach channel eviction")
.expect("eviction signal");
assert_eq!(
lock_peer_health_for_test(addr).detached_rpcs,
1,
"eviction waits with a detached RPC owner"
);
grant_tx
.send(true)
.expect("timed-out stream remains available for a late grant");
assert!(
timeout(Duration::from_secs(1), released_rx)
.await
.expect("late grant must be cleaned up before eviction resumes")
.expect("late cleanup result")
);
assert!(!caller.is_finished(), "channel eviction is still paused");
assert_eq!(
lock_peer_health_for_test(addr).detached_rpcs,
0,
"late completion returns detached admission"
);
resume_tx.send(()).expect("resume channel eviction");
let result = caller.await.expect("timed-out caller completes without panicking");
assert!(
matches!(result, Err(LockError::Timeout { .. })),
"caller preserves its RPC timeout result"
);
reset_lock_peer_health_for_test(addr);
}
#[tokio::test]
async fn test_remote_client_success_polls_rpc_on_callers_task() {
tokio::spawn(async {
let client = RemoteClient::new("http://direct-poll-success".to_string());
let caller_id = tokio::task::id();
let rpc_id = client
.execute_rpc(
"ping",
"health-lock-online",
Duration::from_secs(1),
async { Ok(tokio::task::id()) },
RpcCleanup::new(None, None, None),
)
.await
.expect("successful RPC must return its result");
assert_eq!(rpc_id, caller_id, "successful RPC must not allocate and schedule a separate task");
})
.await
.expect("caller task must complete");
}
#[tokio::test]
async fn test_remote_client_rpc_panic_isolated_and_cleans_up() {
let (released_tx, released_rx) = tokio::sync::oneshot::channel();
let caller = tokio::spawn(async move {
let client = RemoteClient::new("http://direct-poll-panic".to_string());
let late_failure: LateFailure = Some(Box::new(move |()| {
Box::pin(async move {
released_tx.send(()).expect("panic cleanup observer remains alive");
})
}));
client
.execute_rpc::<(), _>(
"lock",
"bucket/object",
Duration::from_secs(1),
async { panic!("test peer RPC panic") },
RpcCleanup::new(None, late_failure, None),
)
.await
});
let result = caller.await.expect("RPC panic must not escape to its caller");
assert!(
matches!(result, Err(LockError::Internal { .. })),
"RPC panic remains an internal lock failure"
);
timeout(Duration::from_secs(1), released_rx)
.await
.expect("panic must trigger ambiguous-grant cleanup")
.expect("cleanup signal");
}
#[tokio::test]
#[serial_test::serial]
async fn test_remote_client_timeout_keeps_channel_of_recently_serving_peer() {
-1
View File
@@ -3506,7 +3506,6 @@ mod tests {
replication_decision: String::new(),
checksum: None,
multipart_completion_replayed: false,
backfilled_old_current_size: None,
}
}
}
@@ -18,7 +18,6 @@ use crate::storage_api_contracts::namespace::NamespaceLocking as _;
use codec::{fixture_inline_pool_meta_replicas as replicas, fixture_split_inline_pool_meta_replicas as split};
use std::path::PathBuf;
use tokio::fs;
use uuid::Uuid;
struct PhysicalFixture {
_directories: Vec<tempfile::TempDir>,
@@ -217,264 +216,6 @@ async fn pool_meta_native_physical_split_without_ordinary_commit_performs_no_mut
"all current heads, identities and complete backups must remain unchanged"
);
assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
state
.ensure_write_safe("ordinary authority temporarily unavailable")
.expect("missing ordinary authority alone must not install a permanent corruption latch");
}
async fn interrupted_first_commit_fixture() -> (PhysicalFixture, PoolMetaCommittedCandidate) {
let fixture = fixture(3).await;
let PoolMetaReplica::Valid {
previous: Some(previous),
committed: false,
..
} = decode_pool_meta_replica(fixture.pending.clone())
else {
panic!("fixture must retain its acknowledged predecessor");
};
let physical = disks(&fixture.store).await;
let prepared = replicas(&fixture.pending, OffsetDateTime::now_utc()).await;
for (index, disk) in physical[0].iter().enumerate() {
fs::write(object_path(disk).join("xl.meta"), &prepared[index])
.await
.expect("one pool must retain an ordinary-readable PREPARE");
}
(fixture, *previous)
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_physical_interrupted_first_commit_restores_proven_predecessor() {
let (mut fixture, previous) = interrupted_first_commit_fixture().await;
let before = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
assert!(
before[1..]
.iter()
.all(|read| matches!(read.replica, PoolMetaReplica::Unreadable(_)))
);
assert!(matches!(before[0].replica, PoolMetaReplica::Valid { committed: false, .. }));
let mut state = clean_state(&fixture);
let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
assert!(
repair(&fixture.store, &mut state, &fence)
.await
.expect("ordinary PREPARE must authorize restoring only its acknowledged predecessor")
);
state
.ensure_write_safe("first commit rollback")
.expect("rollback must finish its transaction");
assert_ne!(
previous.canonical, fixture.canonical,
"the interrupted COMMIT must not become authoritative"
);
fixture.canonical = previous.canonical;
fixture.revision = previous.revision;
assert_committed(&fixture).await;
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_physical_interrupted_rollback_retains_authority_and_resumes() {
let (mut fixture, previous) = interrupted_first_commit_fixture().await;
let pause = Arc::new(NativeApplyPause::default());
let reached = pause.reached.notified();
tokio::pin!(reached);
reached.as_mut().enable();
let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
let mut state = clean_state(&fixture);
let recovery = tokio::spawn({
let store = Arc::clone(&fixture.store);
let pause = Arc::clone(&pause);
async move {
AFTER_NATIVE_POOL_APPLY
.scope(pause, async move { repair(&store, &mut state, &fence).await })
.await
}
});
tokio::time::timeout(std::time::Duration::from_secs(10), reached)
.await
.expect("real rollback should pause after one pool is durably restored");
let partial = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
assert!(matches!(partial[0].replica, PoolMetaReplica::Valid { committed: false, .. }));
assert!(matches!(&partial[1].replica, PoolMetaReplica::Valid { canonical, committed: true, .. }
if canonical == &previous.canonical));
assert!(matches!(partial[2].replica, PoolMetaReplica::Unreadable(_)));
recovery.abort();
assert!(
recovery
.await
.expect_err("the paused repair must be cancelled")
.is_cancelled()
);
let mut state = clean_state(&fixture);
let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
assert!(
repair(&fixture.store, &mut state, &fence)
.await
.expect("restart must resume from the retained ordinary PREPARE")
);
fixture.canonical = previous.canonical;
fixture.revision = previous.revision;
assert_committed(&fixture).await;
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_physical_rollback_resumes_after_last_prepare_loses_read_quorum() {
let (mut fixture, previous) = interrupted_first_commit_fixture().await;
let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
let inspected = inspect(&fixture.store.pools, &fence)
.await
.expect("physical rollback inventory");
let replacements = plan(
&inspected,
&fixture.canonical,
fixture.revision,
&fixture.identity,
Some(&previous.canonical),
)
.await
.expect("the ordinary PREPARE must authorize its predecessor");
for (pool, count) in [(1, 4), (2, 4), (0, 2)] {
for (index, replacement) in replacements[pool].iter().enumerate().take(count) {
let snapshot = &inspected[pool].snapshots[index];
assert_eq!(
inspected[pool].disks[index]
.pool_meta_native_recovery(Request::Apply {
object: PoolMetaNativeObject::PoolBin,
expected_disk_id: snapshot.disk_id,
expected_current_sha256: snapshot.current.as_ref().map(|raw| Sha256::digest(raw).into()),
expected_backup_inventory_sha256: snapshot.inventory_sha256().expect("original inventory hash"),
replacement: replacement.clone(),
})
.await
.expect("partial rollback must use the real conditional publication and sync path"),
Response::Applied
);
}
}
let partial = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
assert!(matches!(partial[0].replica, PoolMetaReplica::Unreadable(_)));
assert!(partial[1..].iter().all(|read| matches!(&read.replica,
PoolMetaReplica::Valid { canonical, committed: true, .. } if canonical == &previous.canonical)));
let mut state = clean_state(&fixture);
assert!(
repair(&fixture.store, &mut state, &fence)
.await
.expect("restored ordinary COMMIT must finish rollback after the last PREPARE becomes split")
);
fixture.canonical = previous.canonical;
fixture.revision = previous.revision;
assert_committed(&fixture).await;
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_physical_rollback_backup_does_not_poison_reused_generation() {
let (mut fixture, previous) = interrupted_first_commit_fixture().await;
let mut state = clean_state(&fixture);
let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
assert!(repair(&fixture.store, &mut state, &fence).await.expect("initial rollback"));
let revision = PoolMetaRevision {
transaction_id: Some(Uuid::new_v4()),
..fixture.revision
};
let mut meta = previous.meta.clone();
meta.pools[0].last_update += Duration::seconds(1);
let canonical = encode_pool_meta_v3_envelope(&meta, revision, true, None).expect("later COMMIT");
let pending = encode_pool_meta_v3_envelope(&meta, revision, false, Some(&previous)).expect("later PREPARE");
let modified = OffsetDateTime::now_utc() + Duration::seconds(1);
let split = split(&pending, &canonical, modified).await;
let commit = replicas(&canonical, modified).await;
let prepared = replicas(&pending, modified).await;
let directory = Uuid::new_v4();
let retained = snapshots(&fixture.store).await;
for (pool_index, pool) in disks(&fixture.store).await.into_iter().enumerate() {
for (disk_index, disk) in pool.iter().enumerate() {
assert!(
!retained[pool_index][disk_index].backups.is_empty(),
"the abandoned PREPARE must remain as real on-disk evidence"
);
let object = object_path(disk);
fs::write(object.join("xl.meta"), &if pool_index < 2 { &split } else { &commit }[disk_index])
.await
.expect("install later partial COMMIT without clearing old backups");
fs::create_dir(object.join(directory.to_string()))
.await
.expect("later backup directory");
fs::write(object.join(directory.to_string()).join("xl.meta.bkp"), &prepared[disk_index])
.await
.expect("later PREPARE backup");
}
}
assert!(
repair(&fixture.store, &mut state, &fence)
.await
.expect("an ordinary later COMMIT must survive the abandoned sibling PREPARE")
);
fixture.canonical = canonical;
fixture.revision = revision;
assert_committed(&fixture).await;
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_physical_rollback_rejects_retained_commit_without_mutation() {
let (fixture, _) = interrupted_first_commit_fixture().await;
let physical = disks(&fixture.store).await;
let commit = replicas(&fixture.canonical, OffsetDateTime::now_utc()).await;
let directory = Uuid::new_v4();
let object = object_path(&physical[0][0]);
fs::create_dir(object.join(directory.to_string()))
.await
.expect("retained COMMIT directory");
fs::write(object.join(directory.to_string()).join("xl.meta.bkp"), &commit[0])
.await
.expect("retain evidence that the interrupted transaction may already have committed");
let before = snapshots(&fixture.store).await;
let mut state = clean_state(&fixture);
let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
assert!(repair(&fixture.store, &mut state, &fence).await.is_err());
assert_eq!(snapshots(&fixture.store).await, before, "ambiguous COMMIT evidence must remain untouched");
assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_physical_rollback_rejects_competing_prepares_without_mutation() {
let (fixture, previous) = interrupted_first_commit_fixture().await;
let pending = encode_pool_meta_v3_envelope(
&previous.meta,
PoolMetaRevision {
transaction_id: Some(Uuid::new_v4()),
..fixture.revision
},
false,
Some(&previous),
)
.expect("competing PREPARE must remain structurally valid");
let physical = disks(&fixture.store).await;
let prepared = replicas(&pending, OffsetDateTime::now_utc()).await;
for (index, disk) in physical[1].iter().enumerate() {
fs::write(object_path(disk).join("xl.meta"), &prepared[index])
.await
.expect("second pool must expose the competing transaction through ordinary reads");
}
let before = snapshots(&fixture.store).await;
let mut state = clean_state(&fixture);
let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
let error = repair(&fixture.store, &mut state, &fence)
.await
.expect_err("two distinct prepared transactions must not authorize a guessed rollback");
assert_eq!(
error
.pool_metadata_failure()
.expect("transaction conflict must be typed")
.kind,
crate::error::PoolMetadataFailure::RecoveryRequired
);
assert_eq!(snapshots(&fixture.store).await, before, "competing transactions must remain untouched");
assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
}
struct IdentityEpochFork {
@@ -726,7 +467,7 @@ async fn pool_meta_native_physical_partial_apply_resumes_after_conditional_rejec
let inspected = inspect(&fixture.store.pools, &fence)
.await
.expect("real native snapshots should be obtained");
let replacements = plan(&inspected, &fixture.canonical, fixture.revision, &fixture.identity, None)
let replacements = plan(&inspected, &fixture.canonical, fixture.revision, &fixture.identity)
.await
.expect("physical recovery plan should validate");
let snapshot = &inspected[0].snapshots[0];
@@ -887,559 +628,3 @@ async fn pool_meta_native_physical_three_plus_one_checkpoint_converges_all_shard
);
}
}
async fn owner_repair_no_backups_fixture() -> PhysicalFixture {
let fixture = fixture(0).await;
for pool in disks(&fixture.store).await {
for disk in pool {
let object = object_path(&disk);
let mut entries = fs::read_dir(&object).await.expect("fixture metadata directory");
while let Some(entry) = entries.next_entry().await.expect("fixture directory entry") {
if entry.file_type().await.expect("fixture entry type").is_dir() {
fs::remove_dir_all(entry.path())
.await
.expect("remove task-owned fixture rollback directory");
}
}
}
}
fixture
}
async fn owner_repair_identity_heads(store: &ECStore) -> Vec<Vec<(Uuid, Bytes)>> {
let mut all = Vec::new();
for pool in &store.pools {
let set = pool.get_disks_by_key(POOL_META_IDENTITY_NAME);
let members = set
.disks
.read()
.await
.iter()
.cloned()
.map(|disk| disk.expect("identity fixture disk must be online"))
.collect::<Vec<_>>();
let mut heads = Vec::new();
for disk in members {
match disk
.pool_meta_native_recovery(Request::InspectIdentity {})
.await
.expect("physical identity inspection")
{
Response::IdentityInspected { disk_id, raw } => heads.push((disk_id, raw)),
_ => panic!("identity inspection must return its physical head"),
}
}
all.push(heads);
}
all
}
async fn legacy_native_owner_fixture(version: u16, stale: bool) -> (PhysicalFixture, Vec<u8>) {
let fixture = owner_repair_no_backups_fixture().await;
let mut meta = match decode_pool_meta_replica(fixture.canonical.clone()) {
PoolMetaReplica::Valid {
meta, committed: true, ..
} => meta,
_ => panic!("fixture must provide valid committed pool metadata"),
};
meta.version = version;
let raw = meta
.encode_config_data_for_v2_gate(version == POOL_META_VERSION)
.expect("legacy fixture should encode in its original version");
assert_eq!(u16::from_le_bytes([raw[2], raw[3]]), version);
let all = disks(&fixture.store).await;
let now = OffsetDateTime::now_utc();
for pool in &all {
let heads = replicas(&raw, now).await;
for (index, disk) in pool.iter().enumerate() {
fs::write(object_path(disk).join("xl.meta"), &heads[index])
.await
.expect("install exact legacy head");
}
}
if stale {
let old = replicas(&raw, now - Duration::seconds(10)).await;
fs::write(object_path(&all[0][0]).join("xl.meta"), &old[0])
.await
.expect("install stale same-raw legacy head");
}
(fixture, raw)
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_heal_v1_with_all_upgrade_gates_disabled_keeps_exact_raw() {
temp_env::async_with_vars(
[
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
],
async {
let (fixture, raw) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
let guard = owner(&fixture.store).await;
let mut state = fixture.store.pool_meta_save_gate.lock().await;
let result = tokio::time::timeout(
std::time::Duration::from_secs(30),
fixture.store.heal_pool_meta_replica_under_fence(0, guard, &mut state),
)
.await
.expect("legacy native heal must terminate");
assert!(result.is_ok(), "fully online legacy native heal should succeed: {result:?}");
drop(state);
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
for read in &reads {
assert!(
matches!(&read.replica, PoolMetaReplica::Valid { raw: bytes, revision, .. }
if bytes == &raw && revision.version == POOL_META_V1_VERSION),
"native heal must not upgrade a V1 raw payload when both upgrade gates are disabled"
);
}
},
)
.await;
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_heal_rejects_legacy_source_with_trailing_bytes_without_writes() {
temp_env::async_with_vars(
[
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
],
async {
for version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
let (fixture, raw) = legacy_native_owner_fixture(version, false).await;
let mut source_raw = raw.clone();
// A second MessagePack value is readable by legacy decoding but
// must not be copied as part of an unverified repair payload.
source_raw.push(0xc0);
assert!(
matches!(decode_pool_meta_replica(source_raw.clone()), PoolMetaReplica::Valid { raw, .. } if raw == source_raw),
"ordinary legacy reads must remain compatible with trailing bytes"
);
let all_disks = disks(&fixture.store).await;
let source_heads = replicas(&source_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
for (index, disk) in all_disks[0].iter().enumerate() {
fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
.await
.expect("install readable legacy source with trailing bytes");
}
let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
.await
.expect("legacy semantic selection must remain readable");
assert_eq!(selection.legacy_raw.as_deref(), Some(source_raw.as_slice()));
let before = snapshots(&fixture.store).await;
let identities = owner_repair_identity_heads(&fixture.store).await;
let guard = owner(&fixture.store).await;
let mut state = fixture.store.pool_meta_save_gate.lock().await;
let result = tokio::time::timeout(
std::time::Duration::from_secs(30),
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
)
.await
.expect("legacy native heal must terminate");
drop(state);
assert!(result.is_err(), "version {version}: unparsed trailing source bytes must not be republished");
assert_eq!(snapshots(&fixture.store).await, before, "version {version}: no pool.bin head or backup may change");
assert_eq!(
owner_repair_identity_heads(&fixture.store).await,
identities,
"version {version}: no identity shard may change"
);
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &source_raw));
assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw: target, .. } if target == &raw));
}
},
)
.await;
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_heal_mixed_v1_v2_preserves_observed_format_floor() {
temp_env::async_with_vars(
[
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
],
async {
for first_version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
for target_pool in [0, 1] {
let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
let mut meta = match decode_pool_meta_replica(raw_v1.clone()) {
PoolMetaReplica::Valid { meta, committed: true, .. } => meta,
_ => panic!("initial V1 metadata must decode"),
};
meta.version = POOL_META_VERSION;
let raw_v2 = meta.encode_config_data_for_v2_gate(true).expect("equivalent V2 payload must encode");
assert_eq!(u16::from_le_bytes([raw_v1[2], raw_v1[3]]), POOL_META_V1_VERSION);
assert_eq!(u16::from_le_bytes([raw_v2[2], raw_v2[3]]), POOL_META_VERSION);
let semantic_v1 = match decode_pool_meta_replica(raw_v1.clone()) {
PoolMetaReplica::Valid { canonical, .. } => canonical,
_ => panic!("V1 payload must remain valid"),
};
let semantic_v2 = match decode_pool_meta_replica(raw_v2.clone()) {
PoolMetaReplica::Valid { canonical, .. } => canonical,
_ => panic!("V2 payload must remain valid"),
};
assert_eq!(semantic_v1, semantic_v2, "the two formats must carry equivalent metadata");
let other_version = if first_version == POOL_META_V1_VERSION { POOL_META_VERSION } else { POOL_META_V1_VERSION };
// Pool two follows pool zero; pools zero and one cover both
// orderings and both target formats in separate runs.
let versions = [first_version, other_version, first_version];
let physical = disks(&fixture.store).await;
let timestamp = OffsetDateTime::now_utc();
for (pool_index, pool) in physical.iter().enumerate() {
let raw = if versions[pool_index] == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
let heads = replicas(raw, timestamp).await;
for (index, disk) in pool.iter().enumerate() {
fs::write(object_path(disk).join("xl.meta"), &heads[index]).await.expect("install mixed legacy pool head");
}
}
let before = snapshots(&fixture.store).await;
let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
.await.expect("equivalent legacy pools should be selectable");
assert_eq!(selection.revision.version, POOL_META_VERSION, "the observed V2 format floor must be retained");
assert_eq!(selection.canonical.as_deref(), Some(semantic_v2.as_slice()));
let guard = owner(&fixture.store).await;
let mut state = fixture.store.pool_meta_save_gate.lock().await;
tokio::time::timeout(std::time::Duration::from_secs(30),
fixture.store.heal_pool_meta_replica_under_fence(target_pool, guard, &mut state))
.await.expect("mixed legacy heal must terminate")
.expect("equivalent legacy metadata should heal without downgrading the format floor");
drop(state);
let after = snapshots(&fixture.store).await;
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
for (pool_index, read) in reads.iter().enumerate() {
let expected_version = if pool_index == target_pool { POOL_META_VERSION } else { versions[pool_index] };
let expected_raw = if expected_version == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
assert!(matches!(&read.replica,
PoolMetaReplica::Valid { raw, canonical, revision, committed: true, .. }
if raw == expected_raw && canonical == &semantic_v2 && revision.version == expected_version
), "pool {pool_index}, first={first_version}, target={target_pool}: preserve equivalent payload and expected legacy version");
for (index, snapshot) in after[pool_index].iter().enumerate() {
codec::validate_inline_pool_meta_shard(snapshot.current.as_ref().expect("physical head"), index, expected_raw)
.await.expect("every physical shard must match its selected legacy payload");
}
if pool_index != target_pool {
assert_eq!(after[pool_index], before[pool_index], "healing one pool must not modify other pools");
}
}
}
}
},
).await;
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_heal_rejects_v1_and_v3_prepare_in_either_pool_order() {
temp_env::async_with_vars(
[
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
],
async {
for prepare_pool in [0, 1] {
let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
let (legacy_meta, legacy_canonical) = match decode_pool_meta_replica(raw_v1.clone()) {
PoolMetaReplica::Valid {
meta,
canonical,
revision,
committed: true,
..
} if revision.version == POOL_META_V1_VERSION => (meta, canonical),
_ => panic!("baseline V1 metadata must be committed and readable"),
};
let previous = PoolMetaCommittedCandidate {
canonical: legacy_canonical,
meta: legacy_meta.clone(),
revision: PoolMetaRevision::legacy(POOL_META_V1_VERSION),
};
let mut next = legacy_meta;
next.version = POOL_META_GENERATION_VERSION;
next.pools[0].last_update += Duration::seconds(1);
let revision = PoolMetaRevision {
version: POOL_META_GENERATION_VERSION,
cluster_id: fixture.revision.cluster_id,
epoch: fixture.revision.epoch,
generation: 1,
transaction_id: Some(Uuid::new_v4()),
};
let pending = encode_pool_meta_v3_envelope(&next, revision, false, Some(&previous))
.expect("V3 PREPARE with V1 predecessor must encode");
assert!(matches!(decode_pool_meta_replica(pending.clone()),
PoolMetaReplica::Valid { committed: false, revision: actual, .. } if actual == revision));
let heads = replicas(&pending, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
let physical = disks(&fixture.store).await;
for (index, disk) in physical[prepare_pool].iter().enumerate() {
fs::write(object_path(disk).join("xl.meta"), &heads[index])
.await
.expect("install task-owned V3 PREPARE evidence");
}
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
assert!(matches!(&reads[prepare_pool].replica,
PoolMetaReplica::Valid { raw, committed: false, revision: actual, .. }
if raw == &pending && *actual == revision));
let other_pool = 1 - prepare_pool;
assert!(matches!(&reads[other_pool].replica,
PoolMetaReplica::Valid { raw, committed: true, revision: actual, .. }
if raw == &raw_v1 && actual.version == POOL_META_V1_VERSION));
let before = snapshots(&fixture.store).await;
let identities = owner_repair_identity_heads(&fixture.store).await;
let guard = owner(&fixture.store).await;
let mut state = fixture.store.pool_meta_save_gate.lock().await;
let result = tokio::time::timeout(
std::time::Duration::from_secs(30),
fixture
.store
.heal_pool_meta_replica_under_fence(prepare_pool, guard, &mut state),
)
.await
.expect("mixed V1/V3 PREPARE heal must terminate");
drop(state);
assert!(
result.is_err(),
"pool {prepare_pool}: native heal must not overwrite a V3 PREPARE with legacy data"
);
assert_eq!(
snapshots(&fixture.store).await,
before,
"pool {prepare_pool}: rejection must retain all physical head and backup evidence"
);
assert_eq!(
owner_repair_identity_heads(&fixture.store).await,
identities,
"pool {prepare_pool}: rejection must not mutate identity evidence"
);
}
},
)
.await;
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_heal_rejects_v1_legacy_dont_save_true_without_writes() {
temp_env::async_with_vars(
[
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
],
async {
let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
let meta = match decode_pool_meta_replica(safe_v1.clone()) {
PoolMetaReplica::Valid {
meta, committed: true, ..
} => meta,
_ => panic!("baseline V1 metadata must decode"),
};
let statuses = meta
.pools
.iter()
.map(|pool| {
assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
LegacyPoolStatus {
id: pool.id,
cmd_line: pool.cmd_line.clone(),
last_update: pool.last_update,
decommission: None,
}
})
.collect();
let legacy = LegacyPoolMeta {
version: POOL_META_V1_VERSION,
pools: statuses,
dont_save: true,
};
let mut dangerous_raw = Vec::new();
dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
let old_reader: LegacyPoolMeta =
rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode source");
assert!(old_reader.dont_save, "the old reader would treat this source as permanently do-not-save");
assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
let source = match decode_pool_meta_replica(dangerous_raw.clone()) {
PoolMetaReplica::Valid {
canonical,
revision,
committed: true,
..
} => {
assert_eq!(revision.version, POOL_META_V1_VERSION);
canonical
}
_ => panic!("current reader must recognize the legacy source"),
};
assert!(matches!(decode_pool_meta_replica(safe_v1.clone()),
PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
if canonical == source && revision.version == POOL_META_V1_VERSION));
let physical = disks(&fixture.store).await;
let source_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
for (index, disk) in physical[0].iter().enumerate() {
fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
.await
.expect("install legacy source on pool zero");
}
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
let before = snapshots(&fixture.store).await;
let identities = owner_repair_identity_heads(&fixture.store).await;
let guard = owner(&fixture.store).await;
let mut state = fixture.store.pool_meta_save_gate.lock().await;
let result = tokio::time::timeout(
std::time::Duration::from_secs(30),
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
)
.await
.expect("unsafe legacy heal must terminate");
drop(state);
let error = result.expect_err("native heal must reject a source whose old-reader dont_save is true");
assert!(
error.to_string().contains("dont_save"),
"rejection must identify the unsafe legacy field: {error}"
);
assert_eq!(
snapshots(&fixture.store).await,
before,
"rejection must preserve every original head and backup"
);
assert_eq!(
owner_repair_identity_heads(&fixture.store).await,
identities,
"rejection must not write identity shards"
);
let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw));
assert!(
matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1),
"the dangerous source bytes must not be copied into the target pool"
);
},
)
.await;
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_heal_rejects_target_v1_dont_save_true_without_writes() {
temp_env::async_with_vars(
[
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
],
async {
let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
let meta = match decode_pool_meta_replica(safe_v1.clone()) {
PoolMetaReplica::Valid {
meta, committed: true, ..
} => meta,
_ => panic!("baseline V1 metadata must decode"),
};
let statuses = meta
.pools
.iter()
.map(|pool| {
assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
LegacyPoolStatus {
id: pool.id,
cmd_line: pool.cmd_line.clone(),
last_update: pool.last_update,
decommission: None,
}
})
.collect();
let legacy = LegacyPoolMeta {
version: POOL_META_V1_VERSION,
pools: statuses,
dont_save: true,
};
let mut dangerous_raw = Vec::new();
dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
let old_reader: LegacyPoolMeta =
rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode target");
assert!(old_reader.dont_save, "the old reader interprets the target as do-not-save");
assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
let source_canonical = match decode_pool_meta_replica(safe_v1.clone()) {
PoolMetaReplica::Valid {
canonical,
revision,
committed: true,
..
} => {
assert_eq!(revision.version, POOL_META_V1_VERSION);
canonical
}
_ => panic!("safe pool-zero source must decode"),
};
assert!(
matches!(decode_pool_meta_replica(dangerous_raw.clone()),
PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
if canonical == source_canonical && revision.version == POOL_META_V1_VERSION),
"the dangerous target must remain semantically equivalent to the safe source"
);
let physical = disks(&fixture.store).await;
let target_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
for (index, disk) in physical[1].iter().enumerate() {
fs::write(object_path(disk).join("xl.meta"), &target_heads[index])
.await
.expect("install legacy dangerous target head");
}
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
let before = snapshots(&fixture.store).await;
let identities = owner_repair_identity_heads(&fixture.store).await;
let guard = owner(&fixture.store).await;
let mut state = fixture.store.pool_meta_save_gate.lock().await;
let result = tokio::time::timeout(
std::time::Duration::from_secs(30),
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
)
.await
.expect("unsafe legacy target heal must terminate");
drop(state);
let error = result.expect_err("native heal must reject a target whose old-reader dont_save is true");
assert!(
error.to_string().contains("dont_save"),
"rejection must identify the unsafe legacy field: {error}"
);
assert_eq!(
snapshots(&fixture.store).await,
before,
"rejection must preserve all physical heads and backups"
);
assert_eq!(
owner_repair_identity_heads(&fixture.store).await,
identities,
"rejection must not write identity shards"
);
let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1));
assert!(
matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw),
"unsafe target evidence must not be overwritten by the safe source"
);
},
)
.await;
}
@@ -30,7 +30,6 @@ pub(super) struct NativeApplyPause {
#[cfg(test)]
tokio::task_local! {
pub(super) static AFTER_NATIVE_APPLY: Arc<NativeApplyPause>;
pub(super) static AFTER_NATIVE_POOL_APPLY: Arc<NativeApplyPause>;
}
struct InspectedPool {
@@ -169,7 +168,6 @@ async fn plan(
canonical: &[u8],
revision: PoolMetaRevision,
identity: &[u8],
rollback: Option<&[u8]>,
) -> Result<Vec<Vec<Bytes>>> {
let mut pending = Vec::<Vec<u8>>::new();
let mut predecessors = Vec::<Vec<u8>>::new();
@@ -225,83 +223,23 @@ async fn plan(
}
}
}
// The last ordinary PREPARE can itself become split during rollback. In
// that case another pool already exposes the restored ordinary COMMIT.
// Its authority permits continuing the rollback, never promoting a child
// transaction reconstructed from disk evidence.
let mut continuation: Option<(Vec<u8>, PoolMetaRevision)> = None;
if rollback.is_none() {
let mut outstanding = Vec::new();
for pool in inspected {
for (index, snapshot) in pool.snapshots.iter().enumerate() {
let raw = snapshot.current.as_ref().ok_or_else(invalid)?;
if codec::validate_inline_pool_meta_shard(raw, index, canonical).await.is_err() {
outstanding.push((index, raw));
}
}
}
for payload in &decoded {
if let PoolMetaReplica::Valid {
let mut obsolete = Vec::<Vec<u8>>::new();
for payload in &decoded {
match decode_pool_meta_replica(payload.clone()) {
PoolMetaReplica::Valid {
canonical: candidate,
revision: candidate_revision,
committed: false,
previous: Some(previous),
..
} = decode_pool_meta_replica(payload.clone())
&& previous.revision == revision
&& previous.canonical == canonical
{
let mut matches = true;
for (index, raw) in &outstanding {
matches &= codec::validate_inline_pool_meta_shard(raw, *index, payload).await.is_ok()
|| codec::validate_inline_pool_meta_shard(raw, *index, &candidate).await.is_ok();
} if candidate_revision == revision && candidate == canonical => {
if !pending.contains(payload) {
pending.push(payload.clone());
}
if matches {
if !outstanding.is_empty()
&& continuation.as_ref().is_some_and(|(selected, selected_revision)| {
selected != &candidate || *selected_revision != candidate_revision
})
{
return Err(known_validation(invalid(), "native_transaction_validation"));
}
continuation.get_or_insert((candidate, candidate_revision));
if !predecessors.contains(&previous.canonical) {
predecessors.push(previous.canonical);
}
}
}
}
let (canonical, revision, rollback) = match &continuation {
Some((transaction, next_revision)) => (transaction.as_slice(), *next_revision, Some(canonical)),
None => (canonical, revision, rollback),
};
for payload in &decoded {
if let PoolMetaReplica::Valid {
canonical: candidate,
revision: candidate_revision,
committed: false,
previous: Some(previous),
..
} = decode_pool_meta_replica(payload.clone())
&& candidate_revision == revision
&& candidate == canonical
{
pending.push(payload.clone());
if !predecessors.contains(&previous.canonical) {
predecessors.push(previous.canonical);
}
}
}
if pending.len() > 1 {
return Err(known_validation(
Error::other("native V3 prepare records disagree on the predecessor"),
"native_transaction_validation",
));
}
let mut obsolete = Vec::<Vec<u8>>::new();
for payload in &decoded {
if pending.contains(payload) {
continue;
}
match decode_pool_meta_replica(payload.clone()) {
PoolMetaReplica::Valid { revision: older, .. }
if older.is_generation_protocol()
&& older.cluster_id == revision.cluster_id
@@ -313,23 +251,6 @@ async fn plan(
// fully reconstructed and bound into the native CAS.
obsolete.push(payload.clone());
}
PoolMetaReplica::Valid {
revision: abandoned,
committed: false,
previous: Some(previous),
..
} if abandoned.is_generation_protocol()
&& abandoned.cluster_id == revision.cluster_id
&& abandoned.epoch == revision.epoch
&& abandoned.generation == revision.generation
&& abandoned.transaction_id != revision.transaction_id
&& predecessors.contains(&previous.canonical) =>
{
// A rollback retains the failed PREPARE as evidence. A later
// transaction may reuse that generation, but only a backup of
// an uncommitted sibling with the same predecessor is obsolete.
obsolete.push(payload.clone());
}
PoolMetaReplica::Valid {
canonical: candidate,
revision: candidate_revision,
@@ -351,7 +272,13 @@ async fn plan(
_ => {}
}
}
if pending.is_empty() || rollback.is_some_and(|previous| !predecessors.iter().any(|candidate| candidate == previous)) {
if pending.len() > 1 {
return Err(known_validation(
Error::other("native V3 prepare records disagree on the predecessor"),
"native_transaction_validation",
));
}
if pending.is_empty() {
return Err(invalid());
}
for payload in &decoded {
@@ -367,25 +294,13 @@ async fn plan(
for (index, snapshot) in pool.snapshots.iter().enumerate() {
let raw = snapshot.current.as_ref().ok_or_else(invalid)?;
let mut matched = false;
for payload in std::iter::once(canonical)
.chain(pending.iter().map(Vec::as_slice))
.chain(rollback)
{
for payload in std::iter::once(canonical).chain(pending.iter().map(Vec::as_slice)) {
matched |= codec::validate_inline_pool_meta_shard(raw, index, payload).await.is_ok();
}
if !matched {
return Err(invalid());
}
for backup in &snapshot.backups {
if rollback.is_some()
&& codec::validate_inline_pool_meta_shard(&backup.data, index, canonical)
.await
.is_ok()
{
// A retained COMMIT might have been authoritative before
// this interruption. Never roll it back from PREPARE alone.
return Err(invalid());
}
let mut matched = false;
for payload in std::iter::once(canonical).chain(
pending
@@ -408,7 +323,7 @@ async fn plan(
let mut replacements = Vec::with_capacity(inspected.len());
for pool in inspected {
replacements.push(
codec::encode_inline_pool_meta_replicas(&current(pool)?, rollback.unwrap_or(canonical), modified)
codec::encode_inline_pool_meta_replicas(&current(pool)?, canonical, modified)
.await
.map_err(|error| codec_validation(error, "native_metadata_validation"))?,
);
@@ -423,12 +338,10 @@ pub(super) async fn repair(
) -> Result<bool> {
fence.ensure_held()?;
let reads = read_pool_meta_replicas(store.pools.clone(), true).await;
let unreadable = reads
if !reads
.iter()
.enumerate()
.filter_map(|(index, read)| matches!(read.replica, PoolMetaReplica::Unreadable(_)).then_some(index))
.collect::<Vec<_>>();
if unreadable.is_empty() {
.any(|read| matches!(read.replica, PoolMetaReplica::Unreadable(_)))
{
return Ok(false);
}
state.ensure_write_safe("native pool metadata recovery")?;
@@ -447,9 +360,8 @@ pub(super) async fn repair(
{
return Err(invalid());
}
// Ordinary reads grant authority: a COMMIT selects its own state, while a
// PREPARE can select only its acknowledged predecessor. Reconstruction
// below proves integrity and never promotes an interrupted transaction.
// Only an ordinary, successfully read COMMIT grants authority. Rebuilding
// two erasure shards below proves integrity, never a commit decision.
let ordinary_commits = reads
.iter()
.filter_map(|read| match &read.replica {
@@ -463,22 +375,7 @@ pub(super) async fn repair(
_ => None,
})
.collect::<Vec<_>>();
let ordinary_prepares = reads
.iter()
.filter_map(|read| match &read.replica {
PoolMetaReplica::Valid {
canonical,
revision,
committed: false,
previous: Some(previous),
..
} if revision.is_generation_protocol() && previous.revision.is_generation_protocol() => {
Some((canonical.clone(), *revision, previous.canonical.clone()))
}
_ => None,
})
.collect::<Vec<_>>();
if ordinary_commits.is_empty() && ordinary_prepares.is_empty() {
if ordinary_commits.is_empty() {
return Err(invalid());
}
let selection = select_pool_meta_replica_reads(reads)
@@ -498,20 +395,10 @@ pub(super) async fn repair(
return Err(invalid());
}
let canonical = selection.canonical.as_ref().ok_or_else(invalid)?;
let mut rollback = None;
for (pending, revision, previous) in &ordinary_prepares {
if previous == canonical {
if rollback.is_some_and(|candidate| candidate != (pending, *revision)) {
return Err(latch_validation(state, known_validation(invalid(), "native_transaction_validation")));
}
rollback = Some((pending, *revision));
}
}
if !selection.revision.is_generation_protocol()
|| (rollback.is_none()
&& !ordinary_commits
.iter()
.any(|(raw, revision)| raw == canonical && *revision == selection.revision))
|| !ordinary_commits
.iter()
.any(|(raw, revision)| raw == canonical && *revision == selection.revision)
{
return Err(invalid());
}
@@ -558,18 +445,13 @@ pub(super) async fn repair(
.await
.map_err(|error| latch_validation(state, error))?;
let inspected = inspect(&store.pools, fence).await?;
let (transaction, revision) = rollback.unwrap_or((canonical, selection.revision));
let replacements = plan(&inspected, transaction, revision, &identity_bytes, rollback.map(|_| canonical.as_slice()))
let replacements = plan(&inspected, canonical, selection.revision, &identity_bytes)
.await
.map_err(|error| latch_validation(state, error))?;
fence.ensure_held()?;
let mut arm = state.arm_transaction();
arm.phase = Some("native_repair_apply");
let mut apply_order = inspected.iter().zip(replacements).enumerate().collect::<Vec<_>>();
// Preserve ordinary authority until all unreadable pools are repaired.
// Cancellation can then resume from a PREPARE or a restored COMMIT.
apply_order.sort_by_key(|(index, _)| !unreadable.contains(index));
for (_, (pool, replacements)) in apply_order {
for (pool, replacements) in inspected.iter().zip(replacements) {
for ((disk, snapshot), replacement) in pool.disks.iter().zip(&pool.snapshots).zip(replacements) {
fence.ensure_held()?;
let request = Request::Apply {
@@ -584,11 +466,6 @@ pub(super) async fn repair(
_ => return Err(invalid()),
}
}
#[cfg(test)]
if let Ok(pause) = AFTER_NATIVE_POOL_APPLY.try_with(Arc::clone) {
pause.reached.notify_one();
pause.resume.notified().await;
}
}
#[cfg(test)]
if let Ok(pause) = AFTER_NATIVE_APPLY.try_with(Arc::clone) {
@@ -750,7 +627,7 @@ mod tests {
#[tokio::test]
async fn pool_meta_native_plan_restores_physical_split_with_fresh_common_identity() {
let f = fixture().await;
let repaired = plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.unwrap();
let repaired = plan(&f.pools, &f.commit, f.revision, &f.identity).await.unwrap();
assert_eq!(repaired.len(), 4);
for pool in repaired {
assert_eq!(
@@ -771,7 +648,7 @@ mod tests {
let mut f = fixture().await;
let wrong = replicas(b"different identity", f.modified).await;
f.pools[0].snapshots[0].identity = Some(wrong[0].clone());
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
}
#[tokio::test]
@@ -794,10 +671,10 @@ mod tests {
.unwrap();
let wrong = replicas(&higher, f.modified + Duration::seconds(2)).await;
f.pools[0].snapshots[0].current = Some(wrong[0].clone());
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
f.pools[0].snapshots[0].current = Some(replicas(&f.pending, f.modified).await[0].clone());
f.pools[0].snapshots[0].backups[0].data = wrong[0].clone();
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
}
#[tokio::test]
@@ -840,17 +717,17 @@ mod tests {
}
}
assert!(older_pending != f.pending);
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_ok());
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_ok());
}
#[tokio::test]
async fn pool_meta_native_plan_rejects_missing_or_corrupt_evidence() {
let mut f = fixture().await;
let before = f.pools[1].snapshots[2].current.take();
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
f.pools[1].snapshots[2].current = before;
f.pools[1].snapshots[2].backups[0].data = Bytes::from_static(b"invalid metadata");
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
}
}
@@ -391,7 +391,7 @@ pub(super) async fn fixture_inline_pool_meta_replicas(payload: &[u8], modified:
writer.shutdown().await.expect("fixture bitrot writer should finish");
let mut info = base.clone();
info.erasure.index = *coding_index;
info.data = Some(writer.into_inline_data().expect("fixture writer should retain inline bytes"));
info.data = Some(Bytes::from(writer.into_inline_data().expect("fixture writer should retain inline bytes")));
let mut meta = FileMeta::new();
meta.add_version(info).expect("test inline FileInfo should serialize");
raw.push(Bytes::from(meta.marshal_msg().expect("test xl.meta should serialize")));
+29 -435
View File
@@ -94,6 +94,7 @@ use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::collections::{HashMap, HashSet};
use std::fmt::Display;
#[cfg(test)]
use std::io::Cursor;
use std::io::Write;
use std::path::PathBuf;
@@ -3719,45 +3720,33 @@ fn is_decommission_copy_quorum_deferral(err: &Error) -> bool {
data_movement::data_movement_stage_source_as::<DecommissionCopyQuorumFailure>(err).is_some()
}
/// A copy whose object namespace lock remained unavailable through its
/// in-place retries, retained as a typed deferral.
/// The storage error of a version copy that kept timing out on a namespace
/// lock through its in-place retries, kept typed so the entry failure policy
/// defers it.
#[derive(Debug, thiserror::Error)]
#[error("{0}")]
struct DecommissionCopyLockUnavailable(#[source] Error);
struct DecommissionCopyLockTimeout(#[source] Error);
fn is_decommission_copy_lock_unavailable_error(err: &Error, bucket: &str, object: &str) -> bool {
let source = data_movement::data_movement_stage_source(err).unwrap_or(err);
match source {
Error::Lock(rustfs_lock::LockError::Timeout { .. }) => true,
Error::NamespaceLockQuorumUnavailable {
mode: "read",
bucket: lock_bucket,
object: lock_object,
..
} => {
data_movement::data_movement_error_is_stage(err, "prepare_put_object")
&& lock_bucket == bucket
&& lock_object == object
}
_ => false,
}
fn is_decommission_copy_lock_timeout_error(err: &Error) -> bool {
let err = data_movement::data_movement_stage_source(err).unwrap_or(err);
matches!(err, Error::Lock(rustfs_lock::LockError::Timeout { .. }))
}
/// A version copy that could not acquire its namespace lock leaves the source
/// A version copy that timed out acquiring a namespace lock leaves the source
/// in place; the next attempt copies the version again or finds an equivalent
/// committed target. The lock is held by another mutation of the key or, when
/// this node restarted mid-copy, by its previous incarnation until that lock's
/// lease expires on the peers. Either clears by waiting, so the entry is
/// deferred instead of being counted as a permanent per-object failure.
fn decommission_copy_lock_unavailable_deferral(stage: &str, bucket: &str, object: &str, err: Error) -> Error {
fn decommission_copy_lock_timeout_deferral(stage: &str, bucket: &str, object: &str, err: Error) -> Error {
data_movement::data_movement_context_error(
format!("decommission {stage} could not acquire a namespace lock for {bucket}/{object}: {err}"),
DecommissionCopyLockUnavailable(err),
format!("decommission {stage} timed out on a namespace lock for {bucket}/{object}: {err}"),
DecommissionCopyLockTimeout(err),
)
}
fn is_decommission_copy_lock_unavailable_deferral(err: &Error) -> bool {
data_movement::data_movement_stage_source_as::<DecommissionCopyLockUnavailable>(err).is_some()
fn is_decommission_copy_lock_timeout_deferral(err: &Error) -> bool {
data_movement::data_movement_stage_source_as::<DecommissionCopyLockTimeout>(err).is_some()
}
/// The deferral of a version copy whose in-place retries ended on `error`, if
@@ -3767,8 +3756,8 @@ fn take_decommission_copy_deferral(stage: &str, bucket: &str, object: &str, erro
return Some(decommission_copy_quorum_deferral(stage, bucket, object, err));
}
error
.take_if(|err| is_decommission_copy_lock_unavailable_error(err, bucket, object))
.map(|err| decommission_copy_lock_unavailable_deferral(stage, bucket, object, err))
.take_if(|err| is_decommission_copy_lock_timeout_error(err))
.map(|err| decommission_copy_lock_timeout_deferral(stage, bucket, object, err))
}
fn resolve_decommission_entry_reload_result(result: Result<()>, bucket: &str, object_name: &str) -> Result<()> {
@@ -5833,13 +5822,13 @@ enum DecommissionEntryOutcome {
/// entry is done and must not be counted as a failure or escalated. A quorum
/// failure of the source cleanup is deferred too: the copied source is still
/// in place and the next attempt cleans it up again. So is a version copy that
/// lost drive quorum or could not acquire an object namespace lock through its in-place
/// lost drive quorum or timed out on a namespace lock through its in-place
/// retries: the source still holds the version and the next attempt copies it
/// again. Every other failure stays terminal.
fn resolve_decommission_entry_failure(err: Error) -> Result<DecommissionEntryOutcome> {
if is_decommission_source_cleanup_retryable_error(&err)
|| is_decommission_copy_quorum_deferral(&err)
|| is_decommission_copy_lock_unavailable_deferral(&err)
|| is_decommission_copy_lock_timeout_deferral(&err)
{
return Ok(DecommissionEntryOutcome::Deferred(err));
}
@@ -6838,47 +6827,11 @@ impl Drop for PoolMetaTransactionArm {
}
}
fn ensure_legacy_pool_meta_republish_safe(data: &[u8]) -> Result<()> {
if data.len() >= 4
&& LittleEndian::read_u16(&data[2..4]) == POOL_META_V1_VERSION
// The selection already validates the payload. The runtime-only
// flag can only appear in the legacy three-field sequence or map.
&& rmp::decode::read_array_len(&mut &data[4..])
.or_else(|_| rmp::decode::read_map_len(&mut &data[4..]))
.ok()
== Some(3)
&& let Ok(legacy) = rmp_serde::from_slice::<LegacyPoolMeta>(&data[4..])
&& legacy.dont_save
{
return Err(Error::other("pool metadata legacy source contains dont_save=true"));
}
Ok(())
}
// Ordinary legacy reads retain their compatibility behavior, but bytes that
// will be copied to another pool must not contain an unparsed extension.
fn ensure_legacy_pool_meta_payload_complete(data: &[u8]) -> Result<()> {
let payload = data
.get(4..)
.ok_or_else(|| Error::other("pool metadata legacy source is too short"))?;
let mut decoder = rmp_serde::Deserializer::new(Cursor::new(payload));
serde::de::IgnoredAny::deserialize(&mut decoder).map_err(|_| Error::other("pool metadata legacy source is invalid"))?;
let payload_len = u64::try_from(payload.len()).map_err(|_| Error::other("pool metadata legacy source is too large"))?;
if decoder.get_ref().position() != payload_len {
return Err(Error::other("pool metadata legacy source has trailing bytes"));
}
Ok(())
}
#[derive(Debug)]
struct PoolMetaSelection {
meta: PoolMeta,
revision: PoolMetaRevision,
canonical: Option<Vec<u8>>,
// Legacy semantic comparison canonicalizes to V2; a heal must instead
// preserve verified raw bytes at the highest observed format floor.
legacy_raw: Option<Vec<u8>>,
legacy_has_unsafe_flag: bool,
replica_state: PoolMetaReplicaState,
cas_tokens: Vec<PoolMetaCasToken>,
absent: bool,
@@ -7269,8 +7222,6 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
meta: PoolMeta::default(),
revision: PoolMetaRevision::legacy(0),
canonical: None,
legacy_raw: None,
legacy_has_unsafe_flag: false,
replica_state: PoolMetaReplicaState {
needs_repair: false,
repair_write_safe: true,
@@ -7325,8 +7276,6 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
meta: selected.meta.clone(),
revision: selected.revision,
canonical: Some(selected.canonical.clone()),
legacy_raw: None,
legacy_has_unsafe_flag: false,
replica_state: PoolMetaReplicaState {
needs_repair,
repair_write_safe,
@@ -7340,10 +7289,6 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
// Legacy V1/V2 has no durable generation. Pool zero remains the commit
// record; divergent backups without it are ambiguous and fail closed.
let legacy_has_unsafe_flag = committed
.iter()
.any(|(_, raw, _)| ensure_legacy_pool_meta_republish_safe(raw).is_err());
let mut floor_raw = None;
let mut selected: Option<(usize, Vec<u8>, PoolMetaCommittedCandidate)> = None;
for (idx, raw, candidate) in committed {
if let Some((selected_idx, selected_raw, selected_candidate)) = selected.as_ref() {
@@ -7357,36 +7302,19 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
}
} else {
needs_repair |= selected_raw != &raw;
if floor_raw.is_none()
&& candidate.revision.version == observed_version
&& raw
.get(2..4)
.is_some_and(|version| LittleEndian::read_u16(version) == observed_version)
{
floor_raw = Some(raw);
}
}
} else {
selected = Some((idx, raw, candidate));
}
}
let (_, selected_raw, mut selected) =
let (_, _, mut selected) =
selected.ok_or_else(|| Error::other("pool metadata recovery required: no valid legacy replica is available"))?;
if selected.revision.version == observed_version
&& selected_raw
.get(2..4)
.is_some_and(|version| LittleEndian::read_u16(version) == observed_version)
{
floor_raw = Some(selected_raw);
}
selected.meta.version = observed_version;
selected.revision.version = observed_version;
Ok(PoolMetaSelection {
meta: selected.meta,
revision: selected.revision,
canonical: Some(selected.canonical),
legacy_raw: floor_raw,
legacy_has_unsafe_flag,
replica_state: PoolMetaReplicaState {
needs_repair,
repair_write_safe,
@@ -8380,9 +8308,8 @@ where
})
});
let result = save_config_with_opts_and_metadata(pool, object, data, &opts).await;
let write_not_started = matches!(&result, Err(Error::Io(inner)) if inner.get_ref().is_some_and(|source| source.is::<crate::set_disk::SystemMetadataWriteNotStarted>()));
let may_have_mutated = !write_not_started && !matches!(&result, Err(Error::PreconditionFailed));
if matches!(&result, Err(Error::PreconditionFailed)) {
let may_have_mutated = !matches!(&result, Err(Error::PreconditionFailed));
if !may_have_mutated {
record_pool_meta_stale_write_rejection(phase);
}
let mut source = None;
@@ -8393,8 +8320,6 @@ where
source = Some(Arc::clone(&original));
if matches!(original.as_ref(), Error::PreconditionFailed) {
Err(Error::PreconditionFailed)
} else if write_not_started {
Err(Error::other(original))
} else {
Err(Error::other(pool_metadata_error(
crate::error::PoolMetadataFailure::TransactionUnknown,
@@ -14609,23 +14534,6 @@ impl ECStore {
.canonical
.as_ref()
.ok_or_else(|| Error::other("pool metadata heal has no committed source"))?;
if selection.generation_protocol_observed && !selection.revision.is_generation_protocol() {
return Err(Error::other("pool metadata heal requires recovery of the observed generation protocol"));
}
if selection.legacy_has_unsafe_flag {
return Err(Error::other("pool metadata legacy replica contains dont_save=true"));
}
let repair_data = if selection.revision.is_generation_protocol() {
canonical.clone()
} else {
let raw = selection
.legacy_raw
.clone()
.ok_or_else(|| Error::other("pool metadata heal cannot prove the legacy format floor"))?;
ensure_legacy_pool_meta_payload_complete(&raw)?;
raw
};
ensure_legacy_pool_meta_republish_safe(&repair_data)?;
let pool = self
.pools
.get(pool_index)
@@ -14636,10 +14544,10 @@ impl ECStore {
.get(pool_index)
.ok_or_else(|| Error::other("pool metadata heal has no target revision"))?;
let mut arm = state.arm_transaction();
save_pool_meta_object_cas(pool.clone(), POOL_META_NAME, repair_data.clone(), token, &fence, "native_heal", &mut arm)
save_pool_meta_object_cas(pool.clone(), POOL_META_NAME, canonical.clone(), token, &fence, "native_heal", &mut arm)
.await?;
let (confirmed, _) = read_config_no_lock_preserve_empty_with_metadata(pool, POOL_META_NAME).await?;
if confirmed != repair_data {
if confirmed != *canonical {
return Err(Error::other("pool metadata native heal readback diverged"));
}
fence.ensure_held()?;
@@ -28445,7 +28353,7 @@ mod tests {
"one version must commit before the next version times out"
);
assert!(
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_unavailable_deferral(err)),
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_timeout_deferral(err)),
"{outcome:?}"
);
assert_eq!(
@@ -28479,88 +28387,6 @@ mod tests {
assert_eq!(actual, b"written after versioning was enabled");
}
/// A target lookup that loses its read-lock quorum must retain the source
/// and resume an entry after an earlier version has already committed.
#[tokio::test]
#[serial_test::serial]
async fn decommission_defers_target_lookup_read_lock_quorum_and_resumes() {
use tokio::io::AsyncReadExt as _;
let fixture = NullVersionDecommission::new(NullVersionTop::NewerVersion).await;
let object = fixture.object;
let attempted = Arc::new(AtomicUsize::new(0));
let fault = DecommissionTestFaultGuard::install_with_error(
Arc::new({
let attempted = Arc::clone(&attempted);
move |stage, _, faulted, _, success| {
stage == "before_target_commit"
&& faulted == object
&& success
&& attempted.fetch_add(1, Ordering::SeqCst) > 0
}
}),
Some(Arc::new(|_, bucket, object| {
data_movement::data_movement_stage_error_for_test(
"decommission_object",
"prepare_put_object",
bucket,
object,
Error::NamespaceLockQuorumUnavailable {
mode: "read",
bucket: bucket.to_owned(),
object: object.to_owned(),
required: 1,
achieved: 0,
},
)
})),
);
let (outcome, _) = fixture.run_entry().await;
assert!(
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_unavailable_deferral(err)),
"{outcome:?}"
);
fixture.assert_source_retained().await;
assert_eq!(
fixture
.target_versions()
.await
.expect("first version committed")
.versions
.len(),
1
);
assert_eq!(
fixture.store.pool_meta.read().await.pools[0]
.decommission
.as_ref()
.expect("active decommission")
.items_decommission_failed,
0
);
drop(fault);
let (outcome, _) = fixture.run_entry().await;
assert!(matches!(outcome, DecommissionEntryOutcome::Complete), "{outcome:?}");
fixture.assert_moved().await;
let mut reader = fixture.store.pools[1]
.get_object_reader(
fixture.bucket,
fixture.object,
None,
HeaderMap::new(),
&ObjectOptions {
version_id: Some(fixture.seeded.versions[0].version_id.expect("newer version").to_string()),
..Default::default()
},
)
.await
.expect("the newer version must remain readable");
let mut actual = Vec::new();
reader.stream.read_to_end(&mut actual).await.expect("read the newer version");
assert_eq!(actual, b"written after versioning was enabled");
}
/// A namespace lock still held on the target key, as by a copy whose node
/// was killed mid-write, makes target selection fail with a typed
/// object-lock timeout. A real held lock, not an injected error, must defer
@@ -28603,7 +28429,7 @@ mod tests {
let (outcome, _) = fixture.run_entry().await;
assert!(
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_unavailable_deferral(err)),
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_timeout_deferral(err)),
"{outcome:?}"
);
assert_eq!(
@@ -29012,17 +28838,17 @@ mod tests {
timeout(),
);
for err in [timeout(), prepare] {
assert!(is_decommission_copy_lock_unavailable_error(&err, "bucket-a", "object-a"), "{err}");
assert!(is_decommission_copy_lock_timeout_error(&err), "{err}");
assert!(!is_decommission_copy_quorum_error(&err), "{err}");
let mut error = Some(err);
let deferred = take_decommission_copy_deferral("migrate_object", "bucket-a", "object-a", &mut error)
.expect("a lock timeout copy must be deferred");
assert!(error.is_none());
assert!(is_decommission_copy_lock_unavailable_deferral(&deferred));
assert!(is_decommission_copy_lock_timeout_deferral(&deferred));
assert!(!is_decommission_copy_quorum_deferral(&deferred));
let failure = data_movement::data_movement_stage_source_as::<DecommissionCopyLockUnavailable>(&deferred)
let failure = data_movement::data_movement_stage_source_as::<DecommissionCopyLockTimeout>(&deferred)
.expect("the deferred error must keep its lock-timeout origin");
assert!(is_decommission_copy_lock_unavailable_error(&failure.0, "bucket-a", "object-a"));
assert!(is_decommission_copy_lock_timeout_error(&failure.0));
assert!(matches!(
resolve_decommission_entry_failure(deferred),
Ok(DecommissionEntryOutcome::Deferred(_))
@@ -29047,43 +28873,6 @@ mod tests {
}
}
#[test]
fn decommission_copy_defers_only_the_matching_object_read_lock_quorum() {
let quorum = |mode, bucket: &str, object: &str| Error::NamespaceLockQuorumUnavailable {
mode,
bucket: bucket.to_owned(),
object: object.to_owned(),
required: 1,
achieved: 0,
};
let wrap = |stage, err| {
data_movement::data_movement_stage_error_for_test("decommission_object", stage, "bucket-a", "object-a", err)
};
let mut error = Some(wrap("prepare_put_object", quorum("read", "bucket-a", "object-a")));
let deferred = take_decommission_copy_deferral("migrate_object", "bucket-a", "object-a", &mut error)
.expect("matching read-lock quorum must defer");
assert!(error.is_none());
assert!(matches!(
resolve_decommission_entry_failure(deferred),
Ok(DecommissionEntryOutcome::Deferred(_))
));
for err in [
quorum("read", "bucket-a", "object-a"),
wrap("source_recheck", quorum("read", "bucket-a", "object-a")),
wrap("prepare_put_object", quorum("write", "bucket-a", "object-a")),
wrap(
"prepare_put_object",
quorum("exact_delete_capacity_reconciliation", "bucket-a", "object-a"),
),
wrap("prepare_put_object", quorum("read", "other-bucket", "object-a")),
wrap("prepare_put_object", quorum("read", "bucket-a", "other-object")),
] {
let mut error = Some(err);
assert!(take_decommission_copy_deferral("migrate_object", "bucket-a", "object-a", &mut error).is_none());
assert!(error.is_some(), "unrelated lock failures must stay terminal");
}
}
#[test]
fn decommission_target_capacity_error_accepts_wrapped_capacity_errors() {
let disk_full = Error::other(format!("decommission_object: put_object failed for bucket/object: {}", Error::DiskFull));
@@ -29562,53 +29351,6 @@ mod tests {
);
}
#[test]
fn legacy_republish_guard_checks_only_three_field_sequence_or_map() {
let frame = |payload: Vec<u8>| {
let mut data = Vec::new();
data.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
data.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
data.extend_from_slice(&payload);
data
};
let two_fields = PersistedPoolMetaV1 {
version: POOL_META_V1_VERSION,
pools: Vec::new(),
};
for payload in [rmp_serde::to_vec(&two_fields), rmp_serde::to_vec_named(&two_fields)] {
let data = frame(payload.expect("two-field V1 payload must serialize"));
assert!(matches!(decode_pool_meta_replica(data.clone()), PoolMetaReplica::Valid { .. }));
assert!(ensure_legacy_pool_meta_republish_safe(&data).is_ok());
assert!(ensure_legacy_pool_meta_payload_complete(&data).is_ok());
let mut trailing = data;
trailing.push(0xc0);
assert!(ensure_legacy_pool_meta_payload_complete(&trailing).is_err());
}
for dont_save in [false, true] {
let legacy = LegacyPoolMeta {
version: POOL_META_V1_VERSION,
pools: Vec::new(),
dont_save,
};
for payload in [rmp_serde::to_vec(&legacy), rmp_serde::to_vec_named(&legacy)] {
let data = frame(payload.expect("three-field V1 payload must serialize"));
assert!(matches!(decode_pool_meta_replica(data.clone()), PoolMetaReplica::Valid { .. }));
assert_eq!(ensure_legacy_pool_meta_republish_safe(&data).is_err(), dont_save);
assert!(ensure_legacy_pool_meta_payload_complete(&data).is_ok());
let mut trailing = data;
trailing.push(0xc0);
assert!(ensure_legacy_pool_meta_payload_complete(&trailing).is_err());
}
}
// This guard is only called for already-decoded replicas; malformed
// containers must fail selection rather than becoming repair sources.
let truncated = frame(vec![0x93, 0x01]);
assert!(matches!(
decode_pool_meta_replica(truncated),
PoolMetaReplica::Corrupt(_) | PoolMetaReplica::Incompatible(_)
));
}
#[test]
fn pool_meta_decode_supports_legacy_payload() {
let start_time = OffsetDateTime::now_utc();
@@ -30829,154 +30571,6 @@ mod pools_tests {
);
}
#[tokio::test]
async fn pool_meta_prewrite_quorum_rejection_allows_retry_after_disks_return() {
let (dirs, disks, pool) = crate::set_disk::hermetic_set_disks_isolated(4).await;
pool.disks.write().await[2..].fill(None);
let state = super::PoolMetaWriteState::default();
let data =
super::initialized_pool_meta_identity_for_test(uuid::Uuid::new_v4(), 1).expect("encode the deployment identity");
let mut transaction = state.arm_transaction();
save_pool_meta_object_cas(
pool.clone(),
POOL_META_IDENTITY_NAME,
data.clone(),
&PoolMetaCasToken::Missing,
&PoolMetaPersistenceFence::OfflineRecovery,
"identity_cas",
&mut transaction,
)
.await
.expect_err("two of four disks cannot satisfy the identity write quorum");
drop(transaction);
for dir in &dirs {
assert!(
!dir.path()
.join(crate::disk::RUSTFS_META_BUCKET)
.join(POOL_META_IDENTITY_NAME)
.exists(),
"the rejected write must not create an identity replica"
);
}
state
.ensure_write_safe("retry after disks return")
.expect("a rejected inline write must not poison the startup transaction");
let ordinary_error = crate::config::com::save_config_with_opts(
pool.clone(),
"ordinary-config",
data.clone(),
&ObjectOptions {
max_parity: true,
no_lock: true,
..Default::default()
},
)
.await
.expect_err("ordinary writes must still fail without quorum");
assert!(
matches!(ordinary_error, Error::InsufficientWriteQuorum(_, _)),
"ordinary callers must retain their existing quorum error"
);
*pool.disks.write().await = disks.into_iter().map(Some).collect();
let mut retry = state.arm_transaction();
save_pool_meta_object_cas(
pool.clone(),
POOL_META_IDENTITY_NAME,
data.clone(),
&PoolMetaCasToken::Missing,
&PoolMetaPersistenceFence::OfflineRecovery,
"identity_cas",
&mut retry,
)
.await
.expect("the same transaction must succeed when the disks return");
retry.disarm();
drop(retry);
assert_eq!(
crate::config::com::read_config(pool, POOL_META_IDENTITY_NAME)
.await
.expect("read back the committed identity"),
data
);
state
.ensure_write_safe("after successful retry")
.expect("retry must leave writes enabled");
}
#[tokio::test]
async fn pool_meta_prewrite_quorum_rejection_keeps_an_earlier_write_armed() {
let (_dirs, disks, pool) = crate::set_disk::hermetic_set_disks_isolated(4).await;
let state = super::PoolMetaWriteState::default();
let data =
super::initialized_pool_meta_identity_for_test(uuid::Uuid::new_v4(), 1).expect("encode the deployment identity");
let mut transaction = state.arm_transaction();
let written = save_pool_meta_object_cas(
pool.clone(),
POOL_META_IDENTITY_NAME,
data.clone(),
&PoolMetaCasToken::Missing,
&PoolMetaPersistenceFence::OfflineRecovery,
"prepare_cas",
&mut transaction,
)
.await
.expect("the earlier phase must really commit");
pool.disks.write().await[2..].fill(None);
save_pool_meta_object_cas(
pool.clone(),
POOL_META_IDENTITY_NAME,
data.clone(),
&PoolMetaCasToken::Existing(written.etag.expect("committed identity has an ETag")),
&PoolMetaPersistenceFence::OfflineRecovery,
"identity_cas",
&mut transaction,
)
.await
.expect_err("the next phase must fail without a write quorum");
drop(transaction);
*pool.disks.write().await = disks.into_iter().map(Some).collect();
assert_eq!(
crate::config::com::read_config(pool, POOL_META_IDENTITY_NAME)
.await
.expect("the earlier commit must survive the rejected write"),
data
);
let error = state
.ensure_write_safe("after a committed phase")
.expect_err("a later rejection cannot discharge the earlier write");
assert_eq!(
error.pool_metadata_failure().expect("retain typed uncertainty").kind,
crate::error::PoolMetadataFailure::TransactionUnknown
);
assert_eq!(error.pool_metadata_failure().expect("retain the earlier mutation").phase, "prepare_cas");
}
#[tokio::test]
async fn pool_meta_prewrite_noninline_failure_keeps_transaction_armed() {
let (_dirs, _disks, pool) = crate::set_disk::hermetic_set_disks_isolated(4).await;
pool.disks.write().await[2..].fill(None);
let state = super::PoolMetaWriteState::default();
let mut transaction = state.arm_transaction();
save_pool_meta_object_cas(
pool,
POOL_META_IDENTITY_NAME,
vec![0x51; 2 * 1024 * 1024],
&PoolMetaCasToken::Missing,
&PoolMetaPersistenceFence::OfflineRecovery,
"identity_cas",
&mut transaction,
)
.await
.expect_err("the noninline write must fail without quorum");
drop(transaction);
assert!(
state.ensure_write_safe("after writer setup").is_err(),
"a failure after disk writers were created must remain fail-closed"
);
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum IdentityRepairFault {
Healthy,
+2 -3
View File
@@ -317,9 +317,8 @@ impl Sets {
&self.ctx
}
/// Stop endpoint reconnection and wait for any in-flight pass before a test owns disk probes.
#[cfg(any(test, feature = "test-util"))]
pub async fn stop_endpoint_monitor_for_test(&self) {
#[cfg(test)]
pub(crate) async fn stop_endpoint_monitor_for_test(&self) {
if let Some(exit_signal) = &self.exit_signal {
let _ = exit_signal.send(());
// Wait for an in-flight reconnect pass as well as its receiver to exit.
+1 -11
View File
@@ -84,7 +84,7 @@ pub(crate) enum CrashPoint {
}
#[cfg(test)]
pub(crate) use armed::{arm, disarm, is_armed, should_crash_at};
pub(crate) use armed::{arm, disarm, should_crash_at};
/// Production no-op: the compiler inlines this to `false`, so armed crash points
/// exist only in `#[cfg(test)]` builds and never add a branch to real commits.
@@ -94,12 +94,6 @@ pub(crate) fn should_crash_at(_point: CrashPoint, _key: &str) -> bool {
false
}
#[cfg(not(test))]
#[inline(always)]
pub(crate) fn is_armed(_point: CrashPoint, _key: &str) -> bool {
false
}
#[cfg(test)]
mod armed {
use super::CrashPoint;
@@ -138,10 +132,6 @@ mod armed {
guard().retain(|(p, k)| !(*p == point && k == key));
}
pub(crate) fn is_armed(point: CrashPoint, key: &str) -> bool {
guard().iter().any(|(p, k)| *p == point && k == key)
}
/// Returns `true` (consuming the arm) iff a crash is armed for exactly this
/// `point` and `key`.
pub(crate) fn should_crash_at(point: CrashPoint, key: &str) -> bool {
@@ -31,8 +31,6 @@ use tokio::fs;
use tokio::time::Instant;
pub(crate) const BARRIER_DIR_ENV: &str = "RUSTFS_E2E_DATA_MOVEMENT_BARRIER_DIR";
/// Rebalance holds its admission fence; no version of this entry has moved.
pub(crate) const STAGE_AFTER_ENTRY_ADMISSION: &str = "after_entry_admission";
/// Every target part is staged and the target version is not yet published.
pub(crate) const STAGE_BEFORE_TARGET_COMMIT: &str = "before_target_commit";
/// The target commit is durable and the source preflight matched; the source

Some files were not shown because too many files have changed in this diff Show More