Compare commits

..

3 Commits

Author SHA1 Message Date
overtrue 4480d7efca ci: rename the workspace lint-and-build job key and evidence path
The job key test-and-lint named a job that runs no tests: it does Clippy, the
test-binary archive, doctests and the log-rule anchors, while the tests live in
workspace-tests. Anyone reading needs: [test-and-lint] would conclude tests had
already gated that dependency. The key becomes workspace-lint-and-build to
match its display name, and the shared evidence directory becomes
artifacts/workspace so the path stops asserting the same thing. The aggregate's
display name stays Test and Lint because that is the live ruleset check
context. ci-gates.md also stops attributing the ECStore test run to the wrong
job.
2026-10-31 17:25:11 +08:00
overtrue dcf63411f0 ci: gate the plan on a contract marker
A base planner predating the NIGHTLY_JOBS split still emits the old required
set, and the strict verifier would reject it. Select now stamps plan_contract=2
and the workflow refuses any plan without the current marker, falling back to
the full matrix, so a stale base planner can neither authorize the new narrower
selection nor force a job the new workflow no longer schedules.
2026-10-31 17:02:39 +08:00
overtrue 8adc1a3082 ci: move specialized lanes off the pull-request path
Connect Feature Boundaries, real io_uring and the rio-v2, Swift, SFTP and
HTTP/3 feature lanes no longer gate a merge. They run only in the scheduled
and manual full-validation runs, which report a regression within a day; on
the pull-request path they bought no earlier signal worth the runner
occupancy (Connect alone measured 53.5 median minutes and occasionally set
the wall). Workspace lint and tests, the short core contract, the debug
build, E2E smoke and the S3 lanes stay required on PRs and merge groups.
2026-10-31 16:43:48 +08:00
660 changed files with 30906 additions and 100384 deletions
+2 -2
View File
@@ -1,2 +1,2 @@
sha256-linux=348fa5eed9ae58962c06b33283ff4f15122cff97bce9b4eb1e3ffcb227264b0b
sha256-darwin=348fa5eed9ae58962c06b33283ff4f15122cff97bce9b4eb1e3ffcb227264b0b
sha256-linux=686dcc075e41a75dde3d37e5bd0393d016469f02f83aa3508f92bfc17f26a4d4
sha256-darwin=686dcc075e41a75dde3d37e5bd0393d016469f02f83aa3508f92bfc17f26a4d4
+2 -2
View File
@@ -1,2 +1,2 @@
sha256-darwin=219ac11e720b3f54dfee4d5d5e95171785ba159b2fe01db3037a8f5061c1ad32
sha256-linux=2d34698cc8ec52d5b6c37ac612919a1d21f36cd56a1af539a7522a4eea518e26
sha256-darwin=5d4509d69c3106c1b8a6c353d123440d6b4dbe5dd42612e42dcb864946beae6c
sha256-linux=414e1dae735e6d14821a992cebc09a545576e9a77e10a5e0ed671ab07b89df98
+2 -2
View File
@@ -1,2 +1,2 @@
sha256-darwin=e84e04208f19dc0596b10d4d1b799ce43d6cc431415902f9c5458aa8232bc2d8
sha256-linux=1f9c7add72c44d1c12ef4ac050bb7917431be657c0c90b29f70c3df56f8bbae3
sha256-darwin=3814cdde8cacf4187e18188d342ed70931cc32e268e42e5bb438d565bb4d0f81
sha256-linux=663025a80273579f95ae3f49af02c0afcbfce94df09701c360a13e607b7637db
+1 -1
View File
@@ -1 +1 @@
sha256=5bdd4c2eadbc41625ae3184fff34062e000d328108436259b2b7c4fe2531ecaf
sha256=e9dc896774122a304e2b8b90f28c82d4d4e0af8ded36a82b147b30945ec994e3
-2
View File
@@ -28,7 +28,6 @@ script-tests: ## Run shell script tests
./scripts/test_build_rustfs_options.sh
./scripts/test_docker_runtime_timezone.sh
./scripts/test_entrypoint_credentials.sh
./scripts/test_entrypoint_directories.sh
./scripts/test_internode_grpc_ab_bench.sh
./scripts/test_object_batch_bench_enhanced.sh
./scripts/test_package_service_scripts.sh
@@ -71,7 +70,6 @@ script-tests: ## Run shell script tests
$(RUSTFS_PYTHON_BIN) ./scripts/test_nightly_candidate.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_functional_chain.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_functional_chain_health.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_repro_rejoin_readiness_docker.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_validation_registry.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_inspect_validation_runner.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_retain_validation_evidence.py
+6 -12
View File
@@ -110,7 +110,7 @@ threads-required = "num-test-threads"
# Reserve capacity so sibling disk-heavy tests cannot consume its 2s
# registration budget; retain the queued backup and deadlock assertions.
[[profile.default.overrides]]
filter = 'package(rustfs) & test(=admin::handlers::table_catalog::tests::cold_object_registration_completes_while_other_bucket_backup_waits)'
filter = 'package(rustfs) & test(=admin::handlers::table_catalog::tests::cold_strong_registration_completes_while_other_bucket_backup_waits)'
threads-required = "num-test-threads"
[[profile.default.overrides]]
@@ -234,7 +234,7 @@ test-group = 'embedded-test-ports'
# processes cannot exhaust those budgets. Keep deadlines and assertions intact.
# The short HTTP fixtures are tracked in rustfs/backlog#2714.
[[profile.default.overrides]]
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|cleanup_accepts_missing_object_and_still_deletes_the_bucket|cleanup_diagnostics_identify_each_disconnected_transport_stage|cleanup_diagnostics_identify_each_failed_http_stage_without_private_context|cleanup_diagnostics_identify_timeouts_without_resetting_the_shared_deadline|oversized_response_is_rejected_without_buffering_it)$/)))'
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))'
threads-required = "num-test-threads"
# This cancellation probe hashes the serving executable before acquiring its
@@ -410,7 +410,7 @@ threads-required = "num-test-threads"
# Reserve capacity so sibling disk-heavy tests cannot consume its 2s
# registration budget; retain the queued backup and deadlock assertions.
[[profile.ci.overrides]]
filter = 'package(rustfs) & test(=admin::handlers::table_catalog::tests::cold_object_registration_completes_while_other_bucket_backup_waits)'
filter = 'package(rustfs) & test(=admin::handlers::table_catalog::tests::cold_strong_registration_completes_while_other_bucket_backup_waits)'
threads-required = "num-test-threads"
[[profile.ci.overrides]]
@@ -548,7 +548,7 @@ filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnost
test-group = 'embedded-test-ports'
[[profile.ci.overrides]]
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|cleanup_accepts_missing_object_and_still_deletes_the_bucket|cleanup_diagnostics_identify_each_disconnected_transport_stage|cleanup_diagnostics_identify_each_failed_http_stage_without_private_context|cleanup_diagnostics_identify_timeouts_without_resetting_the_shared_deadline|oversized_response_is_rejected_without_buffering_it)$/)))'
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))'
threads-required = "num-test-threads"
# Preserve the cancellation probe's startup budget and assertions in CI too.
@@ -701,7 +701,7 @@ test-group = 'ecstore-serial-flaky'
[profile.e2e-smoke]
default-filter = """
package(e2e_test) & (
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|multipart_upload_id|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat|data_usage)_test::|^fake_s3_target::/)
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat|data_usage)_test::|^fake_s3_target::/)
| test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/)
| test(/^reliant::lifecycle::/)
| test(/^conditional_delete_objects_test::/)
@@ -827,7 +827,7 @@ slow-timeout = { period = "120s", terminate-after = 12, grace-period = "10s" }
# ---------------------------------------------------------------------------
# e2e-distributed profile — 4-node 4-disk Actions suite
# ---------------------------------------------------------------------------
# Nightly and manual-dispatch lane owned by
# Storage-sensitive PR / nightly / dispatch lane owned by
# .github/workflows/e2e-distributed.yml.
# Each case starts four rustfs processes (and for site replication, two
# clusters). Upgrade cases also require RUSTFS_UPGRADE_SOURCE_BINARY.
@@ -976,12 +976,6 @@ test-group = 'e2e-vault'
filter = 'package(e2e_test) & (test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter)$/) | test(=reliant::tiering::test_hermetic_transition_restore_failure_expiry_and_retry))'
threads-required = "num-test-threads"
# The mc fixture uploads hundreds of objects concurrently before measuring listing.
# Reserve the runner so other server fixtures do not starve its disk operations.
[[profile.e2e-full.overrides]]
filter = 'package(e2e_test) & test(=mc_mirror_small_bucket_test::test_mc_mirror_small_bucket_completes_without_list_timeout)'
threads-required = "num-test-threads"
# Fixed, small contract for every executable server change. These existing
# scenarios exercise write/read, anonymous denial and retained bytes on restart.
[profile.core-contract]
-77
View File
@@ -1,77 +0,0 @@
{
"schema_version": 1,
"plan_id": "rustfs-release-validation",
"description": "Versioned certification plan for every public RustFS release exit (rustfs/backlog#2662, R01). preview, prerelease and stable all require this explicit plan. Scripts/release/release_gate.py fail-closes publication on any missing, duplicate, stale, substituted or non-PASS required item.",
"release_types": ["preview", "prerelease", "stable"],
"targets": [
{"os": "linux", "arch": "x86_64", "abi": "musl"},
{"os": "linux", "arch": "aarch64", "abi": "musl"},
{"os": "linux", "arch": "x86_64", "abi": "gnu"},
{"os": "linux", "arch": "aarch64", "abi": "gnu"},
{"os": "macos", "arch": "aarch64", "abi": "none"},
{"os": "windows", "arch": "x86_64", "abi": "msvc"}
],
"targets_basis": "Platform matrix of .github/workflows/build.yml (prepare-platform-matrix). gnu and macOS targets build with the pyroscope feature; musl and Windows targets build with no extra features.",
"features": ["pyroscope"],
"trusted_workflows": [
{"repository": "rustfs/s3chaos", "workflow_id": "release-gate.yml"}
],
"trusted_workflows_basis": "The shared evidence envelope producer lives in rustfs/s3chaos (G01, fault/release_gate.rs) and its plan/live split workflow is .github/workflows/release-gate.yml there. Only results from these identities may enter a release certification; PR-uploaded JSON never gains release authority. R02 must extend this list when additional verifier workflows land, never widen it to untrusted sources.",
"lanes": [
{
"lane": "identity",
"harness_repository": "rustfs/s3chaos",
"harness_sha": "9a48f55c1d6d58ce63a8fb1ea7317e024f32f11d",
"harness_basis": "Fixed G01 evidence-envelope producer revision (rustfs/s3chaos PR #102 head recorded in rustfs/backlog#2658). R02 re-pins this when the live verifier ships.",
"timeout_seconds": 1800,
"environment_capabilities": ["process-exec"],
"cases": ["artifact-identity", "startup"],
"applies_to": {
"release_types": ["preview", "prerelease", "stable"],
"platforms": "all"
},
"applies_basis": "Artifact identity and startup checks are mandatory for every release type, preview included, on every shipped target (rustfs/backlog#2662 frozen design item 5)."
},
{
"lane": "upgrade",
"harness_repository": "rustfs/s3chaos",
"harness_sha": "9a48f55c1d6d58ce63a8fb1ea7317e024f32f11d",
"harness_basis": "Same fixed envelope producer as the identity lane; the workflow SHA and the product build SHA are bound separately and need not be equal.",
"timeout_seconds": 5400,
"environment_capabilities": ["process-exec", "previous-release-artifact"],
"cases": [
"direct-upgrade",
"mixed-version-upgrade",
"bucket-config-upgrade",
"bucket-config-rollback",
"multipart-layout-upgrade",
"multipart-layout-baseline"
],
"cases_basis": "Case IDs of the pinned previous-release matrix in .github/workflows/e2e-upgrade.yml (source.version/asset/sha256 and the case list mapped there).",
"applies_to": {
"release_types": ["prerelease", "stable"],
"platforms": [{"os": "linux", "arch": "x86_64", "abi": "gnu"}]
},
"applies_basis": "preview releases use the pre-declared smaller set (identity lane only). The upgrade lane is only exercised on linux-x86_64-gnu, the platform of the pinned source assets."
}
],
"upgrade_sources": {
"basis": "Exactly the two previous-release sources pinned in .github/workflows/e2e-upgrade.yml are exercised today (source.version / source.asset / source.sha256). No other N-1/N-2 compatibility is claimed, and these digests are not inferred from tags.",
"sources": [
{
"version": "1.0.1",
"role": "N-1",
"artifact_name": "rustfs-linux-x86_64-gnu-v1.0.1.zip",
"artifact_sha256": "9742e183cb0dfb0d48fa909797af572fe47b7f827569036baa4156e0d3958ee5"
},
{
"version": "1.0.0-rc.5",
"role": "N-2",
"artifact_name": "rustfs-linux-x86_64-gnu-v1.0.0-rc.5.zip",
"artifact_sha256": "3ee8df71e8edcfada533be452c4135868f697bc515460ae97b027313eade7a3d"
}
],
"rollback_supported": false,
"rollback_basis": "Only the bucket-config-rollback case (rollback_to_previous_release_reads_current_bucket_metadata) is exercised. Full data-plane rollback is not certified and is never a default claim."
}
}
+3 -33
View File
@@ -22,9 +22,9 @@ The following files in the project root are essential for Docker operations:
| File | Description | Usage |
| :--- | :--- | :--- |
| **`docker-buildx.sh`** | **Multi-Arch Build Script**<br>Automates building and pushing Docker images for `amd64` and `arm64`. Supports release and dev channels. | `./docker-buildx.sh --push` |
| **`Dockerfile`** | **Production Image (Alpine)**<br>Lightweight image using musl libc. Downloads verified RustFS and `rc` binaries from GitHub Releases and bundles `/etc/rustfs/bootstrap`. | `docker build -t rustfs:latest .` |
| **`Dockerfile.glibc`** | **Production Image (Ubuntu)**<br>Standard image using glibc. Includes the bundled bootstrap configuration. | `docker build -f Dockerfile.glibc .` |
| **`Dockerfile.source`** | **Development Image**<br>Builds RustFS from source code and bundles `rc` and the default bootstrap configuration. | `docker build -f Dockerfile.source .` |
| **`Dockerfile`** | **Production Image (Alpine)**<br>Lightweight image using musl libc. Downloads pre-built binaries from GitHub Releases. | `docker build -t rustfs:latest .` |
| **`Dockerfile.glibc`** | **Production Image (Ubuntu)**<br>Standard image using glibc. Useful if you need specific dynamic libraries. | `docker build -f Dockerfile.glibc .` |
| **`Dockerfile.source`** | **Development Image**<br>Builds RustFS from source code. Includes build tools. Ideal for local development and CI. | `docker build -f Dockerfile.source .` |
### Docker Compose Configurations
@@ -110,36 +110,6 @@ docker compose up -d
---
## Docker Compose environment values
Use map syntax when quoting environment values in YAML:
```yaml
services:
rustfs:
image: rustfs/rustfs:latest
ports:
- "9000:9000"
- "9001:9001"
volumes:
- ./data:/data
- ./logs:/var/log/rustfs
environment:
RUSTFS_ADDRESS: ":9000"
RUSTFS_CONSOLE_ADDRESS: ":9001"
RUSTFS_CONSOLE_ENABLE: "true"
RUSTFS_OBS_LOGGER_LEVEL: error
RUSTFS_OBS_LOG_DIRECTORY: "/var/log/rustfs/"
RUSTFS_ACCESS_KEY: "${RUSTFS_ACCESS_KEY:?Set RUSTFS_ACCESS_KEY}"
RUSTFS_SECRET_KEY: "${RUSTFS_SECRET_KEY:?Set RUSTFS_SECRET_KEY}"
```
Create the host directories and grant the container user write access before starting this example; official images run as UID/GID `10001:10001`. Set both credential variables to your own values, then run `docker compose config` to inspect the resolved configuration locally before `docker compose up -d`. The resolved output includes credentials, so do not share it publicly.
With list syntax, use `- RUSTFS_OBS_LOG_DIRECTORY=/var/log/rustfs/` or quote the whole item as `- "RUSTFS_OBS_LOG_DIRECTORY=/var/log/rustfs/"`. Writing `- RUSTFS_OBS_LOG_DIRECTORY="/var/log/rustfs/"` passes the inner quotes literally, making this a relative path beginning with `"`, which can fail with `mkdir: cannot create directory '"': Permission denied`. This also applies to addresses and credentials: fix every affected list item, rather than only the log directory. The entrypoint preserves configured values, including literal quotes in valid paths and secrets.
---
## 🔧 Common Operations
### Cleaning Up
@@ -28,15 +28,6 @@ processors:
check_interval: 1s
limit_mib: 1024
spike_limit_mib: 256
transform/metrics:
error_mode: ignore
metric_statements:
# RustFS process metrics already expose process_pid as a metric label.
# Remove only the duplicate datapoint label and keep the shared resource
# process.pid available to sibling metrics and lock histograms.
- context: datapoint
statements:
- delete_key(attributes, "process_pid") where IsMatch(metric.name, "^rustfs_system_process_")
transform/logs:
error_mode: ignore
log_statements:
@@ -121,7 +112,7 @@ service:
exporters: [ otlp/tempo, otlp/jaeger ]
metrics:
receivers: [ otlp ]
processors: [ transform/metrics, memory_limiter, batch ]
processors: [ batch ]
exporters: [ prometheus ]
logs:
receivers: [ otlp ]
-40
View File
@@ -1,40 +0,0 @@
# syntax=docker/dockerfile:1.6
# Copyright 2026 RustFS Team
# Licensed under the Apache License, Version 2.0.
# A compatible native runtime with Python 3.11+, getent and stat is required by
# the isolated fixture's staging/provenance checks. This is a test image.
ARG RUNTIME_BASE=python:3.13-slim-trixie
FROM ${RUNTIME_BASE}
ARG SOURCE_REVISION
ARG BINARY_SHA256
ARG BUILD_PROFILE=dev
COPY rustfs /usr/bin/rustfs
RUN python3 - "$SOURCE_REVISION" "$BINARY_SHA256" "$BUILD_PROFILE" <<'PY'
import hashlib
import os
import re
import shutil
import sys
source, digest, profile = sys.argv[1:]
if not re.fullmatch(r"[0-9a-f]{40}", source):
raise SystemExit("invalid source revision")
if not re.fullmatch(r"[0-9a-f]{64}", digest):
raise SystemExit("invalid binary digest")
if profile not in ("dev", "release"):
raise SystemExit("invalid build profile")
with open("/usr/bin/rustfs", "rb") as binary:
if hashlib.file_digest(binary, "sha256").hexdigest() != digest:
raise SystemExit("binary digest mismatch")
for tool in ("getent", "stat"):
if shutil.which(tool) is None:
raise SystemExit(f"missing runtime tool: {tool}")
os.chmod("/usr/bin/rustfs", 0o755)
PY
LABEL io.rustfs.repro.source_revision="${SOURCE_REVISION}" \
io.rustfs.repro.binary_sha256="${BINARY_SHA256}" \
io.rustfs.repro.build_profile="${BUILD_PROFILE}"
USER 10001:10001
ENTRYPOINT ["/usr/bin/rustfs"]
-2
View File
@@ -32,8 +32,6 @@ updates:
- "overtrue"
- "majinghe"
ignore:
- dependency-name: "pulsar"
versions: [ "6.9.x" ]
- dependency-name: "object_store"
versions: [ "0.13.x" ]
- dependency-name: "libunftp"
+5 -28
View File
@@ -296,32 +296,6 @@
"origin": "baseline",
"alert_contract": "scheduled-validation"
},
{
"workflow": ".github/workflows/rustfs-validation-sweep.yml",
"name": "RustFS Validation Sweep",
"roles": [
"periodic",
"manual"
],
"events": [
"schedule",
"workflow_dispatch"
],
"cadence": "nightly",
"timezone": "UTC",
"schedule": [
"30 19 * * *"
],
"owner": "@overtrue",
"policy": "active",
"monitor": "workflow",
"max_age_hours": 60,
"result_owner": "rustfs-validation-sweep.yml",
"disposition": "canonical validation cases executed in rotating batches",
"failure_owner": "workflow",
"origin": "baseline",
"alert_contract": "scheduled-validation"
},
{
"workflow": ".github/workflows/btf-prototype-reference.yml",
"name": "BTF prototype reference",
@@ -769,10 +743,12 @@
"workflow": ".github/workflows/e2e-distributed.yml",
"name": "e2e-distributed",
"roles": [
"pr",
"periodic",
"manual"
],
"events": [
"pull_request",
"workflow_dispatch",
"schedule"
],
@@ -847,11 +823,12 @@
"workflow": ".github/workflows/e2e-upgrade.yml",
"name": "Upgrade Compatibility",
"roles": [
"pr",
"periodic",
"manual",
"release"
"manual"
],
"events": [
"pull_request",
"push",
"schedule",
"workflow_dispatch"
+5 -311
View File
@@ -814,18 +814,13 @@ jobs:
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
export AWS_DEFAULT_REGION="auto"
# Determine upload path based on build type. Release and prerelease
# builds only stage their bytes here: the official
# artifacts/rustfs/release/ path is written solely by the
# certification-gated promote-release-artifacts job, from these same
# staged objects. Development builds keep the uncertified dev channel
# and never touch official or latest channels.
# Determine upload path based on build type
if [[ "$BUILD_TYPE" == "development" ]]; then
R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/dev/"
echo "📤 Uploading development build to R2 dev directory"
else
R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
echo "📤 Staging release build for certification-gated promotion"
R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/release/"
echo "📤 Uploading release build to R2 release directory"
fi
# Upload all rustfs zip files to R2 using S3-compatible API
@@ -874,221 +869,6 @@ jobs:
echo "This run does not trigger Docker publishing."
fi
# Aggregate the parent candidate batch. This job always collects results and
# retains its evidence, including on failure, and every publication job
# below requires its success. Missing, stale, substituted or non-PASS
# evidence fails closed here: only the trusted verification sources
# (rustfs/s3chaos release-gate envelopes, rustfs/backlog#2658) can satisfy
# it, so a PR-uploaded JSON set never gains release authority.
release-certification:
name: Release Certification
needs: [ build-check, build-rustfs ]
if: ${{ !cancelled() && needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') }}
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
timeout-minutes: 30
permissions:
contents: read
actions: read
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Download all build artifacts
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
path: ./artifacts
pattern: rustfs-*
merge-multiple: true
- name: Write candidate and collect trusted validation results
env:
# Cross-repo read of the trusted verification run. Falls back to the
# workflow token, which cannot read rustfs/s3chaos; a missing token
# therefore fail-closes instead of certifying.
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
PRODUCT_COMMIT: ${{ github.sha }}
VERSION: ${{ needs.build-check.outputs.version }}
BUILD_TYPE: ${{ needs.build-check.outputs.build_type }}
FEATURES: ${{ needs.build-check.outputs.build_type == 'development' && '' || 'pyroscope' }}
shell: bash
run: |
set -euo pipefail
mkdir -p results release-gate-output
PLAN_SHA256=$(python3 -c "import hashlib,json,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('.config/release-validation-plan.json'))))")
python3 - "$PLAN_SHA256" > candidate.json <<'PY'
import hashlib, json, os, sys, zipfile
plan_sha256 = sys.argv[1]
profile = {"release": "stable", "prerelease": "prerelease", "preview": "preview"}.get(os.environ["BUILD_TYPE"])
if profile is None:
raise SystemExit("release certification only covers preview, prerelease and stable builds")
def sha256(path):
digest = hashlib.sha256()
with open(path, "rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
deliverables = []
for name in sorted(os.listdir("artifacts")):
if not name.startswith("rustfs-") or not name.endswith(".zip"):
continue
path = os.path.join("artifacts", name)
members = zipfile.ZipFile(path).namelist()
binary_name = "rustfs.exe" if "rustfs.exe" in members else "rustfs"
with zipfile.ZipFile(path) as archive:
binary_bytes = archive.read(binary_name)
import tempfile
with tempfile.NamedTemporaryFile() as handle:
handle.write(binary_bytes)
handle.flush()
binary_sha = sha256(handle.name)
deliverables.append({
"kind": "zip",
"name": name,
"location": "https://github.com/rustfs/rustfs/releases/download/{}/{}".format(os.environ["VERSION"], name),
"sha256": sha256(path),
"size": os.path.getsize(path),
"binary_sha256": binary_sha,
})
if not deliverables:
raise SystemExit("no built deliverables found to certify")
candidate = {
"schema_version": 1,
"product_commit": os.environ["PRODUCT_COMMIT"],
"version": os.environ["VERSION"],
"release_profile": profile,
"features": [f for f in os.environ.get("FEATURES", "").split(",") if f],
"plan_sha256": plan_sha256,
"validation_batch": {"run_id": os.environ["GITHUB_RUN_ID"], "run_attempt": int(os.environ["GITHUB_RUN_ATTEMPT"])},
"deliverables": deliverables,
}
json.dump(candidate, sys.stdout, indent=2, sort_keys=True)
print()
PY
# Trusted verification evidence only: the release-gate envelopes of
# rustfs/s3chaos for this exact product commit. Absence blocks below.
TRUSTED_REPO="rustfs/s3chaos"
TRUSTED_WORKFLOW="release-gate.yml"
TRUSTED_RUN=$(gh api "repos/${TRUSTED_REPO}/actions/workflows/${TRUSTED_WORKFLOW}/runs?head_sha=${PRODUCT_COMMIT}&status=success&per_page=1" --jq '.workflow_runs[0].id // empty' || true)
if [[ -z "$TRUSTED_RUN" ]]; then
echo "❌ No trusted verification run of ${TRUSTED_REPO}/${TRUSTED_WORKFLOW} for ${PRODUCT_COMMIT}; refusing to certify" >&2
else
gh run download "$TRUSTED_RUN" -R "$TRUSTED_REPO" -n release-validation-evidence -D results || \
echo "❌ Trusted verification run ${TRUSTED_RUN} published no release-validation-evidence artifact" >&2
fi
python3 scripts/release/release_gate.py validate \
--plan .config/release-validation-plan.json \
--candidate candidate.json \
--results results \
--output release-gate-output
- name: Upload certification evidence
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: release-certification
path: |
candidate.json
results/
release-gate-output/
if-no-files-found: warn
retention-days: 30
# The only writer of the official R2 release path. Promotion re-checks the
# exact bytes about to be published against the passed certification list
# and copies the staged objects byte-for-byte: no rebuild-after-verify.
promote-release-artifacts:
name: Promote Release Artifacts
needs: [ build-check, build-rustfs, release-certification ]
# Target-channel lock for the official R2 release path.
concurrency:
group: release-channel-r2-release
cancel-in-progress: false
if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
runs-on: ubuntu-latest
timeout-minutes: 30
permissions:
contents: read
actions: read
env:
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
R2_BUCKET: ${{ secrets.R2_BUCKET }}
AWS_EC2_METADATA_DISABLED: true
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Download build and certification artifacts
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
path: ./artifacts
pattern: rustfs-*
merge-multiple: true
- name: Download certification evidence
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: release-certification
path: ./certification
- name: Verify and promote staged release artifacts
shell: bash
run: |
set -euo pipefail
if [[ -z "$R2_ACCESS_KEY_ID" || -z "$R2_SECRET_ACCESS_KEY" || -z "$R2_ENDPOINT" || -z "$R2_BUCKET" ]]; then
echo "❌ R2 credentials missing; refusing to promote to the official channel" >&2
exit 1
fi
for zip_file in artifacts/rustfs-*.zip; do
[[ -f "$zip_file" ]] || continue
name="$(basename "$zip_file")"
python3 scripts/release/release_gate.py verify-promotion \
--plan .config/release-validation-plan.json \
--candidate certification/candidate.json \
--summary certification/release-gate-output \
--artifact "$zip_file" \
--artifact-name "$name"
done
if ! command -v aws >/dev/null 2>&1; then
sudo apt-get update && sudo apt-get install -y awscli
fi
export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
export AWS_DEFAULT_REGION="auto"
STAGING_PATH="s3://${R2_BUCKET}/artifacts/rustfs/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
RELEASE_PATH="s3://${R2_BUCKET}/artifacts/rustfs/release/"
for zip_file in artifacts/rustfs-*.zip; do
[[ -f "$zip_file" ]] || continue
name="$(basename "$zip_file")"
staged="$(mktemp)"
aws s3 cp "${STAGING_PATH}${name}" "$staged" --endpoint-url "$R2_ENDPOINT" --only-show-errors
staged_sha="$(sha256sum "$staged" | awk '{print $1}')"
built_sha="$(sha256sum "$zip_file" | awk '{print $1}')"
if [[ "$staged_sha" != "$built_sha" ]]; then
echo "❌ Staged object ${name} does not match the verified bytes" >&2
exit 1
fi
aws s3 cp "$staged" "${RELEASE_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
rm -f "$staged"
echo "✅ Promoted ${name} (${built_sha})"
done
# Create GitHub Release for every valid release tag, including previews
create-release:
name: Create GitHub Release
@@ -1248,10 +1028,6 @@ jobs:
update-latest-version:
name: Update Latest Version
needs: [ build-check, publish-release ]
# Target-channel lock for the latest.json version pointer.
concurrency:
group: release-channel-oss-latest
cancel-in-progress: false
if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
runs-on: ubuntu-latest
timeout-minutes: 30
@@ -1292,45 +1068,6 @@ jobs:
rm -rf "$OSSUTIL_DIR" "$OSSUTIL_ZIP"
chmod +x "$OSSUTIL_BIN"
# Candidate/pointer ordering: an older candidate must never overwrite
# a newer pointer, even if its promotion job finishes later.
CURRENT_LATEST="${RUNNER_TEMP}/current-latest.json"
if "$OSSUTIL_BIN" cp oss://rustfs-version/latest.json "$CURRENT_LATEST" 2>/dev/null; then
python3 - "$CURRENT_LATEST" "$VERSION" <<'PY'
import json
import re
import sys
def key(version):
match = re.match(r"^(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?$", version)
if not match:
raise SystemExit(3)
core = tuple(int(part) for part in match.groups()[:3])
prerelease = match.group(4)
if prerelease is None:
return core + (1,)
return core + (0, tuple(int(p) if p.isdigit() else p for p in re.split(r"[.-]", prerelease)))
try:
current = json.load(open(sys.argv[1], encoding="utf-8")).get("version", "")
proposed = sys.argv[2]
if current and key(current) > key(proposed):
raise SystemExit(1)
except SystemExit:
raise
except (OSError, ValueError, TypeError):
raise SystemExit(3)
PY
guard_status=$?
if [[ "$guard_status" -eq 1 ]]; then
echo "Current latest.json points at a newer version; refusing to downgrade" >&2
exit 0
elif [[ "$guard_status" -ne 0 ]]; then
echo "Existing latest.json is unparsable; refusing to overwrite it blindly" >&2
exit 1
fi
fi
# Create latest.json
cat > latest.json << EOF
{
@@ -1347,59 +1084,16 @@ jobs:
echo "✅ Updated latest.json for ${RELEASE_TYPE} release $VERSION"
# Publish release (remove draft status). Requires the release-certification
# aggregate: without a complete legal success there is no public write.
# Publish release (remove draft status)
publish-release:
name: Publish Release
needs: [ build-check, create-release, upload-release-assets, release-certification ]
# Target-channel lock: concurrent releases serialize on this channel so an
# older candidate cannot interleave its publish with a newer one.
concurrency:
group: release-channel-github-release
cancel-in-progress: false
needs: [ build-check, create-release, upload-release-assets ]
if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'preview' || needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
timeout-minutes: 30
permissions:
contents: write
actions: read
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Download certification evidence
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: release-certification
path: ./certification
- name: Verify release assets against certification
env:
GH_TOKEN: ${{ github.token }}
shell: bash
run: |
set -euo pipefail
TAG="${{ needs.build-check.outputs.version }}"
mkdir -p "$RUNNER_TEMP/promotion-assets"
gh release download "$TAG" --dir "$RUNNER_TEMP/promotion-assets" --pattern '*.zip'
found=0
for zip_file in "$RUNNER_TEMP"/promotion-assets/*.zip; do
[[ -f "$zip_file" ]] || continue
found=1
python3 scripts/release/release_gate.py verify-promotion \
--plan .config/release-validation-plan.json \
--candidate certification/candidate.json \
--summary certification/release-gate-output \
--artifact "$zip_file" \
--artifact-name "$(basename "$zip_file")"
done
if [[ "$found" -ne 1 ]]; then
echo "No release zip assets to verify against the certification" >&2
exit 1
fi
- name: Publish release
env:
GH_TOKEN: ${{ github.token }}
+2 -62
View File
@@ -113,7 +113,7 @@ jobs:
printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
echo "Base CI policy unavailable; running the full matrix."
fi
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=3$' "$GITHUB_OUTPUT"; then
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=2$' "$GITHUB_OUTPUT"; then
printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'rehearse_shards=false' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
echo "Base policy does not cover this workflow contract; retaining the full matrix."
fi
@@ -215,8 +215,7 @@ jobs:
- name: Archive workspace test binaries
env:
# Workspace test compilation has high per-process memory use.
CARGO_BUILD_JOBS: "2"
CARGO_BUILD_JOBS: ${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && '3' || '2' }}
run: |
./scripts/ci/resource_sampler.sh start workspace-archive
trap './scripts/ci/resource_sampler.sh stop' EXIT
@@ -498,63 +497,6 @@ jobs:
- name: Check offline enrollment E2E root boundary
run: ./scripts/check_offline_enrollment_e2e.sh
fips-provider:
name: FIPS provider boundaries
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
needs: [ quick-checks, classify-changes ]
# This builds the RustFS library test target, which exceeds hosted-runner disk capacity.
runs-on: sm-standard-4
timeout-minutes: 90
strategy:
fail-fast: false
matrix:
mode: [default, fips]
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
CARGO_BUILD_JOBS: "2"
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Setup Rust environment
uses: ./.github/actions/setup
with:
rust-version: stable
cache-shared-key: ci-fips-provider
cache-save-if: 'false'
install-build-packaging-tools: 'false'
- name: Install FIPS build tools
if: matrix.mode == 'fips'
run: sudo apt-get install -y cmake golang clang libclang-dev
- name: Verify provider installation
shell: bash
run: |
args=()
if [[ '${{ matrix.mode }}' == 'fips' ]]; then
args+=(--features fips)
fi
cargo nextest run --locked -p rustfs-tls-runtime --lib "${args[@]}" --no-tests fail -E 'test(provider_policy_isolated)'
cargo nextest run --locked -p rustfs-s3-client --lib "${args[@]}" --no-tests fail -E 'test(overflowing_closed_range_is_rejected_before_network_io)'
cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(validate_pulsar)'
cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(webhook_provider_policy_isolated)'
if [[ '${{ matrix.mode }}' == 'fips' ]]; then
cargo nextest run --locked -p rustfs-protocols --lib --features ftps,webdav,fips --no-tests fail -E 'test(protocols_reject_incompatible_host_provider)'
cargo nextest run --locked -p rustfs-targets --lib --features fips --no-tests fail -E 'test(fips_postgres_rejects_incompatible_host_provider)'
cargo nextest run --locked -p rustfs-s3-client --lib --features fips --no-tests fail -E 'test(fips_transition_rejects_incompatible_host_provider)'
fi
tests=(provider_installation_isolated prepared_generation_shares_crypto_and_serves_sni_with_mtls)
if [[ '${{ matrix.mode }}' == 'fips' ]]; then
tests+=(fips_provider_builds_fips_tls_configs fips_provider_tls_handshake_accepts_and_rejects_algorithms)
fi
for test_name in "${tests[@]}"; do
cargo nextest run --locked -p rustfs --lib "${args[@]}" --no-tests fail -E "test($test_name)"
done
cargo clippy --locked -p rustfs --lib "${args[@]}" -- -D warnings
test-and-lint-rio-v2:
name: Test and Lint (rio-v2)
# Specialized boundary, hardware and alternate-feature lanes run only on
@@ -1455,7 +1397,6 @@ jobs:
- workspace-shard-evidence
- core-contract
- connect-feature-boundaries
- fips-provider
- test-and-lint-rio-v2
- test-and-lint-protocols
- build-rustfs-debug-binary
@@ -1483,7 +1424,6 @@ jobs:
needs:
- classify-changes
- connect-feature-boundaries
- fips-provider
- required-checks
- typos
- quick-checks
+1 -1
View File
@@ -95,7 +95,7 @@ jobs:
- name: Run CLA Bot
if: github.event_name != 'merge_group'
uses: overtrue/cla-bot@4bd1387c19691168c42fcf5edfa17b15819b2ab3 # v0.1.1
uses: overtrue/cla-bot@5e5929846a60cfb42b16d0d38e135ef67d94fc88 # v0.0.9
with:
github-token: ${{ github.token }}
registry-token: ${{ steps.registry-token.outputs.token }}
+2 -182
View File
@@ -329,75 +329,13 @@ jobs:
echo " - Create latest: $create_latest"
echo " - Source ref: $source_ref"
# Consume the trusted release certification for the exact version this
# workflow wraps and verify the certified input digests. A manual dispatch
# without that certification fails here and pushes no official tag; only
# results from the trusted verification sources carry release authority.
release-certification:
name: Release Certification
needs: build-check
if: needs.build-check.outputs.should_build == 'true'
runs-on: ubuntu-latest
timeout-minutes: 30
permissions:
contents: read
actions: read
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
ref: ${{ needs.build-check.outputs.source_ref }}
- name: Download and verify release certification
env:
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
VERSION: ${{ needs.build-check.outputs.version }}
shell: bash
run: |
set -euo pipefail
mkdir -p certification
run_id="$TRIGGERING_RUN_ID"
if [[ -z "$run_id" ]]; then
run_id="$(gh api --method GET \
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
-f branch="$VERSION" -f status=success -F per_page=1 \
--jq '.workflow_runs[0].id // empty' || true)"
fi
if [[ -z "$run_id" ]]; then
echo "No certified Build and Release run found for ${VERSION}; failing closed" >&2
exit 1
fi
if ! gh run download "$run_id" -n release-certification -D certification; then
echo "Build run ${run_id} published no release-certification artifact; failing closed" >&2
exit 1
fi
python3 scripts/release/release_gate.py verify-promotion \
--plan .config/release-validation-plan.json \
--candidate certification/candidate.json \
--summary certification/release-gate-output \
--artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
--artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
# Input digest binding: every zip this image wraps must be the
# certified byte-for-byte content, never a mutable latest pointer.
while IFS=$'\t' read -r name location expected; do
echo "Verifying certified input ${name}"
actual="$(curl -fsSL "$location" | sha256sum | awk '{print $1}')"
if [[ "$actual" != "$expected" ]]; then
echo "Certified input ${name} does not match its recorded digest" >&2
exit 1
fi
done < <(jq -r '.deliverables[] | select(.name | contains("linux")) | [.name, .location, .sha256] | @tsv' certification/candidate.json)
# Build multi-arch Docker images
# Strategy: Build images using pre-built binaries from dl.rustfs.com
# Supports both release and dev channel binaries based on build context
# Only runs when should_build is true (which includes workflow success check)
build-docker:
name: Build Docker Images
needs: [ build-check, release-certification ]
needs: build-check
if: needs.build-check.outputs.should_build == 'true'
runs-on: ubuntu-latest
timeout-minutes: 60
@@ -505,14 +443,6 @@ jobs:
# Output tags
echo "tags=$TAGS" >> "$GITHUB_OUTPUT"
# Staging identity for this run's promotion: official tags are only
# created after the staged image verifies against the certification,
# and they point at this same digest (no rebuild-after-verify).
STAGING_TAG="staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${VARIANT_SUFFIX}"
STAGING_TAGS="${{ env.REGISTRY_DOCKERHUB }}:${STAGING_TAG},${{ env.REGISTRY_GHCR }}:${STAGING_TAG},${{ env.REGISTRY_QUAY }}:${STAGING_TAG}"
echo "staging_tags=$STAGING_TAGS" >> "$GITHUB_OUTPUT"
echo "staging_tag=$STAGING_TAG" >> "$GITHUB_OUTPUT"
# Generate labels
LABELS="org.opencontainers.image.title=RustFS"
LABELS="$LABELS,org.opencontainers.image.description=RustFS distributed object storage system"
@@ -537,9 +467,7 @@ jobs:
file: ${{ matrix.file }}
platforms: ${{ env.DOCKER_PLATFORMS }}
push: ${{ needs.build-check.outputs.should_push == 'true' }}
# Push to staging tags only: official tags are created by the
# certification-gated promotion step below from this same digest.
tags: ${{ steps.meta.outputs.staging_tags }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
# No layer cache. This build compiles nothing — it downloads a
# release zip and runs apk/apt — so the cache could only save the
@@ -565,114 +493,6 @@ jobs:
no-cache: false
pull: true
# A later-built wrapper: the staged image must complete this platform's
# own binary identity verification before any official tag is written.
# It may not inherit an unverified package's conclusion.
- name: Verify staged image against certification
if: needs.build-check.outputs.should_push == 'true'
env:
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
VERSION: ${{ needs.build-check.outputs.version }}
shell: bash
run: |
set -euo pipefail
mkdir -p certification
run_id="$TRIGGERING_RUN_ID"
if [[ -z "$run_id" ]]; then
run_id="$(gh api --method GET \
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
-f branch="$VERSION" -f status=success -F per_page=1 \
--jq '.workflow_runs[0].id // empty' || true)"
fi
gh run download "$run_id" -n release-certification -D certification
STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
case "${{ matrix.variant }}" in
musl) libc="musl" ;;
glibc) libc="gnu" ;;
esac
for platform in linux/amd64 linux/arm64; do
case "$platform" in
linux/amd64) arch_sub="x86_64" ;;
linux/arm64) arch_sub="aarch64" ;;
esac
docker pull --platform "$platform" "$STAGING_REF"
container="$(docker create --platform "$platform" "$STAGING_REF")"
docker cp "$container:/usr/bin/rustfs" "$RUNNER_TEMP/staged-rustfs"
docker rm "$container" >/dev/null
actual="$(sha256sum "$RUNNER_TEMP/staged-rustfs" | awk '{print $1}')"
expected="$(jq -r --arg needle "linux-${arch_sub}-${libc}" \
'[.deliverables[] | select(.name | contains($needle)) | .binary_sha256][0] // empty' \
certification/candidate.json)"
if [[ -z "$expected" || "$actual" != "$expected" ]]; then
echo "Staged image binary for ${platform} does not match the certified binary digest" >&2
exit 1
fi
python3 scripts/release/release_gate.py verify-promotion \
--plan .config/release-validation-plan.json \
--candidate certification/candidate.json \
--summary certification/release-gate-output \
--artifact "$RUNNER_TEMP/staged-rustfs" \
--artifact-role binary
done
- name: Promote staged image to official tags
if: needs.build-check.outputs.should_push == 'true'
shell: bash
run: |
set -euo pipefail
STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
IFS=',' read -r -a official_tags <<< "${{ steps.meta.outputs.tags }}"
for tag in "${official_tags[@]}"; do
docker buildx imagetools create -t "$tag" "$STAGING_REF"
echo "Promoted $tag -> $STAGING_REF"
done
# Downstream consumers (Helm) verify this record instead of inheriting
# an unverified wrapper's conclusion.
- name: Record image promotion
if: needs.build-check.outputs.should_push == 'true'
shell: bash
env:
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
VERSION: ${{ needs.build-check.outputs.version }}
run: |
set -euo pipefail
mkdir -p certification
run_id="$TRIGGERING_RUN_ID"
if [[ -z "$run_id" ]]; then
run_id="$(gh api --method GET \
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
-f branch="$VERSION" -f status=success -F per_page=1 \
--jq '.workflow_runs[0].id // empty' || true)"
fi
gh run download "$run_id" -n release-certification -D certification
STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
digest="$(docker buildx imagetools inspect "$STAGING_REF" --format '{{json .Manifest}}' | jq -r '.digest')"
candidate_sha="$(python3 -c "import json,hashlib,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('certification/candidate.json'))))")"
jq -n \
--arg version "$VERSION" \
--arg digest "$digest" \
--arg staging_ref "$STAGING_REF" \
--arg candidate_sha256 "$candidate_sha" \
--arg tags "${{ steps.meta.outputs.tags }}" \
--slurpfile candidate certification/candidate.json \
'{schema_version: 1, version: $version, image_digest: $digest, staging_ref: $staging_ref,
promoted_tags: ($tags | split(",")), candidate_sha256: $candidate_sha256,
binary_sha256s: ($candidate[0].deliverables | map(.binary_sha256) | unique)}' \
> image-promotion-record.json
- name: Upload image promotion record
if: needs.build-check.outputs.should_push == 'true'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: image-promotion-record${{ matrix.suffix }}
path: image-promotion-record.json
if-no-files-found: error
retention-days: 30
# Note: Manifest creation is no longer needed as we only build one variant
# Multi-arch manifests are automatically created by docker/build-push-action
+28 -2
View File
@@ -17,8 +17,8 @@
# Each selected test starts a real localhost cluster via
# `RustFSTestClusterEnvironment` (4 processes; 4 drives per node unless the
# case is a two-site 4-node 1-drive pair or a 4-node upgrade). Membership is
# `[profile.e2e-distributed]` in `.config/nextest.toml`. The nightly schedule
# and manual dispatches execute the same fail-closed suite.
# `[profile.e2e-distributed]` in `.config/nextest.toml`. Storage-sensitive PRs,
# nightly runs, and manual dispatches all execute the same fail-closed suite.
# Upgrade cases download the same pinned previous release as e2e-upgrade.yml.
#
# Isolated pool filesystems: expand/decommission/rebalance cases require
@@ -33,6 +33,32 @@
name: e2e-distributed
on:
pull_request:
paths:
- "Cargo.lock"
- "Cargo.toml"
- ".config/nextest.toml"
- ".config/e2e-distributed-selection.txt"
- ".github/workflows/e2e-distributed.yml"
- "crates/audit/**"
- "crates/common/**"
- "crates/config/**"
- "crates/e2e_test/**"
- "crates/ecstore/**"
- "crates/filemeta/**"
- "crates/heal/**"
- "crates/iam/**"
- "crates/lock/**"
- "crates/madmin/**"
- "crates/notify/**"
- "crates/replication/**"
- "crates/s3-client/**"
- "crates/s3-ops/**"
- "crates/s3-types/**"
- "crates/scanner/**"
- "crates/storage-api/**"
- "crates/utils/**"
- "rustfs/**"
workflow_dispatch:
inputs:
filter:
+18 -7
View File
@@ -12,19 +12,30 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# Single-node upgrade and rollback lane.
#
# The weekly Monday schedule, version tags, and workflow_dispatch run this
# suite. Distributed upgrade cases stay on the nightly e2e-distributed lane.
name: Upgrade Compatibility
on:
pull_request:
paths:
- ".github/workflows/e2e-upgrade.yml"
- "crates/e2e_test/src/common.rs"
- "crates/e2e_test/src/distributed/**"
- "crates/e2e_test/src/fake_s3_target/**"
- "crates/e2e_test/src/lib.rs"
- "crates/e2e_test/src/replication_extension_test.rs"
- "crates/e2e_test/src/rpc_fault_proxy.rs"
- "crates/e2e_test/src/upgrade_compatibility_test.rs"
- "crates/ecstore/**"
- "crates/filemeta/**"
- "crates/heal/**"
- "crates/kms/**"
- "crates/storage-api/**"
- "rustfs/**"
- "Cargo.lock"
push:
tags:
- "[0-9]*.[0-9]*.[0-9]*"
schedule:
# 03:17 UTC every Monday.
- cron: "17 3 * * 1"
workflow_dispatch:
@@ -33,7 +44,7 @@ permissions:
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: false
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
env:
CARGO_TERM_COLOR: always
-93
View File
@@ -30,102 +30,9 @@ permissions:
contents: read
jobs:
# Consume the same trusted release certification as every other publication
# exit and verify the image promotion record of the Docker workflow. The
# chart wraps no binary of its own, so it may only reference an image whose
# platform verification already passed; it must not inherit an unverified
# package's conclusion. A run without that evidence fails closed here and
# never reaches the chart repository.
release-certification:
runs-on: ubuntu-latest
timeout-minutes: 30
permissions:
contents: read
actions: read
outputs:
version: ${{ steps.version.outputs.raw_tag }}
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Normalize release version
id: version
env:
RAW_INPUT: ${{ github.event.inputs.version }}
RAW_BRANCH: ${{ github.event.workflow_run.head_branch }}
run: |
set -euo pipefail
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
RAW="$RAW_INPUT"
else
RAW="$RAW_BRANCH"
fi
case "$RAW" in
refs/tags/*) RAW_TAG="${RAW#refs/tags/}" ;;
*) RAW_TAG="$RAW" ;;
esac
echo "raw_tag=$RAW_TAG" >> "$GITHUB_OUTPUT"
- name: Download and verify release certification and image record
env:
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
RAW_TAG: ${{ steps.version.outputs.raw_tag }}
shell: bash
run: |
set -euo pipefail
mkdir -p certification image-record
build_run="$TRIGGERING_RUN_ID"
if [[ -z "$build_run" ]]; then
build_run="$(gh api --method GET \
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
-f branch="$RAW_TAG" -f status=success -F per_page=1 \
--jq '.workflow_runs[0].id // empty' || true)"
fi
if [[ -z "$build_run" ]]; then
echo "No certified Build and Release run found for ${RAW_TAG}; failing closed" >&2
exit 1
fi
if ! gh run download "$build_run" -n release-certification -D certification; then
echo "Build run ${build_run} published no release-certification artifact; failing closed" >&2
exit 1
fi
python3 scripts/release/release_gate.py verify-promotion \
--plan .config/release-validation-plan.json \
--candidate certification/candidate.json \
--summary certification/release-gate-output \
--artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
--artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
docker_run="$(gh api --method GET \
"repos/${GITHUB_REPOSITORY}/actions/workflows/docker.yml/runs" \
-f branch="$RAW_TAG" -f status=success -F per_page=1 \
--jq '.workflow_runs[0].id // empty' || true)"
if [[ -z "$docker_run" ]] || ! gh run download "$docker_run" -n image-promotion-record -D image-record; then
echo "No image promotion record for ${RAW_TAG}; a chart may not wrap an unverified image" >&2
exit 1
fi
candidate_sha="$(python3 -c "import json,hashlib,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('certification/candidate.json'))))")"
record_sha="$(jq -r '.candidate_sha256 // empty' image-record/image-promotion-record.json)"
if [[ "$record_sha" != "$candidate_sha" ]]; then
echo "Image promotion record does not belong to this certification" >&2
exit 1
fi
overlap="$(jq -r --slurpfile candidate certification/candidate.json \
'[.binary_sha256s[]] - ($candidate[0].deliverables | map(.binary_sha256)) | length' \
image-record/image-promotion-record.json)"
if [[ "$overlap" -ne 0 ]]; then
echo "Image promotion record carries binary digests outside the certification" >&2
exit 1
fi
build-helm-package:
runs-on: ubuntu-latest
timeout-minutes: 30
needs: [ release-certification ]
if: |
(github.event_name == 'workflow_dispatch' && !contains(github.event.inputs.version, '-preview')) ||
(
+7 -279
View File
@@ -87,206 +87,18 @@ jobs:
install-build-packaging-tools: 'false'
install-test-tools: 'false'
- name: Download static console assets
shell: bash
env:
GITHUB_TOKEN: ${{ github.token }}
run: |
# Packaging tools are off on this lane, so setup does not install unzip.
if ! command -v unzip >/dev/null 2>&1; then
sudo apt-get update -qq
sudo apt-get install -y -qq unzip
fi
if ! command -v unzip >/dev/null 2>&1; then
echo "unzip is required to extract console assets" >&2
exit 1
fi
mkdir -p ./rustfs/static
verify_sha256() {
local expected="$1"
local file="$2"
local actual=""
local actual_line=""
if command -v sha256sum >/dev/null 2>&1; then
actual_line=$(sha256sum -- "$file") || return 1
elif command -v shasum >/dev/null 2>&1; then
actual_line=$(shasum -a 256 -- "$file") || return 1
else
echo "No SHA-256 verification tool found" >&2
return 1
fi
actual="${actual_line%%[[:space:]]*}"
if [ "$actual" = "$expected" ]; then
echo "SHA256 verified OK: $actual"
return 0
else
echo "SHA256 mismatch: expected=$expected actual=$actual" >&2
return 1
fi
}
download_console_assets() {
local curl_bin="curl"
local python_bin="python3"
local console_api="https://api.github.com/repos/rustfs/console/releases/latest"
local console_json="console-release.json"
local console_url
local console_sha256
local curl_auth_args=()
console_tag=""
chmod +w ./rustfs/static/LICENSE || true
if ! command -v "$python_bin" >/dev/null 2>&1; then
python_bin="python"
fi
if ! command -v "$python_bin" >/dev/null 2>&1; then
echo "No Python interpreter found for release metadata parsing" >&2
return 1
fi
if [[ -n "${GITHUB_TOKEN:-}" ]]; then
curl_auth_args=(-H "Authorization: Bearer ${GITHUB_TOKEN}" -H "X-GitHub-Api-Version: 2022-11-28")
fi
"$curl_bin" "${curl_auth_args[@]}" --fail -L "$console_api" \
-o "$console_json" --retry 3 --retry-delay 5 --max-time 300 || return 1
read -r console_tag console_url console_sha256 < <("$python_bin" - "$console_json" <<'PY'
import json
import re
import sys
with open(sys.argv[1], encoding="utf-8") as handle:
release = json.load(handle)
tag = release.get("tag_name", "") or "unknown"
for asset in release.get("assets", []):
name = asset.get("name", "")
digest = asset.get("digest", "")
url = asset.get("browser_download_url", "")
if name.startswith("rustfs-console-") and name.endswith(".zip") and digest.startswith("sha256:") and url:
sha256 = digest.split(":", 1)[1]
if not re.fullmatch(r"[0-9a-fA-F]{64}", sha256):
raise SystemExit(f"console zip asset has invalid sha256 digest: {sha256}")
sys.stdout.buffer.write(f"{tag} {url} {sha256}\n".encode("utf-8"))
break
else:
raise SystemExit("no console zip asset with sha256 digest found")
PY
) || return 1
if [[ -z "$console_url" || -z "$console_sha256" ]]; then
echo "Console release metadata is missing URL or SHA-256 digest" >&2
return 1
fi
echo "Console release: ${console_tag}"
echo "Downloading console asset: ${console_url}"
"$curl_bin" --fail -L "$console_url" -o console.zip --retry 3 --retry-delay 5 --max-time 300 || return 1
verify_sha256 "$console_sha256" console.zip || return 2
unzip -o console.zip -d ./rustfs/static || return 2
}
if download_console_assets; then
rm -f console.zip console-release.json
else
status=$?
rm -f console.zip console-release.json
if [[ "$status" -eq 2 ]]; then
echo "Console asset integrity verification failed" >&2
fi
echo "Failed to download verified console assets" >&2
exit 1
fi
if [[ ! -s ./rustfs/static/index.html ]]; then
echo "Console asset archive is missing static/index.html" >&2
exit 1
fi
asset_count=$(find ./rustfs/static -type f | wc -l | tr -d '[:space:]')
echo "Console assets ready: version=${console_tag:-unknown}, ${asset_count} files extracted to ./rustfs/static"
- name: Build RustFS
# Pin the glibc baseline (`<target>.<glibc>` for cargo-zigbuild) so
# nightlies run on older distros such as RHEL/Rocky 9 (glibc 2.34); #8062.
shell: bash
run: |
# Force rebuild by touching build.rs so a restored cache cannot
# reuse a rustfs artifact compiled before these console assets existed.
touch rustfs/build.rs
cargo zigbuild --release --locked --target x86_64-unknown-linux-gnu.2.31 -p rustfs --bin rustfs
run: cargo zigbuild --release --locked --target x86_64-unknown-linux-gnu.2.31 -p rustfs --bins
- name: Verify glibc symbol baseline
run: |
python3 scripts/release/check_glibc.py 2.31 \
target/x86_64-unknown-linux-gnu/release/rustfs
target/x86_64-unknown-linux-gnu/release/rustfs \
target/x86_64-unknown-linux-gnu/release/rustfs-cli
target/x86_64-unknown-linux-gnu/release/rustfs --version | grep -Fx 'build target : x86_64-unknown-linux-gnu'
- name: Verify console
shell: bash
run: |
set -euo pipefail
find_free_port() {
python3 - <<'PY'
import socket
with socket.socket() as sock:
sock.bind(("127.0.0.1", 0))
print(sock.getsockname()[1])
PY
}
binary="target/x86_64-unknown-linux-gnu/release/rustfs"
data_dir="$(mktemp -d)"
log_file="${RUNNER_TEMP}/rustfs-console-smoke.log"
server_pid=""
trap '
if [[ -n "${server_pid:-}" ]]; then
kill "$server_pid" 2>/dev/null || true
wait "$server_pid" 2>/dev/null || true
fi
rm -rf "$data_dir"
' EXIT
if [[ ! -x "$binary" ]]; then
echo "rustfs binary not found: ${binary}" >&2
exit 1
fi
api_port="$(find_free_port)"
console_port="$(find_free_port)"
console_url="http://127.0.0.1:${console_port}/rustfs/console/"
"$binary" server \
--address "127.0.0.1:${api_port}" \
--console-enable \
--console-address "127.0.0.1:${console_port}" \
--access-key console-smoke \
--secret-key console-smoke-secret \
"$data_dir" >"$log_file" 2>&1 &
server_pid=$!
response=""
for _ in {1..40}; do
response="$(curl --silent --output /dev/null --write-out '%{http_code} %{content_type}' "$console_url" || true)"
if [[ "$response" == 200\ text/html* ]]; then
exit 0
fi
sleep 0.25
done
echo "Console endpoint did not return 200 text/html: $response" >&2
cat "$log_file"
exit 1
- name: Build DEB package
id: deb
shell: bash
@@ -534,12 +346,12 @@ jobs:
path: ${{ steps.publish.outputs.candidate_file }}
if-no-files-found: error
# Publish the deb/rpm pair to the auto-testing repo's `assets` branch so
# engineers can download and install the nightly directly. The branch is
# a single-commit orphan rewritten on every build, which keeps the repo
# small while the latest files stay reachable at stable raw URLs.
# Publish the deb/rpm pair as assets of the rolling `nightly` release on
# rustfs/auto-testing (see scripts/release/publish_nightly_assets.sh).
# The dispatch below runs only after this upload and the immutable R2
# candidate both succeed. It does not replace rustfs-functional-chain.yml,
# which still starts from this workflow's workflow_run completion.
# kms-vault-lane and kms-vault-ha-failover stay parallel with this job.
- name: Publish packages to auto-testing release assets
env:
ASSETS_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
@@ -548,90 +360,6 @@ jobs:
DEB_DATE: ${{ steps.deb.outputs.deb_date }}
BUILD_REF: ${{ env.NIGHTLY_BUILD_REF }}
run: bash scripts/release/publish_nightly_assets.sh
# Cross-repo repository_dispatch cannot use GITHUB_TOKEN. The secret is a
# PAT or GitHub App token for rustfs/auto-testing: classic `repo` scope,
# or fine-grained Contents write (the permission GitHub checks for
# POST /repos/{owner}/{repo}/dispatches). A dispatch failure fails this
# step, and therefore this job, so the scheduled failure alert fires.
# GitHub allows at most 10 top-level client_payload properties, so the
# schema 2 candidate object is nested under `candidate` and the release
# fields sit beside it.
- name: Dispatch nightly candidate to auto-testing
if: ${{ steps.publish.outputs.candidate_file != '' }}
env:
DISPATCH_TOKEN: ${{ secrets.AUTO_TESTING_DISPATCH_TOKEN }}
CANDIDATE_FILE: ${{ steps.publish.outputs.candidate_file }}
DEB_DATE: ${{ steps.deb.outputs.deb_date }}
SOURCE_REPO: ${{ github.repository }}
shell: bash
run: |
set -euo pipefail
: "${DISPATCH_TOKEN:?AUTO_TESTING_DISPATCH_TOKEN is required to dispatch rustfs/auto-testing}"
: "${CANDIDATE_FILE:?nightly candidate manifest is required}"
: "${DEB_DATE:?DEB_DATE is required}"
[[ "${DEB_DATE}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]
[[ "${SOURCE_REPO}" == "rustfs/rustfs" ]]
payload="${RUNNER_TEMP}/nightly-candidate-dispatch-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
jq -n --slurpfile candidate "${CANDIDATE_FILE}" \
--arg event_type "nightly-candidate-ready" \
--arg source_repo "${SOURCE_REPO}" \
--arg latest_alias_url "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb" \
--arg auto_testing_release "https://github.com/rustfs/auto-testing/releases/tag/nightly" \
--arg channel "nightly" \
--arg deb_date "${DEB_DATE}" \
--arg nightly_workflow ".github/workflows/nightly-gnu.yml" \
'{
event_type: $event_type,
source_repo: $source_repo,
latest_alias_url: $latest_alias_url,
auto_testing_release: $auto_testing_release,
channel: $channel,
deb_date: $deb_date,
nightly_workflow: $nightly_workflow,
candidate: $candidate[0]
}' > "${payload}"
if ! jq -e '
.event_type == "nightly-candidate-ready"
and .source_repo == "rustfs/rustfs"
and .channel == "nightly"
and .deb_date == env.DEB_DATE
and .nightly_workflow == ".github/workflows/nightly-gnu.yml"
and .latest_alias_url == "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb"
and .auto_testing_release == "https://github.com/rustfs/auto-testing/releases/tag/nightly"
and .candidate.schema == 2
and ((.candidate | keys | sort) == ["build_run_attempt","build_run_id","package_sha256","package_url","schema","source_ref","source_sha","workflow_sha"])
and ((keys | sort) == ["auto_testing_release","candidate","channel","deb_date","event_type","latest_alias_url","nightly_workflow","source_repo"])
' "${payload}" >/dev/null; then
echo "nightly dispatch payload does not match the nightly-candidate-ready contract" >&2
exit 1
fi
body="${RUNNER_TEMP}/nightly-candidate-dispatch-body-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
jq -n --slurpfile client_payload "${payload}" \
'{event_type: "nightly-candidate-ready", client_payload: $client_payload[0]}' > "${body}"
response="${RUNNER_TEMP}/nightly-candidate-dispatch-response-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.txt"
status="$(curl -sS --retry 3 --connect-timeout 15 --max-time 60 \
-o "${response}" -w '%{http_code}' \
-X POST \
-H "Authorization: Bearer ${DISPATCH_TOKEN}" \
-H "Accept: application/vnd.github+json" \
-H "X-GitHub-Api-Version: 2022-11-28" \
-H "Content-Type: application/json" \
--data-binary @"${body}" \
https://api.github.com/repos/rustfs/auto-testing/dispatches)" || {
echo "repository_dispatch request to rustfs/auto-testing failed before an HTTP status" >&2
exit 1
}
if [[ "${status}" != "204" ]]; then
echo "repository_dispatch to rustfs/auto-testing failed: HTTP ${status}" >&2
if [[ -s "${response}" ]]; then
cat "${response}" >&2
fi
exit 1
fi
echo "dispatched nightly-candidate-ready to rustfs/auto-testing (deb_date=${DEB_DATE}, run=${GITHUB_RUN_ID}, attempt=${GITHUB_RUN_ATTEMPT})"
# Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
#
# RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and
+14 -210
View File
@@ -225,57 +225,10 @@ jobs:
echo " Build run ID: $BUILD_RUN_ID"
echo " Build run number: $RUN_NUMBER"
# Consume the trusted release certification of the resolved build run and
# verify the certified input digests before any package is wrapped. A run
# without that certification fails closed here; packages built later must
# complete their own verification and cannot inherit an unverified
# conclusion.
release-certification:
name: Release Certification
needs: resolve
if: needs.resolve.result == 'success'
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
timeout-minutes: 30
permissions:
contents: read
actions: read
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Download and verify release certification
env:
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
BUILD_RUN_ID: ${{ needs.resolve.outputs.build_run_id }}
BUILD_TYPE: ${{ needs.resolve.outputs.build_type }}
shell: bash
run: |
set -euo pipefail
mkdir -p certification
if [[ "$BUILD_TYPE" == "development" ]]; then
# Development artifacts are explicitly uncertified and never touch
# official or latest channels; only the staging/dev paths remain.
jq -n '{schema_version: 1, certified: false, reason: "development artifacts are explicitly uncertified"}' \
> certification/development-uncertified.json
exit 0
fi
if ! gh run download "$BUILD_RUN_ID" -n release-certification -D certification; then
echo "Build run ${BUILD_RUN_ID} published no release-certification artifact; failing closed" >&2
exit 1
fi
python3 scripts/release/release_gate.py verify-promotion \
--plan .config/release-validation-plan.json \
--candidate certification/candidate.json \
--summary certification/release-gate-output \
--artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
--artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
# Build DEB and RPM packages for each architecture and libc variant
package:
name: Package (${{ matrix.arch }}-${{ matrix.libc }})
needs: [ resolve, release-certification ]
needs: resolve
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
timeout-minutes: 30
strategy:
@@ -541,55 +494,6 @@ jobs:
echo "rpm_file=$RPM_FILE" >> "$GITHUB_OUTPUT"
echo "✅ RPM built: $RPM_FILE"
- name: Download certification evidence
if: needs.resolve.outputs.build_type != 'development'
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: release-certification
path: ./certification
# Later-built wrapper verification: the binary inside each package must
# match the certified binary digest for this platform before anything is
# uploaded, and the package itself stays on staging until promotion.
- name: Verify wrapped binary against certification
if: needs.resolve.outputs.build_type != 'development'
shell: bash
env:
DEB_FILE: ${{ steps.deb.outputs.deb_file }}
RPM_FILE: ${{ steps.rpm.outputs.rpm_file }}
ARTIFACT_NAME: ${{ matrix.artifact_name }}
run: |
set -euo pipefail
expected="$(jq -r --arg needle "$ARTIFACT_NAME" \
'[.deliverables[] | select(.name | contains($needle)) | .binary_sha256][0] // empty' \
certification/candidate.json)"
if [[ -z "$expected" ]]; then
echo "No certified binary digest recorded for ${ARTIFACT_NAME}" >&2
exit 1
fi
if ! command -v rpm2cpio >/dev/null 2>&1; then
sudo apt-get update && sudo apt-get install -y rpm2cpio
fi
if ! command -v cpio >/dev/null 2>&1; then
sudo apt-get update && sudo apt-get install -y cpio
fi
for package_file in "$DEB_FILE" "$RPM_FILE"; do
case "$package_file" in
*.deb)
dpkg-deb --fsys-tarfile "$package_file" | tar -xOf - ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
;;
*.rpm)
rpm2cpio "$package_file" | cpio -i --to-stdout ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
;;
esac
python3 scripts/release/release_gate.py verify-promotion \
--plan .config/release-validation-plan.json \
--candidate certification/candidate.json \
--summary certification/release-gate-output \
--artifact "$RUNNER_TEMP/wrapped-rustfs" \
--artifact-role binary
done
- name: Upload packages to artifacts
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
@@ -627,14 +531,10 @@ jobs:
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
export AWS_DEFAULT_REGION="auto"
# Build-phase writes stay on staging for release types: the official
# packages/release and packages/latest paths are written only by the
# certification-gated promote-packages job, from these same staged
# objects. Development packages keep the uncertified dev channel.
if [[ "$BUILD_TYPE" == "development" ]]; then
R2_PREFIX="artifacts/rustfs/packages/dev"
else
R2_PREFIX="artifacts/rustfs/packages/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}"
R2_PREFIX="artifacts/rustfs/packages/release"
fi
R2_PATH="s3://${R2_BUCKET}/${R2_PREFIX}/"
@@ -649,6 +549,18 @@ jobs:
echo "✅ Upload complete"
# Also upload as latest for release/prerelease
if [[ "$BUILD_TYPE" == "release" || "$BUILD_TYPE" == "prerelease" ]]; then
LATEST_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/latest/"
for f in "$DEB_FILE" "$RPM_FILE"; do
if [[ -n "$f" && -f "$f" ]]; then
echo "Uploading latest: $(basename "$f")"
aws s3 cp "$f" "$LATEST_PATH" --endpoint-url "$R2_ENDPOINT" --only-show-errors
fi
done
echo "✅ Latest packages updated"
fi
- name: Upload packages to GitHub Release
if: needs.resolve.outputs.tag != ''
env:
@@ -702,114 +614,6 @@ jobs:
echo "✅ GitHub release assets updated"
# The only writer of the official packages/release and packages/latest R2
# paths. Promotion re-checks each package's wrapped binary against the
# passed certification list and copies the staged objects byte-for-byte:
# no rebuild-after-verify, no mutable latest resolution downstream.
promote-packages:
name: Promote Packages
needs: [ resolve, package, release-certification ]
# Target-channel lock for the official package paths.
concurrency:
group: release-channel-r2-packages
cancel-in-progress: false
if: needs.resolve.outputs.tag != '' && (needs.resolve.outputs.build_type == 'release' || needs.resolve.outputs.build_type == 'prerelease')
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
timeout-minutes: 30
permissions:
contents: read
actions: read
env:
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
R2_BUCKET: ${{ secrets.R2_BUCKET }}
AWS_EC2_METADATA_DISABLED: true
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Download packages and certification
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
path: ./packages
pattern: packages-*
merge-multiple: true
- name: Download certification evidence
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: release-certification
path: ./certification
- name: Verify and promote staged packages
shell: bash
run: |
set -euo pipefail
if [[ -z "$R2_ACCESS_KEY_ID" || -z "$R2_SECRET_ACCESS_KEY" || -z "$R2_ENDPOINT" || -z "$R2_BUCKET" ]]; then
echo "❌ R2 credentials missing; refusing to promote to the official channel" >&2
exit 1
fi
STAGING_PREFIX="s3://${R2_BUCKET}/artifacts/rustfs/packages/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
RELEASE_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/release/"
LATEST_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/latest/"
if ! command -v aws >/dev/null 2>&1; then
sudo apt-get update && sudo apt-get install -y awscli
fi
export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
export AWS_DEFAULT_REGION="auto"
if ! command -v rpm2cpio >/dev/null 2>&1; then
sudo apt-get install -y rpm2cpio
fi
if ! command -v cpio >/dev/null 2>&1; then
sudo apt-get install -y cpio
fi
promoted=0
for package_file in packages/*.deb packages/*.rpm; do
[[ -f "$package_file" ]] || continue
name="$(basename "$package_file")"
case "$package_file" in
*.deb)
dpkg-deb --fsys-tarfile "$package_file" | tar -xOf - ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
;;
*.rpm)
rpm2cpio "$package_file" | cpio -i --to-stdout ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
;;
esac
python3 scripts/release/release_gate.py verify-promotion \
--plan .config/release-validation-plan.json \
--candidate certification/candidate.json \
--summary certification/release-gate-output \
--artifact "$RUNNER_TEMP/wrapped-rustfs" \
--artifact-role binary
staged="$(mktemp)"
aws s3 cp "${STAGING_PREFIX}${name}" "$staged" --endpoint-url "$R2_ENDPOINT" --only-show-errors
staged_sha="$(sha256sum "$staged" | awk '{print $1}')"
built_sha="$(sha256sum "$package_file" | awk '{print $1}')"
if [[ "$staged_sha" != "$built_sha" ]]; then
echo "❌ Staged package ${name} does not match the verified bytes" >&2
exit 1
fi
aws s3 cp "$staged" "${RELEASE_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
aws s3 cp "$staged" "${LATEST_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
rm -f "$staged"
promoted=$((promoted + 1))
echo "✅ Promoted ${name} (${built_sha})"
done
if [[ "$promoted" -eq 0 ]]; then
echo "No staged packages found to promote" >&2
exit 1
fi
# Summary
summary:
name: Summary
@@ -1,457 +0,0 @@
# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
name: RustFS Validation Sweep
# Executes the migrated canonical validation cases (rustfs/auto-testing#113)
# on the shared functional fleet. Everything with a lane executor is swept in
# rotating nightly batches so the whole set is covered roughly weekly; a
# single case or one module can be dispatched manually. That is the
# implemented catalog (66 today) plus the eight planned cases whose dedicated
# executors already exist and have live PASS evidence - fresh nightly
# evidence feeds their acceptance review. heal.* cases stay on the
# interactive validation flow: they require reviewed native/tool collectors
# built by separate validation-tools runs before fleet execution.
on:
schedule:
# After the nightly functional chain settles (chain starts ~16:48 UTC and
# runs for about two hours).
- cron: "30 19 * * *"
workflow_dispatch:
inputs:
case:
description: 'Single canonical validation case id (overrides module and rotation), e.g. pool.decommission'
type: string
required: false
default: ''
module:
description: 'Restrict the sweep to one catalog module, e.g. bucket-replication'
type: string
required: false
default: ''
limit:
description: 'Maximum cases per sweep (0 = run everything selected)'
type: number
required: false
default: 12
build_run_id:
description: 'Successful nightly package run; empty resolves the latest'
type: string
required: false
default: ''
build_run_attempt:
description: 'Attempt of that package run'
type: string
required: false
default: '1'
permissions:
contents: read
concurrency:
group: rustfs-validation-sweep
cancel-in-progress: false
env:
RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES || 'vm000 vm001 vm002 vm003' }}
RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER || 'azureuser' }}
RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
jobs:
sweep:
if: github.repository == 'rustfs/rustfs'
runs-on: smoke-testing
timeout-minutes: 150
steps:
- name: Checkout chain tooling
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Resolve nightly candidate
id: candidate
env:
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
BUILD_RUN_ID: ${{ inputs.build_run_id }}
BUILD_RUN_ATTEMPT: ${{ inputs.build_run_attempt }}
CHAIN_OUTPUT: ${{ runner.temp }}/validation-sweep-candidate.json
run: |
set -euo pipefail
# Scheduled runs carry no inputs and the resolver only derives a
# build from a workflow_run payload (which a schedule cannot
# provide), so resolve the newest successful nightly build
# explicitly before invoking it.
if [ -z "${BUILD_RUN_ID:-}" ]; then
LATEST="$(gh api "repos/rustfs/rustfs/actions/workflows/nightly-gnu.yml/runs?branch=main&status=success&per_page=1" --jq '.workflow_runs[0]')"
BUILD_RUN_ID="$(printf '%s' "${LATEST}" | jq -r '.id')"
BUILD_RUN_ATTEMPT="$(printf '%s' "${LATEST}" | jq -r '.run_attempt')"
export BUILD_RUN_ID BUILD_RUN_ATTEMPT
echo "scheduled sweep pinned to nightly ${BUILD_RUN_ID} attempt ${BUILD_RUN_ATTEMPT}"
fi
python3 scripts/resolve_functional_candidate.py
python3 - <<'PY'
import json
import os
from pathlib import Path
root = Path(os.environ["RUNNER_TEMP"]) / "validation-sweep"
root.mkdir(parents=True, exist_ok=True)
chain = json.loads((Path(os.environ["RUNNER_TEMP"]) / "validation-sweep-candidate.json").read_text())
manifest = chain["candidate"]["manifest"]
(root / "candidate.json").write_text(json.dumps({
"package_url": manifest["package_url"],
"package_sha256": manifest["package_sha256"],
"source_sha": manifest["source_sha"],
}, sort_keys=True) + "\n")
with open(os.environ["GITHUB_OUTPUT"], "a") as output:
output.write("testing_sha=" + chain["testing_sha"] + "\n")
output.write("package_url=" + manifest["package_url"] + "\n")
output.write("source_sha=" + manifest["source_sha"] + "\n")
PY
- name: Checkout auto-testing scripts
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
repository: rustfs/auto-testing
ref: ${{ steps.candidate.outputs.testing_sha }}
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
path: auto-testing
persist-credentials: false
- name: Prepare validation Python runtime
# The pool / ilm / replication validation runners require a managed
# Python, exactly like the chain's validation job provides.
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.12"
- name: Install validation Python requirements
# Case scripts call python3 and import boto3 at startup. Install the
# functional chain's client into a per-run env so that interpreter
# sees it without changing the runner's shared Python.
run: |
set -euo pipefail
validation_python="${RUNNER_TEMP}/rustfs-validation-python-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
test ! -e "${validation_python}"
python3 -m venv "${validation_python}"
"${validation_python}/bin/python" -m pip install \
-r auto-testing/scripts/validation_ilm/requirements.txt
printf '%s\n' "${validation_python}/bin" >> "${GITHUB_PATH}"
- name: Select sweep cases
id: select
env:
SELECTED_CASE: ${{ inputs.case }}
SELECTED_MODULE: ${{ inputs.module }}
CASE_LIMIT: ${{ inputs.limit }}
SWEEP_EVENT: ${{ github.event_name }}
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
run: |
set -euo pipefail
python3 - <<'PY'
import datetime
import json
import os
from pathlib import Path
sweep_root = Path(os.environ["SWEEP_ROOT"])
catalog = json.loads(Path("auto-testing/validation-cases.json").read_text())
def sweepable(case):
status = case.get("source_status")
if status == "implemented":
return True
# The remaining planned non-heal cases route to the lane
# executors and already have live PASS evidence; sweeping them
# nightly produces fresh evidence for their acceptance review.
return status == "planned" and not case["id"].startswith("heal.")
selected = [(case["id"], module["id"]) for module in catalog["modules"]
for case in module["cases"]
if sweepable(case)]
if not selected:
raise SystemExit("catalog has no sweepable cases")
one_case = os.environ.get("SELECTED_CASE", "").strip()
module_filter = os.environ.get("SELECTED_MODULE", "").strip()
known = {case_id for case_id, _ in selected}
modules = {module_id for _, module_id in selected}
if one_case:
if one_case not in known:
raise SystemExit(f"unknown case or no lane executor for it: {one_case}")
selected = [(one_case, dict(selected)[one_case])]
elif module_filter:
if module_filter not in modules:
raise SystemExit(f"unknown module or no implemented cases: {module_filter}")
selected = [pair for pair in selected if pair[1] == module_filter]
elif os.environ.get("SWEEP_EVENT") == "schedule":
# Stateless rotation: every night take one window of the catalog
# so a full sweep needs len(selected)/limit nights and no state
# has to be carried between runs.
limit = max(1, int(os.environ.get("CASE_LIMIT") or 12))
day = datetime.datetime.now(datetime.timezone.utc).timetuple().tm_yday
start = (day * limit) % len(selected)
selected = (selected[start:] + selected[:start])[:limit]
limit = int(os.environ.get("CASE_LIMIT") or 0)
if one_case or module_filter or (not os.environ.get("SWEEP_EVENT") == "schedule"):
if limit > 0:
selected = selected[:limit]
(sweep_root / "sweep-cases.txt").write_text("".join(f"{case_id}\n" for case_id, _ in selected))
with open(os.environ["GITHUB_OUTPUT"], "a") as output:
output.write(f"count={len(selected)}\n")
print(f"selected {len(selected)} case(s):", ", ".join(c for c, _ in selected))
PY
- name: Admit clean shared environment
id: environment
timeout-minutes: 5
env:
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-validation-sweep.json
run: python3 scripts/functional_environment.py enter --suite heal
- name: Run validation cases
id: run_cases
timeout-minutes: 120
env:
AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
run: |
set -uo pipefail
mkdir -p "${SWEEP_ROOT}/results"
: > "${SWEEP_ROOT}/results.txt"
while read -r CASE_ID; do
[ -n "${CASE_ID}" ] || continue
echo "::group::case ${CASE_ID}"
rm -rf "${SWEEP_ROOT}/run"
if ./auto-testing/scripts/run_validation_case.sh \
"${CASE_ID}" "${SWEEP_ROOT}/candidate.json" \
"${AUTO_TESTING_SHA}" "${SWEEP_ROOT}/run" \
> "${SWEEP_ROOT}/results/${CASE_ID//\//_}.log" 2>&1; then
printf '%s\tPASS\n' "${CASE_ID}" >> "${SWEEP_ROOT}/results.txt"
echo "case ${CASE_ID}: PASS"
else
printf '%s\tFAIL\n' "${CASE_ID}" >> "${SWEEP_ROOT}/results.txt"
echo "case ${CASE_ID}: FAIL (log retained in the sweep artifact)"
fi
if [ -f "${SWEEP_ROOT}/run/report.md" ]; then
cp "${SWEEP_ROOT}/run/report.md" "${SWEEP_ROOT}/results/${CASE_ID//\//_}.report.md" 2>/dev/null || true
fi
echo "::endgroup::"
# Fleet hygiene between cases: the case runners set up their own
# topology, this purge guarantees the next case starts clean even
# when the previous one died mid-run.
for node in ${RUSTFS_NODES}; do
# -n keeps ssh from consuming the case list this loop reads from stdin
ssh -n -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
${SUDO} systemctl stop rustfs 2>/dev/null || true
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
${SUDO} dpkg -P rustfs
fi
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
${SUDO} rm -f /etc/default/rustfs
' || echo "inter-case cleanup failed on ${node} (continuing)"
done
done < "${SWEEP_ROOT}/sweep-cases.txt"
echo "sweep finished: $(wc -l < "${SWEEP_ROOT}/results.txt") case(s) executed"
- name: Generate sweep report
if: ${{ always() && steps.environment.outcome == 'success' }}
env:
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
PACKAGE_URL: ${{ steps.candidate.outputs.package_url }}
SOURCE_SHA: ${{ steps.candidate.outputs.source_sha }}
AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
run: |
set -euo pipefail
python3 - <<'PY'
import json
import os
from pathlib import Path
root = Path(os.environ["SWEEP_ROOT"])
catalog = json.loads(Path("auto-testing/validation-cases.json").read_text())
module_of = {case["id"]: module["id"] for module in catalog["modules"] for case in module["cases"]}
rows = []
for line in (root / "results.txt").read_text().splitlines():
if not line:
continue
case_id, status = line.split("\t")
rows.append((case_id, module_of.get(case_id, "?"), status))
passed = sum(1 for _, _, s in rows if s == "PASS")
failed = len(rows) - passed
run_url = (f"{os.environ.get('GITHUB_SERVER_URL', 'https://github.com')}/"
f"{os.environ.get('GITHUB_REPOSITORY', 'rustfs/rustfs')}/actions/runs/"
f"{os.environ.get('GITHUB_RUN_ID', '')}")
report = [
"# RustFS validation sweep report", "",
f"- Run: {run_url}",
f"- Attempt: {os.environ.get('GITHUB_RUN_ATTEMPT', '')}",
f"- Trigger: {os.environ.get('GITHUB_EVENT_NAME', '')}",
f"- Package: {os.environ.get('PACKAGE_URL', 'nightly (R2 latest)')}",
f"- Source SHA: {os.environ.get('SOURCE_SHA', '')}",
f"- auto-testing: {os.environ.get('AUTO_TESTING_SHA', '')}", "",
"Canonical validation cases (rustfs/auto-testing#113) executed by the sweep, grouped by catalog module.", "",
]
# Group the executed cases by catalog module (the major category:
# pool lifecycle, heal, ILM expiration, bucket/site replication, kms,
# data services, identity, s3 compatibility). Per-module subtotals
# deliberately use "- Subtotal:" so the dashboard overview regexes
# (which anchor on "- Total:") keep reading the grand totals below.
by_module = {}
module_order = []
for case_id, module, status in rows:
if module not in by_module:
by_module[module] = []
module_order.append(module)
by_module[module].append((case_id, status))
report += ["## Case Summary", ""]
for module in module_order:
cases = by_module[module]
mod_pass = sum(1 for _, st in cases if st == "PASS")
mod_fail = len(cases) - mod_pass
report += [f"### {module} ({len(cases)})", "",
"| Case | Status |", "| --- | --- |"]
report += [f"| {case_id} | {status} |" for case_id, status in cases]
report += ["", f"- Subtotal: {mod_pass} passed, {mod_fail} failed", ""]
report += [
"## Totals", "",
f"- Total: {len(rows)}",
f"- PASS: {passed}",
f"- FAIL: {failed}",
"- UNSUPPORTED: 0", "",
"Failing cases keep this workflow red only as a report; the sweep artifact",
"retains every case log and per-case report for diagnosis.", "",
]
(root / "report.md").write_text("\n".join(report))
(root / "cases.md").write_text("\n".join(f"| {c} | {m} | {s} |" for c, m, s in rows) + "\n")
PY
cat "${SWEEP_ROOT}/report.md" >> "${GITHUB_STEP_SUMMARY}"
- name: Upload functional report to dashboard
if: ${{ always() && steps.environment.outcome == 'success' }}
timeout-minutes: 2
continue-on-error: true
env:
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
REPORT_FILE: ${{ runner.temp }}/validation-sweep/report.md
SUITE: validation
run: |
set -euo pipefail
if [ -z "${GH_TOKEN:-}" ]; then
echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
exit 0
fi
DATE="$(date -u +%Y-%m-%d)"
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
B64_FILE="$(mktemp)"
python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
if [ -n "${SHA}" ]; then
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
'{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
else
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
'{message:$msg, content:($content|rtrimstr("\n"))}' \
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
fi
echo "dashboard updated: ${REPORT_PATH}"
- name: Retain sweep evidence
if: ${{ always() }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: rustfs-validation-sweep-${{ github.run_id }}-${{ github.run_attempt }}
path: |
${{ runner.temp }}/validation-sweep/report.md
${{ runner.temp }}/validation-sweep/results.txt
${{ runner.temp }}/validation-sweep/sweep-cases.txt
${{ runner.temp }}/validation-sweep/candidate.json
${{ runner.temp }}/validation-sweep/results/
${{ runner.temp }}/validation-sweep/run/
# warn (not error): an early infra failure legitimately leaves no
# per-case evidence, and the failing step is already red.
if-no-files-found: warn
retention-days: 30
- name: Cleanup environment (after)
id: environment_cleanup
timeout-minutes: 5
if: ${{ always() && steps.environment.outcome == 'success' }}
run: |
set -uo pipefail
cleanup_status=0
for node in ${RUSTFS_NODES}; do
ssh -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
set -euo pipefail
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
${SUDO} systemctl stop rustfs 2>/dev/null || true
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
${SUDO} dpkg -P rustfs
fi
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
${SUDO} rm -f /etc/default/rustfs
' || cleanup_status=$?
done
exit "${cleanup_status}"
- name: Release environment only after successful cleanup
if: ${{ always() && steps.environment.outcome == 'success' }}
timeout-minutes: 5
env:
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-validation-sweep.json
CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
run: python3 scripts/functional_environment.py finish --suite heal --cleanup-outcome "$CLEANUP_OUTCOME"
- name: Notify on failure
if: failure()
run: echo "RustFS validation sweep failed (harness/environment breakdown); see the sweep artifact for case logs."
- name: Enforce sweep result
if: ${{ always() && steps.environment.outcome == 'success' }}
env:
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
run: |
set -euo pipefail
if grep -q $'\tFAIL$' "${SWEEP_ROOT}/results.txt"; then
echo "validation sweep has failing cases:"
grep $'\tFAIL$' "${SWEEP_ROOT}/results.txt"
exit 1
fi
alert-on-failure:
name: Alert on scheduled failure
# A scheduled sweep failure usually means a canonical validation case
# regressed or the fleet environment broke; surface it as a tracking
# issue instead of leaving it in the Actions tab.
needs: [sweep]
if: github.repository == 'rustfs/rustfs' && (always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure'))
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: read
issues: write
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.0
with:
persist-credentials: false
- name: Open or update failure-tracking issue
uses: ./.github/actions/schedule-failure-issue
with:
github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}
-26
View File
@@ -107,32 +107,6 @@ jobs:
shell: pwsh
run: cargo test -p rustfs-ecstore --lib multipart_owned_record_parent_barriers_ -- --nocapture
# Native multipart completion publishes its payload from a different
# parent than its metadata, and abort renames under a precondition; both
# need their own Windows handle discipline.
- name: Test native multipart complete and abort publication
shell: pwsh
run: >-
cargo test -p rustfs-ecstore --lib --
native_abort_missing_state_with_actual_owned_records_never_becomes_legacy_or_missing
native_abort_pending_intent_is_not_a_delete_certificate_and_raw_cleanup_preserves_exact_evidence
--nocapture
# Owned-undo reclaim removes a directory it pinned, copy heal persists
# fault bytes through a handle, and metadata reads meet a directory.
- name: Test native multipart reclaim, copy heal, and directory reads
shell: pwsh
run: >-
cargo test -p rustfs-ecstore --lib --
native_multipart_owned_undo
native_multipart_copy_heal
orphan_multipart_cleanup_rejects_metadata_directory_on_one_disk
--nocapture
- name: Test directory removal fallback
shell: pwsh
run: "cargo test -p rustfs-ecstore --lib disk::fs::tests::test_remove -- --nocapture"
- name: Test Windows path conversion
shell: pwsh
run: "cargo test -p rustfs-utils --no-default-features --features path --lib dunce:: -- --nocapture"
-5
View File
@@ -7,10 +7,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Notifications and audit
- Notify and audit targets that keep the default `queue_dir` now work on native Windows: the defaults `/opt/rustfs/events` and `/opt/rustfs/audit` resolve per node, to `%ProgramData%\RustFS\events` and `%ProgramData%\RustFS\audit` on Windows, instead of failing validation as non-absolute paths. The new `RUSTFS_TARGET_STORE_DIR` (Helm `config.rustfs.target_store_dir`) relocates both default queues on any platform. Linux and macOS behavior is unchanged when it is unset, persisted configuration is unchanged, and pending entries are not migrated when the root changes. See `docs/operations/target-queue-storage.md`.
### Deployment
- Existing volumes created by older root-running container images must be writable by the non-root runtime identity before upgrading. The default container UID/GID is `10001:10001`; stop the affected instances, verify host mount paths, and for ordinary bind mounts run `sudo chown -R 10001:10001 /actual/data /actual/logs` on each RustFS data/log mount. Account for ACLs, NFS, Kubernetes, and rootless UID mappings. Image-build ownership changes do not migrate mounted data. See the Docker Quick Start in [README.md](README.md).
@@ -30,7 +26,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- IAM snapshot/deletion replay, target-assigned delete-marker purges, timestamp ordering, and best-effort peer broadcast now close the control-plane gaps found by the R6 review ([#7195](https://github.com/rustfs/rustfs/pull/7195)).
- A `set-remote-target?update=true&creds=true` request on a site-replication peer's bucket target is now rejected with `400 InvalidRequest`, naming the target and why it counts as peer-owned. It used to return success while keeping the stored endpoint, credentials, and TLS settings, so an endpoint edit from the Console's bucket replication form appeared to apply but did not. Change a peer's address with a site-replication edit instead; see "Changing a site's address" in `docs/operations/site-replication-operations.md`. The `sync`, `bandwidth`, `proxy`, and `path` update groups are unchanged.
- Site-replication edits that omit the site name, such as `mc admin replicate update --deployment-id <ID> --endpoint <URL>`, now keep the stored name on the editing site and on every receiver instead of renaming the site after the new endpoint host. No persisted format changes.
- The persisted replication MRF journal is now replayed every 5 minutes while the server runs, not only at startup. Delete-marker purges whose targets were unreachable, and replications that overflowed the worker queues, now converge within one replay interval after the target returns instead of waiting for a restart. A target that stays down is re-attempted once per retained entry per interval across the cluster: the start time of the last pass is shared through a small `config/replication/mrf.bin.replay` object, so nodes do not each replay once per interval. Older versions ignore that object; existing persisted formats are unchanged.
- Upgrade and rollback: upgrade every node in one site consecutively and verify reconciliation before moving to the next site; do not intentionally run a site mixed-version. Target-version ledger keys are harmless on rollback, although old code cannot use their routing. Before rolling back past [#7307](https://github.com/rustfs/rustfs/pull/7307), drain or repair every pending version purge: older code can free a retained version's data directory before its remote purge is acknowledged. See `docs/operations/site-replication-operations.md`.
### Security
Generated
+195 -406
View File
File diff suppressed because it is too large Load Diff
+17 -15
View File
@@ -147,7 +147,7 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.2" }
async-channel = "2.5.0"
async_zip = { default-features = false, version = "0.0.19" }
mysql_async = { default-features = false, version = "0.37.1" }
async-compression = { version = "0.4.52" }
async-compression = { version = "0.4.51" }
async-trait = "0.1.92"
async-nats = { version = "0.51.0", default-features = false }
axum = "0.8.9"
@@ -168,7 +168,7 @@ http-body-util = "0.1.5"
memchr = "2.8.3"
minlz = "1.2.3"
reqwest = "0.13.5"
rustfs-kafka-async = { version = "1.4.1" }
rustfs-kafka-async = { version = "1.4.0" }
socket2 = { version = "0.6.5" }
tokio = { version = "1.54.1" }
tokio-rustls = { default-features = false, version = "0.26.6" }
@@ -191,7 +191,6 @@ quinn = { version = "0.11.12", default-features = false, features = [
# Serialization and Data Formats
apache-avro = { version = "0.22.0", features = ["snappy", "zstandard"] }
bytes = { version = "1.12.1" }
csv-core = "0.1.13"
bytesize = "2.7.0"
byteorder = "1.5.0"
flatbuffers = "25.12.19"
@@ -213,13 +212,12 @@ serde_urlencoded = "0.7.1"
# releases.
aes-gcm = { version = "0.11.1" }
argon2 = { version = "0.6.0" }
aws-lc-rs = { version = "1.19.0", default-features = false }
blake2 = "0.11.0"
chacha20poly1305 = { version = "0.11.0" }
crc-fast = "1.10.0"
hmac = { version = "0.13.0" }
jsonwebtoken = { version = "11.1.0" }
openidconnect = { default-features = false, version = "4.1.0" }
openidconnect = { default-features = false, version = "4.0" }
pbkdf2 = "0.13.0"
p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
rsa = { version = "=0.10.0-rc.18" }
@@ -236,7 +234,7 @@ proptest = "1"
# Time and Date
chrono = { version = "0.4.45" }
humantime = "2.4.0"
jiff = { version = "0.2.39" }
jiff = { version = "0.2.38" }
time = { version = "0.3.55" }
# Database
@@ -255,9 +253,9 @@ atoi = "3.1.1"
atomic_enum = "0.3.0"
aws-config = { version = "1.13.0" }
aws-credential-types = { version = "1.3.1" }
aws-sdk-kms = { default-features = false, version = "1.129.0" }
aws-sdk-s3 = { default-features = false, version = "1.157.0" }
aws-sdk-sts = { default-features = false, version = "1.124.0" }
aws-sdk-kms = { default-features = false, version = "1.128.0" }
aws-sdk-s3 = { default-features = false, version = "1.156.0" }
aws-sdk-sts = { default-features = false, version = "1.123.0" }
aws-smithy-async = { version = "1.3.1" }
aws-smithy-http-client = { default-features = false, version = "1.5.2" }
aws-smithy-runtime-api = { version = "1.19.2" }
@@ -339,7 +337,7 @@ strum = { version = "0.28.0" }
sysinfo = "0.40.1"
temp-env = "0.3.6"
tempfile = "3.27.0"
test-case = "3.4.1"
test-case = "3.4.0"
thiserror = "2.0.21"
tracing = { version = "0.1.44" }
tracing-appender = "0.2.5"
@@ -358,13 +356,13 @@ winapi-util = "0.1.11"
windows = { version = "0.62.2" }
windows-sys = "0.61.2"
xxhash-rust = { version = "0.8.19" }
zip = "9.3.0"
zip = "9.1.3"
zstd = "0.14.3"
# Observability and Metrics
metrics = "0.24.6"
metrics-util = "0.20"
dial9 = { version = "0.5.7", default-features = false }
dial9 = { version = "0.5.6", default-features = false }
opentelemetry = { version = "0.33.1" }
opentelemetry-appender-tracing = { version = "0.33.1" }
opentelemetry-otlp = { version = "0.33.1" }
@@ -379,16 +377,16 @@ libunftp = { version = "0.24.0" }
unftp-core = "0.1.0"
suppaftp = { version = "12.2.1" }
rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
russh = { version = "0.65.1" }
russh = { version = "0.65.0" }
russh-sftp = "3.0.1"
# WebDAV
dav-server = "0.11.0"
# Performance Analysis and Memory Profiling
rustfs-mimalloc = { version = "0.7.3" }
rustfs-mimalloc = { version = "0.7.2" }
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
hotpath = { version = "0.30.3", default-features = false }
hotpath = { version = "0.30.2", default-features = false }
# Snapshot testing for output format regression detection
insta = { version = "1.49" }
@@ -429,3 +427,7 @@ strip = "none"
# Pin upstream fixes until compatible crates.io releases are available.
[patch.crates-io]
zstd = { path = "crates/zstd-compat" }
# Preserve RustFS OIDC flows while migrating the old elliptic-curve dependency chain.
# Upstream: openidconnect-rs#244 and oauth2-rs#352.
openidconnect = { git = "https://github.com/ramosbugs/openidconnect-rs", rev = "2f77ee73bc4554e1cb6279adb386005429fb4e41" }
oauth2 = { git = "https://github.com/ramosbugs/oauth2-rs", rev = "89585f4a280afd7b285eaf1c9be0068874e03d22" }
+3 -11
View File
@@ -16,7 +16,6 @@ FROM alpine:3.24.2 AS build
ARG TARGETARCH
ARG RELEASE=latest
ARG RC_RELEASE=latest
RUN apk add --no-cache ca-certificates curl jq unzip
WORKDIR /build
@@ -70,9 +69,6 @@ RUN set -eux; \
chmod +x /build/rustfs; \
rm -rf rustfs.zip /build/.tmp || true
COPY scripts/download_rc.sh /usr/local/bin/download_rc.sh
RUN sh /usr/local/bin/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /build/rc
FROM alpine:3.24.2
@@ -99,18 +95,14 @@ RUN apk upgrade --no-cache && \
ca-certificates \
coreutils \
curl \
jq \
tzdata \
&& test "$(TZ=Asia/Kolkata date +%z)" = "+0530"
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
COPY --from=build /build/rustfs /usr/bin/rustfs
COPY --from=build /build/rc /usr/bin/rc
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
COPY bootstrap/ /etc/rustfs/bootstrap/
COPY entrypoint.sh /entrypoint.sh
RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
RUN chmod +x /usr/bin/rustfs /entrypoint.sh
RUN addgroup -g 10001 -S rustfs && \
adduser -u 10001 -G rustfs -S rustfs -D && \
@@ -132,6 +124,6 @@ VOLUME ["/data"]
USER rustfs
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
ENTRYPOINT ["/entrypoint.sh"]
CMD ["rustfs"]
+4 -13
View File
@@ -12,10 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
FROM rust:1.99.1-trixie
ARG TARGETARCH
ARG RC_RELEASE=latest
FROM rust:1.97.1-trixie
RUN set -eux; \
export DEBIAN_FRONTEND=noninteractive; \
@@ -24,7 +21,6 @@ RUN set -eux; \
build-essential \
ca-certificates \
curl \
jq \
git \
pkg-config \
libssl-dev \
@@ -40,7 +36,6 @@ COPY . .
RUN ./scripts/static.sh
RUN cargo run --bin gproto
RUN cargo build --release --locked --bin rustfs
RUN sh /usr/src/rustfs/scripts/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /usr/local/bin/rc
RUN set -eux; \
groupadd -g 10001 rustfs; \
@@ -55,12 +50,8 @@ RUN set -eux; \
chown -R rustfs:rustfs /data /logs /app; \
chmod 0750 /data /logs
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
COPY bootstrap/ /etc/rustfs/bootstrap/
RUN install -m 0755 /usr/src/rustfs/target/release/rustfs /usr/bin/rustfs && \
install -m 0755 /usr/local/bin/rc /usr/bin/rc && \
chmod +x /etc/rustfs/entrypoint.sh && \
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
COPY entrypoint.sh /entrypoint.sh
RUN install -m 0755 /usr/src/rustfs/target/release/rustfs /usr/bin/rustfs && chmod +x /entrypoint.sh
ENV RUSTFS_VOLUMES="/data" \
RUST_LOG="warn" \
@@ -72,5 +63,5 @@ ENV RUSTFS_VOLUMES="/data" \
EXPOSE 9000 9001
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
ENTRYPOINT ["/entrypoint.sh"]
CMD ["/usr/bin/rustfs"]
+3 -11
View File
@@ -16,7 +16,6 @@ FROM ubuntu:26.04 AS build
ARG TARGETARCH
ARG RELEASE=latest
ARG RC_RELEASE=latest
RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates \
@@ -77,9 +76,6 @@ RUN set -eux; \
chmod +x /build/rustfs; \
rm -rf rustfs.zip /build/.tmp || true
COPY scripts/download_rc.sh /usr/local/bin/download_rc.sh
RUN sh /usr/local/bin/download_rc.sh "$TARGETARCH" gnu "$RC_RELEASE" /build/rc
FROM ubuntu:26.04
ARG RELEASE=latest
@@ -103,18 +99,14 @@ RUN apt-get update && apt-get upgrade -y \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
ca-certificates \
curl \
jq \
tzdata \
&& test "$(TZ=Asia/Kolkata date +%z)" = "+0530" \
&& rm -rf /var/lib/apt/lists/*
COPY --from=build /build/rustfs /usr/bin/rustfs
COPY --from=build /build/rc /usr/bin/rc
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
COPY bootstrap/ /etc/rustfs/bootstrap/
COPY entrypoint.sh /entrypoint.sh
RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
RUN chmod +x /usr/bin/rustfs /entrypoint.sh
RUN groupadd -g 10001 rustfs && \
useradd -u 10001 -g rustfs -m -s /sbin/nologin rustfs && \
@@ -139,6 +131,6 @@ VOLUME ["/data"]
USER rustfs
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
ENTRYPOINT ["/entrypoint.sh"]
CMD ["rustfs"]
+7 -19
View File
@@ -28,19 +28,17 @@ ARG TARGETPLATFORM
ARG BUILDPLATFORM
ARG TARGETARCH
ARG RUSTFS_BUILD_FEATURES=""
ARG RC_RELEASE=latest
# -----------------------------
# Build stage
# -----------------------------
FROM rust:1.99.1-trixie AS builder
FROM rust:1.97.1-trixie AS builder
# Re-declare args after FROM
ARG TARGETPLATFORM
ARG BUILDPLATFORM
ARG TARGETARCH
ARG RUSTFS_BUILD_FEATURES
ARG RC_RELEASE
# Debug: print platforms
RUN echo "Build info -> BUILDPLATFORM=${BUILDPLATFORM}, TARGETPLATFORM=${TARGETPLATFORM}"
@@ -54,7 +52,6 @@ RUN set -eux; \
build-essential \
ca-certificates \
curl \
jq \
git \
pkg-config \
libssl-dev \
@@ -169,8 +166,6 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
;; \
esac
RUN sh /usr/src/rustfs/scripts/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /usr/local/bin/rc
# -----------------------------
# Development stage (keeps toolchain)
# -----------------------------
@@ -204,13 +199,10 @@ ENV RUSTFS_VOLUMES="/data" \
# We rely on cargo run to build and run
EXPOSE 9000 9001
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
COPY bootstrap/ /etc/rustfs/bootstrap/
RUN install -m 0755 /usr/local/bin/rc /usr/bin/rc && \
chmod +x /etc/rustfs/entrypoint.sh && \
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
COPY entrypoint.sh /entrypoint.sh
RUN chmod +x /entrypoint.sh
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
ENTRYPOINT ["/entrypoint.sh"]
CMD ["cargo", "run", "--bin", "rustfs", "--"]
# -----------------------------
@@ -235,7 +227,6 @@ RUN set -eux; \
apt-get install -y --no-install-recommends \
ca-certificates \
curl \
jq \
tzdata \
coreutils; \
rm -rf /var/lib/apt/lists/*
@@ -257,11 +248,8 @@ RUN set -eux; \
# Copy the freshly built binary and the entrypoint
COPY --from=builder /usr/local/bin/rustfs /usr/bin/rustfs
COPY --from=builder /usr/local/bin/rc /usr/bin/rc
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
COPY bootstrap/ /etc/rustfs/bootstrap/
RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
COPY entrypoint.sh /entrypoint.sh
RUN chmod +x /usr/bin/rustfs /entrypoint.sh
# Default environment (override in docker run/compose as needed)
ENV RUSTFS_VOLUMES="/data" \
@@ -275,5 +263,5 @@ EXPOSE 9000
VOLUME ["/data"]
# Keep root here; entrypoint will drop privileges using chroot --userspec
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
ENTRYPOINT ["/entrypoint.sh"]
CMD ["/usr/bin/rustfs"]
+1 -1
View File
@@ -199,7 +199,7 @@ docker run -d --name rustfs -p 9000:9000 \
Notes:
- `RUSTFS_NOTIFY_ENABLE=true` enables the global notify module switch.
- For ARN `arn:rustfs:sqs::primary:webhook`, use instance-scoped env vars with `_PRIMARY`.
- If queue dir is omitted, the official image maps `/opt/rustfs/events` into the persistent `/data` volume so it is writable by the runtime user and pending events survive container recreation. Override `queue_dir` when using another deployment layout, or set `RUSTFS_TARGET_STORE_DIR` to move every default queue; native Windows defaults to `%ProgramData%\RustFS`. See [Target queue storage](docs/operations/target-queue-storage.md).
- If queue dir is omitted, the official image maps `/opt/rustfs/events` into the persistent `/data` volume so it is writable by the runtime user and pending events survive container recreation. Override `queue_dir` when using another deployment layout.
- `RUSTFS_NOTIFY_WEBHOOK_SKIP_TLS_VERIFY_PRIMARY` defaults to `false`; enabling it skips webhook TLS certificate verification, allows MITM attacks, and emits a startup warning. Prefer `RUSTFS_NOTIFY_WEBHOOK_CLIENT_CA_PRIMARY` for private CAs.
- Since `1.0.0-beta.11`, webhook endpoints on private or container networks
(`Docker Compose service names`, `host.docker.internal`, RFC 1918 addresses) are
-21
View File
@@ -1,21 +0,0 @@
# RustFS image bootstrap
The official image bundles this directory at `/etc/rustfs/bootstrap`. The default `buckets.json` is an empty JSON array, so the image starts RustFS directly and does not require a bootstrap endpoint or bootstrap credentials.
To create buckets, replace `buckets.json` in a derived image with a JSON array of bucket names. The entrypoint creates each bucket if it does not exist and applies `public-access-block.json`, whose default blocks all four public access paths. Edit that file in the derived image only when public access is an explicit requirement.
Optional per-bucket files use the bucket name as the filename without an extension:
| Directory | File format | Applied operation |
| --- | --- | --- |
| `policies/` | S3 bucket policy JSON | `rc bucket anonymous set-json` |
| `versioning/` | JSON boolean (`true` or `false`) | Enable or suspend bucket versioning |
| `lifecycle/` | `rc bucket lifecycle rule import` JSON | Replace lifecycle rules |
| `cors/` | JSON or XML accepted by `rc bucket cors set` | Replace CORS rules |
| `notifications/` | JSON array of `{ "arn": "...", "events": ["..."] }` | Add or replace event rules by ARN (prefix/suffix filters are not currently accepted) |
Every per-bucket file must correspond to a bucket listed in `buckets.json`. Applying these files is repeatable: bucket creation ignores existing buckets, while each configuration operation replaces or reconciles that bucket's requested state.
When `buckets.json` is non-empty, set `RUSTFS_BOOTSTRAP_ENDPOINT` to the RustFS S3 endpoint reachable from the container, for example `http://rustfs:9000` for a Compose service or the internal service DNS name in Kubernetes. The entrypoint waits on that endpoint's `/health/ready`; it never assumes `localhost`. `RUSTFS_BOOTSTRAP_TIMEOUT` controls the readiness wait in seconds (default `120`, maximum `3600`). `RUSTFS_BOOTSTRAP_DIR` can point to a different in-image configuration directory; it defaults to `/etc/rustfs/bootstrap`. Set `RUSTFS_ACCESS_KEY` and `RUSTFS_SECRET_KEY`, or their `_FILE` variants, for bootstrap authentication. The endpoint and credentials are read at runtime and are not baked into the image.
RustFS built-in IAM policies remain unchanged; the default bootstrap installs no custom bucket policies, lifecycle rules, versioning, CORS rules, or notification rules. The bundled RustFS CLI is resolved from a published release and its release checksum is verified. Set the Docker build argument `RC_RELEASE` to a version to pin it. Build the RustFS image after a CLI release containing the `bucket public-access-block` commands is available. For Kubernetes, the configured service must route traffic to the starting pod before its readiness probe succeeds (for example, with `publishNotReadyAddresses: true`); otherwise the service endpoint cannot reach the pod while the entrypoint waits for readiness.
-1
View File
@@ -1 +0,0 @@
[]
-1
View File
@@ -1 +0,0 @@
-1
View File
@@ -1 +0,0 @@
-1
View File
@@ -1 +0,0 @@
-1
View File
@@ -1 +0,0 @@
-6
View File
@@ -1,6 +0,0 @@
{
"block_public_acls": true,
"ignore_public_acls": true,
"block_public_policy": true,
"restrict_public_buckets": true
}
-1
View File
@@ -1 +0,0 @@
-7
View File
@@ -276,13 +276,6 @@ Drive timeout profile preset:
- values must be positive; `0` or an invalid value falls back to the default, and values above `60` are clamped to `60`.
- the setting is read by the aggregating node only; it does not change the internode RPC wire contract. Any retry shares one round deadline rather than receiving a fresh timeout.
## Target queue directory
- `RUSTFS_TARGET_STORE_DIR`
- absolute root for notify and audit target queues whose `queue_dir` is the default `/opt/rustfs/events` or `/opt/rustfs/audit`; they open `<root>/events` and `<root>/audit` on this node.
- unset keeps those paths on Linux and macOS; native Windows uses `%ProgramData%\RustFS\events` and `%ProgramData%\RustFS\audit`.
- explicit `queue_dir` values are never relocated, and the persisted target configuration is unchanged. Changing the root does not migrate pending entries. See `docs/operations/target-queue-storage.md`.
## Startup filesystem boundary policy
- `RUSTFS_UNSUPPORTED_FS_POLICY` controls startup behavior when RustFS detects local endpoint filesystems that are outside the supported production boundary.
+2 -3
View File
@@ -32,9 +32,8 @@ pub const ENV_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD: &str = "RUSTFS_OBJECT_MEDIUM_
/// Environment variable name for maximum concurrent disk reads for object operations.
/// - Purpose: Limit the number of concurrent disk read operations for object reads to prevent I/O saturation.
/// - Unit: request count (usize) for GET; estimated drive fanout for native LIST.
/// - Semantics: GET bodies and native LIST walks use separate per-instance budgets of this size.
/// A LIST consumes up to one budget, including its disk-producer cleanup. `0` disables both caps.
/// - Unit: request count (usize).
/// - Semantics: Throttling disk reads helps maintain overall system responsiveness under load.
/// - Example: `export RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=16`
/// - Note: This setting may interact with OS-level I/O scheduling and should be tuned based on hardware capabilities.
pub const ENV_OBJECT_MAX_CONCURRENT_DISK_READS: &str = "RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS";
+3 -3
View File
@@ -30,10 +30,10 @@ pub const ENV_TRUSTED_PROXY_VALIDATION_MODE: &str = "RUSTFS_TRUSTED_PROXY_VALIDA
/// Default validation mode is "hop_by_hop".
pub const DEFAULT_TRUSTED_PROXY_VALIDATION_MODE: &str = "hop_by_hop";
/// Select RFC 7239 "Forwarded" instead of "X-Forwarded-For" in legacy mode.
/// Environment variable to enable RFC 7239 "Forwarded" header support.
pub const ENV_TRUSTED_PROXY_ENABLE_RFC7239: &str = "RUSTFS_TRUSTED_PROXY_ENABLE_RFC7239";
/// Legacy mode uses X-Forwarded-For unless RFC 7239 is explicitly selected.
pub const DEFAULT_TRUSTED_PROXY_ENABLE_RFC7239: bool = false;
/// RFC 7239 support is enabled by default.
pub const DEFAULT_TRUSTED_PROXY_ENABLE_RFC7239: bool = true;
/// Environment variable for the maximum allowed proxy hops.
pub const ENV_TRUSTED_PROXY_MAX_HOPS: &str = "RUSTFS_TRUSTED_PROXY_MAX_HOPS";
-12
View File
@@ -144,17 +144,5 @@ pub const POSTGRES_QUEUE_LIMIT: &str = "queue_limit";
/// Applies to both notify and audit target queue stores.
pub const ENV_TARGET_STORE_COMPRESS: &str = "RUSTFS_TARGET_STORE_COMPRESS";
/// Environment variable that relocates the root of the default notify and audit queue directories.
///
/// It applies only to targets whose `queue_dir` is the logical default (`EVENT_DEFAULT_DIR` or
/// `AUDIT_DEFAULT_DIR`); their queues move to `<root>/events` and `<root>/audit` on this node.
pub const ENV_TARGET_STORE_DIR: &str = "RUSTFS_TARGET_STORE_DIR";
/// Queue-store compression is enabled by default to reduce disk footprint.
pub const DEFAULT_TARGET_STORE_COMPRESS: bool = true;
/// Maximum on-disk bytes in each target queue, including failed entries and pending writes.
pub const ENV_TARGET_STORE_MAX_BYTES: &str = "RUSTFS_TARGET_STORE_MAX_BYTES";
/// Queue writes fail at one GiB unless the operator sets another positive byte budget.
pub const DEFAULT_TARGET_STORE_MAX_BYTES: u64 = 1024 * 1024 * 1024;
+1 -6
View File
@@ -175,17 +175,12 @@ pub const ENV_HTTP_SOCKET_RECV_BUFFER_BYTES: &str = "RUSTFS_HTTP_SOCKET_RECV_BUF
pub const DEFAULT_HTTP_SOCKET_RECV_BUFFER_BYTES: usize = 0;
/// Environment variable for the S3 request-body inter-chunk read timeout
/// (seconds). Default: 300. Set to 0 to disable the inactivity timeout.
/// (seconds). Default: 300. Set to 0 to disable.
///
/// This bounds how long a `PutObject`/upload may wait for the *next* body chunk
/// while more bytes are still expected. It resets on every chunk, so it does not
/// penalize slow-but-progressing uploads — it only fires when a peer sends a
/// partial body and then goes silent *without* closing the connection (no EOF).
/// Independently of this knob, `PutObject` and `UploadPart` require at least
/// 64 KiB of body progress per 60 seconds of server wait (the minimum-progress
/// floor, logged as `upload_body_progress_stalled`); it bounds trickle uploads
/// that would otherwise pin upload admission indefinitely, and it stays in
/// force when this timeout is set to 0.
/// A reverse proxy or CDN that forwards a truncated body this way would
/// otherwise make RustFS wait forever for bytes that never arrive; with this
/// timeout the stalled read is aborted and logged with the received/expected
+2 -55
View File
@@ -675,10 +675,6 @@ impl DataUsageInfo {
/// cycle. The timestamp fallback preserves ordering for legacy snapshots that
/// predate those fields.
pub fn data_usage_snapshot_is_newer(candidate: &DataUsageInfo, baseline: &DataUsageInfo) -> bool {
data_usage_identity_is_newer(candidate.snapshot_identity(), baseline.snapshot_identity())
}
fn data_usage_identity_is_newer(candidate: DataUsageSnapshotIdentity, baseline: DataUsageSnapshotIdentity) -> bool {
match (
candidate.scanner_epoch.zip(candidate.scanner_cycle),
baseline.scanner_epoch.zip(baseline.scanner_cycle),
@@ -701,18 +697,10 @@ fn data_usage_identity_is_newer(candidate: DataUsageSnapshotIdentity, baseline:
/// already advance the authoritative timestamp when deleting a bucket, so a
/// rollback delete/recreate fences the previous bucket incarnation too.
pub fn observed_data_usage_is_newer(observed: &DataUsageInfo, authoritative: &DataUsageInfo) -> bool {
observed_data_usage_is_newer_than_identity(observed, authoritative.snapshot_identity())
}
/// Return whether a nonconverged observation may supersede `authoritative`.
///
/// This applies the structural, exact-baseline, and generation checks of
/// [`observed_data_usage_is_newer`] without retaining the baseline's bucket maps.
pub fn observed_data_usage_is_newer_than_identity(observed: &DataUsageInfo, authoritative: DataUsageSnapshotIdentity) -> bool {
observed.usage_snapshot_converged == Some(false)
&& (observed.is_complete_bucket_usage_snapshot() || observed.is_valid_partial_snapshot())
&& observed.usage_snapshot_authoritative_baseline == Some(authoritative)
&& data_usage_identity_is_newer(observed.snapshot_identity(), authoritative)
&& observed.usage_snapshot_authoritative_baseline.as_ref() == Some(&authoritative.snapshot_identity())
&& data_usage_snapshot_is_newer(observed, authoritative)
}
/// Metadata describing the status of a disk-level data usage snapshot.
@@ -3152,17 +3140,12 @@ mod tests {
};
assert!(observed_data_usage_is_newer(&observed, &authoritative));
assert!(observed_data_usage_is_newer_than_identity(&observed, authoritative.snapshot_identity()));
authoritative.last_update = Some(SystemTime::UNIX_EPOCH + Duration::from_secs(601));
assert!(
!observed_data_usage_is_newer(&observed, &authoritative),
"an old-binary namespace mutation must fence the prior bucket incarnation regardless of clock skew"
);
assert!(
!observed_data_usage_is_newer_than_identity(&observed, authoritative.snapshot_identity()),
"a captured identity must preserve the exact namespace fence"
);
}
#[test]
@@ -3189,18 +3172,6 @@ mod tests {
assert!(!observed_data_usage_is_newer(&candidate(2, 9, Some(false), true), &authoritative));
assert!(!observed_data_usage_is_newer(&candidate(2, 11, Some(true), true), &authoritative));
assert!(!observed_data_usage_is_newer(&candidate(2, 11, Some(false), false), &authoritative));
for (observed, expected) in [
(candidate(2, 11, Some(false), true), true),
(candidate(2, 9, Some(false), true), false),
(candidate(2, 11, Some(true), true), false),
(candidate(2, 11, Some(false), false), false),
] {
assert_eq!(
observed_data_usage_is_newer_than_identity(&observed, authoritative.snapshot_identity()),
expected,
"observation identity selection must preserve convergence, completeness, and generation gates"
);
}
let mut partial = candidate(2, 11, Some(false), false);
partial.usage_snapshot_partial = true;
@@ -3215,30 +3186,6 @@ mod tests {
segment_invalidation_proof: None,
}];
assert!(observed_data_usage_is_newer(&partial, &authoritative));
assert!(observed_data_usage_is_newer_than_identity(&partial, authoritative.snapshot_identity()));
}
#[test]
fn observation_identity_selection_keeps_legacy_timestamp_ordering() {
let baseline = DataUsageSnapshotIdentity {
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(10)),
// An epoch without a cycle is not a coordinated position.
scanner_epoch: Some(2),
scanner_cycle: None,
};
let mut observed = DataUsageInfo {
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(11)),
scanner_epoch: Some(1),
usage_snapshot_complete: true,
usage_snapshot_converged: Some(false),
usage_snapshot_authoritative_baseline: Some(baseline),
..Default::default()
};
assert!(observed_data_usage_is_newer_than_identity(&observed, baseline));
observed.last_update = baseline.last_update;
assert!(!observed_data_usage_is_newer_than_identity(&observed, baseline));
observed.last_update = None;
assert!(!observed_data_usage_is_newer_than_identity(&observed, baseline));
}
#[test]
+3 -3
View File
@@ -26,7 +26,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
| **protocols** | [`src/protocols/`](src/protocols) | FTPS, WebDAV, SFTP compliance. Fixed ports, own guide: [`src/protocols/README.md`](src/protocols/README.md) |
| **reliant** | [`src/reliant/`](src/reliant) | Tests that reuse an **externally started** server (SQL/select, conditional writes, lifecycle, deleted-object reads, node-interact). Run via [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh); see [`src/reliant/README.md`](src/reliant/README.md) |
| **cluster** | `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `namespace_lock_quorum_test`, `admin_timeout_regression_test`, `object_lambda_test`, `replication_extension_test`, `tier_stats_cluster_test` | Multi-node scenarios via `RustFSTestClusterEnvironment` |
| **distributed 4×4** | [`src/distributed/`](src/distributed) | Nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
| **distributed 4×4** | [`src/distributed/`](src/distributed) | Storage-sensitive PR and nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
| **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
| **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
@@ -194,9 +194,9 @@ the same profile for membership and execution with one nightly worker.
| `s3s-e2e` black-box | `e2e-tests` + `e2e-tests-rio-v2` jobs | **Active** (external conformance tool) |
| ILM / lifecycle (ignored) | `test-ilm-integration-serial` lane, `-j1` | **Active** (backlog#1148 ilm-1) |
| KMS suite | `e2e-full` job, merge queue + main | **Active** |
| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, release tags + weekly schedule + manual dispatch | **Active** |
| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
| Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (nightly / dispatch) |
| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (storage-sensitive PR / nightly / dispatch) |
| Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
| Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
| Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) |
@@ -260,7 +260,7 @@ async fn iam_policy_requires_if_match(env: &RustFSTestEnvironment, admin: &Clien
let key = "guarded.txt";
admin_create_user(env, &user, user_secret).await?;
let mut policy = serde_json::json!({
let policy = serde_json::json!({
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
@@ -268,8 +268,9 @@ async fn iam_policy_requires_if_match(env: &RustFSTestEnvironment, admin: &Clien
"Resource": [format!("arn:aws:s3:::{bucket}/*")],
"Condition": { "Null": { "s3:if-match": "false" } }
}]
});
admin_add_and_attach_policy(env, &policy_name, &policy.to_string(), &user).await?;
})
.to_string();
admin_add_and_attach_policy(env, &policy_name, &policy, &user).await?;
admin.create_bucket().bucket(&bucket).send().await?;
let original = admin
@@ -292,46 +293,6 @@ async fn iam_policy_requires_if_match(env: &RustFSTestEnvironment, admin: &Clien
.expect_err("PutObject without If-Match must be denied");
assert_error(&denied, "AccessDenied", 403, "unconditional PutObject");
let denied = client
.put_object()
.bucket(&bucket)
.key(key)
.if_match(&etag)
.body(ByteStream::from_static(b"missing-read-permission"))
.send()
.await
.expect_err("If-Match without GetObject must be denied");
assert_error(&denied, "AccessDenied", 403, "conditional PutObject without GetObject");
// S3 requires GetObject as well as PutObject for an ETag-based write.
policy["Statement"]
.as_array_mut()
.expect("policy statements")
.push(serde_json::json!({
"Effect": "Allow",
"Action": ["s3:GetObject"],
"Resource": [format!("arn:aws:s3:::{bucket}/{key}")]
}));
admin_add_canned_policy_via(
AdminTransport::Signed,
&env.url,
&env.access_key,
&env.secret_key,
&policy_name,
&policy.to_string(),
)
.await?;
let denied = client
.put_object()
.bucket(&bucket)
.key(key)
.body(ByteStream::from_static(b"blind-overwrite"))
.send()
.await
.expect_err("GetObject permission must not allow an unconditional PutObject");
assert_error(&denied, "AccessDenied", 403, "unconditional PutObject with GetObject");
client
.put_object()
.bucket(&bucket)
@@ -340,8 +301,6 @@ async fn iam_policy_requires_if_match(env: &RustFSTestEnvironment, admin: &Clien
.body(ByteStream::from_static(b"v2"))
.send()
.await?;
let written = client.get_object().bucket(&bucket).key(key).send().await?;
assert_eq!(written.body.collect().await?.into_bytes().as_ref(), b"v2");
let _ = admin.delete_object().bucket(&bucket).key(key).send().await;
let _ = admin.delete_bucket().bucket(&bucket).send().await;
@@ -502,163 +502,10 @@ mod tests {
assert!(response.contains(if expected == reqwest::StatusCode::FORBIDDEN {
"<Code>AccessDenied</Code>"
} else {
"<Code>BucketAlreadyOwnedByYou</Code>"
"<Code>BucketAlreadyExists</Code>"
}));
}
}
Ok(())
}
/// A deployment owns a single bucket namespace, so an IAM user that recreates
/// an existing bucket is the owner, not another account. In the default region
/// every such request returns the legacy idempotent `200 OK` (AWS CreateBucket
/// semantics), including buckets created by root and by another IAM user.
#[tokio::test]
async fn test_create_bucket_duplicate_in_default_region_is_idempotent_for_iam_user()
-> Result<(), Box<dyn Error + Send + Sync>> {
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec!["--region", "us-east-1"]).await?;
let root = env.create_s3_client();
admin_create_user(&env, "duplicate-iam", "duplicate-iam-secret").await?;
admin_attach_user_policy_via(
AdminTransport::Signed,
&env.url,
&env.access_key,
&env.secret_key,
"readwrite",
"duplicate-iam",
)
.await?;
let iam = Client::from_conf(build_test_s3_config(
&env.url,
"duplicate-iam",
"duplicate-iam-secret",
None,
"duplicate-create",
));
root.create_bucket()
.bucket("duplicate-root-owned")
.send()
.await
.expect("root creates the first bucket");
iam.create_bucket()
.bucket("duplicate-root-owned")
.send()
.await
.expect("an IAM user recreating a root-owned bucket in us-east-1 gets 200 OK");
iam.create_bucket()
.bucket("duplicate-iam-owned")
.send()
.await
.expect("the IAM user creates its own bucket");
iam.create_bucket()
.bucket("duplicate-iam-owned")
.send()
.await
.expect("the same IAM user recreating its bucket gets 200 OK");
root.create_bucket()
.bucket("duplicate-iam-owned")
.send()
.await
.expect("root recreating an IAM-created bucket gets 200 OK");
Ok(())
}
/// Outside the default region the same ownership answer becomes
/// `409 BucketAlreadyOwnedByYou` for every authorized requester. Requests that
/// lack `s3:CreateBucket` still fail authorization before the duplicate check.
#[tokio::test]
async fn test_create_bucket_duplicate_outside_default_region_reports_owned_by_you() -> Result<(), Box<dyn Error + Send + Sync>>
{
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec!["--region", "eu-west-1"]).await?;
let region = "eu-west-1";
let root = Client::from_conf(
env.create_s3_client()
.config()
.to_builder()
.region(Region::new(region))
.build(),
);
admin_create_user(&env, "duplicate-eu-iam", "duplicate-eu-iam-secret").await?;
admin_attach_user_policy_via(
AdminTransport::Signed,
&env.url,
&env.access_key,
&env.secret_key,
"readwrite",
"duplicate-eu-iam",
)
.await?;
let iam = Client::from_conf(
build_test_s3_config(&env.url, "duplicate-eu-iam", "duplicate-eu-iam-secret", None, "duplicate-create")
.to_builder()
.region(Region::new(region))
.build(),
);
admin_create_user(&env, "duplicate-eu-readonly", "duplicate-eu-readonly-secret").await?;
admin_attach_user_policy_via(
AdminTransport::Signed,
&env.url,
&env.access_key,
&env.secret_key,
"readonly",
"duplicate-eu-readonly",
)
.await?;
let readonly = Client::from_conf(
build_test_s3_config(
&env.url,
"duplicate-eu-readonly",
"duplicate-eu-readonly-secret",
None,
"duplicate-create",
)
.to_builder()
.region(Region::new(region))
.build(),
);
for (client, bucket) in [(&root, "duplicate-eu-root"), (&iam, "duplicate-eu-iam")] {
client
.create_bucket()
.bucket(bucket)
.send()
.await
.unwrap_or_else(|error| panic!("{bucket} must be created: {error:?}"));
// Both the creator and the other authenticated identity own the namespace.
for creator in [&root, &iam] {
let error = creator
.create_bucket()
.bucket(bucket)
.send()
.await
.expect_err("a duplicate create outside us-east-1 must report ownership");
assert_eq!(
error.as_service_error().and_then(ProvideErrorMetadata::code),
Some("BucketAlreadyOwnedByYou"),
"{bucket}"
);
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(409), "{bucket}");
}
}
let denied = readonly
.create_bucket()
.bucket("duplicate-eu-root")
.send()
.await
.expect_err("a requester without s3:CreateBucket must be denied before the duplicate check");
assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
Ok(())
}
}
@@ -16,193 +16,8 @@
mod tests {
use crate::common::{RustFSTestEnvironment, init_logging};
use aws_sdk_s3::Client;
use aws_sdk_s3::config::http::HttpResponse;
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration};
fn assert_not_found<E: ProvideErrorMetadata + std::fmt::Debug>(error: &SdkError<E, HttpResponse>) {
assert_eq!(error.raw_response().map(|r| r.status().as_u16()), Some(404), "{error:?}");
assert_eq!(
error.as_service_error().and_then(ProvideErrorMetadata::code),
Some("NoSuchKey"),
"{error:?}"
);
}
#[tokio::test]
async fn conditional_writes_cannot_replace_delete_marker_with_if_match() {
init_logging();
let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
env.start_rustfs_server(vec![]).await.expect("start RustFS");
let client = env.create_s3_client();
let bucket = "conditional-delete-marker-proof";
create_versioned_bucket(&client, bucket).await;
client
.put_object()
.bucket(bucket)
.key("source")
.body(ByteStream::from_static(b"source"))
.send()
.await
.expect("write copy source");
for operation in ["put", "copy", "multipart"] {
let old = client
.put_object()
.bucket(bucket)
.key(operation)
.body(ByteStream::from_static(b"old"))
.send()
.await
.expect("write historical version");
let old_etag = old.e_tag().expect("historical ETag");
let marker = client
.delete_object()
.bucket(bucket)
.key(operation)
.send()
.await
.expect("create delete marker");
let marker_id = marker.version_id().expect("delete marker version");
let (upload_id, completed) = if operation == "multipart" {
let upload = client
.create_multipart_upload()
.bucket(bucket)
.key(operation)
.send()
.await
.expect("initiate multipart upload");
let upload_id = upload.upload_id().expect("upload ID").to_owned();
let part = client
.upload_part()
.bucket(bucket)
.key(operation)
.upload_id(&upload_id)
.part_number(1)
.body(ByteStream::from_static(b"new"))
.send()
.await
.expect("upload part");
let completed = CompletedMultipartUpload::builder()
.parts(
CompletedPart::builder()
.part_number(1)
.set_e_tag(part.e_tag().map(str::to_owned))
.build(),
)
.build();
(Some(upload_id), Some(completed))
} else {
(None, None)
};
for etag in [old_etag, "\"00000000000000000000000000000000\""] {
match operation {
"put" => {
let err = client
.put_object()
.bucket(bucket)
.key(operation)
.if_match(etag)
.body(ByteStream::from_static(b"new"))
.send()
.await
.expect_err("If-Match must reject marker");
assert_not_found(&err);
}
"copy" => {
let err = client
.copy_object()
.bucket(bucket)
.key(operation)
.copy_source(format!("{bucket}/source"))
.if_match(etag)
.send()
.await
.expect_err("If-Match copy must reject marker");
assert_not_found(&err);
}
_ => {
let err = client
.complete_multipart_upload()
.bucket(bucket)
.key(operation)
.upload_id(upload_id.as_deref().expect("upload ID"))
.multipart_upload(completed.clone().expect("completed parts"))
.if_match(etag)
.send()
.await
.expect_err("If-Match completion must reject marker");
assert_not_found(&err);
}
}
let versions = client
.list_object_versions()
.bucket(bucket)
.prefix(operation)
.send()
.await
.expect("list versions after rejected write");
assert_eq!(versions.versions().len(), 1, "{operation}: failed write created a version");
assert_eq!(versions.delete_markers().len(), 1);
assert_eq!(versions.delete_markers()[0].version_id(), Some(marker_id));
assert_eq!(versions.delete_markers()[0].is_latest(), Some(true));
}
match operation {
"put" => {
client
.put_object()
.bucket(bucket)
.key(operation)
.if_none_match("*")
.body(ByteStream::from_static(b"new"))
.send()
.await
.expect("If-None-Match PUT over marker");
}
"copy" => {
client
.copy_object()
.bucket(bucket)
.key(operation)
.copy_source(format!("{bucket}/source"))
.if_none_match("*")
.send()
.await
.expect("If-None-Match copy over marker");
}
_ => {
client
.complete_multipart_upload()
.bucket(bucket)
.key(operation)
.upload_id(upload_id.as_deref().expect("upload ID"))
.multipart_upload(completed.expect("completed parts"))
.if_none_match("*")
.send()
.await
.expect("If-None-Match completion over marker");
}
}
let current = client
.get_object()
.bucket(bucket)
.key(operation)
.send()
.await
.expect("read successful conditional write")
.body
.collect()
.await
.expect("read object body")
.into_bytes();
let expected: &[u8] = if operation == "copy" { b"source" } else { b"new" };
assert_eq!(current.as_ref(), expected, "{operation}: unexpected current object");
}
}
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
async fn create_versioned_bucket(client: &Client, bucket: &str) {
client
@@ -12,249 +12,11 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use super::harness::{DistCluster, DistLayout, TestResult, payload_for, s3_probe_client, sha256_hex, unique_bucket};
use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, unique_bucket};
use crate::common::init_logging;
use aws_sdk_s3::{
Client,
error::{ProvideErrorMetadata, SdkError},
primitives::ByteStream,
};
use std::any::TypeId;
use std::collections::BTreeSet;
use std::fmt::Debug;
use std::future::Future;
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::Barrier;
use tokio::time::Instant;
const LOCK_ADMISSION_BUDGET: Duration = Duration::from_secs(45);
async fn after_lock_contention<T, E, F, Fut>(operation: &str, mut call: F) -> TestResult<Result<T, SdkError<E>>>
where
E: Debug + ProvideErrorMetadata + 'static,
F: FnMut() -> Fut,
Fut: Future<Output = Result<T, SdkError<E>>>,
{
let started = Instant::now();
let mut attempts = 0;
// The workload uses unique unversioned keys. Repeat only explicitly
// advertised lock backpressure; every step still needs its real result.
let outcome = tokio::time::timeout(LOCK_ADMISSION_BUDGET, async {
loop {
attempts += 1;
let result = call().await;
let retry = result.as_ref().err().is_some_and(|error| {
let Some(response) = error.raw_response() else {
return false;
};
let Some(service_error) = error.as_service_error() else {
return false;
};
// HEAD has no XML error body, so its SDK service error can
// lack a code even when the server supplies retry headers.
let namespace_code = service_error.code() == Some("ServiceUnavailable")
|| (service_error.code().is_none()
&& TypeId::of::<E>() == TypeId::of::<aws_sdk_s3::operation::head_object::HeadObjectError>());
response.status().as_u16() == 503
&& namespace_code
&& response.headers().get("x-rustfs-error-cause") == Some("namespace_lock_timeout")
&& response.headers().get("retry-after") == Some("1")
});
if !retry {
return result;
}
tokio::time::sleep(Duration::from_secs(1)).await;
}
})
.await;
if attempts > 1 || !matches!(&outcome, Ok(Ok(_))) {
eprintln!(
"mixed workload request: operation={operation}, attempts={attempts}, elapsed={:?}",
started.elapsed()
);
}
outcome.map_err(|_| {
format!("{operation} exceeded its {LOCK_ADMISSION_BUDGET:?} request budget after {attempts} attempts").into()
})
}
async fn assert_workload_bytes(client: &Client, bucket: &str, key: &str, expected: &[u8]) -> TestResult {
let output = after_lock_contention(&format!("GET {key}"), || client.get_object().bucket(bucket).key(key).send()).await??;
let actual = output.body.collect().await?.into_bytes();
if actual.as_ref() != expected {
return Err(format!(
"object {key} bytes mismatch: expected {} bytes sha256={} got {} bytes sha256={}",
expected.len(),
sha256_hex(expected),
actual.len(),
sha256_hex(&actual)
)
.into());
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use aws_smithy_http_client::test_util::infallible_client_fn;
use aws_smithy_types::body::SdkBody;
use std::sync::atomic::{AtomicUsize, Ordering};
type Reply = (u16, &'static str, Option<&'static str>, Option<&'static str>, &'static str);
fn client(respond: impl Fn(usize) -> Reply + Send + Sync + 'static) -> (Client, Arc<AtomicUsize>) {
let requests = Arc::new(AtomicUsize::new(0));
let seen = Arc::clone(&requests);
let transport = infallible_client_fn(move |_| {
let (status, code, cause, retry_after, body) = respond(seen.fetch_add(1, Ordering::SeqCst));
let body = if code.is_empty() {
body.to_string()
} else {
format!("<Error><Code>{code}</Code></Error>")
};
let mut response = http::Response::builder()
.status(status)
.header("content-type", "application/xml")
.header("content-length", body.len());
if let Some(cause) = cause {
response = response.header("x-rustfs-error-cause", cause);
}
if let Some(retry_after) = retry_after {
response = response.header("retry-after", retry_after);
}
response.body(SdkBody::from(body)).expect("scripted S3 response")
});
let config = crate::common::build_test_s3_config(
"http://contention.example.invalid",
"synthetic-access",
"synthetic-secret",
None,
"contention-test",
);
let client = Client::from_conf(config.to_builder().http_client(transport).build());
(s3_probe_client(&client), requests)
}
fn contention() -> Reply {
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("1"), "")
}
#[tokio::test(start_paused = true)]
async fn advertised_contention_recovers_after_standard_sdk_attempts_are_exhausted() -> TestResult {
let (client, requests) = client(|attempt| {
if attempt < 3 {
contention()
} else {
(200, "", None, None, "")
}
});
let standard = Client::from_conf(
client
.config()
.to_builder()
.retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(3))
.build(),
);
standard
.put_object()
.bucket("bucket")
.key("key")
.body(ByteStream::from_static(b"body"))
.send()
.await
.expect_err("three advertised lock responses exhaust the standard SDK policy");
assert_eq!(requests.load(Ordering::SeqCst), 3);
requests.store(0, Ordering::SeqCst);
let started = Instant::now();
after_lock_contention("PUT recovery", || {
client
.put_object()
.bucket("bucket")
.key("key")
.body(ByteStream::from_static(b"body"))
.send()
})
.await??;
assert_eq!(requests.load(Ordering::SeqCst), 4);
assert!(started.elapsed() >= Duration::from_secs(3), "honor each Retry-After response");
Ok(())
}
#[tokio::test(start_paused = true)]
async fn bodyless_head_contention_uses_explicit_retry_headers() -> TestResult {
let (client, requests) = client(|attempt| {
if attempt == 0 {
(503, "", Some("namespace_lock_timeout"), Some("1"), "")
} else {
(200, "", None, None, "")
}
});
after_lock_contention("HEAD recovery", || client.head_object().bucket("bucket").key("key").send()).await??;
assert_eq!(requests.load(Ordering::SeqCst), 2);
Ok(())
}
#[tokio::test(start_paused = true)]
async fn unrelated_or_malformed_responses_are_not_retried() -> TestResult {
for reply in [
(500, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
(403, "AccessDenied", None, None, ""),
(404, "NoSuchKey", None, None, ""),
(503, "ServiceUnavailable", Some("storage_write_quorum"), None, ""),
(503, "ServiceUnavailable", Some("namespace_lock_unavailable"), Some("1"), ""),
(503, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
(503, "", Some("namespace_lock_timeout"), Some("1"), ""),
(503, "ServiceUnavailable", None, Some("1"), ""),
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), None, ""),
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("2"), ""),
] {
let (client, requests) = client(move |_| reply);
let result = after_lock_contention("PUT fatal response", || {
client
.put_object()
.bucket("bucket")
.key("key")
.body(ByteStream::from_static(b"body"))
.send()
})
.await?;
assert!(result.is_err(), "fatal response cannot become success: {reply:?}");
assert_eq!(requests.load(Ordering::SeqCst), 1, "fatal response must remain visible: {reply:?}");
}
Ok(())
}
#[tokio::test(start_paused = true)]
async fn persistent_lock_contention_exhausts_the_request_budget() {
let (client, requests) = client(|_| contention());
let started = Instant::now();
let result = after_lock_contention("PUT unavailable", || {
client
.put_object()
.bucket("bucket")
.key("key")
.body(ByteStream::from_static(b"body"))
.send()
})
.await;
assert!(result.is_err(), "persistent contention cannot acknowledge an operation");
assert!(started.elapsed() >= LOCK_ADMISSION_BUDGET);
assert!(requests.load(Ordering::SeqCst) > 1);
assert!(requests.load(Ordering::SeqCst) <= LOCK_ADMISSION_BUDGET.as_secs() as usize + 1);
}
#[tokio::test(start_paused = true)]
async fn acknowledged_wrong_bytes_fail_without_retry() {
let (client, requests) = client(|_| (200, "", None, None, "wrong bytes"));
let error = assert_workload_bytes(&client, "bucket", "key", b"expected bytes")
.await
.expect_err("a successful but incorrect read is a consistency failure");
assert!(error.to_string().contains("bytes mismatch"));
assert_eq!(requests.load(Ordering::SeqCst), 1);
}
}
#[tokio::test]
async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node() -> TestResult {
@@ -262,9 +24,7 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
let dist = DistCluster::start(DistLayout::FourByFour).await?;
let bucket = unique_bucket("conc");
dist.create_bucket(&bucket).await?;
// This consistency probe owns its admission policy; unrelated responses
// must not be hidden by the SDK's general-purpose retries.
let clients = Arc::new(dist.clients()?.iter().map(s3_probe_client).collect::<Vec<_>>());
let clients = Arc::new(dist.clients()?);
let worker_count = 24;
let rounds = 4;
let barrier = Arc::new(Barrier::new(worker_count));
@@ -284,46 +44,31 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
let key = format!("source/worker-{idx:02}-round-{round}.bin");
let copy_key = format!("retained/worker-{idx:02}-round-{round}.bin");
let body = payload_for(&key, 64 * 1024);
let context = format!("worker={idx} round={round}");
after_lock_contention(&format!("{context} PUT source"), || {
writer
.put_object()
.bucket(&bucket)
.key(&key)
.body(ByteStream::from(body.clone()))
.send()
})
.await??;
put_object(writer, &bucket, &key, body.clone()).await?;
let head = after_lock_contention(&format!("{context} HEAD source"), || {
reader.head_object().bucket(&bucket).key(&key).send()
})
.await??;
let head = reader.head_object().bucket(&bucket).key(&key).send().await?;
if head.content_length() != Some(body.len() as i64) {
return Err(format!("HEAD returned the wrong size for {key}: {head:?}").into());
}
assert_workload_bytes(reader, &bucket, &key, &body).await?;
assert_object_bytes(reader, &bucket, &key, &body).await?;
after_lock_contention(&format!("{context} COPY retained"), || {
copier
.copy_object()
.bucket(&bucket)
.key(&copy_key)
.copy_source(format!("{bucket}/{key}"))
.send()
})
.await??;
assert_workload_bytes(writer, &bucket, &copy_key, &body).await?;
copier
.copy_object()
.bucket(&bucket)
.key(&copy_key)
.copy_source(format!("{bucket}/{key}"))
.send()
.await?;
assert_object_bytes(writer, &bucket, &copy_key, &body).await?;
after_lock_contention(&format!("{context} DELETE source"), || {
writer.delete_object().bucket(&bucket).key(&key).send()
})
.await??;
let missing = after_lock_contention(&format!("{context} HEAD deleted source"), || {
reader.head_object().bucket(&bucket).key(&key).send()
})
.await?
.expect_err("deleted source key must not remain visible");
writer.delete_object().bucket(&bucket).key(&key).send().await?;
let missing = reader
.head_object()
.bucket(&bucket)
.key(&key)
.send()
.await
.expect_err("deleted source key must not remain visible");
if missing.raw_response().map(|response| response.status().as_u16()) != Some(404) {
return Err(format!("deleted source {key} returned an unexpected result: {missing:?}").into());
}
@@ -337,22 +82,14 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
for handle in handles {
inventory.extend(handle.await??);
}
assert_eq!(
inventory.len(),
worker_count * rounds,
"every worker round must retain its acknowledged copy"
);
let expected_keys: BTreeSet<_> = inventory.iter().map(|(key, _)| key.as_str()).collect();
for (node_idx, client) in clients.iter().enumerate() {
let listed = after_lock_contention(&format!("node={node_idx} LIST retained"), || {
client.list_objects_v2().bucket(&bucket).prefix("retained/").send()
})
.await??;
let listed = client.list_objects_v2().bucket(&bucket).prefix("retained/").send().await?;
let listed_keys: BTreeSet<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
assert_eq!(listed_keys, expected_keys, "node {node_idx} returned a divergent retained-key listing");
for (key, body) in &inventory {
assert_workload_bytes(client, &bucket, key, body)
assert_object_bytes(client, &bucket, key, body)
.await
.map_err(|error| format!("node {node_idx} failed to read {key}: {error}"))?;
}
@@ -25,8 +25,7 @@
use super::harness::{
DistCluster, DistLayout, TestResult, cluster_admin, cluster_admin_ok, decommission_status_json, get_object_bytes,
payload_for, put_object, rebalance_complete, rebalance_status_json, s3_probe_client, s3_probe_error_code, sha256_hex,
unique_bucket, wait_until,
payload_for, put_object, s3_probe_client, s3_probe_error_code, sha256_hex, unique_bucket, wait_until,
};
use crate::chaos::{VersionShardCensus, census_object_version_on_disk};
use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, RustFSTestClusterEnvironment};
@@ -48,7 +47,6 @@ use tokio::time::{Instant, sleep};
use uuid::Uuid;
pub(crate) const MOVEMENT_BARRIER_DIR_ENV: &str = "RUSTFS_E2E_DATA_MOVEMENT_BARRIER_DIR";
pub(crate) const STAGE_AFTER_ENTRY_ADMISSION: &str = "after_entry_admission";
pub(crate) const STAGE_BEFORE_TARGET_COMMIT: &str = "before_target_commit";
pub(crate) const STAGE_BEFORE_SOURCE_CLEANUP: &str = "before_source_cleanup";
pub(crate) const STAGE_AFTER_RESPONSIBILITY_PERSISTED: &str = "after_responsibility_persisted";
@@ -1168,21 +1166,8 @@ pub(crate) async fn clear_decommission(cluster: &RustFSTestClusterEnvironment, p
.await
}
pub(crate) async fn stop_rebalance(cluster: &RustFSTestClusterEnvironment, expected_id: &str) -> TestResult<String> {
let path = "/rustfs/admin/v3/rebalance/stop";
let (status, response) = cluster_admin(cluster, Method::POST, path, None).await?;
if status.is_success() {
return Ok(response);
}
// The run can finish between the active observation and the stop request.
// Only completion of that same run makes a not-started response acceptable.
if status == StatusCode::NOT_FOUND
&& response.contains("<Code>NoSuchResource</Code>")
&& rebalance_complete(&rebalance_status_json(cluster).await?, expected_id)?
{
return Ok(response);
}
Err(format!("POST {path} failed: {status} {response}").into())
pub(crate) async fn stop_rebalance(cluster: &RustFSTestClusterEnvironment) -> TestResult<String> {
cluster_admin_ok(cluster, Method::POST, "/rustfs/admin/v3/rebalance/stop", None).await
}
pub(crate) fn decommission_pool_entry(status: &Value, pool_id: usize) -> TestResult<Value> {
@@ -1600,129 +1585,6 @@ pub(crate) async fn assert_every_node_serves(dist: &DistCluster, bucket: &str, l
mod tests {
use super::*;
const REBALANCE_NOT_STARTED: &str =
"<Error><Code>NoSuchResource</Code><Message>pool rebalance is not started</Message></Error>";
async fn mock_rebalance_stop(
stop_status: StatusCode,
stop_body: &str,
run_status: Value,
) -> TestResult<(TestResult<String>, Vec<String>)> {
use bytes::Bytes;
use http_body_util::Full;
use hyper::service::service_fn;
use hyper_util::rt::TokioIo;
use tokio::net::TcpListener;
use tokio::sync::mpsc;
use tokio::task::JoinSet;
let listener = TcpListener::bind("127.0.0.1:0").await?;
let mut cluster = RustFSTestClusterEnvironment::new(1).await?;
cluster.nodes[0].url = format!("http://{}", listener.local_addr()?);
let stop_body = stop_body.to_string();
let (requests_tx, mut requests_rx) = mpsc::unbounded_channel();
let mut server = JoinSet::new();
server.spawn(async move {
loop {
let (stream, _) = match listener.accept().await {
Ok(connection) => connection,
Err(error) => return Err::<(), Box<dyn std::error::Error + Send + Sync>>(error.into()),
};
let stop_body = stop_body.clone();
let run_status = run_status.clone();
let requests_tx = requests_tx.clone();
let handler = service_fn(move |request: hyper::Request<hyper::body::Incoming>| {
let route = format!("{} {}", request.method(), request.uri().path());
requests_tx.send(route.clone()).expect("record mock admin request");
let (status, body) = match route.as_str() {
"POST /rustfs/admin/v3/rebalance/stop" => (stop_status, stop_body.clone()),
"GET /rustfs/admin/v3/rebalance/status" => (StatusCode::OK, run_status.to_string()),
_ => (StatusCode::BAD_REQUEST, "unexpected admin request".to_string()),
};
async move { http::Response::builder().status(status).body(Full::new(Bytes::from(body))) }
});
hyper::server::conn::http1::Builder::new()
.keep_alive(false)
.serve_connection(TokioIo::new(stream), handler)
.await?;
}
});
let outcome = tokio::time::timeout(Duration::from_secs(5), stop_rebalance(&cluster, "run-1")).await?;
server.abort_all();
let requests = std::iter::from_fn(|| requests_rx.try_recv().ok()).collect();
Ok((outcome, requests))
}
fn completed_rebalance() -> Value {
serde_json::json!({
"id": "run-1",
"pools": [
{ "id": 0, "status": "Completed", "progress": { "versions": 113 } },
{ "id": 1, "status": "None" }
]
})
}
#[tokio::test]
async fn rebalance_stop_success_does_not_query_completion() -> TestResult {
let (outcome, requests) = mock_rebalance_stop(StatusCode::OK, "", Value::Null).await?;
assert_eq!(outcome?, "");
assert_eq!(requests, ["POST /rustfs/admin/v3/rebalance/stop"]);
Ok(())
}
#[tokio::test]
async fn rebalance_stop_accepts_completed_run_after_not_started_response() -> TestResult {
let (outcome, requests) =
mock_rebalance_stop(StatusCode::NOT_FOUND, REBALANCE_NOT_STARTED, completed_rebalance()).await?;
outcome?;
assert_eq!(
requests,
[
"POST /rustfs/admin/v3/rebalance/stop",
"GET /rustfs/admin/v3/rebalance/status"
]
);
Ok(())
}
#[tokio::test]
async fn rebalance_stop_rejects_unproven_completion_and_other_errors() -> TestResult {
let complete = completed_rebalance();
let mut invalid = vec![Value::Null];
for (pointer, value) in [
("/id", serde_json::json!("run-2")),
("/pools/0/status", serde_json::json!("Started")),
("/pools/0/status", serde_json::json!("Failed")),
("/pools/0/status", serde_json::json!("Stopped")),
("/pools/0/progress/versions", serde_json::json!(0)),
] {
let mut status = complete.clone();
*status.pointer_mut(pointer).expect("completion fixture field") = value;
invalid.push(status);
}
let mut pending = complete.clone();
pending["stopPropagation"] = serde_json::json!({ "pendingTerminalReload": true });
invalid.push(pending);
let mut failed = complete.clone();
failed["pools"][0]["lastError"] = serde_json::json!("migration failed");
invalid.push(failed);
for status in invalid {
let (outcome, _) = mock_rebalance_stop(StatusCode::NOT_FOUND, REBALANCE_NOT_STARTED, status.clone()).await?;
assert!(outcome.is_err(), "unproven completion accepted: {status}");
}
for (code, body) in [
(StatusCode::FORBIDDEN, REBALANCE_NOT_STARTED),
(StatusCode::INTERNAL_SERVER_ERROR, REBALANCE_NOT_STARTED),
(StatusCode::NOT_FOUND, "<Error><Code>NoSuchBucket</Code></Error>"),
] {
let (outcome, requests) = mock_rebalance_stop(code, body, complete.clone()).await?;
assert!(outcome.is_err(), "unrelated error accepted: {code} {body}");
assert_eq!(requests, ["POST /rustfs/admin/v3/rebalance/stop"]);
}
Ok(())
}
#[test]
fn version_ids_normalize_null_spellings() {
for spelling in [None, Some(""), Some("null"), Some("00000000-0000-0000-0000-000000000000")] {
@@ -28,7 +28,7 @@ use super::harness::{
wait_for_rebalance_active, wait_for_rebalance_complete,
};
use super::scanner_heal_topology_fixture::{
LiveWorkload, MOVEMENT_ACTIVE_TIMEOUT, MOVEMENT_COMPLETE_TIMEOUT, MovementOperation, SEED_POOL, STAGE_AFTER_ENTRY_ADMISSION,
LiveWorkload, MOVEMENT_ACTIVE_TIMEOUT, MOVEMENT_COMPLETE_TIMEOUT, MovementOperation, SEED_POOL,
STAGE_AFTER_RESPONSIBILITY_PERSISTED, STAGE_BEFORE_SOURCE_CLEANUP, STAGE_BEFORE_TARGET_COMMIT, TopologyManifest, TopologyRun,
assert_bucket_durability_strict, assert_every_node_serves, assert_probe_objects, assert_process_durability_strict,
assert_seed_pool_census_unchanged, assert_single_pool_placement, cancel_decommission, capture_seed_pool_census,
@@ -274,7 +274,6 @@ async fn wait_for_rebalance_stopped_or_completed(run: &TopologyRun, rebalance_id
#[tokio::test]
async fn scanner_heal_topology_rebalance_survives_live_writes_stop_and_restart() -> TestResult {
init_logging();
require_test_hooks();
let mut run = TopologyRun::start("rebalance").await?;
let manifest = seed_manifest(&run).await?;
assert_bucket_durability_strict(&run.dist.cluster, &manifest.buckets()).await?;
@@ -289,52 +288,23 @@ async fn scanner_heal_topology_rebalance_survives_live_writes_stop_and_restart()
.map(|entry| (entry.key.clone(), entry.sha256.clone()))
.collect();
// Keep an admitted entry on the source until stop cancels admission. Start
// propagation may await that entry, so observe and stop before awaiting ACK.
let first_key = manifest.plain_key("external", 0)?;
run.movement_barrier
.arm_movement("rebalance", &manifest.plain_bucket, &first_key, STAGE_AFTER_ENTRY_ADMISSION)?;
let observe_first = async {
let (observed_id, reached) = tokio::time::timeout(MOVEMENT_ACTIVE_TIMEOUT, async {
let reached = run.movement_barrier.wait_reached(MOVEMENT_ACTIVE_TIMEOUT).await?;
let status = rebalance_status_json(&run.dist.cluster).await?;
let observed_id = status
.get("id")
.and_then(Value::as_str)
.ok_or("rebalance status omitted id")?;
wait_for_rebalance_active(&run.dist.cluster, observed_id, MOVEMENT_ACTIVE_TIMEOUT).await?;
Ok::<_, Box<dyn std::error::Error + Send + Sync>>((observed_id.to_string(), reached))
})
.await??;
let probes = put_probe_objects(&run.dist, &scratch, "during-rebalance", 4).await?;
let workload = run_live_workload(LiveWorkload {
clients: run.dist.clients()?,
bucket: scratch.clone(),
prefix: "live".to_string(),
writers: LIVE_WRITERS,
rounds: LIVE_ROUNDS,
baseline_bucket: manifest.plain_bucket.clone(),
baseline: baseline.clone(),
pause: Duration::from_millis(100),
});
let stop = async {
sleep(Duration::from_secs(2)).await;
stop_rebalance(&run.dist.cluster, &observed_id).await?;
let (outcome, status) = wait_for_rebalance_stopped_or_completed(&run, &observed_id).await?;
if outcome != "stopped" {
return Err("rebalance completed before stop; mid-run stop was not exercised".into());
}
Ok::<_, Box<dyn std::error::Error + Send + Sync>>((outcome, status))
};
let (ops, (outcome, status)) = tokio::try_join!(workload, stop)?;
Ok::<_, Box<dyn std::error::Error + Send + Sync>>((observed_id, reached, probes, ops, outcome, status))
};
let (first_id, (observed_id, first_reached, during_rebalance, ops, first_outcome, first_status)) =
tokio::try_join!(start_rebalance(&run.dist.cluster), observe_first)?;
if first_id != observed_id {
return Err(format!("rebalance start acknowledged {first_id}, but the stopped run was {observed_id}").into());
}
run.movement_barrier.disarm()?;
let first_id = start_rebalance(&run.dist.cluster).await?;
wait_for_rebalance_active(&run.dist.cluster, &first_id, MOVEMENT_ACTIVE_TIMEOUT).await?;
let during_rebalance = put_probe_objects(&run.dist, &scratch, "during-rebalance", 4).await?;
let workload = tokio::spawn(run_live_workload(LiveWorkload {
clients: run.dist.clients()?,
bucket: scratch.clone(),
prefix: "live".to_string(),
writers: LIVE_WRITERS,
rounds: LIVE_ROUNDS,
baseline_bucket: manifest.plain_bucket.clone(),
baseline: baseline.clone(),
pause: Duration::from_millis(100),
}));
sleep(Duration::from_secs(2)).await;
stop_rebalance(&run.dist.cluster).await?;
let (first_outcome, first_status) = wait_for_rebalance_stopped_or_completed(&run, &first_id).await?;
let ops = workload.await??;
let live_state = expected_live_state(&ops);
for node_idx in 0..run.dist.cluster.nodes.len() {
@@ -363,40 +333,8 @@ async fn scanner_heal_topology_rebalance_survives_live_writes_stop_and_restart()
verify_from_every_node(&run.dist, &manifest, "after restart following the stop").await?;
verify_live_state(&run.dist.client(0)?, &scratch, &live_state, "after restart following the stop").await?;
// A short second run can finish before the start response reaches us.
// Hold one remaining source entry until its active state is observed.
let mut second_key = None;
for entry in manifest.plain_entries() {
if pools_holding(&run.dist, &manifest.plain_bucket, &entry.key, None)?.contains(&SEED_POOL) {
second_key = Some(entry.key.clone());
break;
}
}
let second_key = second_key.ok_or("no seed-pool object remains for the second rebalance")?;
let second_barrier = run.movement_barrier.next_phase();
second_barrier.arm_movement("rebalance", &manifest.plain_bucket, &second_key, STAGE_BEFORE_SOURCE_CLEANUP)?;
let observe_second = async {
let reached = second_barrier.wait_reached(MOVEMENT_ACTIVE_TIMEOUT).await?;
let status = rebalance_status_json(&run.dist.cluster).await?;
let observed_id = status
.get("id")
.and_then(Value::as_str)
.ok_or("rebalance status omitted id")?;
if observed_id == first_id {
return Err("the second rebalance reused the stopped run id".into());
}
wait_for_rebalance_active(&run.dist.cluster, observed_id, MOVEMENT_ACTIVE_TIMEOUT).await?;
second_barrier.disarm()?;
second_barrier.release_movement()?;
Ok::<_, Box<dyn std::error::Error + Send + Sync>>((observed_id.to_string(), reached))
};
// Worker propagation can hold the start response until movement drains.
let (second_id, (observed_id, second_reached)) = tokio::try_join!(start_rebalance(&run.dist.cluster), async {
tokio::time::timeout(MOVEMENT_ACTIVE_TIMEOUT, observe_second).await?
})?;
if second_id != observed_id {
return Err(format!("rebalance start acknowledged {second_id}, but the active run was {observed_id}").into());
}
let second_id = start_rebalance(&run.dist.cluster).await?;
wait_for_rebalance_active(&run.dist.cluster, &second_id, MOVEMENT_ACTIVE_TIMEOUT).await?;
wait_for_rebalance_complete(&run.dist.cluster, &second_id, MOVEMENT_COMPLETE_TIMEOUT).await?;
verify_from_every_node(&run.dist, &manifest, "after the second rebalance completed").await?;
verify_live_state(&run.dist.client(3)?, &scratch, &live_state, "after the second rebalance completed").await?;
@@ -413,8 +351,8 @@ async fn scanner_heal_topology_rebalance_survives_live_writes_stop_and_restart()
&json!({
"gate": "G-REB",
"manifest": manifest_summary(&manifest),
"first_run": { "id": first_id, "outcome": first_outcome, "status": first_status, "versions_held_by_two_pools_after_stop": retained_after_stop, "barrier_object": first_key, "barrier": first_reached },
"second_run": { "id": second_id, "versions_off_seed_pool": moved, "barrier_object": second_key, "barrier": second_reached },
"first_run": { "id": first_id, "outcome": first_outcome, "status": first_status, "versions_held_by_two_pools_after_stop": retained_after_stop },
"second_run": { "id": second_id, "versions_off_seed_pool": moved },
"live_ops": ops,
"probe_placement_during_rebalance": during_placement,
"process_durability_strict_nodes": strict_nodes,
@@ -1636,53 +1636,20 @@ async fn hot_window_with_old_followers(
let mixed_initial_physical = if mixed { json!(converged_shards(&run)?) } else { Value::Null };
let scheduler_config = run.assert_default_cadence().await?;
let baseline = run.quiet_baseline().await?;
// A leader-lock refresh gap during old-binary replacement can make
// `leader()` briefly observe a different node. Poll until the coordinator
// is stable before comparing against the measured fleet. A persistent
// change to an old-binary node is a genuine election outcome and fails.
let (leader, leader_pid) = if mixed {
let expected_leader = usize::try_from(
mixed_fleet["leader"]
.as_u64()
.ok_or("mixed fleet omitted its measured leader")?,
)?;
let expected_pid = mixed_fleet["leader_pid"]
.as_u64()
.ok_or("mixed fleet omitted its measured leader pid")?;
let mut resolved = None;
for _ in 0..20 {
let candidate = run.leader().await?;
let candidate_pid = u64::from(run.pid(candidate)?);
if Some(u64::try_from(candidate)?) == mixed_fleet["leader"].as_u64() && Some(candidate_pid) == Some(expected_pid) {
resolved = Some((candidate, run.pid(candidate)?));
break;
}
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
}
match resolved {
Some(pair) => pair,
None => {
let current = match run.leader().await {
Ok(node) => Some((node, run.pid(node).ok())),
Err(_) => None,
};
return Err(format!(
"the hot window did not retain its measured new coordinator after stabilization \
(expected node {expected_leader} pid {expected_pid}, observed {current:?})"
)
.into());
}
}
} else {
let leader = run.leader().await?;
(leader, run.pid(leader)?)
};
let leader = run.leader().await?;
let leader_pid = run.pid(leader)?;
let producer = (leader + 1) % 4;
let producers = if fault.is_some() {
vec![producer, (leader + 2) % 4]
} else {
vec![producer]
};
if mixed
&& (mixed_fleet["leader"].as_u64() != Some(u64::try_from(leader)?)
|| mixed_fleet["leader_pid"].as_u64() != Some(u64::from(leader_pid)))
{
return Err("the hot window did not retain its measured new coordinator".into());
}
let producer_pid = run.pid(producer)?;
let producer_address = run.peer_address(producer);
let [cold_a, cold_b, hot] = run.buckets.clone();
@@ -33,7 +33,7 @@ mod tests {
use sha2::{Digest, Sha256};
use std::collections::HashSet;
use std::error::Error;
use std::io::{Read, Seek, SeekFrom, Write};
use std::io::{Read, Write};
use std::net::SocketAddr;
use std::path::{Path, PathBuf};
use std::process::Command;
@@ -1654,89 +1654,6 @@ mod tests {
TargetEndpointBlackhole,
}
async fn release_heal_commit_barrier_after_shutdown(
log: &Path,
log_offset: u64,
barrier: &Path,
) -> Result<serde_json::Value, Box<dyn Error + Send + Sync>> {
// Observe within the existing graceful process-stop budget.
timeout(Duration::from_secs(45), async {
loop {
let mut file = std::fs::File::open(log)?;
file.seek(SeekFrom::Start(log_offset))?;
let mut tail = String::new();
file.read_to_string(&mut tail)?;
if let Some(event) = tail
.lines()
.filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
.find(|event| {
event["event"] == "server_shutdown_state"
&& event["component"] == "main"
&& event["subsystem"] == "startup"
&& event["state"] == "stopping"
})
{
// Local heal has stopped and HTTP shutdown has been signalled.
// Let admitted peer RPCs finish; keeping this synthetic barrier
// until process exit would make the fixture block its own drain.
std::fs::remove_file(barrier)?;
return Ok(event);
}
sleep(Duration::from_millis(10)).await;
}
})
.await?
}
#[tokio::test(start_paused = true)]
async fn heal_commit_barrier_waits_for_new_shutdown_event() -> Result<(), Box<dyn Error + Send + Sync>> {
let directory = tempfile::tempdir()?;
let log = directory.path().join("server.log");
let barrier = directory.path().join("barrier");
let event = serde_json::json!({
"event": "server_shutdown_state", "component": "main", "subsystem": "startup", "state": "stopping"
});
std::fs::write(&log, format!("{event}\n"))?;
std::fs::write(&barrier, "bucket/prefix/")?;
let log_offset = log.metadata()?.len();
let mut writer = std::fs::OpenOptions::new().append(true).open(&log)?;
writeln!(writer, "{}", serde_json::json!({"event": "shutdown_signal_received"}))?;
writeln!(writer, "{}", serde_json::json!({"event": "server_shutdown_state", "state": "stopping"}))?;
let release = release_heal_commit_barrier_after_shutdown(&log, log_offset, &barrier);
tokio::pin!(release);
assert!(
futures::poll!(release.as_mut()).is_pending(),
"old or unrelated events must not release the barrier"
);
assert!(barrier.is_file(), "the partial-rebuild barrier must remain armed before shutdown");
tokio::time::advance(Duration::from_secs(6)).await;
assert!(
futures::poll!(release.as_mut()).is_pending(),
"background shutdown may take longer than five seconds within the process-stop budget"
);
writeln!(writer, "{event}")?;
tokio::time::advance(Duration::from_millis(10)).await;
assert_eq!(release.await?, event, "retain the actual new shutdown witness");
assert!(!barrier.exists(), "shutdown must release the synthetic hold");
Ok(())
}
#[tokio::test(start_paused = true)]
async fn heal_commit_barrier_missing_shutdown_fails_without_releasing() -> Result<(), Box<dyn Error + Send + Sync>> {
let directory = tempfile::tempdir()?;
let log = directory.path().join("server.log");
let barrier = directory.path().join("barrier");
std::fs::write(&log, "")?;
std::fs::write(&barrier, "bucket/prefix/")?;
let error = release_heal_commit_barrier_after_shutdown(&log, 0, &barrier)
.await
.expect_err("missing shutdown evidence must fail the fixture");
assert!(error.is::<tokio::time::error::Elapsed>());
assert!(barrier.is_file(), "a missing shutdown witness must not release repairs");
Ok(())
}
async fn run_cluster_root_heal_interruption(scenario: InterruptionScenario) -> Result<(), Box<dyn Error + Send + Sync>> {
let server_binary = rustfs_binary_path();
let evidence_case = match scenario {
@@ -1874,7 +1791,6 @@ mod tests {
let mut created_online_objects = Vec::with_capacity(online_object_count);
let mut expected_manifests = Vec::with_capacity(online_object_count);
let mut unclean_shutdown_marker_observed = None;
let mut shutdown_barrier_release = None;
let mut attempt_count = 0usize;
let max_online_attempts = online_object_count.saturating_mul(topology.total_drives().max(1));
while expected_manifests.len() < online_object_count {
@@ -2457,21 +2373,7 @@ mod tests {
| InterruptionScenario::BackgroundTargetRestartEc84MultiSet
| InterruptionScenario::BackgroundCoordinatorRestart
) {
if interruption_node == 1
&& let Some(barrier) = &commit_barrier
{
let target_log = PathBuf::from(format!("{log_dir}/node1.log"));
let log_offset = target_log.metadata()?.len();
let (stopped, released) = tokio::join!(
cluster.stop_node_gracefully(interruption_node),
release_heal_commit_barrier_after_shutdown(&target_log, log_offset, barrier),
);
// Await both branches even on error so the child is reaped.
stopped?;
shutdown_barrier_release = Some(released?);
} else {
cluster.stop_node_gracefully(interruption_node).await?;
}
cluster.stop_node_gracefully(interruption_node).await?;
} else {
cluster.stop_node(interruption_node)?;
}
@@ -2513,9 +2415,7 @@ mod tests {
}
}
}
if shutdown_barrier_release.is_none()
&& let Some(barrier) = &commit_barrier
{
if let Some(barrier) = &commit_barrier {
std::fs::remove_file(barrier)?;
}
cluster.start_node_from_binary(interruption_node, &server_binary).await?;
@@ -2846,7 +2746,6 @@ mod tests {
"all_peers_bound_to_generation_window": true,
"pid_before": target_pid, "pid_after": restarted_pid,
"unclean_shutdown_marker": unclean_shutdown_marker_observed.unwrap_or(false),
"graceful_barrier_release": shutdown_barrier_release,
"objects": evidence_objects, "node_listings": node_listings,
});
if let Some(outage_write) = outage_write_diagnostic {
-3
View File
@@ -283,9 +283,6 @@ mod ssec_copy_test;
#[cfg(test)]
mod multipart_storage_class_test;
#[cfg(test)]
mod multipart_upload_id_test;
#[cfg(test)]
mod storage_class_capability_test;
@@ -1,175 +0,0 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Multipart upload ID regression tests.
//!
//! Every operation that addresses an existing multipart upload must report an
//! upload ID that does not name a live upload as `404 NoSuchUpload`, whether
//! the ID is malformed or is a well-formed ID of an aborted upload.
#[cfg(test)]
mod tests {
use crate::common::{RustFSTestEnvironment, init_logging};
use aws_sdk_s3::Client;
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
const BUCKET: &str = "multipart-upload-id";
const KEY: &str = "target.bin";
const COPY_SOURCE_KEY: &str = "copy-source.bin";
// RustFS upload IDs are URL-safe base64; '.' is outside that alphabet, so
// the object layer rejects this ID as malformed instead of looking it up.
const MALFORMED_UPLOAD_ID: &str = "malformed.upload.id";
const PART_NUMBER: i32 = 1;
const PART_BODY: &[u8] = b"multipart upload id regression part";
const PART_ETAG: &str = "\"00000000000000000000000000000000\"";
const NO_SUCH_UPLOAD_STATUS: u16 = 404;
const NO_SUCH_UPLOAD_CODE: &str = "NoSuchUpload";
const OBJECT_NOT_FOUND_STATUS: u16 = 404;
/// Describes how `result` differs from `404 NoSuchUpload`, or returns `None` when it matches.
fn no_such_upload_mismatch<T, E>(operation: &str, result: Result<T, SdkError<E>>) -> Option<String>
where
E: ProvideErrorMetadata + std::fmt::Debug,
{
let Err(error) = result else {
return Some(format!("{operation}: succeeded"));
};
let status = error.raw_response().map(|response| response.status().as_u16());
let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
(status != Some(NO_SUCH_UPLOAD_STATUS) || code != Some(NO_SUCH_UPLOAD_CODE))
.then(|| format!("{operation}: status {status:?}, code {code:?}: {error:?}"))
}
async fn start_server_with_copy_source() -> Result<(RustFSTestEnvironment, Client), Box<dyn std::error::Error + Send + Sync>>
{
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(Vec::new()).await?;
env.create_test_bucket(BUCKET).await?;
let client = env.create_s3_client();
client
.put_object()
.bucket(BUCKET)
.key(COPY_SOURCE_KEY)
.body(ByteStream::from_static(PART_BODY))
.send()
.await?;
Ok((env, client))
}
/// Sends every operation that addresses an existing upload and asserts that all of them report `404 NoSuchUpload`.
async fn assert_every_upload_operation_reports_no_such_upload(client: &Client, upload_id: &str) {
let upload_part = client
.upload_part()
.bucket(BUCKET)
.key(KEY)
.upload_id(upload_id)
.part_number(PART_NUMBER)
.body(ByteStream::from_static(PART_BODY))
.send()
.await;
let upload_part_copy = client
.upload_part_copy()
.bucket(BUCKET)
.key(KEY)
.upload_id(upload_id)
.part_number(PART_NUMBER)
.copy_source(format!("/{BUCKET}/{COPY_SOURCE_KEY}"))
.send()
.await;
let list_parts = client.list_parts().bucket(BUCKET).key(KEY).upload_id(upload_id).send().await;
let complete = client
.complete_multipart_upload()
.bucket(BUCKET)
.key(KEY)
.upload_id(upload_id)
.multipart_upload(
CompletedMultipartUpload::builder()
.parts(CompletedPart::builder().part_number(PART_NUMBER).e_tag(PART_ETAG).build())
.build(),
)
.send()
.await;
let abort = client
.abort_multipart_upload()
.bucket(BUCKET)
.key(KEY)
.upload_id(upload_id)
.send()
.await;
let mismatches: Vec<String> = [
no_such_upload_mismatch("UploadPart", upload_part),
no_such_upload_mismatch("UploadPartCopy", upload_part_copy),
no_such_upload_mismatch("ListParts", list_parts),
no_such_upload_mismatch("CompleteMultipartUpload", complete),
no_such_upload_mismatch("AbortMultipartUpload", abort),
]
.into_iter()
.flatten()
.collect();
assert!(
mismatches.is_empty(),
"upload ID {upload_id:?}: operations not reporting {NO_SUCH_UPLOAD_STATUS} {NO_SUCH_UPLOAD_CODE}:\n{}",
mismatches.join("\n")
);
}
/// Asserts that the rejected operations left neither a multipart upload nor an object under `KEY`.
async fn assert_no_upload_or_object_published(client: &Client) -> TestResult {
let uploads = client.list_multipart_uploads().bucket(BUCKET).prefix(KEY).send().await?;
assert!(uploads.uploads().is_empty(), "rejected operations must not leave an upload: {uploads:?}");
let head = client.head_object().bucket(BUCKET).key(KEY).send().await;
let status = head
.as_ref()
.err()
.and_then(|error| error.raw_response())
.map(|response| response.status().as_u16());
assert_eq!(
status,
Some(OBJECT_NOT_FOUND_STATUS),
"rejected operations must not publish {KEY}: {head:?}"
);
Ok(())
}
#[tokio::test]
async fn malformed_upload_id_reports_no_such_upload_on_every_upload_operation() -> TestResult {
let (_env, client) = start_server_with_copy_source().await?;
assert_every_upload_operation_reports_no_such_upload(&client, MALFORMED_UPLOAD_ID).await;
assert_no_upload_or_object_published(&client).await
}
#[tokio::test]
async fn aborted_upload_id_reports_no_such_upload_on_every_upload_operation() -> TestResult {
let (_env, client) = start_server_with_copy_source().await?;
let created = client.create_multipart_upload().bucket(BUCKET).key(KEY).send().await?;
let upload_id = created.upload_id().ok_or("CreateMultipartUpload returned no upload ID")?;
client
.abort_multipart_upload()
.bucket(BUCKET)
.key(KEY)
.upload_id(upload_id)
.send()
.await?;
assert_every_upload_operation_reports_no_such_upload(&client, upload_id).await;
assert_no_upload_or_object_published(&client).await
}
}
+3 -1
View File
@@ -77,7 +77,9 @@ core protocol surface plus cross-protocol consistency:
existing path returns an error, bad-password authentication is rejected
- Cross-protocol via aws-sdk-s3: SFTP write then S3 read with SHA256 match,
S3 write then SFTP read with SHA256 match
- Cross-API directory visibility: SFTP-created sub-directory visible via S3 ListObjectsV2, and an S3 trailing-slash directory object visible via SFTP readdir as a directory entry
- Cross-API directory visibility: SFTP-created sub-directory visible via S3
ListObjectsV2, S3-created `__XLDIR__` marker visible via SFTP readdir as a
directory entry
#### sftp_compliance (`test_sftp_compliance_suite`)
+13 -14
View File
@@ -394,9 +394,9 @@ pub async fn test_sftp_core_operations() -> Result<()> {
info!("PASS: S3-written object matches via S3 GetObject and SFTP READ");
// --- Cross-API directory visibility: SFTP mkdir, S3 ListObjectsV2 ---
// SFTP mkdir writes a zero-byte object at the trailing-slash key.
// Storage encodes that key to the __XLDIR__ marker, and the S3
// listing path decodes the marker back to a trailing-slash key.
// SFTP mkdir writes a __XLDIR__ marker. The rustfs S3 listing path
// decodes that marker back to a trailing-slash key, so the asserted
// pattern is "subdir_sftp/".
info!("Testing SFTP: SFTP-created sub-directory visible via S3 ListObjectsV2");
let sftp_subdir_name = "subdir_sftp";
let sftp_subdir_path = format!("/{bucket}/{sftp_subdir_name}");
@@ -422,21 +422,20 @@ pub async fn test_sftp_core_operations() -> Result<()> {
);
info!("PASS: SFTP mkdir visible to S3 ListObjectsV2");
// --- Cross-API directory visibility: S3 directory object, SFTP readdir ---
// aws-sdk-s3 PutObject writes a zero-byte object at the trailing-slash
// key. The __XLDIR__ suffix is reserved on the client-key boundary, so
// a direct put of that suffix is rejected. Storage encodes the slash
// key, and SFTP readdir must show a directory entry.
info!("Testing SFTP: S3-created directory object visible via SFTP readdir");
// --- Cross-API directory visibility: S3 marker, SFTP readdir ---
// aws-sdk-s3 PutObject writes a zero-byte marker keyed with the
// __XLDIR__ suffix. SFTP readdir must decode the marker back to a
// bare directory entry whose file_type reports as a directory.
info!("Testing SFTP: S3-created __XLDIR__ marker visible via SFTP readdir");
let s3_subdir_name = "subdir_s3";
let s3_subdir_key = format!("{s3_subdir_name}/");
let s3_subdir_marker_key = format!("{s3_subdir_name}{XLDIR_SUFFIX}");
s3.put_object()
.bucket(bucket)
.key(&s3_subdir_key)
.key(&s3_subdir_marker_key)
.body(ByteStream::from_static(b""))
.send()
.await
.map_err(|e| anyhow!("S3 PutObject {}/{} failed: {:?}", bucket, s3_subdir_key, e))?;
.map_err(|e| anyhow!("S3 PutObject {}/{} failed: {:?}", bucket, s3_subdir_marker_key, e))?;
let bucket_entries: Vec<(String, bool)> = sftp
.read_dir(&bucket_path)
.await?
@@ -445,9 +444,9 @@ pub async fn test_sftp_core_operations() -> Result<()> {
let visible_via_sftp = bucket_entries.iter().any(|(name, is_dir)| name == s3_subdir_name && *is_dir);
assert!(
visible_via_sftp,
"S3-created directory object must appear as a directory in SFTP readdir: entries were {bucket_entries:?}"
"S3-created marker must appear as a directory in SFTP readdir: entries were {bucket_entries:?}"
);
info!("PASS: S3 directory object visible to SFTP readdir as a directory");
info!("PASS: S3 marker visible to SFTP readdir as a directory");
// --- Pre-cleanup of cross-protocol fixtures ---
// Removes the new files and sub-directories so the existing rmdir
+4 -34
View File
@@ -412,38 +412,6 @@ mod integration_tests {
);
}
async fn upload_after_quota_reconciliation(
env: &QuotaTestEnv,
key: &str,
size_bytes: usize,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let upload = async {
loop {
let result = env
.client
.put_object()
.bucket(&env.bucket_name)
.key(key)
.body(aws_sdk_s3::primitives::ByteStream::from(vec![0u8; size_bytes]))
.send()
.await;
match result {
Ok(_) => break,
Err(error) => assert_quota_rejection(
error.raw_response().map(|response| response.status().as_u16()),
error.as_service_error(),
&error,
),
}
sleep(Duration::from_secs(1)).await;
}
};
// The durable V1 ledger can retain the old charge for its 30-second reconciliation cooldown.
timeout(Duration::from_secs(45), upload)
.await
.map_err(|_| format!("PUT {key} did not succeed after quota reconciliation within 45 seconds").into())
}
async fn assert_put_rejected_by_quota(env: &QuotaTestEnv, key: &str, size_bytes: usize) {
let error = env
.client
@@ -605,7 +573,8 @@ mod integration_tests {
assert!(!env.object_exists("file1.txt").await?);
upload_after_quota_reconciliation(&env, "file3.txt", 256 * 1024).await?;
// Now we should be able to upload again (quota freed up)
env.upload_object("file3.txt", 256 * 1024).await?;
assert!(env.object_exists("file3.txt").await?);
env.cleanup_bucket().await?;
@@ -1012,7 +981,8 @@ mod integration_tests {
);
assert_eq!(delete_result.deleted().len(), 2);
upload_after_quota_reconciliation(&env, "file3.txt", 256 * 1024).await?;
// Now should be able to upload again (quota freed up)
env.upload_object("file3.txt", 256 * 1024).await?;
assert!(env.object_exists("file3.txt").await?);
env.cleanup_bucket().await?;
@@ -42,7 +42,6 @@ fn lock_result_from_response(response: rustfs_lock::LockResponse) -> GenerallyLo
success: response.success,
error_info: response.error,
lock_info: response.lock_info.and_then(|info| serde_json::to_string(&info).ok()),
..Default::default()
}
}
@@ -51,7 +50,6 @@ fn lock_result_from_error(error: impl Into<String>) -> GenerallyLockResult {
success: false,
error_info: Some(error.into()),
lock_info: None,
..Default::default()
}
}
@@ -61,7 +59,6 @@ fn lock_result_from_release(lock_id: &rustfs_lock::LockId, success: bool) -> Gen
success: true,
error_info: None,
lock_info: None,
..Default::default()
}
} else {
lock_result_from_error(format!("lock not found for release: {lock_id}"))
@@ -167,7 +164,6 @@ impl NodeService for MinimalLockNodeService {
success: false,
error_info: Some(format!("can not decode args, err: {err}")),
lock_info: None,
..Default::default()
}));
}
};
@@ -179,7 +175,6 @@ impl NodeService for MinimalLockNodeService {
success: result.success,
error_info: result.error,
lock_info: lock_info_json,
..Default::default()
}))
}
Err(err) => Ok(Response::new(GenerallyLockResponse {
@@ -189,7 +184,6 @@ impl NodeService for MinimalLockNodeService {
args.resource, args.owner, err
)),
lock_info: None,
..Default::default()
})),
}
}
@@ -203,7 +197,6 @@ impl NodeService for MinimalLockNodeService {
success: false,
error_info: Some(format!("can not decode args, err: {err}")),
lock_info: None,
..Default::default()
}));
}
};
@@ -215,7 +208,6 @@ impl NodeService for MinimalLockNodeService {
success: result.success,
error_info: result.error_info,
lock_info: None,
..Default::default()
}))
}
Err(err) => Ok(Response::new(GenerallyLockResponse {
@@ -225,7 +217,6 @@ impl NodeService for MinimalLockNodeService {
args.resource, args.owner, err
)),
lock_info: None,
..Default::default()
})),
}
}
@@ -239,7 +230,6 @@ impl NodeService for MinimalLockNodeService {
success: false,
error_info: Some(format!("can not decode args, err: {err}")),
lock_info: None,
..Default::default()
}));
}
};
@@ -251,7 +241,6 @@ impl NodeService for MinimalLockNodeService {
success: result.success,
error_info: result.error_info,
lock_info: None,
..Default::default()
}))
}
Err(err) => Ok(Response::new(GenerallyLockResponse {
@@ -261,7 +250,6 @@ impl NodeService for MinimalLockNodeService {
args.resource, args.owner, err
)),
lock_info: None,
..Default::default()
})),
}
}
@@ -275,7 +263,6 @@ impl NodeService for MinimalLockNodeService {
success: false,
error_info: Some(format!("can not decode args, err: {err}")),
lock_info: None,
..Default::default()
}));
}
};
@@ -285,13 +272,11 @@ impl NodeService for MinimalLockNodeService {
success,
error_info: None,
lock_info: None,
..Default::default()
})),
Err(err) => Ok(Response::new(GenerallyLockResponse {
success: false,
error_info: Some(format!("can not refresh, err: {err}")),
lock_info: None,
..Default::default()
})),
}
}
@@ -31,8 +31,8 @@ use aws_sdk_s3::error::ProvideErrorMetadata;
use aws_sdk_s3::operation::list_object_versions::ListObjectVersionsOutput;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Delete, DeleteMarkerEntry, ObjectIdentifier, ObjectVersion,
ServerSideEncryption, VersioningConfiguration,
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, DeleteMarkerEntry, ObjectVersion, ServerSideEncryption,
VersioningConfiguration,
};
use base64_simd::STANDARD as BASE64_STANDARD;
use bytes::Bytes;
@@ -3729,8 +3729,6 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
init_logging();
let mut source_env = RustFSTestEnvironment::new().await?;
let source_log = format!("{}/source.log", source_env.temp_dir);
source_env.capture_log_path = Some(source_log.clone());
let mut source_env_vars = replication_fast_env();
source_env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
source_env.start_rustfs_server_with_env(vec![], &source_env_vars).await?;
@@ -3793,66 +3791,19 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
"target did not preserve the source delete-marker version ID"
);
let wait_for_source_writeback = || async {
let metrics_url = format!("{}/{source_bucket}?replication-metrics", source_env.url);
timeout(Duration::from_secs(30), async {
loop {
let response = signed_request(
http::Method::GET,
&metrics_url,
&source_env.access_key,
&source_env.secret_key,
None,
None,
)
.await?;
let metrics: serde_json::Value = response.error_for_status()?.json().await?;
let queued = metrics["queued"]["curr"]["count"]
.as_f64()
.ok_or("missing replication queue count")?;
if metrics["provider_available"] == true && queued == 0.0 {
return Ok::<(), Box<dyn Error + Send + Sync>>(());
}
sleep(Duration::from_millis(50)).await;
}
})
.await?
};
wait_for_source_writeback().await?;
let purged = source_client
.delete_objects()
source_client
.delete_object()
.bucket(source_bucket)
.delete(
Delete::builder()
.objects(
ObjectIdentifier::builder()
.key(object_key)
.version_id(&purged_version_id)
.build()?,
)
.objects(
ObjectIdentifier::builder()
.key(object_key)
.version_id(delete_marker_version_id)
.build()?,
)
.build()?,
)
.key(object_key)
.version_id(&purged_version_id)
.send()
.await?;
assert!(purged.errors().is_empty(), "batch version purge failed: {:?}", purged.errors());
assert_eq!(purged.deleted().len(), 2, "both the data version and marker must be purged");
assert_replication_converged(&source_client, source_bucket, &target_client, target_bucket).await?;
let target_state = list_replication_state(&target_client, target_bucket).await?;
assert!(
target_state.iter().all(|entry| entry.version_id != purged_version_id),
"target retained the explicitly purged object version"
);
assert!(
target_state.iter().all(|entry| entry.version_id != delete_marker_version_id),
"target retained the explicitly purged delete marker"
);
assert!(
target_state
.iter()
@@ -3868,19 +3819,6 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
.await?;
assert_eq!(retained.body.collect().await?.into_bytes().as_ref(), b"versioned replication payload v2");
// Wait for the source workers to finish their final writeback, which runs
// after the remote purge has already become visible (rustfs#9121).
wait_for_source_writeback().await?;
let logs = fs::read_to_string(source_log).await?;
let writeback_failures: Vec<_> = logs
.lines()
.filter(|line| line.contains("apply_replication_delete_state"))
.collect();
assert!(
writeback_failures.is_empty(),
"a completed batch purge must not report a source writeback failure: {writeback_failures:?}"
);
Ok(())
}
-1
View File
@@ -2146,4 +2146,3 @@ mod tests {
);
}
}
// event-drill: isolated harness-only change
-9
View File
@@ -94,12 +94,3 @@ pub(crate) mod scanner_peer_bucket {
pub(crate) use ecstore_erasure::{ReedSolomonEncoder, calc_shard_size};
pub(crate) use ecstore_rpc::PeerRestClient;
}
/// Storage handles used to seed a pre-validation bucket record while the fixture server is stopped.
#[cfg(test)]
pub(crate) mod bucket_tag_fixture {
pub(crate) use rustfs_ecstore::api::bucket::metadata::load_bucket_metadata;
pub(crate) use rustfs_ecstore::api::layout::{DisksLayout, EndpointServerPools};
pub(crate) use rustfs_ecstore::api::runtime::InstanceContext;
pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks_with_instance_ctx};
}
+29 -147
View File
@@ -360,26 +360,12 @@ impl OpaMock {
pub(crate) async fn expect_bucket_tags(&mut self, action: &str, bucket: &str, department: Option<&str>) -> TestResult {
let payload = self.next_request().await?;
Self::assert_bucket_tags(&payload, action, bucket, department);
Ok(())
}
async fn expect_versioned_tag_permission(&mut self, bucket: &str, object: &str, version: &str) -> TestResult {
let payload = self.next_request().await?;
Self::assert_bucket_tags(&payload, "s3:GetObjectVersionTagging", bucket, Some("finance"));
assert_eq!(payload["input"]["resource"]["object"], object);
assert_eq!(payload["input"]["context"]["conditions"]["versionid"], serde_json::json!([version]));
Ok(())
}
fn assert_bucket_tags(payload: &Value, action: &str, bucket: &str, department: Option<&str>) {
let input = &payload["input"];
assert_eq!(input["identity"]["account"], "opabuckettags");
assert_eq!(input["identity"]["is_owner"], false);
assert_eq!(input["action"], action);
assert_eq!(input["resource"]["bucket"], bucket);
let expected = department.map(|value| serde_json::json!([value]));
assert_eq!(input["context"]["conditions"].get("ExistingBucketTag/department"), expected.as_ref());
Ok(())
}
async fn wait_for_validation(&mut self) -> TestResult {
@@ -692,61 +678,22 @@ async fn test_list_buckets_opa_contract() -> TestResult {
Ok(())
}
/// Stop and reap the directory's owning server before opening the fixture store.
async fn seed_legacy_duplicate_bucket_tags(env: &mut RustFSTestEnvironment, bucket: &str) -> TestResult {
use crate::storage_api::bucket_tag_fixture::{
DisksLayout, ECStore, EndpointServerPools, InstanceContext, init_local_disks_with_instance_ctx, load_bucket_metadata,
};
let server = env
.process
.as_mut()
.ok_or("missing server process before offline fixture write")?;
server.kill()?;
server.wait()?;
env.process = None;
let layout = DisksLayout::from_volumes(std::slice::from_ref(&env.temp_dir))?;
let (pools, _) = EndpointServerPools::create_server_endpoints(&env.address, &layout).await?;
let context = std::sync::Arc::new(InstanceContext::new());
init_local_disks_with_instance_ctx(&context, pools.clone()).await?;
let store = ECStore::new_with_instance_ctx(env.address.parse()?, pools, Default::default(), context.clone()).await?;
let _cancel = context
.background_cancel_token()
.expect("store binds its shutdown token")
.drop_guard();
let mut metadata = load_bucket_metadata(store.clone(), bucket).await?;
let incarnation = metadata.bucket_incarnation_id;
let policy = metadata.policy_config_json.clone();
assert!(!incarnation.is_nil());
assert!(!policy.is_empty(), "the bucket-policy Allow must survive legacy seeding");
// Encode the duplicate tag set with the same bucket record and erasure writer used by the store.
let xml = b"<Tagging><TagSet><Tag><Key>department</Key><Value>finance</Value></Tag><Tag><Key>department</Key><Value>engineering</Value></Tag></TagSet></Tagging>";
metadata.tagging_config_xml = xml.to_vec();
metadata.save_with_store(store.clone()).await?;
let persisted = load_bucket_metadata(store, bucket).await?;
assert_eq!(persisted.bucket_incarnation_id, incarnation);
assert_eq!(persisted.policy_config_json, policy);
assert_eq!(persisted.tagging_config_xml.as_slice(), xml);
Ok(())
}
#[tokio::test]
async fn test_bucket_tags_opa_contract() -> TestResult {
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Tag, Tagging, TaggingDirective, VersioningConfiguration,
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Tag, Tagging, VersioningConfiguration,
};
init_logging();
let mut opa = OpaMock::start().await?;
let mut env = RustFSTestEnvironment::new().await?;
let opa_url = opa.url.clone();
let server_env = [
("RUSTFS_POLICY_PLUGIN_URL", opa_url.as_str()),
env.start_rustfs_server_without_cleanup_with_env(&[
("RUSTFS_POLICY_PLUGIN_URL", opa.url.as_str()),
("RUSTFS_POLICY_PLUGIN_AUTH_TOKEN", OPA_AUTH_TOKEN),
("NO_PROXY", "127.0.0.1,localhost"),
];
env.start_rustfs_server_without_cleanup_with_env(&server_env).await?;
])
.await?;
let secret = uuid::Uuid::new_v4().to_string();
create_user(&env, "opabuckettags", &secret).await?;
let client = aws_sdk_s3::Client::from_conf(build_test_s3_config(&env.url, "opabuckettags", &secret, None, "e2e-bucket-tags"));
@@ -786,7 +733,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
.await?;
assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
let head = client
client
.head_object()
.bucket(source)
.key("report")
@@ -794,8 +741,6 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
.send()
.await?;
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
assert_eq!(head.tag_count(), Some(1));
opa.expect_versioned_tag_permission(source, "report", version).await?;
client.list_objects_v2().bucket(source).send().await?;
opa.expect_bucket_tags("s3:ListBucket", source, Some("finance")).await?;
client.list_object_versions().bucket(source).send().await?;
@@ -805,65 +750,28 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
admin.create_bucket().bucket(destination).send().await?;
set_department(&admin, destination, "engineering").await?;
let copy_source = format!("{source}/report?versionId={version}");
for directive in [TaggingDirective::Copy, TaggingDirective::Replace] {
let error = client
.copy_object()
.bucket(destination)
.key("copied")
.copy_source(&copy_source)
.set_tagging_directive((directive == TaggingDirective::Replace).then_some(TaggingDirective::Replace))
.set_tagging((directive == TaggingDirective::Replace).then(|| "department=legal".to_owned()))
.send()
.await
.expect_err("source tags must not authorize a different destination");
assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
if directive == TaggingDirective::Copy {
opa.expect_versioned_tag_permission(source, "report", version).await?;
}
opa.expect_bucket_tags("s3:PutObject", destination, Some("engineering"))
.await?;
let missing = admin
.head_object()
.bucket(destination)
.key("copied")
.send()
.await
.expect_err("denied COPY and REPLACE must leave no destination object");
assert_eq!(missing.raw_response().map(|response| response.status().as_u16()), Some(404));
}
let error = client
.copy_object()
.bucket(destination)
.key("copied")
.copy_source(&copy_source)
.send()
.await
.expect_err("source tags must not authorize a different destination");
assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
opa.expect_bucket_tags("s3:PutObject", destination, Some("engineering"))
.await?;
set_department(&admin, destination, "finance").await?;
for directive in [TaggingDirective::Copy, TaggingDirective::Replace] {
client
.copy_object()
.bucket(destination)
.key("copied")
.copy_source(&copy_source)
.set_tagging_directive((directive == TaggingDirective::Replace).then_some(TaggingDirective::Replace))
.set_tagging((directive == TaggingDirective::Replace).then(|| "department=legal".to_owned()))
.send()
.await?;
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
// The access hook checks preliminary tags, then the copy use case checks
// the exact source reader's tags before publishing the destination.
for _ in 0..2 {
if directive == TaggingDirective::Copy {
opa.expect_versioned_tag_permission(source, "report", version).await?;
}
opa.expect_bucket_tags("s3:PutObject", destination, Some("finance")).await?;
}
let tags = admin.get_object_tagging().bucket(destination).key("copied").send().await?;
assert_eq!(tags.tag_set().len(), 1);
assert_eq!(tags.tag_set()[0].key(), "department");
assert_eq!(
tags.tag_set()[0].value(),
if directive == TaggingDirective::Copy {
"engineering"
} else {
"legal"
}
);
}
client
.copy_object()
.bucket(destination)
.key("copied")
.copy_source(&copy_source)
.send()
.await?;
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
opa.expect_bucket_tags("s3:PutObject", destination, Some("finance")).await?;
let upload = client
.create_multipart_upload()
@@ -939,14 +847,6 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("engineering"))
.await?;
let missing = admin
.head_object()
.bucket(destination)
.key("denied-source")
.send()
.await
.expect_err("a denied source must leave no destination object");
assert_eq!(missing.raw_response().map(|response| response.status().as_u16()), Some(404));
set_department(&client, source, "finance")
.await
.expect_err("requested tags must not authorize their own mutation");
@@ -985,7 +885,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
opa.expect_bucket_tags("s3:GetObject", destination, Some("engineering"))
.await?;
let invalid_write = admin
admin
.put_bucket_tagging()
.bucket(destination)
.tagging(
@@ -995,25 +895,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
.build()?,
)
.send()
.await
.expect_err("new duplicate tags must be rejected before publication");
assert_eq!(invalid_write.as_service_error().and_then(ProvideErrorMetadata::code), Some("InvalidTag"));
assert_eq!(invalid_write.raw_response().map(|response| response.status().as_u16()), Some(400));
set_department(&admin, source, "finance").await?;
seed_legacy_duplicate_bucket_tags(&mut env, destination).await?;
env.start_rustfs_server_without_cleanup_with_env(&server_env).await?;
// Prove this process can authorize the same caller and read a healthy bucket's persisted tags.
let read = client
.get_object()
.bucket(source)
.key("report")
.version_id(version)
.send()
.await?;
assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
let error = client
.get_object()
.bucket(destination)
@@ -114,10 +114,10 @@ fn staged_commit(metadata: &Value, location: &str) -> Value {
})
}
async fn location_claim_preserves_other_users_objects_after_drop_and_restart() -> TestResult {
async fn location_claim_preserves_other_users_objects_after_drop_and_restart(backing: &str) -> TestResult {
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
let server_env = [("RUSTFS_TABLE_CATALOG_BACKING", "object")];
let server_env = [("RUSTFS_TABLE_CATALOG_BACKING", backing)];
env.start_rustfs_server_without_cleanup_with_env(&[("RUSTFS_TABLE_CATALOG_BACKING", "object")])
.await?;
env.wait_for_server_ready().await?;
@@ -147,6 +147,39 @@ async fn location_claim_preserves_other_users_objects_after_drop_and_restart() -
StatusCode::OK,
)
.await?;
if backing == "durable-strong" {
catalog_request(
&env,
&root,
Method::POST,
&format!("/iceberg/v1/{bucket}/catalog/warehouse-index/backfill"),
None,
StatusCode::NO_CONTENT,
)
.await?;
let preflight = catalog_request(
&env,
&root,
Method::GET,
&format!("/iceberg/v1/{bucket}/catalog/migration"),
None,
StatusCode::OK,
)
.await?;
assert_eq!(preflight["blockers"], json!([]), "{preflight}");
let migration = catalog_request(
&env,
&root,
Method::POST,
&format!("/iceberg/v1/{bucket}/catalog/migration"),
None,
StatusCode::OK,
)
.await?;
assert_eq!(migration["ready_to_enable_durable_strong"], json!(true), "{migration}");
env.restart_server_preserving_data(vec![], &server_env).await?;
env.wait_for_server_ready().await?;
}
admin_create_user(&env, &creator.0, &creator.1).await?;
admin_create_user(&env, &reader.0, &reader.1).await?;
let create_statement = json!({
@@ -308,7 +341,7 @@ async fn location_claim_preserves_other_users_objects_after_drop_and_restart() -
StatusCode::FORBIDDEN,
)
.await?;
{
if backing == "object" {
catalog_request(
&env,
&creator,
@@ -375,5 +408,10 @@ async fn location_claim_preserves_other_users_objects_after_drop_and_restart() -
#[tokio::test]
async fn object_catalog_location_claim_preserves_objects_after_drop_and_restart() -> TestResult {
location_claim_preserves_other_users_objects_after_drop_and_restart().await
location_claim_preserves_other_users_objects_after_drop_and_restart("object").await
}
#[tokio::test]
async fn durable_strong_catalog_location_claim_preserves_objects_after_drop_and_restart() -> TestResult {
location_claim_preserves_other_users_objects_after_drop_and_restart("durable-strong").await
}
@@ -48,7 +48,6 @@ const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY";
const G09_EVIDENCE_DIR_ENV: &str = "RUSTFS_SCANNER_HEAL_G09_EVIDENCE_DIR";
const RC5_COMMIT: &str = "40a2470feb567201165a5b809b7598bb4b1f68f5";
const STABLE_1_0_1_COMMIT: &str = "6de965ae3c965a78ff819fbcd7acd4aa44177d92";
const STABLE_1_0_4_COMMIT: &str = "46337942d55f3a8e6a97b147ca98d9ea8115555d";
const SSE_MASTER_KEY_ENV: &str = "RUSTFS_SSE_S3_MASTER_KEY";
const SSE_MASTER_KEY: &str = "QkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkI=";
const PLAIN_BUCKET: &str = "upgrade-plain-data";
@@ -955,8 +954,7 @@ async fn exercise_mixed_cluster(
phase: &str,
current_node: usize,
previous_node: usize,
previous_has_delete_recovery: bool,
) -> Result<Option<String>, BoxError> {
) -> TestResult {
let clients = cluster.create_all_clients()?;
wait_for_upgrade_write_readiness(&clients, MIXED_BUCKET, phase, LISTING_CONVERGENCE_TIMEOUT).await?;
wait_for_drive_readmission(&clients, MIXED_BUCKET, phase, DRIVE_READMISSION_TIMEOUT).await?;
@@ -1011,30 +1009,6 @@ async fn exercise_mixed_cluster(
vec![u8::try_from(MULTIPART_WORKERS - 1)?; 64 * 1024]
);
// 1.0.4 already implements the delete decision protocol. Use a separate
// old-writer key so the original seed still spans the entire rollout.
if previous_has_delete_recovery {
let key = format!(".upgrade-delete/{phase}/previous-writer");
previous_client
.put_object()
.bucket(MIXED_BUCKET)
.key(&key)
.body(ByteStream::from_static(UPGRADE_READINESS_BODY))
.send()
.await?;
for client in &clients {
assert_eq!(read_object(client, MIXED_BUCKET, &key, None).await?.1, UPGRADE_READINESS_BODY);
}
upgrade_probe_client(current_client)
.delete_object()
.bucket(MIXED_BUCKET)
.key(&key)
.send()
.await?;
assert_mixed_deletes_visible(&clients, std::slice::from_ref(&key)).await?;
return Ok(Some(key));
}
// Every online participant must understand the delete decision protocol.
// An upgrade phase can serve reads/writes while intentionally refusing a
// modern DELETE; rejecting it must leave the old writer's seed intact.
@@ -1055,31 +1029,6 @@ async fn exercise_mixed_cluster(
);
}
Ok(None)
}
async fn assert_mixed_deletes_visible(clients: &[Client], keys: &[String]) -> TestResult {
for (node, client) in clients.iter().enumerate() {
for key in keys {
let error = upgrade_probe_client(client)
.get_object()
.bucket(MIXED_BUCKET)
.key(key)
.send()
.await
.expect_err("a successful mixed-version DELETE must remain visible through every reader");
assert_eq!(
error.raw_response().map(|response| response.status().as_u16()),
Some(404),
"node {node}: {key}"
);
assert_eq!(
error.as_service_error().and_then(ProvideErrorMetadata::code),
Some("NoSuchKey"),
"node {node}: {key}"
);
}
}
Ok(())
}
@@ -2291,9 +2240,6 @@ async fn direct_upgrade_from_previous_release_preserves_object_contracts() -> Te
async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts() -> TestResult {
init_logging();
let previous_binary = source_binary()?;
// Bind the expected protocol to the verified release source, never to the
// DELETE result itself. The older CI source pins retain their 503 contract.
let previous_has_delete_recovery = binary_source_revision(&previous_binary).await? == STABLE_1_0_4_COMMIT;
let evidence_context = g09_evidence_context(&previous_binary).await?;
let current_binary = rustfs_binary_path();
let mut cluster = RustFSTestClusterEnvironment::new(MIXED_NODE_COUNT).await?;
@@ -2314,10 +2260,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
cluster.stop_node(0)?;
cluster.start_node_from_binary(0, &current_binary).await?;
let mut mixed_deleted_keys = Vec::new();
if let Some(key) = exercise_mixed_cluster(&cluster, "one-current-node", 0, 1, previous_has_delete_recovery).await? {
mixed_deleted_keys.push(key);
}
exercise_mixed_cluster(&cluster, "one-current-node", 0, 1).await?;
// One controlled out-of-policy attempt proves fail-closed behavior below
// native write quorum; positive MPU writers remain paused while mixed.
assert_mixed_pending_refusal(&cluster, &pending, &sources).await?;
@@ -2325,9 +2268,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
for (node_idx, phase) in [(1, "two-current-nodes"), (2, "one-previous-node")] {
cluster.stop_node(node_idx)?;
cluster.start_node_from_binary(node_idx, &current_binary).await?;
if let Some(key) = exercise_mixed_cluster(&cluster, phase, 0, 3, previous_has_delete_recovery).await? {
mixed_deleted_keys.push(key);
}
exercise_mixed_cluster(&cluster, phase, 0, 3).await?;
}
cluster.stop_node(3)?;
@@ -2337,7 +2278,6 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
let completed_pending = finish_pending_multipart(&clients, &pending, pending_parts.concat()).await?;
exercise_homogeneous_multipart(&cluster, "homogeneous-current").await?;
verify_load_bodies(&clients, "homogeneous-previous", "multipart").await?;
assert_mixed_deletes_visible(&clients, &mixed_deleted_keys).await?;
for (node_idx, client) in clients.iter().enumerate() {
assert_eq!(
@@ -2386,7 +2326,6 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
cluster.start_with_binary(&previous_binary).await?;
prepare_release_baseline(&mut cluster, &previous_binary, "rollback").await?;
let old_clients = cluster.create_all_clients()?;
assert_mixed_deletes_visible(&old_clients, &mixed_deleted_keys).await?;
assert_resumed_multipart(&old_clients, &completed_pending).await?;
for phase in ["homogeneous-previous", "homogeneous-current"] {
verify_load_bodies(&old_clients, phase, "multipart").await?;
@@ -2451,8 +2390,6 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
"phases": phases,
"objects_per_phase": objects_per_phase,
"mixed_load": "plain-put",
"mixed_delete_protocol_supported": previous_has_delete_recovery,
"mixed_deleted_keys": mixed_deleted_keys,
"homogeneous_multipart_phases": ["homogeneous-previous", "homogeneous-current", "homogeneous-rollback"],
"multipart_objects_per_homogeneous_phase": MULTIPART_WORKERS * MULTIPART_UPLOADS_PER_WORKER,
"assertions": [
@@ -2485,11 +2422,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
"16x16 multipart load runs in homogeneous previous, current and rollback phases",
"one-current-node Complete returns typed 503 and preserves actual source authority and parts",
"the same legacy upload completes and replays once fully upgraded without adding a version",
if previous_has_delete_recovery {
"mixed-version DELETE succeeds and stays absent through every reader after upgrade and rollback"
} else {
"modern DELETE fails closed and preserves every byte while old peers remain"
}
"modern DELETE fails closed and preserves every byte while old peers remain"
],
}),
)?;
-4
View File
@@ -31,7 +31,6 @@ workspace = true
[features]
default = []
fips = ["rustfs-s3-client/fips", "rustfs-utils/fips", "rustfs-tls-runtime/fips"]
diagnostic-read-version-caller = ["rustfs-io-metrics/diagnostic-read-version-caller"]
# Sampled wall/CPU diagnostics only; does not change the read backend.
diagnostic-local-batch = ["rustix/time"]
@@ -203,9 +202,6 @@ libc.workspace = true
# (backlog#1178); "thread" provides cleanup affinity; "fs" is a workspace default.
rustix = { workspace = true, features = ["process", "fs", "thread"] }
rustfs-madmin.workspace = true
rustfs-signer.workspace = true
rustfs-tls-runtime.workspace = true
quick-xml.workspace = true
reqwest = { workspace = true }
aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
urlencoding = { workspace = true }
+16 -32
View File
@@ -191,11 +191,7 @@ pub mod bucket {
OBJECT_LOCK_CONFIG, UnreadableBucketConfig, is_unreadable_config_error, load_bucket_metadata,
table_catalog_path_hash, unreadable_config_refusal,
};
pub use crate::bucket::metadata::{
BUCKET_DURABILITY_CONFIG, BUCKET_ON_DEMAND_MIGRATION_CONFIG, MAX_SWIFT_METADATA_COUNT, MAX_SWIFT_METADATA_VALUE_SIZE,
SWIFT_ACCOUNT_META_TAG_PREFIX, SWIFT_CONTAINER_META_TAG_PREFIX, SWIFT_TAGGING_MARKER,
};
pub use crate::bucket::metadata::{MAX_BUCKET_TAGS, is_swift_bucket_metadata_tag_key, validate_tag_set_entries};
pub use crate::bucket::metadata::{BUCKET_DURABILITY_CONFIG, BUCKET_ON_DEMAND_MIGRATION_CONFIG};
}
pub mod config_parse_mode {
@@ -213,20 +209,19 @@ pub mod bucket {
pub mod metadata_sys {
pub use crate::bucket::metadata_sys::{
BUCKET_CONFIG_PUBLISH_HOOK, BucketConfigPublishHook, BucketMetadataMutationGuard, BucketMetadataSys,
ObjectLockConfigState, ScannerBucketIncarnationReadGuard, acquire_bucket_metadata_transaction_lock,
ObjectLockConfigState, acquire_bucket_metadata_transaction_lock,
acquire_bucket_metadata_transaction_lock_for_incarnation, acquire_scanner_bucket_incarnation_fence,
acquire_scanner_bucket_incarnation_read_fence, capture_bucket_metadata_incarnation, delete, delete_if_incarnation,
delete_if_incarnation_at, delete_under_transaction_lock, get, get_accelerate_config, get_bucket_policy,
get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
capture_bucket_metadata_incarnation, delete, delete_if_incarnation, delete_if_incarnation_at,
delete_under_transaction_lock, get, get_accelerate_config, get_bucket_policy, get_bucket_policy_raw,
get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config,
get_on_demand_migration_config_in, get_public_access_block_config, get_quota_config, get_replication_config,
get_request_payment_config, get_s3_tagging_config, get_sse_config, get_tagging_config, get_versioning_config,
get_website_config, init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata,
get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config, get_website_config,
init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata,
remove_bucket_metadata_if_absent, set_bucket_metadata, update, update_bucket_targets_under_transaction_lock,
update_config_with, update_if_incarnation, update_if_incarnation_at, update_quota_if_incarnation,
update_quota_if_incarnation_at, update_s3_tagging_config, update_under_transaction_lock,
update_under_transaction_lock_at,
update_quota_if_incarnation_at, update_under_transaction_lock, update_under_transaction_lock_at,
};
#[cfg(feature = "test-util")]
pub use crate::bucket::metadata_sys::{ConfigWriteLockProbe, test_support};
@@ -336,8 +331,8 @@ pub mod bucket {
pub mod utils {
pub use crate::bucket::utils::{
check_bucket_and_object_names, check_list_objs_args, check_object_name_for_length_and_slash,
check_valid_bucket_name_strict, deserialize, has_bad_path_component, has_xml_invalid_control_characters,
is_meta_bucketname, is_valid_object_prefix, serialize,
check_valid_bucket_name_strict, deserialize, has_bad_path_component, is_meta_bucketname, is_valid_object_prefix,
serialize,
};
}
@@ -454,18 +449,11 @@ pub mod data_usage {
pub mod disk {
pub use crate::cluster::rpc::background_pipe::{BackgroundPipeReader, BackgroundPipeWriter, background_pipe};
pub use crate::cluster::rpc::background_reader_stream::BackgroundReaderStream;
pub use crate::disk::BOUNDED_DELETE_PREPARE_PROTOCOL_VERSION;
pub use crate::disk::background_memory::BackgroundEncodeBuffer;
pub use crate::disk::disk_store::get_object_disk_read_timeout;
#[cfg(feature = "test-util")]
pub use crate::disk::disk_store::wait_for_multipart_index_bootstrap;
pub use crate::disk::local::{ReplacementExecutionLease, ScanGuard};
#[cfg(feature = "test-util")]
pub use crate::disk::local::{local_durability_modes, set_local_rename_data_fail_after_metadata_commit};
pub use crate::disk::multipart_index::{
MULTIPART_INDEX_RPC_MAX_BYTES, MULTIPART_INDEX_RPC_MAX_REQUEST_BYTES, MultipartIndexPage, MultipartIndexRpcReply,
MultipartIndexRpcRequest,
};
pub use crate::disk::multipart_source::{
MULTIPART_SOURCE_PROTOCOL_VERSION, MultipartAbortCertificate, MultipartAbortProof, MultipartAbortProofKind,
MultipartAbortRound, MultipartSourceOperation, MultipartSourcePhase, MultipartSourceRequest, MultipartSourceResponse,
@@ -478,15 +466,13 @@ pub mod disk {
pub use crate::disk::os::{LocalPublicationPause, LocalPublicationStage};
pub use crate::disk::{
BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp,
BoundedDeleteRecoveryDiscovery, BoundedDeleteRecoveryInspection, BoundedDeleteRecoverySettlement,
BoundedDeleteRecoverySettlementReply, CheckPartsResp, ConditionalFileUpdate, CurrentDeleteGeneration,
CurrentDeleteTransition, DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES, DELETE_RECOVERY_MAX_INSPECTION_ENTRIES,
CheckPartsResp, ConditionalFileUpdate, CurrentDeleteGeneration, CurrentDeleteTransition,
DELETE_RECOVERY_PROTOCOL_VERSION, DeleteOptions, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest,
DeleteRecoveryInspection, DeleteRecoveryPage, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, DeleteRecoveryState,
Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore, FileInfoVersions, FileReader, FileWriter,
GuardedDeleteVersionsPayload, GuardedFileInfoVersions, HEAL_REPLACEMENT_RECOVERY_DIR, HEALING_MARKER_PATH,
NsScannerOpenRequest, OldCurrentSize, POOL_META_NATIVE_CAS_METADATA_SUFFIX, PartTransactionAction, PoolMetaNativeBackup,
PoolMetaNativeInspection, PoolMetaNativeObject, PoolMetaNativeRecoveryRequest, PoolMetaNativeRecoveryResponse,
GuardedDeleteVersionsPayload, GuardedFileInfoVersions, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize,
POOL_META_NATIVE_CAS_METADATA_SUFFIX, PartTransactionAction, PoolMetaNativeBackup, PoolMetaNativeInspection,
PoolMetaNativeObject, PoolMetaNativeRecoveryRequest, PoolMetaNativeRecoveryResponse,
QUOTA_MUTATION_FENCE_METADATA_SUFFIX, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_TMP_BUCKET,
ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, SnapshotLeaseToken,
UpdateMetadataOpts, VolumeInfo, WalkDirOptions, is_quota_mutation_fence_path, new_disk,
@@ -631,7 +617,7 @@ pub mod rebalance {
pub mod rio {
pub use crate::io_support::rio::{
DecryptReader, DynReader, EncryptReader, HardLimitReader, HashReader, ReadStream, Reader, WriteEncryption, WritePlan,
boxed_reader, compression_metadata_value, validate_compression_scheme, wrap_reader,
boxed_reader, compression_metadata_value, wrap_reader,
};
}
@@ -668,9 +654,7 @@ pub mod set_disk {
#[cfg(feature = "test-util")]
pub mod test_util {
pub use crate::bucket::quota::reservation::{
fail_next_quota_ledger_save_for_test, initialize_quota_ledger_for_test, retry_quota_reconciliation_for_test,
};
pub use crate::bucket::quota::reservation::fail_next_quota_ledger_save_for_test;
pub use crate::set_disk::ConditionalReadBenchmarkMetadataGuard;
pub use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, PutObjectCommitPause};
+5 -325
View File
@@ -16,8 +16,7 @@ use crate::bucket::metadata::BucketMetadata;
use crate::bucket::metadata_sys::get_bucket_targets_config;
use crate::bucket::metadata_sys::get_replication_config;
use crate::bucket::remote_s3_client::{
PathStyle, REPLICATION_TARGET_RETRY_POLICY, RemoteCredentials, RemoteS3EndpointSpec, RemoteS3RetryPolicy,
UploadResponseDeadline, build_remote_s3_client,
PathStyle, REPLICATION_TARGET_RETRY_POLICY, RemoteCredentials, RemoteS3EndpointSpec, build_remote_s3_client,
};
use crate::bucket::replication::{ObjectLockIntegrity, object_lock_put_integrity, replication_etags_match};
use crate::bucket::replication::{ReplicationStatusType, ReplicationTargetConfigBridge};
@@ -120,7 +119,6 @@ impl From<&BucketTarget> for RemoteS3EndpointSpec {
ca_cert_pem: (!target.ca_cert_pem.trim().is_empty()).then(|| target.ca_cert_pem.clone()),
connect_timeout: None,
read_timeout: None,
operation_timeout: Some(TargetRequestTimeouts::from_env().request),
// Replication has no retry budget of its own on the request path,
// so it keeps the SDK's standard three attempts; stating it here
// pins the behaviour to this line instead of an SDK default.
@@ -1193,7 +1191,6 @@ impl BucketTargetSys {
health_check_duration: target.health_check_duration,
replicate_sync: target.replication_sync,
client: Arc::new(client),
timeouts: TargetRequestTimeouts::from_env(),
})
}
@@ -1925,83 +1922,6 @@ impl std::fmt::Display for S3ClientError {
}
}
/// Replication request budgets, snapshotted when a target client is built.
/// Streamed GET bodies are not subject to the upload-response budget.
#[derive(Debug, Clone, Copy)]
pub struct TargetRequestTimeouts {
request: Duration,
complete: Duration,
minimum_transfer_rate: u64,
}
impl Default for TargetRequestTimeouts {
fn default() -> Self {
Self {
request: Duration::from_secs(60),
complete: Duration::from_secs(15 * 60),
minimum_transfer_rate: 1024 * 1024,
}
}
}
impl TargetRequestTimeouts {
fn from_env() -> Self {
let defaults = Self::default();
let seconds = |name: &str, fallback: Duration| {
let value = rustfs_utils::get_env_u64(name, fallback.as_secs());
Duration::from_secs(if (1..=86400).contains(&value) {
value
} else {
fallback.as_secs()
})
};
let minimum_transfer_rate =
rustfs_utils::get_env_u64("RUSTFS_REPLICATION_MIN_TRANSFER_RATE_BYTES_PER_SEC", defaults.minimum_transfer_rate);
Self {
request: seconds("RUSTFS_REPLICATION_REQUEST_TIMEOUT_SECS", defaults.request),
complete: seconds("RUSTFS_REPLICATION_COMPLETE_TIMEOUT_SECS", defaults.complete),
minimum_transfer_rate: if minimum_transfer_rate == 0 {
defaults.minimum_transfer_rate
} else {
minimum_transfer_rate
},
}
}
fn upload_timeout(self, size: i64) -> Result<Duration, S3ClientError> {
let size = u64::try_from(size).map_err(|_| S3ClientError::new("replication upload size must be non-negative"))?;
// The SDK can make three attempts. Leave transmission headroom for
// all three; the separate response deadline still bounds a trickling
// target after each upload finishes.
let attempts = match REPLICATION_TARGET_RETRY_POLICY {
RemoteS3RetryPolicy::Disabled => 1,
RemoteS3RetryPolicy::Standard { max_attempts } => u64::from(max_attempts.max(1)),
};
let seconds = size
.div_ceil(self.minimum_transfer_rate)
.checked_mul(attempts)
.ok_or_else(|| S3ClientError::new("replication upload timeout exceeds supported duration"))?;
let timeout = self
.request
.checked_add(Duration::from_secs(seconds))
.ok_or_else(|| S3ClientError::new("replication upload timeout exceeds supported duration"))?;
if Instant::now().checked_add(timeout).is_none() {
return Err(S3ClientError::new("replication upload timeout exceeds supported duration"));
}
Ok(timeout)
}
fn upload_override(self, size: i64) -> Result<aws_sdk_s3::config::Builder, S3ClientError> {
Ok(aws_sdk_s3::config::Builder::new()
.timeout_config(
aws_smithy_types::timeout::TimeoutConfig::builder()
.operation_timeout(self.upload_timeout(size)?)
.build(),
)
.interceptor(UploadResponseDeadline(self.request)))
}
}
#[derive(Debug)]
pub struct TargetClient {
pub endpoint: String,
@@ -2015,7 +1935,6 @@ pub struct TargetClient {
pub health_check_duration: Duration,
pub replicate_sync: bool,
pub client: Arc<S3Client>,
pub timeouts: TargetRequestTimeouts,
}
impl TargetClient {
@@ -2441,11 +2360,12 @@ impl TargetClient {
Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
});
let mut config = self.timeouts.upload_override(size)?;
if forwards_source_checksum {
config = config.request_checksum_calculation(aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired);
operation = operation.config_override(
aws_sdk_s3::config::Builder::new()
.request_checksum_calculation(aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired),
);
}
operation = operation.config_override(config);
match operation.send().await {
Ok(output) => {
// Under SSE-KMS/DSSE or SSE-C the target's ETag is not the MD5
@@ -2581,7 +2501,6 @@ impl TargetClient {
}
Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
})
.config_override(self.timeouts.upload_override(size)?)
.send()
.await
{
@@ -2619,13 +2538,6 @@ impl TargetClient {
}
Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
})
.config_override(
aws_sdk_s3::config::Builder::new().timeout_config(
aws_smithy_types::timeout::TimeoutConfig::builder()
.operation_timeout(self.timeouts.complete)
.build(),
),
)
.send()
.await
{
@@ -3054,241 +2966,11 @@ mod tests {
health_check_duration: Duration::from_secs(5),
replicate_sync: false,
client: Arc::new(client),
timeouts: TargetRequestTimeouts::default(),
},
request_headers,
)
}
#[derive(Debug, Clone)]
struct TimedResponseConnector {
upload_delay: Duration,
response_delay: Duration,
prefix_bytes: usize,
response: bytes::Bytes,
frames: Arc<std::sync::atomic::AtomicUsize>,
dropped: Arc<std::sync::atomic::AtomicUsize>,
}
struct ResponseDrop(Arc<std::sync::atomic::AtomicUsize>);
impl Drop for ResponseDrop {
fn drop(&mut self) {
self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
}
}
impl SmithyHttpConnector for TimedResponseConnector {
fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
let settings = self.clone();
HttpConnectorFuture::new(async move {
http_body_util::BodyExt::collect(request.into_body())
.await
.map_err(aws_smithy_runtime_api::client::result::ConnectorError::io)?;
tokio::time::sleep(settings.upload_delay).await;
let guard = ResponseDrop(Arc::clone(&settings.dropped));
let stream = futures::stream::unfold(
(settings.prefix_bytes, Some(settings.response), true, guard),
move |(remaining, mut body, first, guard)| {
let frames = Arc::clone(&settings.frames);
async move {
if remaining == 0 && body.is_none() {
return None;
}
if !first {
tokio::time::sleep(settings.response_delay).await;
}
let bytes = if remaining > 0 {
bytes::Bytes::from_static(b" ")
} else {
body.take()?
};
frames.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
Some((
Ok::<_, std::io::Error>(http_body::Frame::data(bytes)),
(remaining.saturating_sub(1), body, false, guard),
))
}
},
)
.fuse();
let body = SdkBody::from_body_1_x(http_body_util::StreamBody::new(stream));
let mut response =
HttpResponse::new(aws_smithy_runtime_api::http::StatusCode::try_from(200u16).expect("status"), body);
response.headers_mut().insert("etag", "\"test-etag\"");
Ok(response)
})
}
}
fn timed_response_target(
upload_delay: Duration,
response_delay: Duration,
prefix_bytes: usize,
response: &'static [u8],
) -> (TargetClient, Arc<std::sync::atomic::AtomicUsize>, Arc<std::sync::atomic::AtomicUsize>) {
let frames = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let dropped = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let connector = SharedHttpConnector::new(TimedResponseConnector {
upload_delay,
response_delay,
prefix_bytes,
response: bytes::Bytes::from_static(response),
frames: Arc::clone(&frames),
dropped: Arc::clone(&dropped),
});
let http_client = http_client_fn(move |_settings, _components| connector.clone());
let (mut target, _) = header_recording_target_client(Vec::new());
target.timeouts = TargetRequestTimeouts {
request: Duration::from_secs(2),
complete: Duration::from_secs(10),
minimum_transfer_rate: 1024 * 1024,
};
target.client = Arc::new(S3Client::from_conf(
target
.client
.config()
.to_builder()
.http_client(http_client)
.retry_config(aws_sdk_s3::config::retry::RetryConfig::disabled())
.timeout_config(
aws_smithy_types::timeout::TimeoutConfig::builder()
.operation_timeout(target.timeouts.request)
.build(),
)
.build(),
));
(target, frames, dropped)
}
#[test]
fn replication_upload_budgets_preserve_large_transfers_and_reject_invalid_sizes() {
let policy = TargetRequestTimeouts::default();
assert_eq!(policy.upload_timeout(0).expect("empty upload"), Duration::from_secs(60));
assert_eq!(
policy.upload_timeout(5 * 1024 * 1024 * 1024).expect("maximum single PUT"),
Duration::from_secs(15420)
);
assert!(policy.upload_timeout(-1).is_err());
let slow = TargetRequestTimeouts {
minimum_transfer_rate: 1,
..policy
};
assert!(slow.upload_timeout(i64::MAX).is_err());
}
#[tokio::test(start_paused = true)]
async fn replication_upload_response_deadline_is_independent_of_size_and_checksum_override() {
for operation in ["put", "forwarded-checksum", "part"] {
let (target, frames, dropped) = timed_response_target(Duration::from_secs(4), Duration::from_millis(500), 100, b"");
let body = vec![b'x'; 2 * 1024 * 1024];
let size = i64::try_from(body.len()).expect("size");
let started = tokio::time::Instant::now();
let error = if operation == "part" {
target
.put_object_part(
"bucket",
"key",
"upload",
1,
size,
ByteStream::from(body),
&PutObjectPartOptions::default(),
)
.await
.expect_err("response deadline")
} else {
let mut opts = PutObjectOptions::default();
if operation == "forwarded-checksum" {
opts.user_metadata.insert(
"x-amz-checksum-sha256".to_owned(),
"OoJ3yNhRwv3wwtZoGqEIPrPX9xwTnfLl+ka0wStN1g0=".to_owned(),
);
}
target
.put_object("bucket", "key", size, ByteStream::from(body), &opts)
.await
.expect_err("response deadline")
};
assert!(
started.elapsed() >= Duration::from_secs(6),
"upload gets its full independent window: {error}"
);
assert!(
started.elapsed() < target.timeouts.upload_timeout(size).expect("budget"),
"response must end before the larger total deadline"
);
assert!(
frames.load(std::sync::atomic::Ordering::SeqCst) >= 2,
"trickle delivered progress before failure"
);
assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1, "transport body released");
}
}
#[tokio::test(start_paused = true)]
async fn replication_slow_upload_preserves_transfer_window_and_streaming_get_body() {
let (target, _, _) = timed_response_target(Duration::from_secs(4), Duration::from_millis(100), 5, b"");
let body = vec![b'x'; 2 * 1024 * 1024];
let size = i64::try_from(body.len()).expect("size");
target
.put_object("bucket", "key", size, ByteStream::from(body), &PutObjectOptions::default())
.await
.expect("slow upload, prompt response");
let (target, _, dropped) = timed_response_target(Duration::ZERO, Duration::from_secs(1), 5, b"payload");
let response = target
.get_object("bucket", "key", None, None, None, HeaderMap::new())
.await
.expect("GET headers");
let started = tokio::time::Instant::now();
let body = response
.body
.collect()
.await
.expect("streamed GET remains outside upload deadline")
.into_bytes();
assert!(started.elapsed() > target.timeouts.request);
assert_eq!(&body[..], b" payload");
assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
}
#[tokio::test(start_paused = true)]
async fn replication_complete_processing_whitespace_has_a_separate_bounded_window() {
const RESULT: &[u8] = b"<CompleteMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><ETag>\"etag\"</ETag></CompleteMultipartUploadResult>";
for prefix in [5, 100] {
let (target, frames, dropped) = timed_response_target(Duration::ZERO, Duration::from_secs(1), prefix, RESULT);
let started = tokio::time::Instant::now();
let result = target
.complete_multipart_upload(
"bucket",
"key",
"upload",
vec![CompletedPart::builder().part_number(1).e_tag("etag").build()],
&PutObjectOptions::default(),
)
.await;
if prefix == 5 {
assert!(result.is_ok(), "legal processing whitespace: {result:?}");
} else {
assert!(result.is_err());
assert!(started.elapsed() <= Duration::from_secs(11));
}
assert!(started.elapsed() > target.timeouts.request);
assert!(frames.load(std::sync::atomic::Ordering::SeqCst) >= 2);
assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
}
}
#[tokio::test(start_paused = true)]
async fn replication_control_operation_deadline_covers_response_trickle() {
let (target, frames, dropped) =
timed_response_target(Duration::ZERO, Duration::from_millis(500), 100, b"<VersioningConfiguration/>");
let started = tokio::time::Instant::now();
assert!(target.get_bucket_versioning("bucket").await.is_err());
assert!(started.elapsed() <= Duration::from_secs(3));
assert!(frames.load(std::sync::atomic::Ordering::SeqCst) >= 2);
assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
}
fn streaming_test_body(payload: &'static [u8]) -> ByteStream {
let stream = tokio_util::io::ReaderStream::new(std::io::Cursor::new(payload));
let body = http_body_util::StreamBody::new(futures::StreamExt::map(stream, |r| r.map(http_body::Frame::data)));
@@ -3597,7 +3279,6 @@ mod tests {
health_check_duration: Duration::from_secs(5),
replicate_sync: false,
client: Arc::new(client),
timeouts: TargetRequestTimeouts::default(),
},
request_uris,
)
@@ -3910,7 +3591,6 @@ mod tests {
health_check_duration: Duration::from_secs(5),
replicate_sync: false,
client: Arc::new(client),
timeouts: TargetRequestTimeouts::default(),
})
}
@@ -5487,14 +5487,6 @@ pub async fn put_restore_opts(
if !oi.user_tags.is_empty() {
meta.insert(AMZ_OBJECT_TAGGING.to_string(), (*oi.user_tags).clone());
}
if let Some(expires) = oi.expires {
meta.insert(
"expires".to_string(),
expires
.format(&time::format_description::well_known::Rfc3339)
.map_err(std::io::Error::other)?,
);
}
let restore_expiry = lifecycle::expected_expiry_time(OffsetDateTime::now_utc(), rreq.days.unwrap_or(1));
meta.insert(
metadata_keys::RESTORE.to_string(),
@@ -5514,6 +5506,7 @@ pub async fn put_restore_opts(
// computed MD5 is not the object's public plaintext ETag.
preserve_etag: oi.etag.clone(),
shard_integrity_write_mode: Some(oi.shard_integrity_write_mode()),
//expires: oi.expires,
..Default::default()
})
}
@@ -7151,25 +7144,6 @@ mod tests {
0,
"tier generation lease should release after EOF"
);
let unread = get_transitioned_object_reader_with_tier_manager(
&object_info.bucket,
&object_info.name,
&None,
&HeaderMap::new(),
&object_info,
&ObjectOptions::default(),
&manager,
None,
)
.await
.expect("open an unread transitioned response");
assert_eq!(TierConfigMgr::active_operation_lease_count(&manager, &tier).await, 1);
drop(unread);
assert_eq!(
TierConfigMgr::active_operation_lease_count(&manager, &tier).await,
0,
"canceling an unread response must release the tier lease"
);
}
#[cfg(feature = "test-util")]
-188
View File
@@ -256,128 +256,6 @@ pub const BUCKET_POLICY_CONFIG: &str = "policy.json";
pub const BUCKET_NOTIFICATION_CONFIG: &str = "notification.xml";
pub const BUCKET_LIFECYCLE_CONFIG: &str = "lifecycle.xml";
pub const BUCKET_SSECONFIG: &str = "bucket-encryption.xml";
/// Validate tag entries supplied to the S3 API.
pub fn validate_tag_set_entries(tag_set: &[s3s::dto::Tag]) -> std::result::Result<(), &'static str> {
validate_tag_entries(tag_set, validate_s3_tag_entry)
}
fn validate_tag_entries(
tag_set: &[s3s::dto::Tag],
mut validate_entry: impl FnMut(&str, &str) -> std::result::Result<(), &'static str>,
) -> std::result::Result<(), &'static str> {
let mut keys = std::collections::HashSet::with_capacity(tag_set.len());
for tag in tag_set {
let key = tag
.key
.as_deref()
.filter(|key| !key.is_empty())
.ok_or("Tag key cannot be empty")?;
let value = tag.value.as_deref().ok_or("Tag value cannot be null")?;
validate_entry(key, value)?;
if !keys.insert(key) {
return Err("Cannot provide multiple Tags with the same key");
}
}
Ok(())
}
fn validate_s3_tag_entry(key: &str, value: &str) -> std::result::Result<(), &'static str> {
fn is_valid_tag_text(value: &str) -> bool {
static VALID_TAG_TEXT: std::sync::OnceLock<std::result::Result<regex::Regex, regex::Error>> = std::sync::OnceLock::new();
VALID_TAG_TEXT
.get_or_init(|| regex::Regex::new(r"^[\p{L}\p{N}\p{Z}+\-=._:/@]*$"))
.as_ref()
.is_ok_and(|pattern| pattern.is_match(value))
}
if key.encode_utf16().count() > 128 || !is_valid_tag_text(key) {
return Err("The TagKey you have provided is invalid");
}
if value.encode_utf16().count() > 256 || !is_valid_tag_text(value) {
return Err("The TagValue you have provided is invalid");
}
Ok(())
}
pub const MAX_BUCKET_TAGS: usize = 50;
/// Maximum number of metadata headers on a Swift resource.
pub const MAX_SWIFT_METADATA_COUNT: usize = 90;
/// Maximum byte length of a Swift metadata value.
pub const MAX_SWIFT_METADATA_VALUE_SIZE: usize = 256;
/// Prefix for Swift account metadata persisted in tagging.xml.
pub const SWIFT_ACCOUNT_META_TAG_PREFIX: &str = "swift-account-meta-";
/// Prefix for Swift container metadata persisted in tagging.xml.
pub const SWIFT_CONTAINER_META_TAG_PREFIX: &str = "swift-meta-";
/// Marks a bucket whose shared tag config contains Swift-owned metadata.
/// The key cannot be supplied through S3's tag alphabet.
pub const SWIFT_TAGGING_MARKER: &str = "!rustfs-swift-metadata";
/// Whether a persisted tag belongs to Swift's internal metadata namespace.
pub fn is_swift_bucket_metadata_tag_key(key: &str) -> bool {
matches!(
key,
"swift-acl-read" | "swift-acl-write" | "swift-versions-location" | SWIFT_TAGGING_MARKER
) || key.starts_with(SWIFT_ACCOUNT_META_TAG_PREFIX)
|| key.starts_with(SWIFT_CONTAINER_META_TAG_PREFIX)
}
fn validate_persisted_bucket_tags(tags: &[s3s::dto::Tag]) -> std::result::Result<(), &'static str> {
const SWIFT_ACL_MAX_BYTES: usize = 8192;
const SWIFT_META_NAME_MAX_BYTES: usize = 128;
const MAX_PERSISTED_TAGS: usize = MAX_BUCKET_TAGS + 2 * MAX_SWIFT_METADATA_COUNT + 4;
if tags.len() > MAX_PERSISTED_TAGS {
return Err("Too many persisted bucket tags");
}
// Swift stores ACLs, versioning and metadata in the same XML as S3 tags.
// Only those known keys may carry values outside the S3 tag alphabet.
let mut account_metadata = 0;
let mut container_metadata = 0;
let mut s3_tags = 0;
validate_tag_entries(tags, |key, value| {
let swift_metadata = key
.strip_prefix(SWIFT_ACCOUNT_META_TAG_PREFIX)
.map(|name| (name, &mut account_metadata))
.or_else(|| {
key.strip_prefix(SWIFT_CONTAINER_META_TAG_PREFIX)
.map(|name| (name, &mut container_metadata))
});
if let Some((name, count)) = swift_metadata
&& name.len() <= SWIFT_META_NAME_MAX_BYTES
&& http::HeaderName::from_bytes(name.as_bytes()).is_ok()
&& value.len() <= MAX_SWIFT_METADATA_VALUE_SIZE
&& http::HeaderValue::from_str(value).is_ok()
{
*count += 1;
if *count > MAX_SWIFT_METADATA_COUNT {
return Err("Too many Swift metadata tags");
}
return Ok(());
}
if key == SWIFT_TAGGING_MARKER && value == "1" {
return Ok(());
}
let valid_swift_header = match key {
"swift-acl-read" | "swift-acl-write" => value.len() <= SWIFT_ACL_MAX_BYTES,
"swift-versions-location" => value.len() <= 256 && !value.is_empty() && !value.contains('/'),
_ => false,
};
if valid_swift_header && http::HeaderValue::from_str(value).is_ok() {
return Ok(());
}
s3_tags += 1;
if s3_tags > MAX_BUCKET_TAGS {
return Err("Cannot have more than 50 tags per bucket");
}
validate_s3_tag_entry(key, value)
})
}
pub const BUCKET_TAGGING_CONFIG: &str = "tagging.xml";
pub const BUCKET_QUOTA_CONFIG_FILE: &str = "quota.json";
pub const OBJECT_LOCK_CONFIG: &str = "object-lock.xml";
@@ -1112,13 +990,6 @@ impl BucketMetadata {
self.encryption_config_updated_at = updated;
}
BUCKET_TAGGING_CONFIG => {
if !data.is_empty() {
let tagging = deserialize::<Tagging>(&data).map_err(|_| {
Error::InvalidArgument(self.name.clone(), String::new(), "Invalid bucket tagging XML".to_owned())
})?;
validate_persisted_bucket_tags(&tagging.tag_set)
.map_err(|message| Error::InvalidArgument(self.name.clone(), String::new(), message.to_owned()))?;
}
self.tagging_config_xml = data;
// Drop the parsed form (like lifecycle above) so clearing the
// payload can't leave stale parsed tags to be cached.
@@ -1910,68 +1781,9 @@ mod test {
assert_eq!(metadata.bucket_incarnation_id, incarnation);
}
#[test]
fn persisted_bucket_tags_keep_separate_s3_and_swift_limits() {
use s3s::dto::Tag;
let tag = |key: String, value: &str| Tag {
key: Some(key),
value: Some(value.to_owned()),
};
let acl = tag("swift-acl-read".to_owned(), ".r:*,AUTH_other");
assert_eq!(
validate_tag_set_entries(std::slice::from_ref(&acl)),
Err("The TagValue you have provided is invalid")
);
let mut tags = vec![acl.clone()];
tags.extend((0..MAX_BUCKET_TAGS).map(|i| tag(format!("team-{i}"), "blue")));
tags.extend((0..MAX_SWIFT_METADATA_COUNT).map(|i| tag(format!("{SWIFT_CONTAINER_META_TAG_PREFIX}item!{i}"), "a,b")));
assert_eq!(validate_persisted_bucket_tags(&tags), Ok(()));
tags.push(tag(format!("{SWIFT_CONTAINER_META_TAG_PREFIX}overflow"), "a,b"));
assert_eq!(validate_persisted_bucket_tags(&tags), Err("Too many Swift metadata tags"));
tags.pop();
tags.push(acl);
assert_eq!(
validate_persisted_bucket_tags(&tags),
Err("Cannot provide multiple Tags with the same key")
);
tags.pop();
tags[0].value = Some(".r:*,".repeat(1700));
assert_eq!(validate_persisted_bucket_tags(&tags), Err("The TagValue you have provided is invalid"));
}
/// backlog#2292: a replicated config is stamped with the source
/// `updated_at` it was given, not the local clock, while the plain
/// `update_config` entry keeps stamping the local clock.
#[test]
fn bucket_tag_updates_reject_unreadable_shapes_without_replacing_metadata() {
let mut metadata = BucketMetadata::new("validated-tags");
let valid = br#"<Tagging><TagSet><Tag><Key>team</Key><Value></Value></Tag></TagSet></Tagging>"#.to_vec();
metadata
.update_config(BUCKET_TAGGING_CONFIG, valid.clone())
.expect("empty value is valid");
let updated = metadata.tagging_config_updated_at;
for invalid in [
"<Tagging><TagSet><Tag><Key>team</Key></Tag></TagSet></Tagging>".to_owned(),
"<Tagging><TagSet><Tag><Key></Key><Value>x</Value></Tag></TagSet></Tagging>".to_owned(),
"<Tagging><TagSet><Tag><Key>team</Key><Value>x</Value></Tag><Tag><Key>team</Key><Value>y</Value></Tag></TagSet></Tagging>".to_owned(),
format!("<Tagging><TagSet>{}</TagSet></Tagging>", (0..51).map(|i| format!("<Tag><Key>k{i}</Key><Value>v</Value></Tag>")).collect::<String>()),
] {
assert!(metadata.update_config_at(BUCKET_TAGGING_CONFIG, invalid.into_bytes(), updated + time::Duration::hours(1)).is_err());
assert_eq!(metadata.tagging_config_xml, valid);
assert_eq!(metadata.tagging_config_updated_at, updated);
}
metadata.tagging_config_xml = b"legacy corrupt tags".to_vec();
metadata
.update_config(BUCKET_TAGGING_CONFIG, Vec::new())
.expect("deletion repairs legacy corruption");
assert!(metadata.tagging_config_xml.is_empty());
}
#[test]
fn update_config_at_stamps_the_given_time_and_update_config_stamps_now() {
let source_time = OffsetDateTime::now_utc() - time::Duration::hours(3);
+50 -494
View File
@@ -22,7 +22,6 @@ use super::target::BucketTargets;
use crate::bucket::bucket_target_sys::BucketTargetSys;
use crate::bucket::metadata::{load_bucket_metadata_parse, load_bucket_metadata_parse_with_presence};
use crate::bucket::utils::is_meta_bucketname;
use crate::diagnostics::object_lock::ObjectLockAttempt;
use crate::disk::RUSTFS_META_BUCKET;
use crate::error::{Error, Result, is_err_bucket_not_found, is_err_strict_volume_not_found};
use crate::object_api::ObjectOptions;
@@ -600,9 +599,7 @@ pub(crate) async fn get_bucket_incarnation_id_for_options_in(
}
let sys = bucket_metadata_sys_of(ctx)?;
let sys = sys.read().await.clone();
let guard =
acquire_bucket_metadata_transaction_read_lock_for_options_in(ctx, bucket, opts, "bucket_incarnation_options_read")
.await?;
let guard = acquire_bucket_metadata_transaction_read_lock_for_options_in(ctx, bucket, opts).await?;
sys.get_bucket_incarnation_id_under_transaction_lock(bucket, &guard).await
}
@@ -686,7 +683,6 @@ pub(crate) async fn update_validated_in<F>(
bucket: &str,
config_file: &str,
data: Vec<u8>,
expected_incarnation_id: Option<Uuid>,
validate: F,
) -> Result<OffsetDateTime>
where
@@ -694,7 +690,7 @@ where
{
Box::pin(async {
let sys = require_bucket_metadata_sys_in(ctx)?;
let guard = acquire_config_write_guard_for_incarnation(sys.clone(), bucket, expected_incarnation_id).await?;
let guard = acquire_config_write_guard(sys.clone(), bucket).await?;
validate()?;
update_under_config_write_guard(sys, &guard, config_file, data, None).await
})
@@ -869,30 +865,6 @@ impl BucketMetadataMutationGuard {
}
}
/// Read-only scanner authority for one bucket incarnation.
///
/// Lock order is bucket lifecycle (read), then metadata transaction (read).
/// This guard cannot authorize a configuration mutation or scanner ACK write.
pub struct ScannerBucketIncarnationReadGuard {
bucket: String,
incarnation_id: Uuid,
lifecycle_guard: rustfs_lock::NamespaceLockGuard,
transaction_guard: rustfs_lock::NamespaceLockGuard,
}
impl ScannerBucketIncarnationReadGuard {
/// Returns the verified identity while both read fences remain valid.
///
/// # Errors
/// Returns an error if either lock has lost its lease.
pub fn checked_bucket_incarnation(&self) -> Result<(&str, Uuid)> {
if self.lifecycle_guard.is_lock_lost() || self.transaction_guard.is_lock_lost() {
return Err(Error::other("scanner bucket incarnation read lock was lost"));
}
Ok((&self.bucket, self.incarnation_id))
}
}
async fn acquire_config_write_guard(sys: Arc<RwLock<BucketMetadataSys>>, bucket: &str) -> Result<BucketMetadataMutationGuard> {
acquire_config_write_guard_for_incarnation(sys, bucket, None).await
}
@@ -919,68 +891,6 @@ pub async fn acquire_scanner_bucket_incarnation_fence(
acquire_config_write_guard_with_migration(sys, bucket, Some(expected_incarnation_id), false).await
}
/// Validate scanner read authority without serializing compatible metadata readers.
///
/// The returned guard retains both read locks. It never migrates metadata or
/// creates an incarnation; mutation callers must use the exclusive scanner fence.
///
/// # Errors
/// Returns an error for invalid bucket/owner identities, unavailable locks or
/// storage, absent incarnation metadata, or a deleted/recreated bucket.
pub async fn acquire_scanner_bucket_incarnation_read_fence(
bucket: &str,
expected_incarnation_id: Uuid,
expected_owner_id: Uuid,
) -> Result<ScannerBucketIncarnationReadGuard> {
acquire_scanner_bucket_incarnation_read_fence_with_sys(
get_bucket_metadata_sys()?,
bucket,
expected_incarnation_id,
expected_owner_id,
)
.await
}
async fn acquire_scanner_bucket_incarnation_read_fence_with_sys(
sys: Arc<RwLock<BucketMetadataSys>>,
bucket: &str,
expected_incarnation_id: Uuid,
expected_owner_id: Uuid,
) -> Result<ScannerBucketIncarnationReadGuard> {
super::utils::check_valid_bucket_name(bucket)?;
let metadata_sys = sys.read().await.clone();
if expected_owner_id.is_nil() || metadata_sys.object_store().id != expected_owner_id || expected_incarnation_id.is_nil() {
return Err(Error::other("scanner bucket incarnation owner does not match"));
}
let lifecycle_guard = metadata_sys.object_store().acquire_bucket_lifecycle_read_lock(bucket).await?;
let transaction_guard = await_bucket_namespace_operation(
Some(&lifecycle_guard),
bucket,
"scanner incarnation read transaction lock acquisition",
acquire_metadata_transaction_guard(
metadata_sys.object_store(),
bucket,
"scanner_incarnation_probe",
rustfs_lock::LockType::Shared,
),
)
.await?;
let incarnation_id = validate_bucket_metadata_guard_incarnation(
&metadata_sys,
bucket,
Some(expected_incarnation_id),
&lifecycle_guard,
&transaction_guard,
)
.await?;
Ok(ScannerBucketIncarnationReadGuard {
bucket: bucket.to_owned(),
incarnation_id,
lifecycle_guard,
transaction_guard,
})
}
async fn acquire_config_write_guard_with_migration(
sys: Arc<RwLock<BucketMetadataSys>>,
bucket: &str,
@@ -1006,47 +916,15 @@ async fn acquire_config_write_guard_with_migration(
Some(&lifecycle_guard),
bucket,
"bucket config transaction lock acquisition",
acquire_transaction_lock_with_sys_for_purpose(
&sys,
bucket,
if migrate {
"bucket_config_update"
} else {
"scanner_incarnation_validation"
},
),
acquire_transaction_lock_with_sys(&sys, bucket),
)
.await?;
let current_incarnation_id = validate_bucket_metadata_guard_incarnation(
&metadata_sys,
bucket,
expected_incarnation_id,
&lifecycle_guard,
&transaction_guard,
)
.await?;
Ok(BucketMetadataMutationGuard {
bucket: bucket.to_string(),
incarnation_id: current_incarnation_id,
lifecycle_guard,
transaction_guard,
})
}
async fn validate_bucket_metadata_guard_incarnation(
metadata_sys: &BucketMetadataSys,
bucket: &str,
expected_incarnation_id: Option<Uuid>,
lifecycle_guard: &rustfs_lock::NamespaceLockGuard,
transaction_guard: &rustfs_lock::NamespaceLockGuard,
) -> Result<Uuid> {
await_bucket_namespace_operation(
Some(lifecycle_guard),
Some(&lifecycle_guard),
bucket,
"bucket config existence validation",
await_bucket_namespace_operation(
Some(transaction_guard),
Some(&transaction_guard),
bucket,
"bucket config existence transaction validation",
async {
@@ -1064,11 +942,11 @@ async fn validate_bucket_metadata_guard_incarnation(
)
.await?;
let current_incarnation_id = await_bucket_namespace_operation(
Some(lifecycle_guard),
Some(&lifecycle_guard),
bucket,
"bucket config incarnation validation",
await_bucket_namespace_operation(
Some(transaction_guard),
Some(&transaction_guard),
bucket,
"bucket config incarnation transaction validation",
load_bucket_incarnation(metadata_sys.object_store(), bucket),
@@ -1081,7 +959,12 @@ async fn validate_bucket_metadata_guard_incarnation(
return Err(Error::BucketNotFound(bucket.to_string()));
}
Ok(current_incarnation_id)
Ok(BucketMetadataMutationGuard {
bucket: bucket.to_string(),
incarnation_id: current_incarnation_id,
lifecycle_guard,
transaction_guard,
})
}
/// Rewrite one config file while the caller already holds this bucket's
@@ -1290,23 +1173,11 @@ async fn delete_under_config_write_guard(
/// targets one, so a writer on another node cannot land a whole-file save in
/// the middle of this read-modify-write.
pub async fn update_config_with<F>(bucket: &str, config_file: &str, mutate: F) -> Result<OffsetDateTime>
where
F: FnOnce(&BucketMetadata) -> Result<Vec<u8>> + Send,
{
update_config_with_expected(bucket, config_file, None, mutate).await
}
async fn update_config_with_expected<F>(
bucket: &str,
config_file: &str,
expected_incarnation_id: Option<Uuid>,
mutate: F,
) -> Result<OffsetDateTime>
where
F: FnOnce(&BucketMetadata) -> Result<Vec<u8>> + Send,
{
let sys = get_bucket_metadata_sys()?;
let guard = Box::pin(acquire_config_write_guard_for_incarnation(sys.clone(), bucket, expected_incarnation_id)).await?;
let guard = Box::pin(acquire_config_write_guard(sys.clone(), bucket)).await?;
guard.ensure_valid(bucket)?;
let metadata_sys = sys.read().await.clone();
let updated = await_bucket_namespace_operation(
@@ -1325,70 +1196,6 @@ where
Ok(updated)
}
fn swift_manages_bucket_tags(tagging: &Tagging) -> bool {
tagging
.tag_set
.iter()
.any(|tag| tag.key.as_deref() == Some(super::metadata::SWIFT_TAGGING_MARKER) && tag.value.as_deref() == Some("1"))
}
/// Read the S3 tag set while keeping Swift's internal tags private.
pub async fn get_s3_tagging_config(bucket: &str) -> Result<(Tagging, OffsetDateTime)> {
let (mut tagging, updated) = get_tagging_config(bucket).await?;
if swift_manages_bucket_tags(&tagging) {
tagging.tag_set.retain(|tag| {
!tag.key
.as_deref()
.is_some_and(super::metadata::is_swift_bucket_metadata_tag_key)
});
}
Ok((tagging, updated))
}
/// Replace or clear S3 tags atomically while retaining Swift metadata.
///
/// The expected incarnation, when present, is checked under the bucket
/// lifecycle and metadata guards. A failed write leaves the previous config
/// intact. Repeated writes replace the same S3 set.
pub async fn update_s3_tagging_config(
bucket: &str,
tagging: Option<Tagging>,
expected_incarnation_id: Option<Uuid>,
) -> Result<OffsetDateTime> {
let tags = tagging.map(|tagging| tagging.tag_set).unwrap_or_default();
let invalid = |message: &str| Error::InvalidArgument(bucket.to_owned(), String::new(), message.to_owned());
if tags.len() > super::metadata::MAX_BUCKET_TAGS {
return Err(invalid("Cannot have more than 50 tags per bucket"));
}
super::metadata::validate_tag_set_entries(&tags).map_err(invalid)?;
update_config_with_expected(bucket, BUCKET_TAGGING_CONFIG, expected_incarnation_id, |metadata| {
let mut merged = metadata.tagging_config.clone().unwrap_or_default();
if swift_manages_bucket_tags(&merged) {
if tags.iter().any(|tag| {
tag.key
.as_deref()
.is_some_and(super::metadata::is_swift_bucket_metadata_tag_key)
}) {
return Err(invalid("Tag key is reserved for Swift metadata on this bucket"));
}
merged.tag_set.retain(|tag| {
tag.key
.as_deref()
.is_some_and(super::metadata::is_swift_bucket_metadata_tag_key)
});
} else {
merged.tag_set.clear();
}
merged.tag_set.extend(tags);
if merged.tag_set.is_empty() {
Ok(Vec::new())
} else {
super::utils::serialize(&merged).map_err(Error::other)
}
})
.await
}
/// Acquire a bucket's metadata transaction lock, held across a whole
/// read-modify-write of its metadata file.
///
@@ -1427,18 +1234,13 @@ pub(crate) async fn acquire_bucket_metadata_transaction_lock_in(
pub(crate) async fn acquire_bucket_metadata_transaction_read_lock_in(
ctx: &crate::runtime::instance::InstanceContext,
bucket: &str,
) -> Result<rustfs_lock::NamespaceLockGuard> {
acquire_bucket_metadata_transaction_read_lock_in_for_purpose(ctx, bucket, "bucket_metadata_read").await
}
pub(crate) async fn acquire_bucket_metadata_transaction_read_lock_in_for_purpose(
ctx: &crate::runtime::instance::InstanceContext,
bucket: &str,
op: &'static str,
) -> Result<rustfs_lock::NamespaceLockGuard> {
let sys = bucket_metadata_sys_of(ctx)?;
let api = sys.read().await.object_store();
acquire_metadata_transaction_guard(api, bucket, op, rustfs_lock::LockType::Shared).await
let lock = api
.new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
.await?;
Ok(lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?)
}
/// Readers already holding an Object Lock snapshot must share its guard:
@@ -1447,7 +1249,6 @@ pub(crate) async fn acquire_bucket_metadata_transaction_read_lock_for_options_in
ctx: &crate::runtime::instance::InstanceContext,
bucket: &str,
opts: &ObjectOptions,
op: &'static str,
) -> Result<Arc<rustfs_lock::NamespaceLockGuard>> {
if let Some(snapshot) = opts.object_lock_config_snapshot.as_ref() {
let store = object_store_in(ctx).await?;
@@ -1457,22 +1258,12 @@ pub(crate) async fn acquire_bucket_metadata_transaction_read_lock_for_options_in
Error::other("Object Lock snapshot does not hold a valid metadata transaction fence for this bucket")
});
}
Ok(Arc::new(
acquire_bucket_metadata_transaction_read_lock_in_for_purpose(ctx, bucket, op).await?,
))
Ok(Arc::new(acquire_bucket_metadata_transaction_read_lock_in(ctx, bucket).await?))
}
async fn acquire_transaction_lock_with_sys(
sys: &Arc<RwLock<BucketMetadataSys>>,
bucket: &str,
) -> Result<rustfs_lock::NamespaceLockGuard> {
acquire_transaction_lock_with_sys_for_purpose(sys, bucket, "bucket_metadata_transaction_write").await
}
async fn acquire_transaction_lock_with_sys_for_purpose(
sys: &Arc<RwLock<BucketMetadataSys>>,
bucket: &str,
op: &'static str,
) -> Result<rustfs_lock::NamespaceLockGuard> {
// Resolve the store under a short-lived read guard: this runs before the
// write guard in `acquire_config_write_guards`, and must not still hold a
@@ -1481,22 +1272,12 @@ async fn acquire_transaction_lock_with_sys_for_purpose(
let lock = api
.new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
.await?;
let timeout = crate::set_disk::get_lock_acquire_timeout();
let mut attempt = ObjectLockAttempt::start(
op,
RUSTFS_META_BUCKET,
&bucket_metadata_transaction_lock_key(bucket),
None,
&lock,
"write",
timeout,
);
let acquire = lock.get_write_lock(timeout);
let acquire = lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout());
#[cfg(any(test, feature = "test-util"))]
{
tokio::pin!(acquire);
let mut notified = false;
let result = futures::future::poll_fn(|cx| match std::future::Future::poll(acquire.as_mut(), cx) {
let guard = futures::future::poll_fn(|cx| match std::future::Future::poll(acquire.as_mut(), cx) {
std::task::Poll::Pending => {
if !notified {
notify_config_write_lock_attempt(bucket);
@@ -1506,38 +1287,11 @@ async fn acquire_transaction_lock_with_sys_for_purpose(
}
std::task::Poll::Ready(result) => std::task::Poll::Ready(result),
})
.await;
attempt.observe_acquisition(&result);
Ok(result?)
.await?;
Ok(guard)
}
#[cfg(not(any(test, feature = "test-util")))]
{
let result = acquire.await;
attempt.observe_acquisition(&result);
Ok(result?)
}
}
pub(crate) async fn acquire_metadata_transaction_guard(
api: Arc<ECStore>,
bucket: &str,
op: &'static str,
lock_type: rustfs_lock::LockType,
) -> Result<rustfs_lock::NamespaceLockGuard> {
let object = bucket_metadata_transaction_lock_key(bucket);
let lock = api.new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
let timeout = crate::set_disk::get_lock_acquire_timeout();
let mode = match lock_type {
rustfs_lock::LockType::Shared => "read",
rustfs_lock::LockType::Exclusive => "write",
};
let mut attempt = ObjectLockAttempt::start(op, RUSTFS_META_BUCKET, &object, None, &lock, mode, timeout);
let result = match lock_type {
rustfs_lock::LockType::Shared => lock.get_read_lock(timeout).await,
rustfs_lock::LockType::Exclusive => lock.get_write_lock(timeout).await,
};
attempt.observe_acquisition(&result);
Ok(result?)
Ok(acquire.await?)
}
/// The lock resource name is deliberately still the `bucket-targets` one it
@@ -2147,15 +1901,6 @@ impl BucketMetadataSys {
self.api.upgrade()
}
/// Whether the backing `ECStore` has been dropped.
///
/// Used by `InstanceContext::clear_bucket_metadata_sys_if_stale` during
/// `ECStore::drop` so a shared bootstrap context does not retain a
/// `BucketMetadataSys` whose weak handle is dangling.
pub(crate) fn is_store_stale(&self) -> bool {
self.api.upgrade().is_none()
}
fn metadata_publish_lock(&self, bucket: &str) -> Arc<Mutex<MetadataPublishLockState>> {
Self::bucket_lock_in(&self.metadata_publish_locks, bucket)
}
@@ -3039,13 +2784,13 @@ impl BucketMetadataSys {
}
async fn find_bucket_incarnation_id_from_disk(&self, bucket: &str) -> Result<Option<Uuid>> {
let _transaction_guard = acquire_metadata_transaction_guard(
self.object_store(),
bucket,
"bucket_incarnation_disk_read",
rustfs_lock::LockType::Shared,
)
.await?;
let transaction_lock = self
.object_store()
.new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
.await?;
let _transaction_guard = transaction_lock
.get_read_lock(crate::set_disk::get_lock_acquire_timeout())
.await?;
self.find_bucket_incarnation_id_under_transaction_lock(bucket, &_transaction_guard)
.await
}
@@ -3121,13 +2866,13 @@ impl BucketMetadataSys {
/// revalidated at write quorum, so a stale snapshot can neither revert an
/// acknowledged configuration update nor outlive a delete/recreate.
async fn migrate_bucket_creation_commit(&self, bucket: &str) -> Result<BucketMetadataAuthority> {
let transaction_guard = acquire_metadata_transaction_guard(
self.object_store(),
bucket,
"bucket_creation_commit_migration",
rustfs_lock::LockType::Exclusive,
)
.await?;
let transaction_lock = self
.object_store()
.new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
.await?;
let transaction_guard = transaction_lock
.get_write_lock(crate::set_disk::get_lock_acquire_timeout())
.await?;
let namespace_lock = self.object_store().new_ns_lock(bucket, bucket).await?;
let namespace_guard = namespace_lock
.get_read_lock(crate::set_disk::get_lock_acquire_timeout())
@@ -3177,13 +2922,13 @@ impl BucketMetadataSys {
}
async fn migrate_legacy_metadata(&self, bucket: &str) -> Result<BucketMetadataAuthority> {
let _transaction_guard = acquire_metadata_transaction_guard(
self.object_store(),
bucket,
"bucket_metadata_legacy_migration",
rustfs_lock::LockType::Exclusive,
)
.await?;
let transaction_lock = self
.object_store()
.new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
.await?;
let _transaction_guard = transaction_lock
.get_write_lock(crate::set_disk::get_lock_acquire_timeout())
.await?;
let authority = self
.load_authoritative_metadata_from_disk_under_transaction_lock(bucket)
@@ -3673,7 +3418,7 @@ mod tests {
),
(
BUCKET_TAGGING_CONFIG,
r#"<Tagging><TagSet><Tag><Key>environment</Key><Value>测试-rust</Value></Tag></TagSet></Tagging>"#.as_bytes(),
r#"<Tagging><TagSet><Tag><Key>environment</Key><Value>测试-🦀</Value></Tag></TagSet></Tagging>"#.as_bytes(),
),
(BUCKET_REPLICATION_CONFIG, NEW_WRITER_REPLICATION_XML),
(
@@ -4222,12 +3967,6 @@ mod tests {
.await
.is_err()
);
assert!(
acquire_scanner_bucket_incarnation_read_fence_with_sys(sys.clone(), bucket, Uuid::new_v4(), store.id)
.await
.is_err(),
"read-only scanner validation must not migrate an absent incarnation"
);
assert!(load_bucket_incarnation(store, bucket).await.expect("read sidecar").is_none());
assert!(
sys.read()
@@ -4284,128 +4023,6 @@ mod tests {
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn scanner_incarnation_read_fence_shares_reads_and_blocks_mutations() {
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let sys = bucket_metadata_sys_of(&store.ctx).expect("metadata owner");
let bucket = "scanner-read-fence";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
let incarnation = store
.bucket_incarnation_id_from_disk(bucket)
.await
.expect("bucket incarnation");
let guard = acquire_scanner_bucket_incarnation_read_fence_with_sys(sys.clone(), bucket, incarnation, store.id)
.await
.expect("read-only scanner fence");
assert_eq!(guard.checked_bucket_incarnation().expect("valid read fences"), (bucket, incarnation));
let transaction_lock = store
.new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
.await
.expect("metadata transaction namespace");
assert!(
transaction_lock.get_write_lock(Duration::from_millis(20)).await.is_err(),
"scanner read authority must exclude metadata writers"
);
let lifecycle_lock = store
.new_ns_lock(bucket, crate::storage_api_contracts::bucket::BUCKET_LIFECYCLE_LOCK_OBJECT)
.await
.expect("bucket lifecycle namespace");
assert!(
lifecycle_lock.get_write_lock(Duration::from_millis(20)).await.is_err(),
"scanner read authority must exclude delete and recreation"
);
let another = timeout(
Duration::from_secs(1),
acquire_scanner_bucket_incarnation_read_fence_with_sys(sys.clone(), bucket, incarnation, store.id),
)
.await
.expect("scanner readers must not serialize")
.expect("second read-only scanner fence");
let compatible = timeout(
Duration::from_secs(1),
acquire_bucket_metadata_transaction_read_lock_in(&store.ctx, bucket),
)
.await
.expect("metadata readers must remain compatible")
.expect("metadata transaction reader");
drop(compatible);
drop(another);
let mut mutation = Box::pin(acquire_config_write_guard_with_migration(sys.clone(), bucket, Some(incarnation), false));
assert!(matches!(futures::poll!(&mut mutation), std::task::Poll::Pending));
let mut deletion = Box::pin(store.acquire_bucket_lifecycle_write_lock(bucket));
assert!(matches!(futures::poll!(&mut deletion), std::task::Poll::Pending));
drop(deletion);
drop(guard);
let mutation = timeout(Duration::from_secs(1), mutation)
.await
.expect("metadata mutation must progress after scanner read release")
.expect("exclusive mutation fence");
drop(mutation);
let lifecycle = timeout(Duration::from_secs(1), store.acquire_bucket_lifecycle_write_lock(bucket))
.await
.expect("bucket delete fence must progress after readers release")
.expect("exclusive lifecycle fence");
drop(lifecycle);
for (owner, expected) in [
(Uuid::new_v4(), incarnation),
(Uuid::nil(), incarnation),
(store.id, Uuid::nil()),
] {
assert!(
acquire_scanner_bucket_incarnation_read_fence_with_sys(sys.clone(), bucket, expected, owner)
.await
.is_err(),
"invalid owner or nil incarnation must not receive authority"
);
}
assert!(
acquire_scanner_bucket_incarnation_read_fence_with_sys(sys.clone(), bucket, Uuid::new_v4(), store.id)
.await
.is_err(),
"a stale incarnation must not receive authority"
);
store
.delete_bucket(bucket, &DeleteBucketOptions::default())
.await
.expect("delete bucket");
assert!(
acquire_scanner_bucket_incarnation_read_fence_with_sys(sys.clone(), bucket, incarnation, store.id)
.await
.is_err(),
"deleted bucket must not receive authority"
);
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("recreate bucket");
let replacement = store
.bucket_incarnation_id_from_disk(bucket)
.await
.expect("replacement incarnation");
assert_ne!(replacement, incarnation);
assert!(
acquire_scanner_bucket_incarnation_read_fence_with_sys(sys.clone(), bucket, incarnation, store.id)
.await
.is_err(),
"recreation must reject the previous scanner identity"
);
let guard = acquire_scanner_bucket_incarnation_read_fence_with_sys(sys, bucket, replacement, store.id)
.await
.expect("replacement read-only scanner fence");
assert_eq!(
guard.checked_bucket_incarnation().expect("replacement remains fenced"),
(bucket, replacement)
);
}
#[tokio::test]
async fn old_node_metadata_rewrite_cannot_replace_bucket_incarnation_sidecar() {
let (dirs, ecstore) = isolated_store_over_temp_disks().await;
@@ -4929,13 +4546,8 @@ mod tests {
std::fs::create_dir_all(dir.path().join(bucket)).expect("bucket volume should be created");
}
let mut bm = BucketMetadata::new(bucket);
if config_file == crate::bucket::metadata::BUCKET_TAGGING_CONFIG {
// Model legacy persisted corruption without passing the write-time validator.
bm.tagging_config_xml = raw.to_vec();
} else {
bm.update_config(config_file, raw.to_vec())
.expect("raw config should be stored");
}
bm.update_config(config_file, raw.to_vec())
.expect("raw config should be stored");
sys.persist_new_and_set(bm).await.expect("initial metadata should persist");
}
@@ -5588,61 +5200,6 @@ mod tests {
);
}
#[tokio::test]
#[serial]
async fn validated_config_update_rejects_a_different_bucket_incarnation() {
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = "catalog-cleanup-incarnation";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
let incarnation = store
.bucket_incarnation_id_from_disk(bucket)
.await
.expect("read bucket incarnation");
let called = std::sync::atomic::AtomicBool::new(false);
let result = store
.update_bucket_metadata_config_for_incarnation_validated(
bucket,
"quota.json",
br#"{"quota":1}"#.to_vec(),
Uuid::new_v4(),
|| {
called.store(true, std::sync::atomic::Ordering::SeqCst);
Ok(())
},
)
.await;
assert!(matches!(result, Err(Error::BucketNotFound(name)) if name == bucket));
assert!(
!called.load(std::sync::atomic::Ordering::SeqCst),
"incarnation must be checked before the external fence"
);
let (metadata, _) = get_config_from_disk_with_presence_in(&store.ctx, bucket)
.await
.expect("read unchanged bucket metadata");
assert!(
metadata.quota_config_json.is_empty(),
"a stale cleanup must not modify the replacement bucket"
);
store
.update_bucket_metadata_config_for_incarnation_validated(
bucket,
"quota.json",
br#"{"quota":1}"#.to_vec(),
incarnation,
|| Ok(()),
)
.await
.expect("the current incarnation permits the validated update");
let (metadata, _) = get_config_from_disk_with_presence_in(&store.ctx, bucket)
.await
.expect("read updated bucket metadata");
assert_eq!(metadata.quota_config_json, br#"{"quota":1}"#);
}
#[tokio::test]
#[serial]
async fn quota_admission_rejects_stale_cached_no_quota() {
@@ -6116,8 +5673,7 @@ mod tests {
// update only needs one interleaving to show up.
const ROUNDS: usize = 8;
for round in 0..ROUNDS {
let tagging =
format!("<Tagging><TagSet><Tag><Key>round</Key><Value>{round}</Value></Tag></TagSet></Tagging>").into_bytes();
let tagging = format!("<Tagging><Round>{round}</Round></Tagging>").into_bytes();
let policy = format!(r#"{{"Version":"2012-10-17","Round":{round}}}"#).into_bytes();
let start = Arc::new(tokio::sync::Barrier::new(2));
@@ -6190,7 +5746,7 @@ mod tests {
let blocked = tokio::spawn({
let node_b = node_b.clone();
async move { update_with_sys(node_b, bucket, BUCKET_TAGGING_CONFIG, b"<Tagging><TagSet/></Tagging>".to_vec()).await }
async move { update_with_sys(node_b, bucket, BUCKET_TAGGING_CONFIG, b"<Tagging/>".to_vec()).await }
});
// Long enough for the write to have finished had it not waited: the
@@ -119,21 +119,6 @@ async fn durable_quota_reporting_retires_degraded_floors_without_changing_ledger
// Exercise the real storage entry point before testing the new
// reporting semantics: the same ledger must admit and refuse
// these writes even when a scanner floor claims a million bytes.
if protocol == 1 {
let mut bootstrap = PutObjReader::from_vec(vec![0x5a; 512]);
assert!(
matches!(
store
.put_object(&bucket, "bootstrap", &mut bootstrap, &ObjectOptions::default())
.await,
Err(StorageError::SlowDown)
),
"a missing authoritative ledger must not admit a write from scanner reporting"
);
crate::bucket::quota::reservation::retry_quota_reconciliation_for_test(store.clone(), &bucket)
.await
.expect("bootstrap authoritative usage through the production worker");
}
let mut too_large = PutObjReader::from_vec(vec![0x5a; 8192]);
assert!(
matches!(
File diff suppressed because it is too large Load Diff
+2 -87
View File
@@ -40,9 +40,7 @@ use aws_smithy_runtime_api::client::http::{
HttpConnector as SmithyHttpConnector, HttpConnectorFuture, SharedHttpConnector, http_client_fn,
};
use aws_smithy_runtime_api::client::interceptors::Intercept;
use aws_smithy_runtime_api::client::interceptors::context::{
BeforeDeserializationInterceptorContextMut, BeforeTransmitInterceptorContextMut,
};
use aws_smithy_runtime_api::client::interceptors::context::BeforeTransmitInterceptorContextMut;
use aws_smithy_runtime_api::client::orchestrator::{HttpRequest, HttpResponse};
use aws_smithy_runtime_api::client::result::ConnectorError;
use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents;
@@ -157,8 +155,6 @@ pub struct RemoteS3EndpointSpec {
pub ca_cert_pem: Option<String>,
pub connect_timeout: Option<Duration>,
pub read_timeout: Option<Duration>,
/// Entire SDK call, including non-streaming response bodies and retries.
pub operation_timeout: Option<Duration>,
/// How many wire requests one logical call may cost. Every consumer
/// declares it; see [`RemoteS3RetryPolicy`].
pub retry: RemoteS3RetryPolicy,
@@ -256,78 +252,6 @@ pub(crate) fn remote_sdk_credentials(credentials: &RemoteCredentials, now: Syste
Ok(builder.build())
}
/// Only installed on non-streaming upload operations. The budget begins at
/// response headers, independently of the potentially long request upload.
#[derive(Debug)]
pub(crate) struct UploadResponseDeadline(pub Duration);
impl Intercept for UploadResponseDeadline {
fn name(&self) -> &'static str {
"RustfsUploadResponseDeadline"
}
fn modify_before_deserialization(
&self,
context: &mut BeforeDeserializationInterceptorContextMut<'_>,
_runtime_components: &RuntimeComponents,
_cfg: &mut ConfigBag,
) -> Result<(), BoxError> {
let response = context.response_mut();
let inner = std::mem::replace(response.body_mut(), SdkBody::taken());
*response.body_mut() = SdkBody::from_body_1_x(UploadResponseBody {
inner: Some(inner),
deadline: Box::pin(tokio::time::sleep(self.0)),
});
Ok(())
}
}
struct UploadResponseBody {
inner: Option<SdkBody>,
deadline: std::pin::Pin<Box<tokio::time::Sleep>>,
}
impl http_body::Body for UploadResponseBody {
type Data = bytes::Bytes;
type Error = BoxError;
fn poll_frame(
mut self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
) -> std::task::Poll<Option<Result<http_body::Frame<Self::Data>, Self::Error>>> {
use std::future::Future as _;
use std::task::Poll;
if self.inner.is_none() {
return Poll::Ready(None);
}
// The absolute response budget never renews on data, empty frames,
// or trailers. A retry gets its own bounded response window.
if self.deadline.as_mut().poll(cx).is_ready() {
self.inner.take();
return Poll::Ready(Some(Err(std::io::Error::new(
std::io::ErrorKind::TimedOut,
"replication response body deadline exceeded",
)
.into())));
}
let Some(inner) = self.inner.as_mut() else {
return Poll::Ready(None);
};
let result = std::pin::Pin::new(inner).poll_frame(cx);
if matches!(result, Poll::Ready(None) | Poll::Ready(Some(Err(_)))) {
self.inner.take();
}
result
}
fn is_end_stream(&self) -> bool {
self.inner.as_ref().is_none_or(http_body::Body::is_end_stream)
}
fn size_hint(&self) -> http_body::SizeHint {
self.inner.as_ref().map(http_body::Body::size_hint).unwrap_or_default()
}
}
/// Appends a caller-identifying token to the SDK `User-Agent`. Runs after
/// signing: SigV4 excludes `user-agent` from the canonical request, so the
/// signature stays valid.
@@ -385,7 +309,7 @@ pub async fn build_remote_s3_config(spec: &RemoteS3EndpointSpec) -> Result<aws_s
config_builder = config_builder.http_client(http_client);
}
if spec.connect_timeout.is_some() || spec.read_timeout.is_some() || spec.operation_timeout.is_some() {
if spec.connect_timeout.is_some() || spec.read_timeout.is_some() {
let mut timeouts = TimeoutConfig::builder();
if let Some(connect_timeout) = spec.connect_timeout {
timeouts = timeouts.connect_timeout(connect_timeout);
@@ -393,9 +317,6 @@ pub async fn build_remote_s3_config(spec: &RemoteS3EndpointSpec) -> Result<aws_s
if let Some(read_timeout) = spec.read_timeout {
timeouts = timeouts.read_timeout(read_timeout);
}
if let Some(operation_timeout) = spec.operation_timeout {
timeouts = timeouts.operation_timeout(operation_timeout);
}
config_builder = config_builder.timeout_config(timeouts.build());
}
@@ -753,7 +674,6 @@ mod tests {
ca_cert_pem: None,
connect_timeout: None,
read_timeout: None,
operation_timeout: None,
retry: RemoteS3RetryPolicy::Disabled,
user_agent_suffix: "",
}
@@ -1107,16 +1027,11 @@ mod tests {
spec.user_agent_suffix = "RustFS-Test/0.0";
spec.connect_timeout = Some(Duration::from_secs(5));
spec.read_timeout = Some(Duration::from_secs(5));
spec.operation_timeout = Some(Duration::from_secs(12));
let config = build_remote_s3_config(&spec)
.await
.expect("spec should build")
.http_client(http_client)
.build();
assert_eq!(
config.timeout_config().and_then(TimeoutConfig::operation_timeout),
Some(Duration::from_secs(12))
);
S3Client::from_conf(config)
.head_bucket()
.bucket("bucket")
@@ -33,7 +33,6 @@ pub(crate) struct ReplicationMetadataStore;
impl ReplicationMetadataStore {
pub(crate) const MRF_REPLICATION_FILE: &'static str = "config/replication/mrf.bin";
pub(crate) const MRF_REPLICATION_RECOVERY_LOCK: &'static str = "config/replication/mrf.bin.recovery";
pub(crate) const MRF_REPLICATION_REPLAY_STAMP: &'static str = "config/replication/mrf.bin.replay";
pub(crate) const FORCE_DELETE_REPLICATION_FILE: &'static str = "config/replication/force-delete.bin";
pub(crate) const FORCE_DELETE_REPLICATION_TRANSACTION_LOCK: &'static str = "config/replication/force-delete.bin.transaction";
@@ -117,10 +116,6 @@ mod tests {
ReplicationMetadataStore::MRF_REPLICATION_RECOVERY_LOCK,
"config/replication/mrf.bin.recovery"
);
assert_eq!(
ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
"config/replication/mrf.bin.replay"
);
assert_eq!(
ReplicationMetadataStore::FORCE_DELETE_REPLICATION_FILE,
"config/replication/force-delete.bin"
@@ -16,8 +16,8 @@ use super::replication_config_store::ReplicationConfigStore;
use super::replication_error_boundary::{Error as EcstoreError, is_err_object_not_found, is_err_version_not_found};
use super::replication_filemeta_boundary::{
MrfOpKind, MrfReplicateEntry, REPLICATE_HEAL_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicatedTargetInfo,
ReplicationGenerationSnapshot, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision,
replicate_decision_for_admitted_targets, replication_statuses_map, version_purge_statuses_map,
ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision, replicate_decision_for_admitted_targets,
replication_statuses_map, version_purge_statuses_map,
};
use super::replication_lock_boundary::ReplicationLockTiming;
use super::replication_logging::{EVENT_REPLICATION_CONFIG_LOOKUP_SKIPPED, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REPLICATION};
@@ -86,10 +86,6 @@ const MRF_CORRUPT_FILE_PREFIX: &str = "config/replication/mrf.corrupt";
const MRF_PENDING_CAP: usize = 200_000;
const MRF_RETRY_INITIAL_DELAY: Duration = Duration::from_millis(100);
const MRF_RETRY_MAX_DELAY: Duration = Duration::from_secs(5);
/// Interval between runtime replays of the MRF journal, cluster-wide. A pass
/// re-attempts every retained entry, so a target that stays down costs one
/// attempt per entry per interval.
const MRF_REPLAY_INTERVAL: Duration = Duration::from_secs(300);
#[derive(Debug, Default)]
pub struct DurableMrfBacklog {
@@ -619,7 +615,7 @@ async fn read_mrf_entries<S: ReplicationObjectIO>(storage: Arc<S>) -> Result<Vec
}
/// Acknowledge only the generation read by the recovery leader. Entries appended
/// after that generation are retained as a suffix and replayed on the next pass.
/// after that generation are retained as a suffix and replayed on the next startup.
/// Lock order: recovery leader lock -> MRF journal object lock.
async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
storage: Arc<S>,
@@ -671,7 +667,7 @@ async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
Err(EcstoreError::PreconditionFailed)
}
/// Acquires the MRF recovery leader lock for one journal replay pass.
/// Acquires the MRF recovery leader lock for the startup replay.
/// Returns `None` (after logging) when the lock cannot be created or another
/// node is already processing the backlog.
async fn acquire_mrf_recovery_guard<S: ReplicationStorage>(storage: &Arc<S>) -> Option<rustfs_lock::NamespaceLockGuard> {
@@ -1022,7 +1018,7 @@ fn admitted_mrf_replicate_object(oi: ObjectInfo, entry: &MrfReplicateEntry, op_t
}
/// Acknowledges the replayed MRF prefix and returns the retained backlog.
/// On acknowledgement failure the backlog is preserved for the next replay pass and
/// On acknowledgement failure the backlog is preserved for the next startup and
/// re-read (falling back to the replayed snapshot) so the published summary stays accurate.
async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
storage: &Arc<S>,
@@ -1037,7 +1033,7 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
error = %error,
"Failed to acknowledge the MRF recovery prefix; preserving it for the next replay pass"
"Failed to acknowledge the MRF recovery prefix; preserving it for the next startup"
);
match read_mrf_entries(storage.clone()).await {
Ok(current) => current,
@@ -1055,120 +1051,6 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
}
}
fn unix_millis_now() -> i64 {
i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000_000).unwrap_or(i64::MAX)
}
/// Whether a periodic pass may run, given the start time of the last pass on
/// any node. A stamp more than one interval ahead of this node's clock is skew
/// or a bad value, so it does not suppress replay.
fn mrf_replay_due(last_started_ms: Option<i64>, now_ms: i64, interval: Duration) -> bool {
let Some(last_started_ms) = last_started_ms else {
return true;
};
let interval_ms = i64::try_from(interval.as_millis()).unwrap_or(i64::MAX);
now_ms.saturating_sub(last_started_ms) >= interval_ms || last_started_ms.saturating_sub(now_ms) >= interval_ms
}
async fn read_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>) -> Option<i64> {
let data = ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP)
.await
.ok()?;
std::str::from_utf8(&data).ok()?.trim().parse().ok()
}
async fn write_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>, started_ms: i64) {
if let Err(error) = ReplicationConfigStore::save(
storage.clone(),
ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
started_ms.to_string().into_bytes(),
)
.await
{
warn!(
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
error = %error,
"Failed to record the MRF replay start time"
);
}
}
/// One MRF journal replay pass. The recovery leader reads the journal,
/// re-drives every entry, and acknowledges the replayed prefix; entries that
/// could not be admitted, and entries appended meanwhile, stay for the next pass.
///
/// Every node runs the periodic loop, and the leader lock only serializes
/// passes. With `min_interval`, a pass is skipped when a pass on any node
/// started within that interval, read from the shared replay stamp, so a
/// cluster replays once per interval rather than once per node. The startup
/// pass passes `None` and always runs. A pass that finds entries records its
/// start time. Returns whether the journal was read.
/// Lock order: recovery leader lock -> replay stamp object lock.
async fn replay_mrf_journal<S: ReplicationStorage>(storage: &Arc<S>, min_interval: Option<Duration>) -> bool {
let Some(recovery_guard) = acquire_mrf_recovery_guard(storage).await else {
return false;
};
let started_ms = unix_millis_now();
if let Some(interval) = min_interval
&& !mrf_replay_due(read_mrf_replay_stamp(storage).await, started_ms, interval)
{
return false;
}
let Some(entries) = load_mrf_recovery_entries(storage).await else {
return true;
};
if !entries.is_empty() {
write_mrf_replay_stamp(storage, started_ms).await;
}
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
let total = entries.len();
let mut queued_count = 0usize;
let mut retry_entries = Vec::new();
for entry in entries.iter() {
let Some(admission) = replay_mrf_entry(entry, storage, &mut retry_entries).await else {
continue;
};
if admission == ReplicationQueueAdmission::Missed {
retry_entries.push(entry.clone());
} else if admission == ReplicationQueueAdmission::Queued {
queued_count += 1;
}
}
let retained = resolve_retained_mrf_entries(storage, &recovery_guard, &entries, &retry_entries).await;
let retained_count = retained.len();
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
if queued_count > 0 {
info!(
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
recovered = queued_count,
total,
retained = retained_count,
"Replayed MRF entries admitted for retry"
);
}
true
}
/// Re-runs [`replay_mrf_journal`] every `period`. The recovery leader lock keeps
/// one pass at a time across the cluster, including the startup pass, and the
/// shared replay stamp spaces passes from different nodes by `period`.
fn spawn_mrf_replay_loop<S: ReplicationStorage>(storage: Arc<S>, period: Duration) -> JoinHandle<()> {
tokio::spawn(async move {
loop {
tokio::time::sleep(period).await;
replay_mrf_journal(&storage, Some(period)).await;
}
})
}
#[derive(Debug, thiserror::Error)]
#[error("replication resync {active_resync_id} is already active for {bucket}/{arn}")]
struct ResyncActiveConflictError {
@@ -1289,7 +1171,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
// Start background tasks
pool.start_mrf_persister().await;
pool.start_mrf_processor().await;
pool.start_mrf_replayer().await;
pool.start_force_delete_processor().await;
pool
@@ -1581,7 +1462,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
/// Queues a replica task
pub async fn queue_replica_task(&self, ri: ReplicateObjectInfo) -> ReplicationQueueAdmission {
// A source generation already queued or being uploaded is not driven a
// A version that is already queued or being uploaded is not driven a
// second time: the scanner heal pass sees it as PENDING until the
// first upload lands and would otherwise re-queue it every cycle
// (backlog#2362). The key is released when the worker finishes, or
@@ -1758,23 +1639,52 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
/// Starts the MRF processor — one-shot at startup.
///
/// Replays the on-disk MRF journal once; [`Self::start_mrf_replayer`] repeats
/// the pass while the server runs.
/// Reads the on-disk MRF file, re-injects admitted entries as Heal operations, and
/// rewrites any entries that could not be admitted for a later startup retry.
async fn start_mrf_processor(&self) {
let storage = self.storage.clone();
let handle = tokio::spawn(async move {
replay_mrf_journal(&storage, None).await;
});
self.task_handles.lock().await.push(handle);
}
/// Starts the runtime MRF replayer — ongoing background task.
///
/// Entries reach the journal while the server runs (worker-queue overflow,
/// delete-marker purge intents whose watch window drained with targets still
/// dirty). Replaying only at startup left them waiting for a restart.
async fn start_mrf_replayer(&self) {
let handle = spawn_mrf_replay_loop(self.storage.clone(), MRF_REPLAY_INTERVAL);
let handle = tokio::spawn(async move {
let Some(recovery_guard) = acquire_mrf_recovery_guard(&storage).await else {
return;
};
let Some(entries) = load_mrf_recovery_entries(&storage).await else {
return;
};
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
let total = entries.len();
let mut queued_count = 0usize;
let mut retry_entries = Vec::new();
for entry in entries.iter() {
let Some(admission) = replay_mrf_entry(entry, &storage, &mut retry_entries).await else {
continue;
};
if admission == ReplicationQueueAdmission::Missed {
retry_entries.push(entry.clone());
} else if admission == ReplicationQueueAdmission::Queued {
queued_count += 1;
}
}
let retained = resolve_retained_mrf_entries(&storage, &recovery_guard, &entries, &retry_entries).await;
let retained_count = retained.len();
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
if queued_count > 0 {
info!(
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
recovered = queued_count,
total,
retained = retained_count,
"Replayed MRF entries admitted for retry"
);
}
});
self.task_handles.lock().await.push(handle);
}
@@ -1845,7 +1755,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
/// writes them to the on-disk MRF file every flush interval (default 10s,
/// overridable via `RUSTFS_REPL_MRF_FLUSH_INTERVAL_MS`) or when 1 000 new
/// entries accumulate. Runtime writers append under the journal lock; only the
/// recovery leader of a replay pass may remove an acknowledged prefix.
/// startup recovery leader may remove an acknowledged prefix.
async fn start_mrf_persister(&self) {
let Some(mut rx) = self.mrf_save_rx.lock().await.take() else {
return;
@@ -2404,47 +2314,28 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
}
/// Object versions currently queued or being uploaded, keyed by bucket,
/// object name, version and source generation. `queue_replica_task` admits a generation only once
/// object name and version. `queue_replica_task` admits a version only once
/// while it is in flight; the scanner heal pass and MRF replays that arrive
/// in the meantime are `Skipped` instead of driving a second complete upload
/// (backlog#2362). Entries are removed when the worker finishes the task or
/// when no worker accepted it.
#[derive(Debug, Default)]
pub(crate) struct ReplicationInFlight {
keys: std::sync::Mutex<std::collections::HashSet<ReplicationInFlightKey>>,
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
struct ReplicationInFlightKey {
bucket: String,
object: String,
version_id: Option<uuid::Uuid>,
generation: ReplicationGenerationSnapshot,
}
impl From<&ReplicateObjectInfo> for ReplicationInFlightKey {
fn from(info: &ReplicateObjectInfo) -> Self {
Self {
bucket: info.bucket.clone(),
object: info.name.clone(),
version_id: info.version_id,
generation: info.replication_generation.clone(),
}
}
keys: std::sync::Mutex<std::collections::HashSet<(String, String, Option<uuid::Uuid>)>>,
}
impl ReplicationInFlight {
fn lock(&self) -> std::sync::MutexGuard<'_, std::collections::HashSet<ReplicationInFlightKey>> {
fn lock(&self) -> std::sync::MutexGuard<'_, std::collections::HashSet<(String, String, Option<uuid::Uuid>)>> {
self.keys.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
}
/// Claim `ri`; a metadata mutation is independent of its earlier generation.
/// Claim `ri`; `false` when the same version is already in flight.
fn try_begin(&self, ri: &ReplicateObjectInfo) -> bool {
self.lock().insert(ReplicationInFlightKey::from(ri))
self.lock().insert((ri.bucket.clone(), ri.name.clone(), ri.version_id))
}
fn finish(&self, ri: &ReplicateObjectInfo) {
self.lock().remove(&ReplicationInFlightKey::from(ri));
self.lock().remove(&(ri.bucket.clone(), ri.name.clone(), ri.version_id));
}
#[cfg(test)]
@@ -2457,21 +2348,26 @@ impl ReplicationInFlight {
/// including when replication panics.
struct ReplicationInFlightGuard {
in_flight: Arc<ReplicationInFlight>,
key: ReplicationInFlightKey,
key: ReplicateObjectInfo,
}
impl ReplicationInFlightGuard {
fn new(in_flight: Arc<ReplicationInFlight>, ri: &ReplicateObjectInfo) -> Self {
Self {
in_flight,
key: ReplicationInFlightKey::from(ri),
key: ReplicateObjectInfo {
bucket: ri.bucket.clone(),
name: ri.name.clone(),
version_id: ri.version_id,
..Default::default()
},
}
}
}
impl Drop for ReplicationInFlightGuard {
fn drop(&mut self) {
self.in_flight.lock().remove(&self.key);
self.in_flight.finish(&self.key);
}
}
@@ -3525,8 +3421,6 @@ mod tests {
allow_first_read: Notify,
read_count: AtomicUsize,
delete_storage_calls: AtomicUsize,
delete_error: StdMutex<EcstoreError>,
replay_stamp: StdMutex<Option<Vec<u8>>>,
reported_size: StdMutex<Option<i64>>,
stream_read_bytes: Arc<AtomicUsize>,
write_count: AtomicUsize,
@@ -3594,24 +3488,6 @@ mod tests {
_h: Self::HeaderMap,
_opts: &Self::ObjectOptions,
) -> Result<Self::GetObjectReader, Self::Error> {
if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
let stamp = self
.shared
.replay_stamp
.lock()
.expect("test replay stamp lock should not be poisoned")
.clone()
.ok_or(EcstoreError::FileNotFound)?;
return Ok(Self::GetObjectReader {
object_info: ObjectInfo {
size: i64::try_from(stamp.len()).expect("test stamp length should fit i64"),
..Default::default()
},
stream: Box::new(Cursor::new(stamp)),
buffered_body: None,
body_source: Default::default(),
});
}
if !object.ends_with("/.replication/resync.bin")
&& !object.ends_with("config/replication/mrf.bin")
&& !object.ends_with("config/replication/force-delete.bin")
@@ -3675,16 +3551,6 @@ mod tests {
let lock = self.new_ns_lock(bucket, object).await?;
Some(lock.get_write_lock(Duration::from_secs(10)).await?)
};
if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
let mut stamp = Vec::new();
data.stream.read_to_end(&mut stamp).await.map_err(EcstoreError::from)?;
*self
.shared
.replay_stamp
.lock()
.expect("test replay stamp lock should not be poisoned") = Some(stamp);
return Ok(ObjectInfo::default());
}
if opts.http_preconditions.is_some()
&& let Some(replacement) = self
.shared
@@ -3806,12 +3672,7 @@ mod tests {
_opts: Self::ObjectOptions,
) -> Result<Self::ObjectInfo, Self::Error> {
self.shared.delete_storage_calls.fetch_add(1, Ordering::SeqCst);
Err(self
.shared
.delete_error
.lock()
.expect("delete error lock should not be poisoned")
.clone())
Err(EcstoreError::NotImplemented)
}
async fn delete_objects(
@@ -4068,48 +3929,6 @@ mod tests {
assert_eq!(current_queue(&pool, "in-flight-bucket").await, (3, 3 * 4096));
}
#[tokio::test]
async fn queue_replica_task_admits_metadata_changes_during_an_initial_upload() {
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
let (tx, mut rx) = mpsc::channel(4);
pool.workers.write().await.push(tx);
let initial = ReplicateObjectInfo {
bucket: "pending-tags-bucket".to_string(),
name: "object".to_string(),
version_id: Some(uuid::Uuid::new_v4()),
size: 4096,
replication_generation: ReplicationGenerationSnapshot {
mutation_id: Some(uuid::Uuid::new_v4().to_string()),
..Default::default()
},
..Default::default()
};
let metadata = ReplicateObjectInfo {
op_type: ReplicationType::Metadata,
user_tags: String::new(),
replication_generation: ReplicationGenerationSnapshot {
mutation_id: Some(uuid::Uuid::new_v4().to_string()),
..Default::default()
},
..initial.clone()
};
assert_eq!(pool.queue_replica_task(initial.clone()).await, ReplicationQueueAdmission::Queued);
assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Queued);
assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Skipped);
assert_eq!(pool.in_flight.len(), 2);
drop(ReplicationInFlightGuard::new(pool.in_flight.clone(), &initial));
assert_eq!(pool.in_flight.len(), 1);
assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Skipped);
for expected in [&initial, &metadata] {
let ReplicationOperation::Object(queued) = rx.recv().await.expect("queued operation") else {
panic!("expected object replication");
};
assert_eq!(queued.replication_generation, expected.replication_generation);
}
drop(ReplicationInFlightGuard::new(pool.in_flight.clone(), &metadata));
assert_eq!(pool.in_flight.len(), 0);
}
#[tokio::test]
async fn queue_replica_task_releases_the_version_when_no_worker_accepts_it() {
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
@@ -4352,8 +4171,6 @@ mod tests {
allow_first_read: Notify::new(),
read_count: AtomicUsize::new(0),
delete_storage_calls: AtomicUsize::new(0),
delete_error: StdMutex::new(EcstoreError::NotImplemented),
replay_stamp: StdMutex::new(None),
reported_size: StdMutex::new(None),
stream_read_bytes: Arc::new(AtomicUsize::new(0)),
write_count: AtomicUsize::new(0),
@@ -4404,103 +4221,6 @@ mod tests {
}
}
#[tokio::test]
async fn delete_worker_already_purged_source_completes_without_retry() {
let arn = "arn:test";
for delete_marker_version in [false, true] {
for version_id in [Uuid::new_v4(), Uuid::nil()] {
for error in [
EcstoreError::VersionNotFound("source".to_string(), "purged".to_string(), version_id.to_string()),
EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
EcstoreError::FileVersionNotFound,
EcstoreError::FileNotFound,
] {
let shared = empty_resync_shared_state();
*shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
let storage = Arc::new(LoadResyncNodeStore::new("purged-source", shared.clone()));
let delete = DeletedObjectReplicationInfo {
bucket: "source".to_string(),
delete_object: ReplicationDeletedObject {
object_name: "purged".to_string(),
version_id: (!delete_marker_version).then_some(version_id),
delete_marker_version_id: delete_marker_version.then_some(version_id),
replication_state: Some(ReplicationState {
replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
purge_targets: HashMap::from([(arn.to_string(), VersionPurgeStatusType::Complete)]),
..Default::default()
}),
..Default::default()
},
op_type: ReplicationType::Delete,
..Default::default()
};
assert!(
replicate_delete_with_outcome(delete, storage).await,
"a confirmed target purge must complete when the source is already absent (marker={delete_marker_version}, version={version_id})"
);
assert_eq!(
shared.delete_storage_calls.load(Ordering::SeqCst),
2 + usize::from(!delete_marker_version && !version_id.is_nil()),
"the worker must lock and attempt writeback, reading a non-null data version first"
);
}
}
}
}
#[tokio::test]
async fn delete_worker_missing_source_does_not_hide_unfinished_replication() {
let arn = "arn:test";
for (version_purge, target_status, error) in [
(
true,
VersionPurgeStatusType::Pending,
EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
),
(true, VersionPurgeStatusType::Complete, EcstoreError::ErasureWriteQuorum),
(
false,
VersionPurgeStatusType::Empty,
EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
),
] {
let shared = empty_resync_shared_state();
*shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
let storage = Arc::new(LoadResyncNodeStore::new("unfinished-delete", shared));
let delete = DeletedObjectReplicationInfo {
bucket: "source".to_string(),
delete_object: ReplicationDeletedObject {
object_name: "purged".to_string(),
version_id: version_purge.then(Uuid::new_v4),
delete_marker: !version_purge,
replication_state: Some(ReplicationState {
replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
targets: if version_purge {
HashMap::new()
} else {
HashMap::from([(arn.to_string(), ReplicationStatusType::Completed)])
},
purge_targets: if version_purge {
HashMap::from([(arn.to_string(), target_status)])
} else {
HashMap::new()
},
..Default::default()
}),
..Default::default()
},
op_type: ReplicationType::Delete,
..Default::default()
};
assert!(
!replicate_delete_with_outcome(delete, storage).await,
"unfinished target work, source I/O failures, and marker creation must remain retryable"
);
}
}
#[tokio::test]
async fn delete_worker_preserves_persisted_admission_without_live_config() {
let arn = "arn:test";
@@ -5584,8 +5304,6 @@ mod tests {
allow_first_read: Notify::new(),
read_count: AtomicUsize::new(0),
delete_storage_calls: AtomicUsize::new(0),
delete_error: StdMutex::new(EcstoreError::NotImplemented),
replay_stamp: StdMutex::new(None),
reported_size: StdMutex::new(None),
stream_read_bytes: Arc::new(AtomicUsize::new(0)),
write_count: AtomicUsize::new(0),
@@ -6550,108 +6268,6 @@ mod tests {
assert_eq!(retained.target_arns, entry.target_arns);
}
#[tokio::test]
async fn mrf_replay_loop_drains_entries_persisted_after_startup() {
let shared = empty_resync_shared_state();
let storage = Arc::new(LoadResyncNodeStore::new("mrf-runtime-replay", shared.clone()));
// A force-delete intent from a superseded generation replays as
// Skipped, so a pass that reaches it acknowledges it off the journal.
let stale_intent = |object: &str| MrfReplicateEntry {
bucket: "mrf-runtime-replay".to_string(),
object: object.to_string(),
op: MrfOpKind::Delete,
force_delete_id: Some(uuid::Uuid::new_v4()),
force_delete_local_commit: false,
target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
..Default::default()
};
let journal_drained = || async {
tokio::time::timeout(Duration::from_secs(5), async {
while !shared.data.lock().expect("test data lock should not be poisoned").is_empty() {
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.is_ok()
};
let handle = spawn_mrf_replay_loop(storage, Duration::from_millis(20));
for object in ["first", "second"] {
*shared.data.lock().expect("test data lock should not be poisoned") =
encode_mrf_file(&[stale_intent(object)]).expect("MRF entry should encode");
assert!(
journal_drained().await,
"an MRF entry persisted while the server runs must be replayed without a restart ({object})"
);
}
handle.abort();
}
#[tokio::test]
async fn mrf_periodic_replay_runs_once_per_interval_across_nodes() {
let shared = empty_resync_shared_state();
let node_a = Arc::new(LoadResyncNodeStore::new("mrf-node-a", shared.clone()));
let node_b = Arc::new(LoadResyncNodeStore::new("mrf-node-b", shared.clone()));
let interval = Duration::from_secs(1);
let persist_backlog = |object: &str| {
let entry = MrfReplicateEntry {
bucket: "mrf-node-replay".to_string(),
object: object.to_string(),
op: MrfOpKind::Delete,
force_delete_id: Some(uuid::Uuid::new_v4()),
force_delete_local_commit: false,
target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
..Default::default()
};
*shared.data.lock().expect("test data lock should not be poisoned") =
encode_mrf_file(&[entry]).expect("MRF entry should encode");
};
let journal_is_empty = || shared.data.lock().expect("test data lock should not be poisoned").is_empty();
persist_backlog("first");
assert!(replay_mrf_journal(&node_a, Some(interval)).await, "node A's periodic pass should replay");
assert!(journal_is_empty(), "node A's pass should acknowledge the replayed entry");
// Node B's timer fires within the same interval with a backlog present.
persist_backlog("second");
assert!(
!replay_mrf_journal(&node_b, Some(interval)).await,
"a node must not replay within one interval of another node's pass"
);
assert!(!journal_is_empty(), "the backlog should wait for the next interval");
assert!(
replay_mrf_journal(&node_b, None).await,
"the startup pass should replay regardless of the shared stamp"
);
assert!(journal_is_empty(), "the startup pass should acknowledge the replayed entry");
persist_backlog("third");
tokio::time::sleep(interval).await;
assert!(
replay_mrf_journal(&node_a, Some(interval)).await,
"a periodic pass should replay once the interval has elapsed"
);
assert!(journal_is_empty(), "the due pass should acknowledge the replayed entry");
}
#[test]
fn mrf_replay_due_tolerates_missing_and_skewed_stamps() {
let interval = Duration::from_secs(300);
let now = 1_000_000_000;
assert!(mrf_replay_due(None, now, interval));
assert!(!mrf_replay_due(Some(now - 299_999), now, interval));
assert!(mrf_replay_due(Some(now - 300_000), now, interval));
assert!(
!mrf_replay_due(Some(now + 1_000), now, interval),
"small clock skew should still space passes"
);
assert!(
mrf_replay_due(Some(now + 300_000), now, interval),
"a stamp an interval ahead must not suppress replay"
);
}
#[test]
fn mrf_entry_delete_marker_roundtrip() {
let dm_vid = Uuid::new_v4();
@@ -2809,22 +2809,17 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
true
}
Err(e) => {
// A batch purge or another worker may have removed this version
// already. Remote failures still control the final outcome below.
let source_already_purged = is_version_purge && (is_err_object_not_found(&e) || is_err_version_not_found(&e));
if !source_already_purged {
error!(
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %dobj.target_arn,
object = %dobj.delete_object.object_name,
operation = "apply_replication_delete_state",
error = %e,
"Replication target operation failed"
);
}
error!(
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %dobj.target_arn,
object = %dobj.delete_object.object_name,
operation = "apply_replication_delete_state",
error = %e,
"Replication target operation failed"
);
send_event(EventArgs {
event_name,
bucket_name: bucket.clone(),
@@ -2837,7 +2832,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
},
..Default::default()
});
source_already_purged
false
}
};
@@ -3015,7 +3010,7 @@ const DELETE_MARKER_PURGE_WATCH_INTERVAL: TokioDuration = TokioDuration::from_se
/// If the marker disappears (deleted before or while the replica landed),
/// purge the replicated marker from the targets, retrying failed targets on
/// later rounds. When the window drains with targets still dirty, persist the
/// purge intent as a durable MRF entry so the next MRF replay pass re-drives it through
/// purge intent as a durable MRF entry so the next startup replays it through
/// `purge_stale_delete_marker_targets`.
async fn watch_and_purge_source_delete_marker<S: ReplicationStorage>(
bucket: String,
@@ -3106,7 +3101,7 @@ async fn enqueue_delete_marker_purge_mrf(dobj: &DeletedObjectReplicationInfo, fa
/// Heal callers retry for the full watch window because the startup MRF
/// processor runs before bucket metadata (and thus target clients) finishes
/// initializing — the first pass can see `target_client_missing` and a later
/// round resolves the client; replay passes are serial background work, so
/// round resolves the client; the replay loop is serial and startup-only, so
/// blocking it for up to the window per dirty entry is acceptable. Live
/// callers run on replication workers where a down target would pin a worker
/// for the whole window, so they attempt once and lean on the durable intent
@@ -6609,7 +6604,6 @@ mod tests {
health_check_duration: std::time::Duration::from_secs(5),
replicate_sync: false,
client: Arc::new(aws_sdk_s3::Client::from_conf(config)),
timeouts: Default::default(),
})
}
-12
View File
@@ -259,10 +259,6 @@ pub fn check_object_name_for_length_and_slash(bucket: &str, object: &str) -> Res
pub fn check_new_object_args(bucket: &str, object: &str, data_movement: bool) -> Result<()> {
check_put_object_args(bucket, object)?;
if !data_movement && !is_meta_bucketname(bucket) && has_xml_invalid_control_characters(object) {
return Err(StorageError::ObjectNameInvalid(bucket.to_owned(), object.to_owned()));
}
if cfg!(target_os = "windows") && !data_movement && !is_meta_bucketname(bucket) && object.contains('\\') {
return Err(StorageError::InvalidArgument(
bucket.to_owned(),
@@ -274,14 +270,6 @@ pub fn check_new_object_args(bucket: &str, object: &str, data_movement: bool) ->
Ok(())
}
/// New public keys must remain representable in an unencoded XML listing.
/// Existing keys still use the read/delete validators so they can be removed.
pub fn has_xml_invalid_control_characters(object: &str) -> bool {
object
.chars()
.any(|character| character < ' ' && !matches!(character, '\t' | '\n' | '\r'))
}
pub fn check_copy_obj_args(bucket: &str, object: &str) -> Result<()> {
check_bucket_and_object_names(bucket, object)
}
+36 -294
View File
@@ -34,7 +34,7 @@ use tokio::sync::Mutex as TokioMutex;
use tokio::task::JoinSet;
use tokio::time::timeout;
use tokio_util::sync::CancellationToken;
use tracing::{Instrument, debug, error, warn};
use tracing::{debug, error, warn};
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_METACACHE: &str = "metacache";
@@ -217,8 +217,6 @@ impl Drop for NotifyOnDrop {
#[derive(Default)]
pub struct ListPathRawOptions {
/// Retain native listing admission through producer cancellation/draining.
pub read_budget: Option<Arc<tokio::sync::OwnedSemaphorePermit>>,
pub disks: Vec<Option<DiskStore>>,
pub fallback_disks: Vec<Option<DiskStore>>,
pub bucket: String,
@@ -226,8 +224,6 @@ pub struct ListPathRawOptions {
pub recursive: bool,
pub incl_deleted: bool,
pub skip_hidden_prefix_check: bool,
pub hidden_entry_budget: usize,
pub pending_prefix: Option<String>,
pub filter_prefix: Option<String>,
pub forward_to: Option<String>,
pub min_disks: usize,
@@ -257,7 +253,6 @@ pub struct ListPathRawOptions {
impl Clone for ListPathRawOptions {
fn clone(&self) -> Self {
Self {
read_budget: self.read_budget.clone(),
disks: self.disks.clone(),
fallback_disks: self.fallback_disks.clone(),
bucket: self.bucket.clone(),
@@ -265,8 +260,6 @@ impl Clone for ListPathRawOptions {
recursive: self.recursive,
incl_deleted: self.incl_deleted,
skip_hidden_prefix_check: self.skip_hidden_prefix_check,
hidden_entry_budget: self.hidden_entry_budget,
pending_prefix: self.pending_prefix.clone(),
filter_prefix: self.filter_prefix.clone(),
forward_to: self.forward_to.clone(),
min_disks: self.min_disks,
@@ -296,8 +289,6 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
recursive: opts.recursive,
incl_deleted: opts.incl_deleted,
skip_hidden_prefix_check: opts.skip_hidden_prefix_check,
hidden_entry_budget: opts.hidden_entry_budget,
pending_prefix: opts.pending_prefix.clone(),
report_notfound: opts.report_not_found,
filter_prefix: opts.filter_prefix.clone(),
forward_to: opts.forward_to.clone(),
@@ -310,15 +301,13 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
}
/// How a merged raw listing stream ended.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ListPathRawOutcome {
/// At least one producer stopped at its per-disk limit instead of at the
/// end of its namespace, so the closed stream is not proof of exhaustion.
/// Carried in-band by the metacache stream trailer, so remote producers
/// report it as well.
pub limit_reached: bool,
/// Inclusive sorted boundary proved by every reader before a bounded stop.
pub scan_cut: Option<String>,
}
pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> disk::error::Result<()> {
@@ -374,15 +363,10 @@ async fn list_path_raw_inner(
let producer_errs: Arc<[OnceLock<DiskError>]> = (0..opts.disks.len()).map(|_| OnceLock::new()).collect::<Vec<_>>().into();
let cancel_rx = CancellationToken::new();
let walk_options = Arc::new(walk_dir_options(&opts));
for (disk_idx, disk) in opts.disks.iter().enumerate() {
let opdisk = disk.clone();
let walk_options = walk_options.clone();
#[cfg(test)]
let test_reader_behavior = opts.test_reader_behaviors.get(disk_idx).cloned();
#[cfg(test)]
let test_disk_io_context_observed = opts.test_disk_io_context_observed.clone();
let opts_clone = opts.clone();
let fallback_claim_tracker = fallback_claim_tracker.clone();
let fds_clone = fds.clone();
#[cfg(test)]
@@ -392,19 +376,17 @@ async fn list_path_raw_inner(
let disk_io_context_clone = disk_io_context.clone();
let (rd, wr) = tokio::io::duplex(64 * 1024);
readers.push(MetacacheReader::new(rd));
let read_budget = opts.read_budget.clone();
let producer = with_optional_disk_io_context(disk_io_context_clone, async move {
let _read_budget = read_budget;
jobs.spawn(with_optional_disk_io_context(disk_io_context_clone, async move {
let mut wr = PublishedBytesWriter::new(wr);
#[cfg(test)]
if let Some(observer) = &test_disk_io_context_observed {
if let Some(observer) = &opts_clone.test_disk_io_context_observed {
observer.store(
crate::disk::current_disk_io_context().is_some_and(|context| context.class == DiskIoClass::Background),
std::sync::atomic::Ordering::Release,
);
}
#[cfg(test)]
let test_primary_error = if let Some(behavior) = test_reader_behavior {
let test_primary_error = if let Some(behavior) = opts_clone.test_reader_behaviors.get(disk_idx).cloned() {
match behavior {
TestReaderBehavior::Eof => return Ok(()),
TestReaderBehavior::Entries(entries) => {
@@ -461,6 +443,8 @@ async fn list_path_raw_inner(
None
};
let wakl_opts = walk_dir_options(&opts_clone);
let mut need_fallback = false;
let mut last_err = None;
#[cfg(test)]
@@ -475,7 +459,7 @@ async fn list_path_raw_inner(
if !need_fallback && let Some(disk) = opdisk {
let primary_walk_started = std::time::Instant::now();
match disk.walk_dir((*walk_options).clone(), &mut wr).await {
match disk.walk_dir(wakl_opts, &mut wr).await {
Ok(_res) => {
rustfs_io_metrics::record_stage_duration(
"metacache_walk_dir_primary",
@@ -499,8 +483,8 @@ async fn list_path_raw_inner(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
bucket = %walk_options.bucket,
path = %walk_options.base_dir,
bucket = %opts_clone.bucket,
path = %opts_clone.path,
disk_index = disk_idx,
state = "walk_dir_missing_path",
error = ?err,
@@ -511,8 +495,8 @@ async fn list_path_raw_inner(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
bucket = %walk_options.bucket,
path = %walk_options.base_dir,
bucket = %opts_clone.bucket,
path = %opts_clone.path,
disk_index = disk_idx,
state = "walk_dir_failed",
error = ?err,
@@ -614,8 +598,8 @@ async fn list_path_raw_inner(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
bucket = %walk_options.bucket,
path = %walk_options.base_dir,
bucket = %opts_clone.bucket,
path = %opts_clone.path,
disk_index = disk_idx,
state = "fallback_disk_missing_for_path",
error = ?err,
@@ -626,8 +610,8 @@ async fn list_path_raw_inner(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
bucket = %walk_options.bucket,
path = %walk_options.base_dir,
bucket = %opts_clone.bucket,
path = %opts_clone.path,
disk_index = disk_idx,
state = "fallback_disk_missing",
error = ?err,
@@ -642,7 +626,7 @@ async fn list_path_raw_inner(
}
let fallback_walk_started = std::time::Instant::now();
match disk.as_ref().walk_dir((*walk_options).clone(), &mut wr).await {
match disk.as_ref().walk_dir(walk_dir_options(&opts_clone), &mut wr).await {
Ok(_r) => {
rustfs_io_metrics::record_stage_duration(
"metacache_walk_dir_fallback",
@@ -668,8 +652,8 @@ async fn list_path_raw_inner(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
bucket = %walk_options.bucket,
path = %walk_options.base_dir,
bucket = %opts_clone.bucket,
path = %opts_clone.path,
disk_index = disk_idx,
state = "fallback_walk_dir_missing_path",
error = ?err,
@@ -680,8 +664,8 @@ async fn list_path_raw_inner(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
bucket = %walk_options.bucket,
path = %walk_options.base_dir,
bucket = %opts_clone.bucket,
path = %opts_clone.path,
disk_index = disk_idx,
state = "fallback_walk_dir_failed",
error = ?err,
@@ -703,13 +687,12 @@ async fn list_path_raw_inner(
// warn!("list_path_raw: while need_fallback done");
Ok(())
});
jobs.spawn(producer.in_current_span());
}));
}
let revjob_rx = rx.clone();
let mut revjobs = JoinSet::new();
let merge = async move {
revjobs.spawn(async move {
// Consumer-side peek timeout, and a caveat worth understanding
// (rustfs/backlog#1217).
//
@@ -739,8 +722,6 @@ async fn list_path_raw_inner(
// Sticky across rounds: once any reader's trailer says its producer
// stopped at the per-disk limit, the merged stream may hide more keys.
let mut limit_reached = false;
let mut scan_cut = None;
let mut last_scanned_key: Option<String> = None;
loop {
let mut current = MetaCacheEntry::default();
@@ -752,7 +733,7 @@ async fn list_path_raw_inner(
// );
if revjob_rx.is_cancelled() {
return Ok(ListPathRawOutcome { limit_reached, scan_cut });
return Ok(ListPathRawOutcome { limit_reached });
}
let mut top_entries: Vec<Option<MetaCacheEntry>> = vec![None; readers.len()];
@@ -766,16 +747,6 @@ async fn list_path_raw_inner(
// Start every missing head read in the same round so one stalled
// disk cannot multiply the wait budget by the erasure-set width.
// Outcomes are still consumed below in stable disk-index order.
if !revjob_rx.is_cancelled() {
for (i, reader) in readers.iter_mut().enumerate() {
if errs[i].is_none()
&& pending_entries[i].is_none()
&& let Some(Ok(Some(entry))) = reader.try_peek_buffered()
{
pending_entries[i] = Some(entry);
}
}
}
let concurrent_peeks = readers.iter_mut().enumerate().filter_map(|(i, reader)| {
if errs[i].is_some() || pending_entries[i].is_some() {
return None;
@@ -812,12 +783,7 @@ async fn list_path_raw_inner(
continue;
}
// eof
if r.limit_reached() == Some(true) {
limit_reached = true;
scan_cut = Some(last_scanned_key.clone().ok_or_else(|| {
DiskError::other("bounded listing reader ended without a scan boundary")
})?);
}
limit_reached |= r.limit_reached() == Some(true);
at_eof += 1;
// warn!("list_path_raw: peek eof, disk: {}", i);
continue;
@@ -891,7 +857,7 @@ async fn list_path_raw_inner(
*r = MetacacheReader::new(detached_rd);
continue;
}
PeekOutcome::Cancelled => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
PeekOutcome::Cancelled => return Ok(ListPathRawOutcome { limit_reached }),
}
};
@@ -912,7 +878,7 @@ async fn list_path_raw_inner(
}
if entry.name == current.name {
// If exact match, we agree.
if current.matches_strict_borrowed(&entry) {
if current.matches_strict(&entry) {
top_entries[i] = Some(entry);
agree += 1;
} else {
@@ -950,7 +916,7 @@ async fn list_path_raw_inner(
if let Some(finished_fn) = opts.finished.as_ref() {
tokio::select! {
biased;
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
_ = finished_fn(&errs) => {}
}
}
@@ -1000,14 +966,6 @@ async fn list_path_raw_inner(
return Err(classify_listing_quorum_failure(&failures));
}
// The next head from a bounded reader is unknown. The previous
// globally consumed group is a safe cut; consuming another reader's
// later head would turn this absence of evidence into a skipped key.
// Check quorum failures above before accepting the bounded result.
if limit_reached {
break;
}
// Break if all at EOF or error.
if at_eof + has_err == readers.len() {
if has_err > 0
@@ -1015,7 +973,7 @@ async fn list_path_raw_inner(
{
tokio::select! {
biased;
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
_ = finished_fn(&errs) => {}
}
}
@@ -1026,19 +984,16 @@ async fn list_path_raw_inner(
break;
}
last_scanned_key = Some(current.name.clone());
if agree == readers.len() && !opts.preserve_replica_metadata {
for r in readers.iter_mut() {
if !r.skip_current() {
let _ = r.skip(1).await;
}
let _ = r.skip(1).await;
}
if let Some(agreed_fn) = opts.agreed.as_ref() {
// warn!("list_path_raw: agreed_fn start, current: {:?}", &current.name);
tokio::select! {
biased;
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
result = agreed_fn(current) => result?
}
// warn!("list_path_raw: agreed_fn done");
@@ -1054,7 +1009,7 @@ async fn list_path_raw_inner(
}
for (i, r) in readers.iter_mut().enumerate() {
if top_entries[i].is_some() && !r.skip_current() {
if top_entries[i].is_some() {
let _ = r.skip(1).await;
}
}
@@ -1062,14 +1017,13 @@ async fn list_path_raw_inner(
if let Some(partial_fn) = opts.partial.as_ref() {
tokio::select! {
biased;
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
_ = partial_fn(MetaCacheEntries(top_entries), &errs) => {}
}
}
}
Ok(ListPathRawOutcome { limit_reached, scan_cut })
};
revjobs.spawn(merge.in_current_span());
Ok(ListPathRawOutcome { limit_reached })
});
let merge_started = std::time::Instant::now();
let Some(revjob) = revjobs.join_next().await else {
@@ -1233,24 +1187,6 @@ mod tests {
assert!(!options.skip_total_timeout);
}
#[test]
fn cloned_reader_options_preserve_hidden_scan_budget_and_pending_prefix() {
let request = ListPathRawOptions {
bucket: "versioned".to_owned(),
hidden_entry_budget: 100,
pending_prefix: Some("pending/".to_owned()),
forward_to: Some("pending/0099".to_owned()),
..Default::default()
};
// Both walks are built once from the shared options; the per-job clone
// must carry every field that the walk derives from the request.
let job = request.clone();
let walk = walk_dir_options(&job);
assert_eq!(walk.hidden_entry_budget, 100);
assert_eq!(walk.pending_prefix.as_deref(), request.pending_prefix.as_deref());
assert_eq!(walk.forward_to.as_deref(), Some("pending/0099"));
}
#[tokio::test]
async fn list_path_raw_empty_disks_returns_read_quorum() {
let err = list_path_raw(CancellationToken::new(), ListPathRawOptions::default())
@@ -1301,163 +1237,6 @@ mod tests {
);
}
#[tokio::test]
async fn list_path_raw_fanout_retains_only_linear_disk_references() {
let root = tempfile::tempdir().expect("create disk root");
let endpoint = crate::disk::endpoint::Endpoint::try_from(root.path().to_str().expect("UTF-8 disk path"))
.expect("parse disk endpoint");
let local = Arc::new(
crate::disk::local::LocalDisk::new(&endpoint, false)
.await
.expect("open local disk"),
);
let wrapper = crate::disk::disk_store::LocalDiskWrapper::new(local, false);
let disk = Arc::new(crate::disk::Disk::Local(Box::new(wrapper)));
let baseline = Arc::strong_count(&disk);
for primary_count in [1, 4, 16] {
for fallback_count in [0, 2] {
let entered: Vec<_> = (0..primary_count).map(|_| Arc::new(tokio::sync::Notify::new())).collect();
let stopped: Vec<_> = (0..primary_count).map(|_| Arc::new(tokio::sync::Notify::new())).collect();
let behaviors = entered
.iter()
.zip(&stopped)
.map(|(entered, stopped)| TestReaderBehavior::StallWithSignals {
entered: entered.clone(),
stopped: stopped.clone(),
})
.collect();
let cancel = CancellationToken::new();
let listing = tokio::spawn(list_path_raw(
cancel.clone(),
ListPathRawOptions {
disks: vec![Some(disk.clone()); primary_count],
fallback_disks: vec![Some(disk.clone()); fallback_count],
min_disks: primary_count,
test_reader_behaviors: behaviors,
peek_timeout: Some(Duration::from_secs(60)),
..Default::default()
},
));
for signal in &entered {
timeout(Duration::from_secs(2), signal.notified())
.await
.expect("every producer must enter the real fanout before counting references");
}
// The merge owns the configured primary/fallback slots, each producer
// owns its primary, and the shared fallback queue owns each fallback.
let expected = baseline + 2 * (primary_count + fallback_count);
let retained = Arc::strong_count(&disk);
cancel.cancel();
timeout(Duration::from_secs(2), listing)
.await
.expect("cancellation must stop the listing")
.expect("listing task must join")
.expect("external cancellation must finish cleanly");
for signal in &stopped {
timeout(Duration::from_secs(2), signal.notified())
.await
.expect("cancellation must drop every producer");
}
assert_eq!(Arc::strong_count(&disk), baseline, "cancellation must release every fanout reference");
assert_eq!(
retained, expected,
"fanout must retain linear references for {primary_count} primary and {fallback_count} fallback slots"
);
}
}
}
#[tokio::test(flavor = "current_thread")]
async fn list_path_raw_producer_and_merge_preserve_request_span() {
use tracing_subscriber::{Layer, layer::SubscriberExt};
struct ProducerSpan(Arc<AtomicBool>);
#[derive(Default)]
struct StateField(bool);
impl tracing::field::Visit for StateField {
fn record_debug(&mut self, _: &tracing::field::Field, _: &dyn std::fmt::Debug) {}
fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
self.0 |= field.name() == "state" && value == "walk_dir_missing_path";
}
}
impl<S> Layer<S> for ProducerSpan
where
S: tracing::Subscriber + for<'a> tracing_subscriber::registry::LookupSpan<'a>,
{
fn on_event(&self, event: &tracing::Event<'_>, context: tracing_subscriber::layer::Context<'_, S>) {
let mut state = StateField::default();
event.record(&mut state);
if state.0
&& context
.lookup_current()
.is_some_and(|span| span.name() == "flink_list_request")
{
self.0.store(true, std::sync::atomic::Ordering::Release);
}
}
}
// The subscriber is thread-local; keep the spawned tasks on this runtime thread.
let producer_observed = Arc::new(AtomicBool::new(false));
let subscriber = tracing_subscriber::registry().with(ProducerSpan(producer_observed.clone()));
let _subscriber = tracing::subscriber::set_default(subscriber);
let request = tracing::info_span!("flink_list_request", request_id = "list-span-regression");
let root = tempfile::tempdir().expect("create disk root");
let endpoint = crate::disk::endpoint::Endpoint::try_from(root.path().to_str().expect("UTF-8 disk path"))
.expect("parse disk endpoint");
let local = Arc::new(
crate::disk::local::LocalDisk::new(&endpoint, false)
.await
.expect("open local disk"),
);
let wrapper = crate::disk::disk_store::LocalDiskWrapper::new(local, false);
let disk = Arc::new(crate::disk::Disk::Local(Box::new(wrapper)));
let error = list_path_raw(
CancellationToken::new(),
ListPathRawOptions {
disks: vec![Some(disk)],
bucket: "missing-bucket".to_owned(),
min_disks: 1,
..Default::default()
},
)
.instrument(request.clone())
.await
.expect_err("missing bucket must fail the real disk producer");
assert_eq!(error, DiskError::VolumeNotFound);
assert!(
producer_observed.load(std::sync::atomic::Ordering::Acquire),
"disk error must retain the request span"
);
let merge_observed = Arc::new(AtomicBool::new(false));
let merge_sink = merge_observed.clone();
let request_id = request.id();
list_path_raw(
CancellationToken::new(),
ListPathRawOptions {
disks: vec![None],
min_disks: 1,
test_reader_behaviors: vec![TestReaderBehavior::Entries(vec![fallback_test_entry()])],
agreed: Some(Box::new(move |_| {
merge_sink.store(tracing::Span::current().id() == request_id, std::sync::atomic::Ordering::Release);
Box::pin(async { Ok(()) })
})),
..Default::default()
},
)
.instrument(request)
.await
.expect("merge should deliver the test entry");
assert!(
merge_observed.load(std::sync::atomic::Ordering::Acquire),
"merge callback must retain the request span"
);
}
#[tokio::test]
async fn scanner_joinset_walk_uses_the_local_disk_background_gate() {
let root = tempfile::tempdir().expect("local disk root should be created");
@@ -1525,7 +1304,6 @@ mod tests {
outcome.limit_reached,
"a reader whose trailer reports a limit stop must mark the merged stream as possibly incomplete"
);
assert_eq!(outcome.scan_cut.as_deref(), Some("bucket/object"));
}
#[tokio::test]
@@ -1902,42 +1680,6 @@ mod tests {
assert_eq!(seen.lock().expect("seen mutex poisoned").as_slice(), &["bucket/object".to_string()]);
}
#[tokio::test]
async fn list_read_budget_is_retained_by_raw_producer_until_cancellation() {
let semaphore = Arc::new(tokio::sync::Semaphore::new(4));
let grant = Arc::new(semaphore.clone().acquire_many_owned(4).await.expect("listing grant"));
let weak_grant = Arc::downgrade(&grant);
let entered = Arc::new(tokio::sync::Notify::new());
let stopped = Arc::new(tokio::sync::Notify::new());
let worker = tokio::spawn(list_path_raw(
CancellationToken::new(),
ListPathRawOptions {
read_budget: Some(grant),
disks: vec![None],
min_disks: 1,
test_reader_behaviors: vec![TestReaderBehavior::StallWithSignals {
entered: entered.clone(),
stopped: stopped.clone(),
}],
..Default::default()
},
));
timeout(Duration::from_secs(5), entered.notified())
.await
.expect("producer entered");
assert!(weak_grant.strong_count() >= 2, "producer must retain a grant independently of the parent");
assert_eq!(semaphore.available_permits(), 0);
worker.abort();
assert!(worker.await.expect_err("parent cancelled").is_cancelled());
timeout(Duration::from_secs(5), stopped.notified())
.await
.expect("producer stopped");
let _next = timeout(Duration::from_secs(5), semaphore.acquire_many(4))
.await
.expect("cancelled listing releases all permits")
.expect("budget stays open");
}
#[tokio::test]
async fn list_path_raw_aborts_unresponsive_producer_after_quorum_eof() {
let result = timeout(
+22 -330
View File
@@ -32,6 +32,7 @@ use crate::storage_api_contracts::internode::{
PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_CAPABILITY_VERSION,
};
use hmac::{Hmac, KeyInit, Mac};
use http::uri::Authority;
use http::{HeaderMap, HeaderValue, Method, Uri};
#[cfg(test)]
@@ -46,7 +47,6 @@ use rustfs_io_metrics::internode_metrics::{
global_internode_metrics,
};
use rustfs_object_data_cache::{MemoryBasis, resolve_effective_memory};
use rustfs_utils::crypto::{HmacSha256, KeyInit, Mac};
use rustfs_utils::get_env_bool;
use sha2::Digest as _;
use sha2::Sha256;
@@ -58,6 +58,8 @@ use time::OffsetDateTime;
use tracing::{error, info, warn};
use uuid::Uuid;
type HmacSha256 = Hmac<Sha256>;
const SIGNATURE_HEADER: &str = "x-rustfs-signature";
pub const RPC_BACKGROUND_ADMISSION_HEADER: &str = "x-rustfs-rpc-background-admission";
const RPC_BACKGROUND_ADMISSION_V1: &str = "v1";
@@ -127,14 +129,9 @@ const HTTP_PUT_FILE_CAPABILITY_AUTH_DOMAIN: &[u8] = b"rustfs-http-put-file-capab
const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
const UNSIGNED_PAYLOAD_NONCE: &str = "unsigned";
const SIGNATURE_VALID_DURATION: i64 = 300; // 5 minutes
// Timestamp validity and the wall-clock high-water mark still gate eviction,
// including future-dated signatures and replay attempts after clock rollback.
const REPLAY_CACHE_RETENTION: Duration = Duration::from_secs(301);
// Preserve capacity for the full validity window at maximum accepted clock skew.
const REPLAY_CACHE_CAPACITY_WINDOW_SECS: usize = 601;
// A reserved index avoids overlapping old/new tables during a rehash. Include
// index slack, compact expiry storage, and page/allocator bookkeeping.
const REPLAY_CACHE_ENTRY_BYTES_ESTIMATE: usize = 2 * std::mem::size_of::<Uuid>() + std::mem::size_of::<RpcNonceExpiration>() + 16;
const REPLAY_CACHE_RETENTION: Duration = Duration::from_secs(601);
const REPLAY_CACHE_RETENTION_SECS: usize = 601;
const REPLAY_CACHE_ENTRY_BYTES_ESTIMATE: u64 = 128;
// Keep 16 CPU / 32 GiB field nodes at the 32M cap without requiring an env override.
const REPLAY_CACHE_AUTO_MEMORY_PERCENT: u64 = 13;
const REPLAY_CACHE_AUTO_RPC_RPS_PER_CPU: usize = 4096;
@@ -227,11 +224,11 @@ fn replay_cache_capacity_from_resources(cpu_count: usize, memory_limit_bytes: Op
let cpu_count = cpu_count.max(1);
let cpu_based_capacity = cpu_count
.saturating_mul(REPLAY_CACHE_AUTO_RPC_RPS_PER_CPU)
.saturating_mul(REPLAY_CACHE_CAPACITY_WINDOW_SECS);
.saturating_mul(REPLAY_CACHE_RETENTION_SECS);
let memory_based_capacity = memory_limit_bytes
.map(|bytes| {
let budget = bytes.saturating_mul(REPLAY_CACHE_AUTO_MEMORY_PERCENT) / 100;
saturating_usize_from_u64(budget / u64::try_from(REPLAY_CACHE_ENTRY_BYTES_ESTIMATE).unwrap_or(u64::MAX))
saturating_usize_from_u64(budget / REPLAY_CACHE_ENTRY_BYTES_ESTIMATE)
})
.unwrap_or(REPLAY_CACHE_AUTO_MAX_CAPACITY);
let capacity = memory_based_capacity
@@ -371,84 +368,10 @@ fn resolve_replay_cache_capacity() -> usize {
decision.capacity
}
const RPC_NONCE_EXPIRATIONS_PER_PAGE: usize = 1_024;
struct RpcNonceExpiration {
nonce: Uuid,
expiry_offset_ns: u32,
valid_until_offset_secs: i32,
}
struct RpcNonceExpirationPage {
expires_at: Instant,
valid_until: i64,
entries: VecDeque<RpcNonceExpiration>,
}
#[derive(Default)]
struct RpcNonceExpirations {
pages: VecDeque<RpcNonceExpirationPage>,
}
impl RpcNonceExpirations {
fn push_back(&mut self, expires_at: Instant, valid_until: i64, nonce: Uuid) {
// Use exact offsets and start another page if either offset is too large.
// Fully drained pages can be freed while newer nonces are still valid.
if let Some(page) = self.pages.back_mut()
&& page.entries.len() < RPC_NONCE_EXPIRATIONS_PER_PAGE
&& let Some(offset) = expires_at.checked_duration_since(page.expires_at)
&& let Ok(expiry_offset_ns) = u32::try_from(offset.as_nanos())
&& let Some(offset) = valid_until.checked_sub(page.valid_until)
&& let Ok(valid_until_offset_secs) = i32::try_from(offset)
{
page.entries.push_back(RpcNonceExpiration {
nonce,
expiry_offset_ns,
valid_until_offset_secs,
});
} else {
self.pages.push_back(RpcNonceExpirationPage {
expires_at,
valid_until,
entries: VecDeque::from([RpcNonceExpiration {
nonce,
expiry_offset_ns: 0,
valid_until_offset_secs: 0,
}]),
});
}
}
fn pop_expired(&mut self, now: Instant, wall_time: i64) -> Option<Uuid> {
let page = self.pages.front_mut()?;
let entry = page.entries.front()?;
let elapsed = now.checked_duration_since(page.expires_at)?;
let valid_until = page.valid_until + i64::from(entry.valid_until_offset_secs);
if elapsed <= Duration::from_nanos(u64::from(entry.expiry_offset_ns)) || wall_time <= valid_until {
return None;
}
let nonce = entry.nonce;
page.entries.pop_front();
if page.entries.is_empty() {
self.pages.pop_front();
if self.pages.is_empty() {
self.pages.shrink_to_fit();
}
}
Some(nonce)
}
#[cfg(test)]
fn capacity(&self) -> usize {
self.pages.iter().map(|page| page.entries.capacity()).sum()
}
}
#[derive(Default)]
struct RpcNonceCache {
nonces: HashSet<Uuid>,
expirations: RpcNonceExpirations,
expirations: VecDeque<(Instant, i64, Uuid)>,
max_wall_time: i64,
}
@@ -502,21 +425,18 @@ fn publish_nonce_cache_metrics(metrics: Option<RpcNonceCacheMetrics<'_>>) {
}
impl RpcNonceCache {
fn reserve_index(&mut self, capacity: usize) -> std::io::Result<()> {
self.nonces
.try_reserve(capacity)
.map_err(|error| std::io::Error::new(std::io::ErrorKind::OutOfMemory, error))
}
fn remove_expired(&mut self, now: Instant, wall_time: i64) -> usize {
let mut removed = 0;
while let Some(nonce) = self.expirations.pop_expired(now, wall_time) {
while matches!(
self.expirations.front(),
Some((expires_at, valid_until, _)) if *expires_at < now && *valid_until < wall_time
) {
let Some((_, _, nonce)) = self.expirations.pop_front() else {
break;
};
self.nonces.remove(&nonce);
removed += 1;
}
if removed > 0 && self.nonces.is_empty() {
self.nonces.shrink_to_fit();
}
removed
}
@@ -548,14 +468,9 @@ impl RpcNonceCache {
}),
);
}
if self.nonces.is_empty()
&& let Err(error) = self.reserve_index(record.capacity.min(REPLAY_CACHE_AUTO_MAX_CAPACITY))
{
return (Err(error), Some(metrics));
}
self.nonces.insert(record.nonce);
self.expirations
.push_back(record.expires_at, record.signed_at.saturating_add(SIGNATURE_VALID_DURATION), record.nonce);
.push_back((record.expires_at, record.signed_at.saturating_add(SIGNATURE_VALID_DURATION), record.nonce));
(
Ok(()),
Some(RpcNonceCacheMetrics {
@@ -1675,13 +1590,7 @@ fn verify_tonic_replay_scope_signature(audience: &str, path: &str, headers: &Hea
if boot_epoch != tonic_rpc_boot_epoch() {
return Err(std::io::Error::other("RPC boot epoch is stale"));
}
check_and_record_tonic_nonce(nonce, signed_at, path)?;
// A body-bound request also carries a usable v2 signature. Consume both
// nonces so stripping the replay-scope headers cannot authorize it again.
if content_sha256 != UNSIGNED_PAYLOAD {
verify_tonic_rpc_signature_with_strictness(audience, path, headers, true)?;
}
Ok(())
check_and_record_tonic_nonce(nonce, signed_at, path)
}
/// Verify gRPC authentication, preferring v2 without downgrade on malformed v2 metadata.
@@ -1715,9 +1624,6 @@ pub fn verify_tonic_rpc_signature_with_bootstrap(
}
pub fn tonic_rpc_auth_failure_reason(error: &std::io::Error) -> &'static str {
if error.kind() == std::io::ErrorKind::OutOfMemory {
return "replay_cache_allocation";
}
match error.to_string().as_str() {
"Missing RPC audience" => "missing_audience",
"Invalid RPC request path" => "invalid_request_path",
@@ -2675,43 +2581,6 @@ mod tests {
assert_eq!(path_error.to_string(), "Invalid RPC replay scope signature");
}
#[test]
fn replay_scope_consumes_the_body_bound_v2_nonce() {
ensure_test_rpc_secret();
let audience = "node-a:9000";
let path = "/node_service.NodeService/RecoverDeleteTransaction";
let digest = "a".repeat(64);
for v3_first in [true, false] {
let mut headers =
gen_tonic_signature_headers(audience, "node_service.NodeService", "RecoverDeleteTransaction", Some(&digest))
.expect("body-bound v2 headers should build");
let timestamp = headers[TIMESTAMP_HEADER].to_str().expect("timestamp").to_string();
headers.extend(
gen_tonic_replay_scope_headers(audience, path, &timestamp, &digest, tonic_rpc_boot_epoch())
.expect("replay-scope headers should build"),
);
let mut downgraded = headers.clone();
for header in [
RPC_REPLAY_SCOPE_VERSION_HEADER,
RPC_REPLAY_SCOPE_SIGNATURE_HEADER,
RPC_REPLAY_SCOPE_NONCE_HEADER,
RPC_BOOT_EPOCH_HEADER,
] {
downgraded.remove(header);
}
let (first, replay) = if v3_first {
(&headers, &downgraded)
} else {
(&downgraded, &headers)
};
verify_tonic_rpc_signature_with_policy(audience, path, first, true, false, false)
.expect("the first delivery must be accepted");
let error = verify_tonic_rpc_signature_with_policy(audience, path, replay, true, false, false)
.expect_err("a mutation cannot replay through the other signature generation");
assert_eq!(error.to_string(), "RPC request replay detected");
}
}
#[test]
fn replay_scope_rejects_partial_metadata_and_stale_epoch_without_fallback() {
ensure_test_rpc_secret();
@@ -3157,9 +3026,9 @@ mod tests {
assert_eq!(decision.source, ReplayCacheCapacitySource::Auto);
assert_eq!(decision.memory_basis, Some(MemoryBasis::Host));
assert_eq!(decision.memory_based_capacity, 31_019_208);
assert_eq!(decision.memory_based_capacity, 17_448_304);
assert_eq!(decision.cpu_based_capacity, 19_693_568);
assert_eq!(decision.capacity, 19_693_568);
assert_eq!(decision.capacity, 17_448_304);
}
#[test]
@@ -3169,13 +3038,13 @@ mod tests {
replay_cache_capacity_decision(rustfs_utils::EnvParseOutcome::Absent, 16, Some(32 * gib), Some(MemoryBasis::Host));
assert_eq!(decision.source, ReplayCacheCapacitySource::Auto);
assert_eq!(decision.memory_based_capacity, 62_038_416);
assert_eq!(decision.memory_based_capacity, 34_896_609);
assert_eq!(decision.cpu_based_capacity, 39_387_136);
assert_eq!(decision.capacity, REPLAY_CACHE_AUTO_MAX_CAPACITY);
let observed_field_node =
replay_cache_capacity_decision(rustfs_utils::EnvParseOutcome::Absent, 16, Some(31 * gib), Some(MemoryBasis::Host));
assert_eq!(observed_field_node.memory_based_capacity, 60_099_715);
assert_eq!(observed_field_node.memory_based_capacity, 33_806_090);
assert_eq!(observed_field_node.cpu_based_capacity, 39_387_136);
assert_eq!(observed_field_node.capacity, REPLAY_CACHE_AUTO_MAX_CAPACITY);
}
@@ -3248,138 +3117,6 @@ mod tests {
}
}
#[test]
fn nonce_cache_releases_expired_timestamp_capacity_without_allowing_clock_rollback_replay() {
let now = Instant::now();
let expiry = now + REPLAY_CACHE_RETENTION;
let later = now + Duration::from_secs(302);
let nonce = Uuid::new_v4();
let replacement = Uuid::new_v4();
let mut cache = RpcNonceCache::default();
cache
.check_and_record(test_nonce_record(nonce, 1_000, now, 1_000, expiry, 1))
.0
.expect("record initial nonce");
let boundary = cache
.check_and_record(test_nonce_record(nonce, 1_000, later, 1_300, later + REPLAY_CACHE_RETENTION, 1))
.0
.expect_err("a signature at the inclusive freshness boundary must remain protected");
assert_eq!(boundary.to_string(), "RPC request replay detected");
cache
.check_and_record(test_nonce_record(replacement, 1_302, later, 1_302, later + REPLAY_CACHE_RETENTION, 1))
.0
.expect("an expired signature must release bounded cache capacity");
assert!(!cache.nonces.contains(&nonce));
assert!(cache.nonces.contains(&replacement));
let rollback = cache
.check_and_record(test_nonce_record(nonce, 1_000, later, 1_000, later + REPLAY_CACHE_RETENTION, 1))
.0
.expect_err("clock rollback must not resurrect an evicted signature");
assert_eq!(rollback.to_string(), "RPC request timestamp expired after clock regression");
}
#[test]
fn nonce_cache_keeps_future_timestamp_reserved_until_its_full_validity_window_closes() {
let now = Instant::now();
let expiry = now + REPLAY_CACHE_RETENTION;
let later = now + Duration::from_secs(302);
let nonce = Uuid::new_v4();
let mut cache = RpcNonceCache::default();
cache
.check_and_record(test_nonce_record(nonce, 1_300, now, 1_000, expiry, 1))
.0
.expect("record nonce signed at maximum allowed forward skew");
let replay = cache
.check_and_record(test_nonce_record(nonce, 1_300, later, 1_302, later + REPLAY_CACHE_RETENTION, 1))
.0
.expect_err("monotonic expiry must not remove a still-valid future signature");
assert_eq!(replay.to_string(), "RPC request replay detected");
let full = cache
.check_and_record(test_nonce_record(Uuid::new_v4(), 1_302, later, 1_302, later + REPLAY_CACHE_RETENTION, 1))
.0
.expect_err("a still-valid nonce must continue consuming capacity");
assert_eq!(full.to_string(), "RPC replay cache capacity exceeded");
let expired = now + Duration::from_secs(602);
cache
.check_and_record(test_nonce_record(
Uuid::new_v4(),
1_602,
expired,
1_602,
expired + REPLAY_CACHE_RETENTION,
1,
))
.0
.expect("release capacity only once future timestamp validity has closed");
}
#[test]
fn nonce_index_rejects_unallocatable_capacity_with_typed_error() {
let mut cache = RpcNonceCache::default();
let error = cache
.reserve_index(usize::MAX)
.expect_err("unallocatable capacity must fail without a panic");
assert_eq!(error.kind(), std::io::ErrorKind::OutOfMemory);
assert!(
error
.get_ref()
.is_some_and(|source| source.is::<std::collections::TryReserveError>())
);
assert_eq!(tonic_rpc_auth_failure_reason(&error), "replay_cache_allocation");
assert!(cache.nonces.is_empty());
assert!(cache.expirations.pages.is_empty());
}
#[test]
fn nonce_cache_reserves_the_bounded_index_before_sustained_traffic() {
let now = Instant::now();
let expiry = now + REPLAY_CACHE_RETENTION;
let mut cache = RpcNonceCache::default();
let capacity = 4096;
cache
.check_and_record(test_nonce_record(Uuid::from_u128(1), 1000, now, 1000, expiry, capacity))
.0
.expect("record first nonce");
let allocated = cache.nonces.capacity();
assert!(allocated >= capacity, "the hash index must not rehash while serving the bounded window");
for n in 2..=capacity {
cache
.check_and_record(test_nonce_record(
Uuid::from_u128(u128::try_from(n).expect("nonce index")),
1000,
now,
1000,
expiry,
capacity,
))
.0
.expect("record within bounded capacity");
}
assert_eq!(cache.nonces.capacity(), allocated);
let full = cache
.check_and_record(test_nonce_record(Uuid::from_u128(9999), 1000, now, 1000, expiry, capacity))
.0
.expect_err("preallocation must not relax the logical capacity fence");
assert_eq!(full.to_string(), "RPC replay cache capacity exceeded");
assert_eq!(cache.nonces.len(), capacity);
}
#[test]
fn replay_cache_compact_storage_fits_two_cpu_window_in_2800_mib() {
let decision = replay_cache_capacity_decision(
rustfs_utils::EnvParseOutcome::Absent,
2,
Some(2800 * 1024 * 1024),
Some(MemoryBasis::Cgroup),
);
assert_eq!(decision.cpu_based_capacity, 4_923_392);
assert_eq!(decision.capacity, decision.cpu_based_capacity);
assert!(decision.memory_based_capacity >= decision.capacity);
assert_eq!(decision.source, ReplayCacheCapacitySource::Auto);
}
#[test]
fn nonce_cache_expires_by_monotonic_deadline_and_fails_closed_at_capacity() {
let now = Instant::now();
@@ -3400,51 +3137,6 @@ mod tests {
assert!(cache.nonces.contains(&nonce_b));
}
#[test]
fn nonce_cache_releases_expiration_storage_after_a_burst() {
let start = Instant::now();
let expiry = start + REPLAY_CACHE_RETENTION;
let mut cache = RpcNonceCache::default();
for nonce in 1..=4_096 {
let record = test_nonce_record(Uuid::from_u128(nonce), 100, start, 100, expiry, 4_096);
cache.check_and_record(record).0.expect("first use of nonce");
}
assert!(cache.expirations.capacity() >= 4_096);
assert_eq!(cache.remove_expired(expiry, 701), 0);
assert_eq!(cache.remove_expired(expiry + Duration::from_nanos(1), 400), 0);
assert_eq!(cache.remove_expired(expiry + Duration::from_nanos(1), 701), 4_096);
assert!(cache.nonces.is_empty());
assert_eq!(cache.nonces.capacity(), 0);
assert_eq!(cache.expirations.capacity(), 0, "expired entries must release their expiration storage");
}
#[test]
fn nonce_expirations_preserve_exact_deadlines_across_pages() {
let expiry = Instant::now();
let first = Uuid::from_u128(1);
let second = Uuid::from_u128(2);
let third = Uuid::from_u128(3);
let fourth = Uuid::from_u128(4);
let mut queue = super::RpcNonceExpirations::default();
let next = expiry + Duration::from_nanos(1);
let later = expiry + Duration::from_secs(5);
queue.push_back(expiry, 501, first);
queue.push_back(next, 500, second);
queue.push_back(later, 505, third);
queue.push_back(later, i64::MAX, fourth);
assert_eq!(queue.pages.len(), 3);
assert_eq!(queue.pop_expired(next, 501), None);
assert_eq!(queue.pop_expired(next, 502), Some(first));
assert_eq!(queue.pop_expired(next, 502), None);
assert_eq!(queue.pop_expired(later, 502), Some(second));
assert_eq!(queue.pages.len(), 2);
assert_eq!(queue.pop_expired(later, 506), None);
assert_eq!(queue.pop_expired(later + Duration::from_nanos(1), 506), Some(third));
assert_eq!(queue.pop_expired(later + Duration::from_nanos(1), i64::MAX), None);
}
#[test]
fn nonce_cache_metrics_mark_successful_records_only() {
let now = Instant::now();
@@ -2014,21 +2014,9 @@ impl PeerRestClient {
}
result
})
.await;
self.settle_bucket_metadata_reload(bucket, result).await
}
async fn settle_bucket_metadata_reload(
&self,
bucket: &str,
result: std::result::Result<Result<()>, tokio::time::error::Elapsed>,
) -> Result<()> {
match result {
Ok(result) => self.finalize_result(result).await,
// A local operation deadline does not establish that the peer is
// unreachable. In particular, it must not gate IAM revocations.
Err(_) => Err(Error::other(format!("load_bucket_metadata({bucket}) timed out"))),
}
.await
.unwrap_or_else(|_| Err(Error::other(format!("load_bucket_metadata({bucket}) timed out"))));
self.finalize_result(result).await
}
async fn load_bucket_metadata_once(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
@@ -4765,31 +4753,6 @@ mod tests {
assert!(client.offline.load(Ordering::Acquire));
}
#[tokio::test]
async fn bucket_metadata_deadline_does_not_gate_healthy_peer_notifications() {
let client = test_peer_client();
let elapsed = tokio::time::timeout(std::time::Duration::ZERO, std::future::pending::<Result<()>>()).await;
let error = client
.settle_bucket_metadata_reload("slow-policy", elapsed)
.await
.expect_err("local budget expires");
assert!(error.to_string().contains("timed out"));
assert!(
!client.offline.load(Ordering::Acquire),
"local work must not suppress subsequent IAM RPCs"
);
client
.settle_bucket_metadata_reload("healthy", Ok(Ok(())))
.await
.expect("peer reply");
let error = client
.settle_bucket_metadata_reload("unreachable", Ok(Err(Error::from(tonic::Status::unavailable("connection refused")))))
.await
.expect_err("real transport failure");
assert!(error.to_string().contains("connection refused"));
assert!(client.offline.load(Ordering::Acquire), "transport failures must still enter recovery");
}
#[tokio::test]
async fn peer_rest_client_finalize_result_keeps_online_for_business_errors() {
let client = test_peer_client();
+25 -565
View File
@@ -27,13 +27,13 @@ use crate::disk::multipart_source::{
MULTIPART_SOURCE_PROTOCOL_VERSION, MultipartSourceOperation, MultipartSourceRequest, MultipartSourceResponse,
};
use crate::disk::{
BOUNDED_DELETE_PREPARE_PROTOCOL_VERSION, BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, ConditionalFileUpdate,
DELETE_RECOVERY_PROTOCOL_VERSION, DeleteOptions, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest,
DeleteRecoveryInspection, DeleteRecoveryPage, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, DiskAPI, DiskInfo,
DiskInfoOptions, DiskLocation, DiskOption, FileInfoVersions, FileReader, FileWriter, GuardedDeleteVersionsPayload,
GuardedFileInfoVersions, PartTransactionAction, PoolMetaNativeRecoveryRequest, PoolMetaNativeRecoveryResponse,
ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo,
WalkDirOptions, batch_read_version_one_by_one,
BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, ConditionalFileUpdate, DELETE_RECOVERY_PROTOCOL_VERSION,
DeleteOptions, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest, DeleteRecoveryInspection,
DeleteRecoveryPage, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation,
DiskOption, FileInfoVersions, FileReader, FileWriter, GuardedDeleteVersionsPayload, GuardedFileInfoVersions,
PartTransactionAction, PoolMetaNativeRecoveryRequest, PoolMetaNativeRecoveryResponse, ReadMultipleReq, ReadMultipleResp,
ReadOptions, RenameDataResp, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
batch_read_version_one_by_one,
disk_store::{
DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING, ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, SKIP_IF_SUCCESS_BEFORE,
get_drive_active_check_interval, get_drive_active_check_timeout, get_drive_disk_info_timeout, get_drive_list_dir_timeout,
@@ -376,21 +376,6 @@ struct WalkCopyError {
source: io::Error,
}
impl WalkCopyError {
fn into_disk_error(self) -> DiskError {
// Only a local output write/flush proves the metacache consumer closed.
// An identical I/O kind from the remote input remains a transport failure.
if matches!(self.reason, RemoteWalkFailureReason::Write | RemoteWalkFailureReason::Flush)
&& self.source.kind() == io::ErrorKind::BrokenPipe
&& self.source.get_ref().is_none()
{
DiskError::metacache_output_stream_closed()
} else {
DiskError::Io(self.source)
}
}
}
async fn copy_stream_with_buffer<R, W>(
reader: &mut R,
writer: &mut W,
@@ -1011,32 +996,21 @@ fn internode_idempotent_read_retries() -> usize {
static PREWARMED_PEERS: std::sync::LazyLock<std::sync::Mutex<std::collections::HashSet<String>>> =
std::sync::LazyLock::new(|| std::sync::Mutex::new(std::collections::HashSet::new()));
fn claim_control_channel_prewarm(addr: &str) -> bool {
let Ok(mut prewarmed) = PREWARMED_PEERS.lock() else {
return false;
};
prewarmed.insert(addr.to_owned())
}
fn release_control_channel_prewarm(addr: &str) {
if let Ok(mut prewarmed) = PREWARMED_PEERS.lock() {
prewarmed.remove(addr);
}
}
/// Best-effort background prewarm of a peer's control channel (grpc-optimization P3-1). Deduped per
/// peer; a dial failure releases the marker so a later disk construction can retry the prewarm.
/// peer; a dial failure just falls through to the existing lazy connect and recovery monitor.
fn spawn_control_channel_prewarm(addr: String) {
if !claim_control_channel_prewarm(&addr) {
return;
{
let Ok(mut prewarmed) = PREWARMED_PEERS.lock() else {
return;
};
if !prewarmed.insert(addr.clone()) {
return;
}
}
tokio::spawn(async move {
match node_service_time_out_client_no_auth(&addr).await {
Ok(_) => debug!(addr = %addr, "internode control channel prewarmed"),
Err(err) => {
release_control_channel_prewarm(&addr);
debug!(addr = %addr, error = %err, "internode control channel prewarm failed (best-effort)");
}
Err(err) => debug!(addr = %addr, error = %err, "internode control channel prewarm failed (best-effort)"),
}
});
}
@@ -1047,7 +1021,6 @@ impl RemoteDisk {
volume: &str,
payload: Vec<u8>,
probe_only: bool,
bounded_prepare: bool,
) -> Result<DeleteVersionsResponse> {
if payload.len() > rustfs_protos::DELETE_RECOVERY_MAX_PAYLOAD_BYTES {
return Err(DiskError::FileCorrupt);
@@ -1072,8 +1045,6 @@ impl RemoteDisk {
.map_err(|status| {
if status.code() == Code::Unimplemented {
DiskError::GuardedDeleteUnsupported
} else if bounded_prepare && status.code() == Code::InvalidArgument {
DiskError::UnsupportedDisk
} else {
status.into()
}
@@ -1107,22 +1078,11 @@ impl RemoteDisk {
volume: &str,
operation: DeleteRecoveryRpcOperation,
) -> Result<DeleteRecoveryRpcReply> {
let protocol_version = match &operation {
DeleteRecoveryRpcOperation::ApplyVersion { opts, .. } | DeleteRecoveryRpcOperation::ApplyVersions { opts, .. }
if opts.requires_bounded_delete_recovery =>
{
BOUNDED_DELETE_PREPARE_PROTOCOL_VERSION
}
_ => DELETE_RECOVERY_PROTOCOL_VERSION,
};
let payload = rmp_serde::to_vec_named(&operation)?;
if payload.len() > operation.max_wire_payload_bytes() {
return Err(Error::other("delete recovery operation exceeds payload limit"));
}
let settle_timeout = if matches!(
&operation,
DeleteRecoveryRpcOperation::Settle { .. } | DeleteRecoveryRpcOperation::SettleBounded(_)
) {
let settle_timeout = if matches!(&operation, DeleteRecoveryRpcOperation::Settle { .. }) {
Some(crate::disk::get_delete_recovery_timeout()?)
} else {
None
@@ -1145,7 +1105,7 @@ impl RemoteDisk {
let mut request = Request::new(RecoverDeleteTransactionRequest {
disk: self.endpoint.to_string(),
volume: volume.to_owned(),
protocol_version,
protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION,
operation: payload.into(),
});
let canonical_body = rustfs_protos::canonical_recover_delete_transaction_request_body(request.get_ref())
@@ -1163,7 +1123,7 @@ impl RemoteDisk {
}
})?
.into_inner();
if response.protocol_version != protocol_version {
if response.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION {
return Err(DiskError::UnsupportedDisk);
}
if !response.success {
@@ -1677,7 +1637,6 @@ impl RemoteDisk {
let initial_probe_ok = Self::perform_connectivity_check(&addr).await.is_ok();
if initial_probe_ok {
health.record_operation_success(&endpoint, "connectivity_probe_success");
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
} else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
warn!(
event = EVENT_REMOTE_DISK_HEALTH,
@@ -1749,7 +1708,6 @@ impl RemoteDisk {
// Perform basic connectivity check
if Self::perform_connectivity_check(&addr).await.is_ok() {
health.record_operation_success(&endpoint, "connectivity_probe_success");
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
} else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
warn!(
event = EVENT_REMOTE_DISK_HEALTH,
@@ -1820,7 +1778,6 @@ impl RemoteDisk {
}
_ = interval.tick() => {
if Self::perform_recovery_probe(&addr, &endpoint).await.is_ok() {
rustfs_io_metrics::internode_metrics::record_peer_reachable(&addr);
let became_online = health.mark_recovery_success(&endpoint, "disk_info_probe_success");
debug!(
event = EVENT_REMOTE_DISK_HEALTH,
@@ -2052,7 +2009,6 @@ impl RemoteDisk {
let operation_result = operation().await;
if operation_result.is_ok() {
self.health.record_operation_success(&self.endpoint, "operation_success");
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
}
self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
.await;
@@ -2067,7 +2023,6 @@ impl RemoteDisk {
// Log success; the waiting guard balances every exit path.
if operation_result.is_ok() {
self.health.record_operation_success(&self.endpoint, "operation_success");
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
}
self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
.await;
@@ -2111,7 +2066,6 @@ impl RemoteDisk {
failure_health_action: FailureHealthAction,
) {
if let Err(err) = operation_result
&& !err.is_metacache_output_stream_closed()
&& is_network_like_disk_error(err)
{
counter!(
@@ -2757,56 +2711,8 @@ impl RemoteDisk {
#[async_trait::async_trait]
impl DiskAPI for RemoteDisk {
async fn list_multipart_index_roots(
&self,
bucket: &str,
marker: Option<&str>,
) -> Result<crate::disk::multipart_index::MultipartIndexPage> {
use crate::disk::multipart_index::{MULTIPART_INDEX_RPC_MAX_BYTES, MultipartIndexRpcReply, MultipartIndexRpcRequest};
let expected_disk = self
.id
.lock()
.await
.filter(|id| !id.is_nil())
.ok_or(DiskError::DiskNotFound)?;
let query = MultipartIndexRpcRequest::new(bucket, marker);
let endpoint = self.endpoint.to_string();
let canonical = query.canonical_request(&endpoint)?;
let opts = serde_json::to_string(&query)?;
self.execute_with_timeout(
|| async {
let mut client = self.get_client().await?;
let mut request = Request::new(DiskInfoRequest {
disk: endpoint.clone(),
opts: opts.clone(),
});
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
let response = client.disk_info(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
if response.disk_info.len() > MULTIPART_INDEX_RPC_MAX_BYTES {
return Err(DiskError::FileCorrupt);
}
// An old server ignores the additive query and returns ordinary DiskInfo.
// It cannot satisfy the dedicated reply schema or the fresh challenge.
let reply: MultipartIndexRpcReply =
serde_json::from_str(&response.disk_info).map_err(|_| DiskError::MethodNotAllowed)?;
reply.validate(&query, expected_disk)?;
crate::cluster::rpc::verify_tonic_rpc_response_proof(
&query.canonical_response(&endpoint, &reply.page)?,
&reply.proof,
)
.map_err(Error::other)?;
Ok(reply.page)
},
get_drive_disk_info_timeout(),
)
.await
}
async fn supports_guarded_delete_versions(&self) -> Result<()> {
let response = self.guarded_delete_versions_rpc("", Vec::new(), true, false).await?;
let response = self.guarded_delete_versions_rpc("", Vec::new(), true).await?;
if !response.errors.is_empty() || !response.item_errors.is_empty() {
return Err(DiskError::FileCorrupt);
}
@@ -2819,12 +2725,7 @@ impl DiskAPI for RemoteDisk {
opts: DeleteOptions,
) -> Vec<Option<DiskError>> {
let count = versions.len();
let bounded_prepare = opts.requires_bounded_delete_recovery;
let payload = GuardedDeleteVersionsPayload {
versions,
opts,
bounded_recovery_protocol: bounded_prepare.then_some(BOUNDED_DELETE_PREPARE_PROTOCOL_VERSION),
};
let payload = GuardedDeleteVersionsPayload { versions, opts };
if let Err(error) = payload.validate() {
return vec![Some(error); count];
}
@@ -2832,10 +2733,7 @@ impl DiskAPI for RemoteDisk {
Ok(encoded) => encoded,
Err(error) => return vec![Some(error.into()); count],
};
match self
.guarded_delete_versions_rpc(volume, encoded, false, bounded_prepare)
.await
{
match self.guarded_delete_versions_rpc(volume, encoded, false).await {
Ok(response) => {
// New guarded peers must return typed errors, never lossy strings.
if response.item_errors.len() != count {
@@ -3726,13 +3624,7 @@ impl DiskAPI for RemoteDisk {
);
}
let rpc_started = read_version_stage_timer(batch_read_version_attribution_enabled);
let response_result = client.batch_read_version(request).await;
if let Some(rpc_started) = rpc_started {
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_duration(
rpc_started.elapsed(),
);
}
let response = match response_result {
let response = match client.batch_read_version(request).await {
Ok(response) => {
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
response.into_inner()
@@ -4072,7 +3964,7 @@ impl DiskAPI for RemoteDisk {
Ok(_) => return Ok(()),
Err(error) => {
failure_reason = Some(error.reason);
return Err(error.into_disk_error());
return Err(DiskError::Io(error.source));
}
}
}
@@ -4402,38 +4294,6 @@ impl DiskAPI for RemoteDisk {
}
}
async fn discover_delete_transactions_bounded(
&self,
volume: &str,
object: &str,
max_entries: u32,
) -> Result<crate::disk::BoundedDeleteRecoveryDiscovery> {
// Older peers reject this variant. Do not retry through the unbounded protocol;
// they are reported as unsupported participants (rolling-upgrade contract).
match self
.recover_delete_transaction(
volume,
DeleteRecoveryRpcOperation::DiscoverBounded {
object: object.to_owned(),
max_entries,
},
)
.await
{
Ok(DeleteRecoveryRpcReply::DiscoveredBounded(page)) => Ok(page),
Ok(_) => Err(Error::FileCorrupt),
Err(error)
if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
|| matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
.is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
{
// No paginated fallback through the unbounded protocol; the peer
// is an unsupported participant until it is upgraded.
Err(Error::UnsupportedDisk)
}
Err(error) => Err(error),
}
}
async fn discover_delete_transactions(
&self,
volume: &str,
@@ -4469,62 +4329,6 @@ impl DiskAPI for RemoteDisk {
_ => Err(Error::other("unexpected delete discovery reply")),
}
}
async fn settle_delete_transaction_bounded(
&self,
volume: &str,
request: crate::disk::BoundedDeleteRecoverySettlement,
_namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> Result<crate::disk::BoundedDeleteRecoverySettlementReply> {
match self
.recover_delete_transaction(volume, DeleteRecoveryRpcOperation::SettleBounded(request))
.await
{
Ok(DeleteRecoveryRpcReply::SettledBounded(reply)) => Ok(reply),
Ok(_) => Err(Error::FileCorrupt),
Err(error)
if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
|| matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
.is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
{
// No paginated fallback through the unbounded protocol; the peer
// is an unsupported participant until it is upgraded.
Err(Error::UnsupportedDisk)
}
Err(error) => Err(error),
}
}
async fn inspect_delete_transaction_bounded(
&self,
volume: &str,
object: &str,
transaction: Uuid,
max_entries: u32,
) -> Result<crate::disk::BoundedDeleteRecoveryInspection> {
match self
.recover_delete_transaction(
volume,
DeleteRecoveryRpcOperation::InspectBounded {
object: object.to_owned(),
transaction,
max_entries,
},
)
.await
{
Ok(DeleteRecoveryRpcReply::InspectedBounded(page)) => Ok(page),
Ok(_) => Err(Error::FileCorrupt),
Err(error)
if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
|| matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
.is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
{
// No paginated fallback through the unbounded protocol; the peer
// is an unsupported participant until it is upgraded.
Err(Error::UnsupportedDisk)
}
Err(error) => Err(error),
}
}
async fn inspect_delete_transaction(
&self,
volume: &str,
@@ -4747,10 +4551,7 @@ impl DiskAPI for RemoteDisk {
read_multiple_req_bin: read_multiple_req_bin.into(),
});
let rpc_started = Instant::now();
let response = client.read_multiple(request).await;
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_multiple_duration(rpc_started.elapsed());
let response = response?.into_inner();
let response = client.read_multiple(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
@@ -4801,7 +4602,6 @@ impl DiskAPI for RemoteDisk {
attach_mutation_body_digest(&mut request, canonical_body, "write_all")?;
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_request();
let rpc_started = Instant::now();
let response = match client.write_all(request).await {
Ok(response) => response.into_inner(),
Err(err) => {
@@ -4810,7 +4610,6 @@ impl DiskAPI for RemoteDisk {
}
};
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_duration(rpc_started.elapsed());
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_sent_bytes(data_len);
if !response.success {
@@ -4908,7 +4707,6 @@ impl DiskAPI for RemoteDisk {
});
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_all_request();
let rpc_started = Instant::now();
let response = match client.read_all(request).await {
Ok(response) => response.into_inner(),
Err(err) => {
@@ -4917,8 +4715,6 @@ impl DiskAPI for RemoteDisk {
}
};
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_all_duration(rpc_started.elapsed());
if !response.success {
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_all_error();
return Err(response.error.unwrap_or_default().into());
@@ -5053,113 +4849,6 @@ mod tests {
}
}
#[tokio::test]
async fn bounded_delete_discovery_lost_remote_reply_leaves_no_cursor() {
runtime_sources::ensure_test_rpc_secret();
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
.await
.expect("test peer must bind");
let directory = tempfile::tempdir().expect("temporary disk");
let endpoint = Endpoint::try_from(directory.path().to_str().expect("disk path")).expect("endpoint");
let disk = Arc::new(
crate::disk::local::LocalDisk::new(&endpoint, false)
.await
.expect("local disk"),
);
disk.make_volume("bucket").await.expect("bucket");
let prefix = directory.path().join("bucket/prefix");
std::fs::create_dir(&prefix).expect("prefix");
for index in 0..64 {
std::fs::create_dir(prefix.join(format!("child-{index}"))).expect("legitimate child");
}
let fixture = Arc::new(TestBoundedDiscovery {
disk: Arc::clone(&disk),
response_gate: TestFormatReadGate::default(),
});
*peer.peer.bounded_discovery.lock().expect("bounded fixture") = Some(Arc::clone(&fixture));
let remote = remote_disk_for_addr(&peer.addr).await;
let task = tokio::spawn(async move { remote.discover_delete_transactions_bounded("bucket", "prefix", 8).await });
tokio::time::timeout(Duration::from_secs(5), fixture.response_gate.entered.notified())
.await
.expect("disk completed bounded scan");
task.abort();
assert!(task.await.expect_err("caller cancelled").is_cancelled());
assert_eq!(disk.public_delete_discovery_sessions_for_test(), 0);
fixture.response_gate.release.cancel();
peer.stop().await;
assert_eq!(disk.public_delete_discovery_sessions_for_test(), 0);
assert!(prefix.join("child-63").is_dir());
}
#[tokio::test]
async fn multipart_index_old_disk_info_handler_is_not_a_ready_empty_index() {
runtime_sources::ensure_test_rpc_secret();
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
.await
.expect("old peer must bind");
let remote = remote_disk_for_addr(&peer.addr).await;
remote
.set_disk_id(Some(Uuid::new_v4()))
.await
.expect("expected disk identity");
assert!(matches!(
remote.list_multipart_index_roots("bucket", None).await,
Err(DiskError::MethodNotAllowed)
));
assert_eq!(
peer.peer.disk_info_calls.load(Ordering::Acquire),
1,
"exercise the actual legacy JSON handler"
);
peer.stop().await;
}
#[tokio::test]
async fn bounded_delete_discovery_old_peer_rejects_without_paginated_fallback() {
runtime_sources::ensure_test_rpc_secret();
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
.await
.expect("old peer must bind");
let remote = remote_disk_for_addr(&peer.addr).await;
assert_eq!(
remote
.discover_delete_transactions_bounded("bucket", "prefix", 4096)
.await
.expect_err("old peer cannot prove a bounded inventory"),
DiskError::UnsupportedDisk
);
let transaction = Uuid::new_v4();
assert_eq!(
remote
.inspect_delete_transaction_bounded("bucket", "prefix", transaction, 65536)
.await
.expect_err("old peer cannot inspect with a budget"),
DiskError::UnsupportedDisk
);
assert_eq!(
remote
.settle_delete_transaction_bounded(
"bucket",
crate::disk::BoundedDeleteRecoverySettlement {
object: "prefix".to_owned(),
transaction,
expected_fingerprint: [0; 32],
action: DeleteRecoveryAction::PublishCommit,
max_entries: 65536,
},
None
)
.await
.expect_err("old peer cannot settle with a budget"),
DiskError::UnsupportedDisk
);
assert_eq!(
*peer.peer.unsupported_methods.lock().expect("observed methods"),
vec!["/node_service.NodeService/RecoverDeleteTransaction"; 3]
);
peer.stop().await;
}
#[tokio::test]
async fn multipart_source_recovery_old_peer_rejects_without_generic_fallback() {
runtime_sources::ensure_test_rpc_secret();
@@ -6051,18 +5740,10 @@ mod tests {
object_read_all_disks: Arc<StdMutex<Vec<String>>>,
format_data: Bytes,
read_all_data: Bytes,
fail_format_reads: Arc<AtomicBool>,
next_format_read_gate: Arc<StdMutex<Option<Arc<TestFormatReadGate>>>>,
bounded_discovery: Arc<StdMutex<Option<Arc<TestBoundedDiscovery>>>>,
unsupported_methods: Arc<StdMutex<Vec<String>>>,
}
#[derive(Debug)]
struct TestBoundedDiscovery {
disk: Arc<crate::disk::local::LocalDisk>,
response_gate: TestFormatReadGate,
}
#[derive(Debug, Default)]
struct TestFormatReadGate {
entered: tokio::sync::Notify,
@@ -6078,9 +5759,7 @@ mod tests {
object_read_all_disks: Arc::default(),
format_data,
read_all_data,
fail_format_reads: Arc::default(),
next_format_read_gate: Arc::default(),
bounded_discovery: Arc::default(),
unsupported_methods: Arc::default(),
}
}
@@ -6180,56 +5859,6 @@ mod tests {
Ok(grpc.unary(method, request).await)
})
}
"/node_service.NodeService/RecoverDeleteTransaction"
if self.peer.bounded_discovery.lock().expect("bounded fixture").is_some() =>
{
#[derive(Clone)]
struct BoundedSvc(Arc<TestBoundedDiscovery>);
impl tonic::server::UnaryService<RecoverDeleteTransactionRequest> for BoundedSvc {
type Response = rustfs_protos::proto_gen::node_service::RecoverDeleteTransactionResponse;
type Future = Pin<Box<dyn Future<Output = std::result::Result<Response<Self::Response>, Status>> + Send>>;
fn call(&mut self, request: Request<RecoverDeleteTransactionRequest>) -> Self::Future {
let fixture = Arc::clone(&self.0);
Box::pin(async move {
let request = request.into_inner();
let operation: DeleteRecoveryRpcOperation =
rustfs_protos::decode_delete_recovery_payload(&request.operation)
.map_err(|_| Status::invalid_argument("invalid bounded request"))?;
let DeleteRecoveryRpcOperation::DiscoverBounded { object, max_entries } = operation else {
return Err(Status::invalid_argument("unexpected fallback"));
};
let page = fixture
.disk
.discover_delete_transactions_bounded(&request.volume, &object, max_entries)
.await
.map_err(|_| Status::internal("bounded discovery failed"))?;
// Lose the response only after the real disk operation has completed.
fixture.response_gate.entered.notify_one();
fixture.response_gate.release.cancelled().await;
let result = rmp_serde::to_vec_named(&DeleteRecoveryRpcReply::DiscoveredBounded(page))
.map_err(|_| Status::internal("encode bounded reply"))?;
Ok(Response::new(rustfs_protos::proto_gen::node_service::RecoverDeleteTransactionResponse {
success: true,
protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION,
result: result.into(),
error: None,
}))
})
}
}
let fixture = self
.peer
.bounded_discovery
.lock()
.expect("bounded fixture")
.clone()
.expect("configured fixture");
Box::pin(async move {
let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec);
Ok(grpc.unary(BoundedSvc(fixture), request).await)
})
}
"/node_service.NodeService/ReadAll" => {
#[derive(Clone)]
struct ReadAllSvc(AuthenticatedReadPeer);
@@ -6253,9 +5882,6 @@ mod tests {
gate.entered.notify_one();
gate.release.cancelled().await;
}
if peer.fail_format_reads.load(Ordering::Acquire) {
return Err(Status::unavailable("format read temporarily unavailable"));
}
peer.format_data.clone()
} else {
peer.object_read_all_disks
@@ -9221,124 +8847,6 @@ mod tests {
peer.stop().await;
}
#[tokio::test]
#[serial(remote_disk_recovery_probe)]
async fn endpoint_reconnect_failure_preserves_registered_remote_recovery() {
temp_env::async_with_vars(
[
(crate::disk::disk_store::ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, Some("true")),
(rustfs_config::ENV_DRIVE_RETURNING_SUCCESS_THRESHOLD, Some("1")),
],
async {
runtime_sources::ensure_test_rpc_secret();
let mut reference = crate::layout::format::FormatV3::new(1, 1);
reference.erasure.this = reference.erasure.sets[0][0];
let peer = TestGrpcPeer::spawn(
Bytes::from(reference.to_json().expect("format should serialize")),
Bytes::from_static(b"recovered-peer-data"),
)
.await
.expect("authenticated regression peer is required");
let set = missing_read_set(&peer, reference).await;
let disk = crate::disk::new_disk(
&set.set_endpoints[0],
&DiskOption {
cleanup: false,
health_check: true,
},
)
.await
.expect("registered remote disk should construct");
let crate::disk::Disk::Remote(remote) = disk.as_ref() else {
panic!("fixture must create a remote disk");
};
assert!(remote.health_check);
remote.force_runtime_state_for_test(RuntimeDriveHealthState::Offline);
set.disks.write().await[0] = Some(Arc::clone(&disk));
peer.peer.fail_format_reads.store(true, Ordering::Release);
set.connect_disks().await;
peer.peer.fail_format_reads.store(false, Ordering::Release);
let cancelled_after_failure = remote.cancel_token.is_cancelled();
assert!(peer.peer.read_all_calls() > 0, "reconnect must attempt the failing format RPC");
assert!(
Arc::ptr_eq(&disk, set.disks.read().await[0].as_ref().expect("existing slot must remain")),
"a failed reconnect must not publish its unvalidated candidate"
);
remote.spawn_recovery_monitor_if_needed();
let recovery = time::timeout(Duration::from_secs(5), async {
while !disk.is_online().await {
tokio::task::yield_now().await;
}
disk.read_all("bucket", "object").await
})
.await;
disk.close().await.expect("stop the registered disk's monitoring");
peer.stop().await;
assert!(!cancelled_after_failure, "failed replacement must retain the registered monitor");
assert_eq!(
recovery
.expect("failed reconnect must leave the registered disk able to recover without another sweep")
.expect("the recovered registered disk must serve real I/O"),
Bytes::from_static(b"recovered-peer-data")
);
},
)
.await;
}
#[tokio::test]
#[serial(remote_disk_recovery_probe)]
async fn endpoint_reconnect_stops_old_remote_monitor_only_after_publication() {
runtime_sources::ensure_test_rpc_secret();
let mut reference = crate::layout::format::FormatV3::new(1, 1);
reference.erasure.this = reference.erasure.sets[0][0];
let peer = TestGrpcPeer::spawn(
Bytes::from(reference.to_json().expect("format should serialize")),
Bytes::from_static(b"replacement-peer-data"),
)
.await
.expect("authenticated regression peer is required");
let set = missing_read_set(&peer, reference).await;
let disk = crate::disk::new_disk(
&set.set_endpoints[0],
&DiskOption {
cleanup: false,
health_check: true,
},
)
.await
.expect("registered remote disk should construct");
let crate::disk::Disk::Remote(remote) = disk.as_ref() else {
panic!("fixture must create a remote disk");
};
remote.force_runtime_state_for_test(RuntimeDriveHealthState::Offline);
set.disks.write().await[0] = Some(Arc::clone(&disk));
let gate = peer.peer.pause_next_format_read();
let pending_set = Arc::clone(&set);
let pending = tokio::spawn(async move { pending_set.connect_disks().await });
time::timeout(Duration::from_secs(5), gate.entered.notified())
.await
.expect("replacement must reach the format validation barrier");
let cancelled_before_publication = remote.cancel_token.is_cancelled();
gate.release.cancel();
pending.await.expect("validated replacement must finish publication");
let replacement = set.disks.read().await[0].clone().expect("replacement must occupy the slot");
let readback = replacement.read_all("bucket", "object").await;
replacement.close().await.expect("stop replacement monitoring");
peer.stop().await;
assert!(
!cancelled_before_publication,
"the registered monitor must survive pending format validation"
);
assert!(!Arc::ptr_eq(&disk, &replacement), "validated replacement must be published");
assert!(remote.cancel_token.is_cancelled(), "the displaced monitor must stop after publication");
assert_eq!(
readback.expect("replacement must serve real I/O"),
Bytes::from_static(b"replacement-peer-data")
);
}
#[tokio::test]
#[serial(remote_disk_recovery_probe)]
async fn endpoint_reconnect_admits_ready_drive_while_another_format_probe_is_blocked() {
@@ -9703,43 +9211,6 @@ mod tests {
}
}
#[tokio::test]
async fn remote_walk_preserves_local_metacache_consumer_close() {
let transport =
RetryingWalkDirInternodeDataTransport::with_steps(vec![WalkDirTestStep::Data(b"metacache payload".to_vec())]);
let disk = new_remote_disk_with_transport(Arc::new(transport.clone())).await;
let (mut writer, reader) = tokio::io::duplex(1);
drop(reader);
let error = disk
.walk_dir(WalkDirOptions::default(), &mut writer)
.await
.expect_err("a dropped metacache consumer must stop the producer");
assert!(
error.is_metacache_output_stream_closed(),
"preserve the benign completion marker: {error:?}"
);
assert_eq!(transport.calls().len(), 1, "local consumer closure is not a remote open retry");
assert!(!disk.health.is_faulty());
}
#[tokio::test]
async fn remote_walk_keeps_remote_read_broken_pipe_as_transport_failure() {
let transport = RetryingWalkDirInternodeDataTransport::with_steps(vec![WalkDirTestStep::PartialDataThenError {
data: b"partial metacache".to_vec(),
error: io::ErrorKind::BrokenPipe.into(),
}]);
let disk = new_remote_disk_with_transport(Arc::new(transport.clone())).await;
let mut writer = Vec::new();
let error = disk
.walk_dir(WalkDirOptions::default(), &mut writer)
.await
.expect_err("an upstream close after partial output is a failed walk");
assert!(!error.is_metacache_output_stream_closed());
assert!(error.contains_io_error_kind(io::ErrorKind::BrokenPipe));
assert_eq!(writer, b"partial metacache");
assert_eq!(transport.calls().len(), 1, "partial output must not be replayed");
}
#[tokio::test]
async fn walk_stream_backpressure_limits_read_ahead_and_is_cancellable() {
let mut reader = Cursor::new(vec![5; 4 * DEFAULT_READ_BUFFER_SIZE]);
@@ -12072,15 +11543,4 @@ mod tests {
accept_task.abort();
}
#[test]
fn prewarm_claim_is_deduplicated_and_released_after_failure() {
let addr = "http://prewarm-retry-test.invalid:9000";
release_control_channel_prewarm(addr);
assert!(claim_control_channel_prewarm(addr));
assert!(!claim_control_channel_prewarm(addr));
release_control_channel_prewarm(addr);
assert!(claim_control_channel_prewarm(addr));
release_control_channel_prewarm(addr);
}
}
File diff suppressed because it is too large Load Diff
@@ -481,14 +481,6 @@ pub(crate) fn record_remote_disk_grpc_write_all_request() {
.record_outgoing_request_for_operation_and_backend(INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC);
}
pub(crate) fn record_remote_disk_grpc_write_all_duration(duration: Duration) {
global_internode_metrics().record_duration_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_WRITE_ALL,
INTERNODE_TRANSPORT_BACKEND_GRPC,
duration,
);
}
pub(crate) fn record_remote_disk_grpc_write_all_sent_bytes(bytes: usize) {
global_internode_metrics().record_sent_bytes_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_WRITE_ALL,
@@ -529,14 +521,6 @@ pub(crate) fn record_remote_disk_grpc_read_all_request() {
.record_outgoing_request_for_operation_and_backend(INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC);
}
pub(crate) fn record_remote_disk_grpc_read_all_duration(duration: Duration) {
global_internode_metrics().record_duration_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_READ_ALL,
INTERNODE_TRANSPORT_BACKEND_GRPC,
duration,
);
}
#[cfg(feature = "diagnostic-read-version-caller")]
fn read_version_request_mode(opts: &ReadOptions) -> ReadVersionRequestMode {
match (opts.read_data, opts.healing, opts.incl_free_versions) {
@@ -614,17 +598,6 @@ pub(crate) fn record_remote_disk_grpc_batch_read_version_stage(stage: &'static s
);
}
pub(crate) fn record_remote_disk_grpc_batch_read_version_duration(duration: Duration) {
if !rustfs_io_metrics::get_stage_metrics_enabled() {
return;
}
global_internode_metrics().record_duration_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
INTERNODE_TRANSPORT_BACKEND_GRPC,
duration,
);
}
pub(crate) fn record_remote_disk_grpc_batch_read_version_error() {
if !rustfs_io_metrics::get_stage_metrics_enabled() {
return;
@@ -717,14 +690,6 @@ pub(crate) fn record_remote_disk_grpc_read_multiple_recv_bytes(bytes: usize) {
record_grpc_payload_size(INTERNODE_OPERATION_GRPC_READ_MULTIPLE, bytes);
}
pub(crate) fn record_remote_disk_grpc_read_multiple_duration(duration: Duration) {
global_internode_metrics().record_duration_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
INTERNODE_TRANSPORT_BACKEND_GRPC,
duration,
);
}
/// Payload-size threshold (bytes) above which a unary internode gRPC response is counted as a
/// "large payload" for alerting. Env-overridable via `RUSTFS_INTERNODE_RPC_LARGE_PAYLOAD_WARN_BYTES`.
fn internode_rpc_large_payload_warn_bytes() -> usize {
-1
View File
@@ -3506,7 +3506,6 @@ mod tests {
replication_decision: String::new(),
checksum: None,
multipart_completion_replayed: false,
backfilled_old_current_size: None,
}
}
}
@@ -18,7 +18,6 @@ use crate::storage_api_contracts::namespace::NamespaceLocking as _;
use codec::{fixture_inline_pool_meta_replicas as replicas, fixture_split_inline_pool_meta_replicas as split};
use std::path::PathBuf;
use tokio::fs;
use uuid::Uuid;
struct PhysicalFixture {
_directories: Vec<tempfile::TempDir>,
@@ -217,264 +216,6 @@ async fn pool_meta_native_physical_split_without_ordinary_commit_performs_no_mut
"all current heads, identities and complete backups must remain unchanged"
);
assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
state
.ensure_write_safe("ordinary authority temporarily unavailable")
.expect("missing ordinary authority alone must not install a permanent corruption latch");
}
async fn interrupted_first_commit_fixture() -> (PhysicalFixture, PoolMetaCommittedCandidate) {
let fixture = fixture(3).await;
let PoolMetaReplica::Valid {
previous: Some(previous),
committed: false,
..
} = decode_pool_meta_replica(fixture.pending.clone())
else {
panic!("fixture must retain its acknowledged predecessor");
};
let physical = disks(&fixture.store).await;
let prepared = replicas(&fixture.pending, OffsetDateTime::now_utc()).await;
for (index, disk) in physical[0].iter().enumerate() {
fs::write(object_path(disk).join("xl.meta"), &prepared[index])
.await
.expect("one pool must retain an ordinary-readable PREPARE");
}
(fixture, *previous)
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_physical_interrupted_first_commit_restores_proven_predecessor() {
let (mut fixture, previous) = interrupted_first_commit_fixture().await;
let before = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
assert!(
before[1..]
.iter()
.all(|read| matches!(read.replica, PoolMetaReplica::Unreadable(_)))
);
assert!(matches!(before[0].replica, PoolMetaReplica::Valid { committed: false, .. }));
let mut state = clean_state(&fixture);
let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
assert!(
repair(&fixture.store, &mut state, &fence)
.await
.expect("ordinary PREPARE must authorize restoring only its acknowledged predecessor")
);
state
.ensure_write_safe("first commit rollback")
.expect("rollback must finish its transaction");
assert_ne!(
previous.canonical, fixture.canonical,
"the interrupted COMMIT must not become authoritative"
);
fixture.canonical = previous.canonical;
fixture.revision = previous.revision;
assert_committed(&fixture).await;
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_physical_interrupted_rollback_retains_authority_and_resumes() {
let (mut fixture, previous) = interrupted_first_commit_fixture().await;
let pause = Arc::new(NativeApplyPause::default());
let reached = pause.reached.notified();
tokio::pin!(reached);
reached.as_mut().enable();
let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
let mut state = clean_state(&fixture);
let recovery = tokio::spawn({
let store = Arc::clone(&fixture.store);
let pause = Arc::clone(&pause);
async move {
AFTER_NATIVE_POOL_APPLY
.scope(pause, async move { repair(&store, &mut state, &fence).await })
.await
}
});
tokio::time::timeout(std::time::Duration::from_secs(10), reached)
.await
.expect("real rollback should pause after one pool is durably restored");
let partial = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
assert!(matches!(partial[0].replica, PoolMetaReplica::Valid { committed: false, .. }));
assert!(matches!(&partial[1].replica, PoolMetaReplica::Valid { canonical, committed: true, .. }
if canonical == &previous.canonical));
assert!(matches!(partial[2].replica, PoolMetaReplica::Unreadable(_)));
recovery.abort();
assert!(
recovery
.await
.expect_err("the paused repair must be cancelled")
.is_cancelled()
);
let mut state = clean_state(&fixture);
let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
assert!(
repair(&fixture.store, &mut state, &fence)
.await
.expect("restart must resume from the retained ordinary PREPARE")
);
fixture.canonical = previous.canonical;
fixture.revision = previous.revision;
assert_committed(&fixture).await;
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_physical_rollback_resumes_after_last_prepare_loses_read_quorum() {
let (mut fixture, previous) = interrupted_first_commit_fixture().await;
let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
let inspected = inspect(&fixture.store.pools, &fence)
.await
.expect("physical rollback inventory");
let replacements = plan(
&inspected,
&fixture.canonical,
fixture.revision,
&fixture.identity,
Some(&previous.canonical),
)
.await
.expect("the ordinary PREPARE must authorize its predecessor");
for (pool, count) in [(1, 4), (2, 4), (0, 2)] {
for (index, replacement) in replacements[pool].iter().enumerate().take(count) {
let snapshot = &inspected[pool].snapshots[index];
assert_eq!(
inspected[pool].disks[index]
.pool_meta_native_recovery(Request::Apply {
object: PoolMetaNativeObject::PoolBin,
expected_disk_id: snapshot.disk_id,
expected_current_sha256: snapshot.current.as_ref().map(|raw| Sha256::digest(raw).into()),
expected_backup_inventory_sha256: snapshot.inventory_sha256().expect("original inventory hash"),
replacement: replacement.clone(),
})
.await
.expect("partial rollback must use the real conditional publication and sync path"),
Response::Applied
);
}
}
let partial = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
assert!(matches!(partial[0].replica, PoolMetaReplica::Unreadable(_)));
assert!(partial[1..].iter().all(|read| matches!(&read.replica,
PoolMetaReplica::Valid { canonical, committed: true, .. } if canonical == &previous.canonical)));
let mut state = clean_state(&fixture);
assert!(
repair(&fixture.store, &mut state, &fence)
.await
.expect("restored ordinary COMMIT must finish rollback after the last PREPARE becomes split")
);
fixture.canonical = previous.canonical;
fixture.revision = previous.revision;
assert_committed(&fixture).await;
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_physical_rollback_backup_does_not_poison_reused_generation() {
let (mut fixture, previous) = interrupted_first_commit_fixture().await;
let mut state = clean_state(&fixture);
let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
assert!(repair(&fixture.store, &mut state, &fence).await.expect("initial rollback"));
let revision = PoolMetaRevision {
transaction_id: Some(Uuid::new_v4()),
..fixture.revision
};
let mut meta = previous.meta.clone();
meta.pools[0].last_update += Duration::seconds(1);
let canonical = encode_pool_meta_v3_envelope(&meta, revision, true, None).expect("later COMMIT");
let pending = encode_pool_meta_v3_envelope(&meta, revision, false, Some(&previous)).expect("later PREPARE");
let modified = OffsetDateTime::now_utc() + Duration::seconds(1);
let split = split(&pending, &canonical, modified).await;
let commit = replicas(&canonical, modified).await;
let prepared = replicas(&pending, modified).await;
let directory = Uuid::new_v4();
let retained = snapshots(&fixture.store).await;
for (pool_index, pool) in disks(&fixture.store).await.into_iter().enumerate() {
for (disk_index, disk) in pool.iter().enumerate() {
assert!(
!retained[pool_index][disk_index].backups.is_empty(),
"the abandoned PREPARE must remain as real on-disk evidence"
);
let object = object_path(disk);
fs::write(object.join("xl.meta"), &if pool_index < 2 { &split } else { &commit }[disk_index])
.await
.expect("install later partial COMMIT without clearing old backups");
fs::create_dir(object.join(directory.to_string()))
.await
.expect("later backup directory");
fs::write(object.join(directory.to_string()).join("xl.meta.bkp"), &prepared[disk_index])
.await
.expect("later PREPARE backup");
}
}
assert!(
repair(&fixture.store, &mut state, &fence)
.await
.expect("an ordinary later COMMIT must survive the abandoned sibling PREPARE")
);
fixture.canonical = canonical;
fixture.revision = revision;
assert_committed(&fixture).await;
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_physical_rollback_rejects_retained_commit_without_mutation() {
let (fixture, _) = interrupted_first_commit_fixture().await;
let physical = disks(&fixture.store).await;
let commit = replicas(&fixture.canonical, OffsetDateTime::now_utc()).await;
let directory = Uuid::new_v4();
let object = object_path(&physical[0][0]);
fs::create_dir(object.join(directory.to_string()))
.await
.expect("retained COMMIT directory");
fs::write(object.join(directory.to_string()).join("xl.meta.bkp"), &commit[0])
.await
.expect("retain evidence that the interrupted transaction may already have committed");
let before = snapshots(&fixture.store).await;
let mut state = clean_state(&fixture);
let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
assert!(repair(&fixture.store, &mut state, &fence).await.is_err());
assert_eq!(snapshots(&fixture.store).await, before, "ambiguous COMMIT evidence must remain untouched");
assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_physical_rollback_rejects_competing_prepares_without_mutation() {
let (fixture, previous) = interrupted_first_commit_fixture().await;
let pending = encode_pool_meta_v3_envelope(
&previous.meta,
PoolMetaRevision {
transaction_id: Some(Uuid::new_v4()),
..fixture.revision
},
false,
Some(&previous),
)
.expect("competing PREPARE must remain structurally valid");
let physical = disks(&fixture.store).await;
let prepared = replicas(&pending, OffsetDateTime::now_utc()).await;
for (index, disk) in physical[1].iter().enumerate() {
fs::write(object_path(disk).join("xl.meta"), &prepared[index])
.await
.expect("second pool must expose the competing transaction through ordinary reads");
}
let before = snapshots(&fixture.store).await;
let mut state = clean_state(&fixture);
let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
let error = repair(&fixture.store, &mut state, &fence)
.await
.expect_err("two distinct prepared transactions must not authorize a guessed rollback");
assert_eq!(
error
.pool_metadata_failure()
.expect("transaction conflict must be typed")
.kind,
crate::error::PoolMetadataFailure::RecoveryRequired
);
assert_eq!(snapshots(&fixture.store).await, before, "competing transactions must remain untouched");
assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
}
struct IdentityEpochFork {
@@ -726,7 +467,7 @@ async fn pool_meta_native_physical_partial_apply_resumes_after_conditional_rejec
let inspected = inspect(&fixture.store.pools, &fence)
.await
.expect("real native snapshots should be obtained");
let replacements = plan(&inspected, &fixture.canonical, fixture.revision, &fixture.identity, None)
let replacements = plan(&inspected, &fixture.canonical, fixture.revision, &fixture.identity)
.await
.expect("physical recovery plan should validate");
let snapshot = &inspected[0].snapshots[0];
@@ -887,559 +628,3 @@ async fn pool_meta_native_physical_three_plus_one_checkpoint_converges_all_shard
);
}
}
async fn owner_repair_no_backups_fixture() -> PhysicalFixture {
let fixture = fixture(0).await;
for pool in disks(&fixture.store).await {
for disk in pool {
let object = object_path(&disk);
let mut entries = fs::read_dir(&object).await.expect("fixture metadata directory");
while let Some(entry) = entries.next_entry().await.expect("fixture directory entry") {
if entry.file_type().await.expect("fixture entry type").is_dir() {
fs::remove_dir_all(entry.path())
.await
.expect("remove task-owned fixture rollback directory");
}
}
}
}
fixture
}
async fn owner_repair_identity_heads(store: &ECStore) -> Vec<Vec<(Uuid, Bytes)>> {
let mut all = Vec::new();
for pool in &store.pools {
let set = pool.get_disks_by_key(POOL_META_IDENTITY_NAME);
let members = set
.disks
.read()
.await
.iter()
.cloned()
.map(|disk| disk.expect("identity fixture disk must be online"))
.collect::<Vec<_>>();
let mut heads = Vec::new();
for disk in members {
match disk
.pool_meta_native_recovery(Request::InspectIdentity {})
.await
.expect("physical identity inspection")
{
Response::IdentityInspected { disk_id, raw } => heads.push((disk_id, raw)),
_ => panic!("identity inspection must return its physical head"),
}
}
all.push(heads);
}
all
}
async fn legacy_native_owner_fixture(version: u16, stale: bool) -> (PhysicalFixture, Vec<u8>) {
let fixture = owner_repair_no_backups_fixture().await;
let mut meta = match decode_pool_meta_replica(fixture.canonical.clone()) {
PoolMetaReplica::Valid {
meta, committed: true, ..
} => meta,
_ => panic!("fixture must provide valid committed pool metadata"),
};
meta.version = version;
let raw = meta
.encode_config_data_for_v2_gate(version == POOL_META_VERSION)
.expect("legacy fixture should encode in its original version");
assert_eq!(u16::from_le_bytes([raw[2], raw[3]]), version);
let all = disks(&fixture.store).await;
let now = OffsetDateTime::now_utc();
for pool in &all {
let heads = replicas(&raw, now).await;
for (index, disk) in pool.iter().enumerate() {
fs::write(object_path(disk).join("xl.meta"), &heads[index])
.await
.expect("install exact legacy head");
}
}
if stale {
let old = replicas(&raw, now - Duration::seconds(10)).await;
fs::write(object_path(&all[0][0]).join("xl.meta"), &old[0])
.await
.expect("install stale same-raw legacy head");
}
(fixture, raw)
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_heal_v1_with_all_upgrade_gates_disabled_keeps_exact_raw() {
temp_env::async_with_vars(
[
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
],
async {
let (fixture, raw) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
let guard = owner(&fixture.store).await;
let mut state = fixture.store.pool_meta_save_gate.lock().await;
let result = tokio::time::timeout(
std::time::Duration::from_secs(30),
fixture.store.heal_pool_meta_replica_under_fence(0, guard, &mut state),
)
.await
.expect("legacy native heal must terminate");
assert!(result.is_ok(), "fully online legacy native heal should succeed: {result:?}");
drop(state);
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
for read in &reads {
assert!(
matches!(&read.replica, PoolMetaReplica::Valid { raw: bytes, revision, .. }
if bytes == &raw && revision.version == POOL_META_V1_VERSION),
"native heal must not upgrade a V1 raw payload when both upgrade gates are disabled"
);
}
},
)
.await;
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_heal_rejects_legacy_source_with_trailing_bytes_without_writes() {
temp_env::async_with_vars(
[
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
],
async {
for version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
let (fixture, raw) = legacy_native_owner_fixture(version, false).await;
let mut source_raw = raw.clone();
// A second MessagePack value is readable by legacy decoding but
// must not be copied as part of an unverified repair payload.
source_raw.push(0xc0);
assert!(
matches!(decode_pool_meta_replica(source_raw.clone()), PoolMetaReplica::Valid { raw, .. } if raw == source_raw),
"ordinary legacy reads must remain compatible with trailing bytes"
);
let all_disks = disks(&fixture.store).await;
let source_heads = replicas(&source_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
for (index, disk) in all_disks[0].iter().enumerate() {
fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
.await
.expect("install readable legacy source with trailing bytes");
}
let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
.await
.expect("legacy semantic selection must remain readable");
assert_eq!(selection.legacy_raw.as_deref(), Some(source_raw.as_slice()));
let before = snapshots(&fixture.store).await;
let identities = owner_repair_identity_heads(&fixture.store).await;
let guard = owner(&fixture.store).await;
let mut state = fixture.store.pool_meta_save_gate.lock().await;
let result = tokio::time::timeout(
std::time::Duration::from_secs(30),
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
)
.await
.expect("legacy native heal must terminate");
drop(state);
assert!(result.is_err(), "version {version}: unparsed trailing source bytes must not be republished");
assert_eq!(snapshots(&fixture.store).await, before, "version {version}: no pool.bin head or backup may change");
assert_eq!(
owner_repair_identity_heads(&fixture.store).await,
identities,
"version {version}: no identity shard may change"
);
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &source_raw));
assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw: target, .. } if target == &raw));
}
},
)
.await;
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_heal_mixed_v1_v2_preserves_observed_format_floor() {
temp_env::async_with_vars(
[
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
],
async {
for first_version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
for target_pool in [0, 1] {
let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
let mut meta = match decode_pool_meta_replica(raw_v1.clone()) {
PoolMetaReplica::Valid { meta, committed: true, .. } => meta,
_ => panic!("initial V1 metadata must decode"),
};
meta.version = POOL_META_VERSION;
let raw_v2 = meta.encode_config_data_for_v2_gate(true).expect("equivalent V2 payload must encode");
assert_eq!(u16::from_le_bytes([raw_v1[2], raw_v1[3]]), POOL_META_V1_VERSION);
assert_eq!(u16::from_le_bytes([raw_v2[2], raw_v2[3]]), POOL_META_VERSION);
let semantic_v1 = match decode_pool_meta_replica(raw_v1.clone()) {
PoolMetaReplica::Valid { canonical, .. } => canonical,
_ => panic!("V1 payload must remain valid"),
};
let semantic_v2 = match decode_pool_meta_replica(raw_v2.clone()) {
PoolMetaReplica::Valid { canonical, .. } => canonical,
_ => panic!("V2 payload must remain valid"),
};
assert_eq!(semantic_v1, semantic_v2, "the two formats must carry equivalent metadata");
let other_version = if first_version == POOL_META_V1_VERSION { POOL_META_VERSION } else { POOL_META_V1_VERSION };
// Pool two follows pool zero; pools zero and one cover both
// orderings and both target formats in separate runs.
let versions = [first_version, other_version, first_version];
let physical = disks(&fixture.store).await;
let timestamp = OffsetDateTime::now_utc();
for (pool_index, pool) in physical.iter().enumerate() {
let raw = if versions[pool_index] == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
let heads = replicas(raw, timestamp).await;
for (index, disk) in pool.iter().enumerate() {
fs::write(object_path(disk).join("xl.meta"), &heads[index]).await.expect("install mixed legacy pool head");
}
}
let before = snapshots(&fixture.store).await;
let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
.await.expect("equivalent legacy pools should be selectable");
assert_eq!(selection.revision.version, POOL_META_VERSION, "the observed V2 format floor must be retained");
assert_eq!(selection.canonical.as_deref(), Some(semantic_v2.as_slice()));
let guard = owner(&fixture.store).await;
let mut state = fixture.store.pool_meta_save_gate.lock().await;
tokio::time::timeout(std::time::Duration::from_secs(30),
fixture.store.heal_pool_meta_replica_under_fence(target_pool, guard, &mut state))
.await.expect("mixed legacy heal must terminate")
.expect("equivalent legacy metadata should heal without downgrading the format floor");
drop(state);
let after = snapshots(&fixture.store).await;
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
for (pool_index, read) in reads.iter().enumerate() {
let expected_version = if pool_index == target_pool { POOL_META_VERSION } else { versions[pool_index] };
let expected_raw = if expected_version == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
assert!(matches!(&read.replica,
PoolMetaReplica::Valid { raw, canonical, revision, committed: true, .. }
if raw == expected_raw && canonical == &semantic_v2 && revision.version == expected_version
), "pool {pool_index}, first={first_version}, target={target_pool}: preserve equivalent payload and expected legacy version");
for (index, snapshot) in after[pool_index].iter().enumerate() {
codec::validate_inline_pool_meta_shard(snapshot.current.as_ref().expect("physical head"), index, expected_raw)
.await.expect("every physical shard must match its selected legacy payload");
}
if pool_index != target_pool {
assert_eq!(after[pool_index], before[pool_index], "healing one pool must not modify other pools");
}
}
}
}
},
).await;
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_heal_rejects_v1_and_v3_prepare_in_either_pool_order() {
temp_env::async_with_vars(
[
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
],
async {
for prepare_pool in [0, 1] {
let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
let (legacy_meta, legacy_canonical) = match decode_pool_meta_replica(raw_v1.clone()) {
PoolMetaReplica::Valid {
meta,
canonical,
revision,
committed: true,
..
} if revision.version == POOL_META_V1_VERSION => (meta, canonical),
_ => panic!("baseline V1 metadata must be committed and readable"),
};
let previous = PoolMetaCommittedCandidate {
canonical: legacy_canonical,
meta: legacy_meta.clone(),
revision: PoolMetaRevision::legacy(POOL_META_V1_VERSION),
};
let mut next = legacy_meta;
next.version = POOL_META_GENERATION_VERSION;
next.pools[0].last_update += Duration::seconds(1);
let revision = PoolMetaRevision {
version: POOL_META_GENERATION_VERSION,
cluster_id: fixture.revision.cluster_id,
epoch: fixture.revision.epoch,
generation: 1,
transaction_id: Some(Uuid::new_v4()),
};
let pending = encode_pool_meta_v3_envelope(&next, revision, false, Some(&previous))
.expect("V3 PREPARE with V1 predecessor must encode");
assert!(matches!(decode_pool_meta_replica(pending.clone()),
PoolMetaReplica::Valid { committed: false, revision: actual, .. } if actual == revision));
let heads = replicas(&pending, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
let physical = disks(&fixture.store).await;
for (index, disk) in physical[prepare_pool].iter().enumerate() {
fs::write(object_path(disk).join("xl.meta"), &heads[index])
.await
.expect("install task-owned V3 PREPARE evidence");
}
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
assert!(matches!(&reads[prepare_pool].replica,
PoolMetaReplica::Valid { raw, committed: false, revision: actual, .. }
if raw == &pending && *actual == revision));
let other_pool = 1 - prepare_pool;
assert!(matches!(&reads[other_pool].replica,
PoolMetaReplica::Valid { raw, committed: true, revision: actual, .. }
if raw == &raw_v1 && actual.version == POOL_META_V1_VERSION));
let before = snapshots(&fixture.store).await;
let identities = owner_repair_identity_heads(&fixture.store).await;
let guard = owner(&fixture.store).await;
let mut state = fixture.store.pool_meta_save_gate.lock().await;
let result = tokio::time::timeout(
std::time::Duration::from_secs(30),
fixture
.store
.heal_pool_meta_replica_under_fence(prepare_pool, guard, &mut state),
)
.await
.expect("mixed V1/V3 PREPARE heal must terminate");
drop(state);
assert!(
result.is_err(),
"pool {prepare_pool}: native heal must not overwrite a V3 PREPARE with legacy data"
);
assert_eq!(
snapshots(&fixture.store).await,
before,
"pool {prepare_pool}: rejection must retain all physical head and backup evidence"
);
assert_eq!(
owner_repair_identity_heads(&fixture.store).await,
identities,
"pool {prepare_pool}: rejection must not mutate identity evidence"
);
}
},
)
.await;
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_heal_rejects_v1_legacy_dont_save_true_without_writes() {
temp_env::async_with_vars(
[
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
],
async {
let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
let meta = match decode_pool_meta_replica(safe_v1.clone()) {
PoolMetaReplica::Valid {
meta, committed: true, ..
} => meta,
_ => panic!("baseline V1 metadata must decode"),
};
let statuses = meta
.pools
.iter()
.map(|pool| {
assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
LegacyPoolStatus {
id: pool.id,
cmd_line: pool.cmd_line.clone(),
last_update: pool.last_update,
decommission: None,
}
})
.collect();
let legacy = LegacyPoolMeta {
version: POOL_META_V1_VERSION,
pools: statuses,
dont_save: true,
};
let mut dangerous_raw = Vec::new();
dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
let old_reader: LegacyPoolMeta =
rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode source");
assert!(old_reader.dont_save, "the old reader would treat this source as permanently do-not-save");
assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
let source = match decode_pool_meta_replica(dangerous_raw.clone()) {
PoolMetaReplica::Valid {
canonical,
revision,
committed: true,
..
} => {
assert_eq!(revision.version, POOL_META_V1_VERSION);
canonical
}
_ => panic!("current reader must recognize the legacy source"),
};
assert!(matches!(decode_pool_meta_replica(safe_v1.clone()),
PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
if canonical == source && revision.version == POOL_META_V1_VERSION));
let physical = disks(&fixture.store).await;
let source_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
for (index, disk) in physical[0].iter().enumerate() {
fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
.await
.expect("install legacy source on pool zero");
}
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
let before = snapshots(&fixture.store).await;
let identities = owner_repair_identity_heads(&fixture.store).await;
let guard = owner(&fixture.store).await;
let mut state = fixture.store.pool_meta_save_gate.lock().await;
let result = tokio::time::timeout(
std::time::Duration::from_secs(30),
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
)
.await
.expect("unsafe legacy heal must terminate");
drop(state);
let error = result.expect_err("native heal must reject a source whose old-reader dont_save is true");
assert!(
error.to_string().contains("dont_save"),
"rejection must identify the unsafe legacy field: {error}"
);
assert_eq!(
snapshots(&fixture.store).await,
before,
"rejection must preserve every original head and backup"
);
assert_eq!(
owner_repair_identity_heads(&fixture.store).await,
identities,
"rejection must not write identity shards"
);
let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw));
assert!(
matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1),
"the dangerous source bytes must not be copied into the target pool"
);
},
)
.await;
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_heal_rejects_target_v1_dont_save_true_without_writes() {
temp_env::async_with_vars(
[
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
],
async {
let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
let meta = match decode_pool_meta_replica(safe_v1.clone()) {
PoolMetaReplica::Valid {
meta, committed: true, ..
} => meta,
_ => panic!("baseline V1 metadata must decode"),
};
let statuses = meta
.pools
.iter()
.map(|pool| {
assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
LegacyPoolStatus {
id: pool.id,
cmd_line: pool.cmd_line.clone(),
last_update: pool.last_update,
decommission: None,
}
})
.collect();
let legacy = LegacyPoolMeta {
version: POOL_META_V1_VERSION,
pools: statuses,
dont_save: true,
};
let mut dangerous_raw = Vec::new();
dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
let old_reader: LegacyPoolMeta =
rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode target");
assert!(old_reader.dont_save, "the old reader interprets the target as do-not-save");
assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
let source_canonical = match decode_pool_meta_replica(safe_v1.clone()) {
PoolMetaReplica::Valid {
canonical,
revision,
committed: true,
..
} => {
assert_eq!(revision.version, POOL_META_V1_VERSION);
canonical
}
_ => panic!("safe pool-zero source must decode"),
};
assert!(
matches!(decode_pool_meta_replica(dangerous_raw.clone()),
PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
if canonical == source_canonical && revision.version == POOL_META_V1_VERSION),
"the dangerous target must remain semantically equivalent to the safe source"
);
let physical = disks(&fixture.store).await;
let target_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
for (index, disk) in physical[1].iter().enumerate() {
fs::write(object_path(disk).join("xl.meta"), &target_heads[index])
.await
.expect("install legacy dangerous target head");
}
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
let before = snapshots(&fixture.store).await;
let identities = owner_repair_identity_heads(&fixture.store).await;
let guard = owner(&fixture.store).await;
let mut state = fixture.store.pool_meta_save_gate.lock().await;
let result = tokio::time::timeout(
std::time::Duration::from_secs(30),
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
)
.await
.expect("unsafe legacy target heal must terminate");
drop(state);
let error = result.expect_err("native heal must reject a target whose old-reader dont_save is true");
assert!(
error.to_string().contains("dont_save"),
"rejection must identify the unsafe legacy field: {error}"
);
assert_eq!(
snapshots(&fixture.store).await,
before,
"rejection must preserve all physical heads and backups"
);
assert_eq!(
owner_repair_identity_heads(&fixture.store).await,
identities,
"rejection must not write identity shards"
);
let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1));
assert!(
matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw),
"unsafe target evidence must not be overwritten by the safe source"
);
},
)
.await;
}
@@ -30,7 +30,6 @@ pub(super) struct NativeApplyPause {
#[cfg(test)]
tokio::task_local! {
pub(super) static AFTER_NATIVE_APPLY: Arc<NativeApplyPause>;
pub(super) static AFTER_NATIVE_POOL_APPLY: Arc<NativeApplyPause>;
}
struct InspectedPool {
@@ -169,7 +168,6 @@ async fn plan(
canonical: &[u8],
revision: PoolMetaRevision,
identity: &[u8],
rollback: Option<&[u8]>,
) -> Result<Vec<Vec<Bytes>>> {
let mut pending = Vec::<Vec<u8>>::new();
let mut predecessors = Vec::<Vec<u8>>::new();
@@ -225,83 +223,23 @@ async fn plan(
}
}
}
// The last ordinary PREPARE can itself become split during rollback. In
// that case another pool already exposes the restored ordinary COMMIT.
// Its authority permits continuing the rollback, never promoting a child
// transaction reconstructed from disk evidence.
let mut continuation: Option<(Vec<u8>, PoolMetaRevision)> = None;
if rollback.is_none() {
let mut outstanding = Vec::new();
for pool in inspected {
for (index, snapshot) in pool.snapshots.iter().enumerate() {
let raw = snapshot.current.as_ref().ok_or_else(invalid)?;
if codec::validate_inline_pool_meta_shard(raw, index, canonical).await.is_err() {
outstanding.push((index, raw));
}
}
}
for payload in &decoded {
if let PoolMetaReplica::Valid {
let mut obsolete = Vec::<Vec<u8>>::new();
for payload in &decoded {
match decode_pool_meta_replica(payload.clone()) {
PoolMetaReplica::Valid {
canonical: candidate,
revision: candidate_revision,
committed: false,
previous: Some(previous),
..
} = decode_pool_meta_replica(payload.clone())
&& previous.revision == revision
&& previous.canonical == canonical
{
let mut matches = true;
for (index, raw) in &outstanding {
matches &= codec::validate_inline_pool_meta_shard(raw, *index, payload).await.is_ok()
|| codec::validate_inline_pool_meta_shard(raw, *index, &candidate).await.is_ok();
} if candidate_revision == revision && candidate == canonical => {
if !pending.contains(payload) {
pending.push(payload.clone());
}
if matches {
if !outstanding.is_empty()
&& continuation.as_ref().is_some_and(|(selected, selected_revision)| {
selected != &candidate || *selected_revision != candidate_revision
})
{
return Err(known_validation(invalid(), "native_transaction_validation"));
}
continuation.get_or_insert((candidate, candidate_revision));
if !predecessors.contains(&previous.canonical) {
predecessors.push(previous.canonical);
}
}
}
}
let (canonical, revision, rollback) = match &continuation {
Some((transaction, next_revision)) => (transaction.as_slice(), *next_revision, Some(canonical)),
None => (canonical, revision, rollback),
};
for payload in &decoded {
if let PoolMetaReplica::Valid {
canonical: candidate,
revision: candidate_revision,
committed: false,
previous: Some(previous),
..
} = decode_pool_meta_replica(payload.clone())
&& candidate_revision == revision
&& candidate == canonical
{
pending.push(payload.clone());
if !predecessors.contains(&previous.canonical) {
predecessors.push(previous.canonical);
}
}
}
if pending.len() > 1 {
return Err(known_validation(
Error::other("native V3 prepare records disagree on the predecessor"),
"native_transaction_validation",
));
}
let mut obsolete = Vec::<Vec<u8>>::new();
for payload in &decoded {
if pending.contains(payload) {
continue;
}
match decode_pool_meta_replica(payload.clone()) {
PoolMetaReplica::Valid { revision: older, .. }
if older.is_generation_protocol()
&& older.cluster_id == revision.cluster_id
@@ -313,23 +251,6 @@ async fn plan(
// fully reconstructed and bound into the native CAS.
obsolete.push(payload.clone());
}
PoolMetaReplica::Valid {
revision: abandoned,
committed: false,
previous: Some(previous),
..
} if abandoned.is_generation_protocol()
&& abandoned.cluster_id == revision.cluster_id
&& abandoned.epoch == revision.epoch
&& abandoned.generation == revision.generation
&& abandoned.transaction_id != revision.transaction_id
&& predecessors.contains(&previous.canonical) =>
{
// A rollback retains the failed PREPARE as evidence. A later
// transaction may reuse that generation, but only a backup of
// an uncommitted sibling with the same predecessor is obsolete.
obsolete.push(payload.clone());
}
PoolMetaReplica::Valid {
canonical: candidate,
revision: candidate_revision,
@@ -351,7 +272,13 @@ async fn plan(
_ => {}
}
}
if pending.is_empty() || rollback.is_some_and(|previous| !predecessors.iter().any(|candidate| candidate == previous)) {
if pending.len() > 1 {
return Err(known_validation(
Error::other("native V3 prepare records disagree on the predecessor"),
"native_transaction_validation",
));
}
if pending.is_empty() {
return Err(invalid());
}
for payload in &decoded {
@@ -367,25 +294,13 @@ async fn plan(
for (index, snapshot) in pool.snapshots.iter().enumerate() {
let raw = snapshot.current.as_ref().ok_or_else(invalid)?;
let mut matched = false;
for payload in std::iter::once(canonical)
.chain(pending.iter().map(Vec::as_slice))
.chain(rollback)
{
for payload in std::iter::once(canonical).chain(pending.iter().map(Vec::as_slice)) {
matched |= codec::validate_inline_pool_meta_shard(raw, index, payload).await.is_ok();
}
if !matched {
return Err(invalid());
}
for backup in &snapshot.backups {
if rollback.is_some()
&& codec::validate_inline_pool_meta_shard(&backup.data, index, canonical)
.await
.is_ok()
{
// A retained COMMIT might have been authoritative before
// this interruption. Never roll it back from PREPARE alone.
return Err(invalid());
}
let mut matched = false;
for payload in std::iter::once(canonical).chain(
pending
@@ -408,7 +323,7 @@ async fn plan(
let mut replacements = Vec::with_capacity(inspected.len());
for pool in inspected {
replacements.push(
codec::encode_inline_pool_meta_replicas(&current(pool)?, rollback.unwrap_or(canonical), modified)
codec::encode_inline_pool_meta_replicas(&current(pool)?, canonical, modified)
.await
.map_err(|error| codec_validation(error, "native_metadata_validation"))?,
);
@@ -423,12 +338,10 @@ pub(super) async fn repair(
) -> Result<bool> {
fence.ensure_held()?;
let reads = read_pool_meta_replicas(store.pools.clone(), true).await;
let unreadable = reads
if !reads
.iter()
.enumerate()
.filter_map(|(index, read)| matches!(read.replica, PoolMetaReplica::Unreadable(_)).then_some(index))
.collect::<Vec<_>>();
if unreadable.is_empty() {
.any(|read| matches!(read.replica, PoolMetaReplica::Unreadable(_)))
{
return Ok(false);
}
state.ensure_write_safe("native pool metadata recovery")?;
@@ -447,9 +360,8 @@ pub(super) async fn repair(
{
return Err(invalid());
}
// Ordinary reads grant authority: a COMMIT selects its own state, while a
// PREPARE can select only its acknowledged predecessor. Reconstruction
// below proves integrity and never promotes an interrupted transaction.
// Only an ordinary, successfully read COMMIT grants authority. Rebuilding
// two erasure shards below proves integrity, never a commit decision.
let ordinary_commits = reads
.iter()
.filter_map(|read| match &read.replica {
@@ -463,22 +375,7 @@ pub(super) async fn repair(
_ => None,
})
.collect::<Vec<_>>();
let ordinary_prepares = reads
.iter()
.filter_map(|read| match &read.replica {
PoolMetaReplica::Valid {
canonical,
revision,
committed: false,
previous: Some(previous),
..
} if revision.is_generation_protocol() && previous.revision.is_generation_protocol() => {
Some((canonical.clone(), *revision, previous.canonical.clone()))
}
_ => None,
})
.collect::<Vec<_>>();
if ordinary_commits.is_empty() && ordinary_prepares.is_empty() {
if ordinary_commits.is_empty() {
return Err(invalid());
}
let selection = select_pool_meta_replica_reads(reads)
@@ -498,20 +395,10 @@ pub(super) async fn repair(
return Err(invalid());
}
let canonical = selection.canonical.as_ref().ok_or_else(invalid)?;
let mut rollback = None;
for (pending, revision, previous) in &ordinary_prepares {
if previous == canonical {
if rollback.is_some_and(|candidate| candidate != (pending, *revision)) {
return Err(latch_validation(state, known_validation(invalid(), "native_transaction_validation")));
}
rollback = Some((pending, *revision));
}
}
if !selection.revision.is_generation_protocol()
|| (rollback.is_none()
&& !ordinary_commits
.iter()
.any(|(raw, revision)| raw == canonical && *revision == selection.revision))
|| !ordinary_commits
.iter()
.any(|(raw, revision)| raw == canonical && *revision == selection.revision)
{
return Err(invalid());
}
@@ -558,18 +445,13 @@ pub(super) async fn repair(
.await
.map_err(|error| latch_validation(state, error))?;
let inspected = inspect(&store.pools, fence).await?;
let (transaction, revision) = rollback.unwrap_or((canonical, selection.revision));
let replacements = plan(&inspected, transaction, revision, &identity_bytes, rollback.map(|_| canonical.as_slice()))
let replacements = plan(&inspected, canonical, selection.revision, &identity_bytes)
.await
.map_err(|error| latch_validation(state, error))?;
fence.ensure_held()?;
let mut arm = state.arm_transaction();
arm.phase = Some("native_repair_apply");
let mut apply_order = inspected.iter().zip(replacements).enumerate().collect::<Vec<_>>();
// Preserve ordinary authority until all unreadable pools are repaired.
// Cancellation can then resume from a PREPARE or a restored COMMIT.
apply_order.sort_by_key(|(index, _)| !unreadable.contains(index));
for (_, (pool, replacements)) in apply_order {
for (pool, replacements) in inspected.iter().zip(replacements) {
for ((disk, snapshot), replacement) in pool.disks.iter().zip(&pool.snapshots).zip(replacements) {
fence.ensure_held()?;
let request = Request::Apply {
@@ -584,11 +466,6 @@ pub(super) async fn repair(
_ => return Err(invalid()),
}
}
#[cfg(test)]
if let Ok(pause) = AFTER_NATIVE_POOL_APPLY.try_with(Arc::clone) {
pause.reached.notify_one();
pause.resume.notified().await;
}
}
#[cfg(test)]
if let Ok(pause) = AFTER_NATIVE_APPLY.try_with(Arc::clone) {
@@ -750,7 +627,7 @@ mod tests {
#[tokio::test]
async fn pool_meta_native_plan_restores_physical_split_with_fresh_common_identity() {
let f = fixture().await;
let repaired = plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.unwrap();
let repaired = plan(&f.pools, &f.commit, f.revision, &f.identity).await.unwrap();
assert_eq!(repaired.len(), 4);
for pool in repaired {
assert_eq!(
@@ -771,7 +648,7 @@ mod tests {
let mut f = fixture().await;
let wrong = replicas(b"different identity", f.modified).await;
f.pools[0].snapshots[0].identity = Some(wrong[0].clone());
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
}
#[tokio::test]
@@ -794,10 +671,10 @@ mod tests {
.unwrap();
let wrong = replicas(&higher, f.modified + Duration::seconds(2)).await;
f.pools[0].snapshots[0].current = Some(wrong[0].clone());
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
f.pools[0].snapshots[0].current = Some(replicas(&f.pending, f.modified).await[0].clone());
f.pools[0].snapshots[0].backups[0].data = wrong[0].clone();
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
}
#[tokio::test]
@@ -840,17 +717,17 @@ mod tests {
}
}
assert!(older_pending != f.pending);
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_ok());
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_ok());
}
#[tokio::test]
async fn pool_meta_native_plan_rejects_missing_or_corrupt_evidence() {
let mut f = fixture().await;
let before = f.pools[1].snapshots[2].current.take();
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
f.pools[1].snapshots[2].current = before;
f.pools[1].snapshots[2].backups[0].data = Bytes::from_static(b"invalid metadata");
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
}
}
@@ -315,7 +315,7 @@ pub(super) async fn encode_inline_pool_meta_replicas(
.encode_with_shard_integrity(
Cursor::new(Bytes::copy_from_slice(canonical)),
&mut writers,
crate::erasure::coding::encode::ShardWritePolicy::object(raw.len()),
raw.len(),
1,
IntegrityEncodeMode::Inline(canonical.len()),
protected,
File diff suppressed because it is too large Load Diff
+4 -35
View File
@@ -1616,7 +1616,7 @@ mod decommission_lock_order_tests {
run_routine_to_a_deferral_pause(
&store,
1,
"target pool 1 no longer fits the selected mutation: target pool 1 reservation requires",
"target pool 1 no longer fits the selected mutation: source pool 0 reservation estimate is exhausted",
)
.await;
});
@@ -1647,7 +1647,7 @@ mod decommission_lock_order_tests {
run_routine_to_a_deferral_pause(
&store,
1,
"target pool 1 no longer fits the selected mutation: target pool 1 reservation requires",
"target pool 1 no longer fits the selected mutation: source pool 0 reservation estimate is exhausted",
)
.await;
});
@@ -6062,12 +6062,6 @@ mod decommission_lock_order_tests {
)
.await
.expect("read the next source object for the multipart capacity oracle");
// Reconciled bytes remain charged. With no physical room for a new
// reservation, a distinct mutation must fail instead of reusing them.
let mut exhausted_capacity = capacity_snapshot();
exhausted_capacity[2] = DecommissionPoolCapacityInfo::for_test(2, layout, 0, target_total, target_total);
set_decommission_capacity_info_overrides_for_test(other_store.id, vec![exhausted_capacity]);
let next_result = tokio::time::timeout(
Duration::from_secs(30),
data_movement::migrate_decommission_object(
@@ -7833,12 +7827,6 @@ mod decommission_lock_order_tests {
.expect("same-mutation retry should publish a readable target");
assert_eq!(retry_info.size, i64::try_from(body.len()).expect("test body size fits i64"));
// Reconciled bytes remain charged. With no physical room for a new
// reservation, a distinct mutation must fail instead of reusing them.
let mut exhausted_capacity = capacity_snapshot();
exhausted_capacity[2] = DecommissionPoolCapacityInfo::for_test(2, layout, 0, target_total, target_total);
set_decommission_capacity_info_overrides_for_test(other_store.id, vec![exhausted_capacity]);
let next_object = "data-movement-put-after-retry.bin";
let next_version_id = uuid::Uuid::new_v4().to_string();
let mut next_opts = ObjectOptions {
@@ -7853,11 +7841,7 @@ mod decommission_lock_order_tests {
let next_result = other_store
.put_object_for_data_movement(RUSTFS_META_BUCKET, next_object, &mut next_data, &next_opts, None)
.await;
let next_error = match next_result {
Err(error) | Ok((_, Err(error))) => error,
Ok((_, Ok(_))) => panic!("a different mutation must not reuse A's recovered capacity"),
};
assert!(next_error.to_string().contains("decommission_capacity_blocked"), "{next_error}");
assert!(next_result.is_err(), "a different mutation must not reuse A's recovered capacity");
let next_target_err = other_store.pools[2]
.get_object_info(
RUSTFS_META_BUCKET,
@@ -9945,12 +9929,6 @@ mod decommission_lock_order_tests {
)
.await
.expect("read the next source object for the capacity oracle");
// Reconciled bytes remain charged. With no physical room for a new
// reservation, a distinct mutation must fail instead of reusing them.
let mut exhausted_capacity = capacity_snapshot();
exhausted_capacity[2] = DecommissionPoolCapacityInfo::for_test(2, layout, 0, target_total, target_total);
set_decommission_capacity_info_overrides_for_test(other_store.id, vec![exhausted_capacity]);
let next_result = tokio::time::timeout(
Duration::from_secs(30),
data_movement::migrate_decommission_object(
@@ -10212,20 +10190,11 @@ mod decommission_lock_order_tests {
assert_eq!(reconciled_reservation.pending_target_physical_bytes, 0);
assert_eq!(reconciled_reservation.consumed_target_physical_bytes, 1);
// Reconciled bytes remain charged. With no physical room for a new
// reservation, a distinct mutation must fail instead of reusing them.
let mut exhausted_capacity = capacity_snapshot();
exhausted_capacity[2] = DecommissionPoolCapacityInfo::for_test(2, layout, 1, 2, 1);
set_decommission_capacity_info_overrides_for_test(other_store.id, vec![exhausted_capacity]);
let next_after_reconcile = other_store
.delete_object(&bucket, next_object, next_opts)
.await
.expect_err("the consumed exact-fit capacity must reject the next marker");
assert!(
next_after_reconcile.to_string().contains("decommission_capacity_blocked"),
"{next_after_reconcile}"
);
assert!(next_after_reconcile.to_string().contains("decommission_capacity_blocked"));
let next_target_err = other_store.pools[2]
.get_object_info(
&bucket,
+2 -3
View File
@@ -317,9 +317,8 @@ impl Sets {
&self.ctx
}
/// Stop endpoint reconnection and wait for any in-flight pass before a test owns disk probes.
#[cfg(any(test, feature = "test-util"))]
pub async fn stop_endpoint_monitor_for_test(&self) {
#[cfg(test)]
pub(crate) async fn stop_endpoint_monitor_for_test(&self) {
if let Some(exit_signal) = &self.exit_signal {
let _ = exit_signal.send(());
// Wait for an in-flight reconnect pass as well as its receiver to exit.
@@ -31,8 +31,6 @@ use tokio::fs;
use tokio::time::Instant;
pub(crate) const BARRIER_DIR_ENV: &str = "RUSTFS_E2E_DATA_MOVEMENT_BARRIER_DIR";
/// Rebalance holds its admission fence; no version of this entry has moved.
pub(crate) const STAGE_AFTER_ENTRY_ADMISSION: &str = "after_entry_admission";
/// Every target part is staged and the target version is not yet published.
pub(crate) const STAGE_BEFORE_TARGET_COMMIT: &str = "before_target_commit";
/// The target commit is durable and the source preflight matched; the source
+1 -17
View File
@@ -605,7 +605,6 @@ fn resolve_data_movement_abort_result(
#[derive(Debug)]
struct DataMovementStageError {
rendered: String,
stage: Option<String>,
source: Box<dyn std::error::Error + Send + Sync>,
}
@@ -627,7 +626,6 @@ where
{
Error::other(DataMovementStageError {
rendered,
stage: None,
source: Box::new(err),
})
}
@@ -636,21 +634,7 @@ fn data_movement_stage_error<E>(op_label: &str, stage: &str, bucket: &str, objec
where
E: std::error::Error + Send + Sync + 'static,
{
Error::other(DataMovementStageError {
rendered: format!("{op_label}: {stage} failed for {bucket}/{object}: {err}"),
stage: Some(stage.to_owned()),
source: Box::new(err),
})
}
pub(crate) fn data_movement_error_is_stage(err: &Error, stage: &str) -> bool {
let Error::Io(io_err) = err else {
return false;
};
io_err
.get_ref()
.and_then(|source| source.downcast_ref::<DataMovementStageError>())
.is_some_and(|source| source.stage.as_deref() == Some(stage))
data_movement_context_error(format!("{op_label}: {stage} failed for {bucket}/{object}: {err}"), err)
}
#[cfg(test)]
+22 -414
View File
@@ -34,7 +34,7 @@ pub use local_snapshot::{LocalUsageSnapshot, read_snapshot as read_local_snapsho
use rustfs_data_usage::{
BucketTargetUsageInfo, BucketUsageInfo, CompressionTotalInfo, DATA_USAGE_OBJECT_NAME, DATA_USAGE_OBSERVED_OBJECT_NAME,
DataUsageCache, DataUsageInfo, DiskUsageStatus, LEGACY_DATA_USAGE_OBJECT_NAME, SizeHistogram, VersionsHistogram,
observed_data_usage_is_newer, observed_data_usage_is_newer_than_identity,
observed_data_usage_is_newer,
};
use rustfs_io_metrics::record_system_path_failure;
use rustfs_utils::path::SLASH_SEPARATOR;
@@ -227,10 +227,7 @@ pub async fn lookup_degraded_bucket_usage_baseline(store: Arc<ECStore>, bucket:
/// endpoints for up to `DATA_USAGE_CACHE_TTL_SECS` between backend reads.
#[derive(Debug, Clone)]
struct CachedDataUsageSnapshot {
/// Last successful load, retained through failures for lineage comparison.
info: Option<DataUsageInfo>,
/// Controls serving independently from the retained ordering evidence.
last_load_failed: bool,
loaded_at: tokio::time::Instant,
/// Pre-discard per-bucket sizes from the same load, retained even when the
/// snapshot is incomplete and its bucket data is discarded. Consumed only
@@ -243,7 +240,6 @@ impl CachedDataUsageSnapshot {
fn result<T>(&self, project: impl FnOnce(&DataUsageInfo) -> T) -> Result<T, Error> {
self.info
.as_ref()
.filter(|_| !self.last_load_failed)
.map(project)
.ok_or_else(|| Error::other("data usage snapshot load recently failed"))
}
@@ -279,17 +275,6 @@ fn cache_data_usage_snapshot_result<T>(
mut degraded_baseline,
observed_unavailable,
}) => {
// A primary read can fall back to an older backup. Refresh the
// retry age without replacing a newer snapshot this process has
// already read; snapshot size itself is not an ordering signal.
if let Some(previous) = cache.as_mut()
&& let Some(cached_info) = previous.info.as_ref()
&& snapshot_predates(&info, cached_info)
{
previous.loaded_at = loaded_at;
previous.last_load_failed = false;
return Some(Ok(project(cached_info)));
}
// A flaky observed read must not shrink quota coverage for a TTL
// window: carry the previous refresh's baseline entries forward,
// letting the fresh (authoritative) values win where they exist.
@@ -301,23 +286,21 @@ fn cache_data_usage_snapshot_result<T>(
let projected = project(&info);
*cache = Some(CachedDataUsageSnapshot {
info: Some(info),
last_load_failed: false,
loaded_at,
degraded_baseline,
});
Ok(projected)
}
Err(e) => {
// A failed load is negatively cached, but cannot erase the last
// good snapshot's ordering proof or grounded quota baseline.
let cached = cache.get_or_insert_with(|| CachedDataUsageSnapshot {
// Keep the previous baseline through a failed refresh: quota
// admission must not lose its last grounded values because one
// backend read errored.
let degraded_baseline = cache.take().map(|cached| cached.degraded_baseline).unwrap_or_default();
*cache = Some(CachedDataUsageSnapshot {
info: None,
last_load_failed: true,
loaded_at,
degraded_baseline: HashMap::new(),
degraded_baseline,
});
cached.loaded_at = loaded_at;
cached.last_load_failed = true;
Err(e)
}
})
@@ -1262,7 +1245,7 @@ async fn load_data_usage_from_backend_with_baseline(store: Arc<ECStore>) -> Resu
let (loaded_snapshot, source) = load_data_usage_snapshot(store.clone()).await?;
// The observed-newness gate below compares against the snapshot as
// persisted, before normalization demotes or discards anything.
let authoritative_as_persisted = loaded_snapshot.snapshot_identity();
let authoritative_as_persisted = loaded_snapshot.clone();
let (info, mut degraded_baseline) = normalize_loaded_data_usage(loaded_snapshot, source.is_authoritative()).await;
// A bucket without a converged scanner cycle behind it — a freshly joined
@@ -1280,7 +1263,7 @@ async fn load_data_usage_from_backend_with_baseline(store: Arc<ECStore>) -> Resu
// read per snapshot refresh.
let mut observed_unavailable = false;
match load_observed_data_usage_snapshot(store).await {
Ok(Some(observed)) if observed_data_usage_is_newer_than_identity(&observed, authoritative_as_persisted) => {
Ok(Some(observed)) if observed_data_usage_is_newer(&observed, &authoritative_as_persisted) => {
backfill_degraded_baseline_from_observed(&mut degraded_baseline, &observed);
}
Ok(_) => {}
@@ -2384,9 +2367,6 @@ fn unknown_bucket_reload_due(cache: &Option<CachedDataUsageSnapshot>, bucket: &s
let Some(cached) = cache.as_ref() else {
return false;
};
if cached.last_load_failed {
return false;
}
let Some(info) = cached.info.as_ref() else {
return false;
};
@@ -2402,35 +2382,11 @@ fn unknown_bucket_reload_due(cache: &Option<CachedDataUsageSnapshot>, bucket: &s
.is_none_or(|attempted_at| now.duration_since(attempted_at) >= refresh_age)
}
/// Compare authoritative lineage by the writer's epoch, cycle, then timestamp
/// order. Within one lineage, compare the observation's coordinated position
/// before its timestamp; a later observation cannot revive an older lineage.
fn snapshot_predates(loaded: &DataUsageInfo, cached: &DataUsageInfo) -> bool {
let loaded_baseline = loaded
.usage_snapshot_authoritative_baseline
.unwrap_or_else(|| loaded.snapshot_identity());
let cached_baseline = cached
.usage_snapshot_authoritative_baseline
.unwrap_or_else(|| cached.snapshot_identity());
let loaded_position = (loaded_baseline.scanner_epoch, loaded_baseline.scanner_cycle);
let cached_position = (cached_baseline.scanner_epoch, cached_baseline.scanner_cycle);
if loaded_position != cached_position {
return loaded_position < cached_position;
}
let (loaded, cached) = if loaded_baseline == cached_baseline {
let loaded_observation_position = loaded.scanner_epoch.zip(loaded.scanner_cycle);
let cached_observation_position = cached.scanner_epoch.zip(cached.scanner_cycle);
if loaded_observation_position != cached_observation_position {
return loaded_observation_position < cached_observation_position;
}
(loaded.last_update, cached.last_update)
} else {
(loaded_baseline.last_update, cached_baseline.last_update)
};
/// Whether a freshly loaded snapshot is older than the cached one, as when a
/// primary read fails and the loader falls back to the previous `.bkp` copy.
fn snapshot_predates(loaded: Option<SystemTime>, cached: Option<SystemTime>) -> bool {
match (loaded, cached) {
(Some(loaded), Some(cached)) => {
loaded < cached && !rustfs_data_usage::usage_last_update_is_untrusted_future(cached, SystemTime::now())
}
(Some(loaded), Some(cached)) => loaded < cached,
(None, Some(_)) => true,
_ => false,
}
@@ -2471,15 +2427,22 @@ where
return false;
};
let mut cache = data_usage_snapshot_cache().write().await;
let cached_last_update = cache
.as_ref()
.and_then(|cached| cached.info.as_ref())
.and_then(|info| info.last_update);
if snapshot_predates(loaded.info.last_update, cached_last_update) {
return false;
}
cache_data_usage_snapshot_result(
&mut cache,
Ok(loaded),
tokio::time::Instant::now(),
refresh_generation,
data_usage_snapshot_generation(),
|info| info.buckets_usage.contains_key(bucket),
|_| (),
)
.is_some_and(|result| matches!(result, Ok(true)))
.is_some()
}
// Calls of `reload_unknown_bucket_usage` per bucket. Unit tests do not run in
@@ -4261,7 +4224,6 @@ mod tests {
let degraded_baseline = info.bucket_sizes.clone();
*data_usage_snapshot_cache().write().await = Some(CachedDataUsageSnapshot {
info: Some(info),
last_load_failed: false,
loaded_at: tokio::time::Instant::now()
.checked_sub(age)
.expect("the monotonic clock must reach back by the requested age"),
@@ -4738,355 +4700,6 @@ mod tests {
assert_snapshot_bucket(&cached, "bucket");
}
#[tokio::test]
#[serial]
async fn expired_usage_snapshot_refresh_keeps_newer_counts_over_an_older_backup() {
use crate::bucket::metadata_sys::test_support::isolated_store_over_temp_disks;
use crate::config::com::save_config;
let (_dirs, store) = isolated_store_over_temp_disks().await;
let mut current = usage_snapshot_with_new_bucket(SystemTime::UNIX_EPOCH + Duration::from_secs(2), 100);
current.scanner_epoch = Some(4);
current.scanner_cycle = Some(11);
current.usage_snapshot_converged = Some(true);
let mut backup = data_usage_info_for_test("existing", 1, 5, SystemTime::UNIX_EPOCH + Duration::from_secs(1));
backup.scanner_epoch = Some(4);
backup.scanner_cycle = Some(10);
backup.usage_snapshot_converged = Some(true);
for (path, info) in [
(DATA_USAGE_OBJ_NAME_PATH.as_str(), &current),
(DATA_USAGE_OBJ_BACKUP_PATH.as_str(), &backup),
] {
save_config(store.clone(), path, serde_json::to_vec(info).expect("encode usage snapshot"))
.await
.expect("persist initial usage snapshot");
}
invalidate_data_usage_snapshot_cache().await;
for loaded in [
load_data_usage_from_backend_cached(store.clone()).await,
load_admin_data_usage_from_backend_cached(store.clone()).await,
] {
assert_eq!(loaded.expect("prime usage cache"), current);
}
// Bucket cleanup can reduce usage without advancing the scanner
// position. Only its publication timestamp advances.
let mut after_delete = data_usage_info_for_test("existing", 1, 3, SystemTime::UNIX_EPOCH + Duration::from_secs(3));
after_delete.scanner_epoch = current.scanner_epoch;
after_delete.scanner_cycle = current.scanner_cycle;
after_delete.usage_snapshot_converged = Some(true);
let mut stale_observation =
data_usage_info_for_test("retired-incarnation", 99, 999, SystemTime::UNIX_EPOCH + Duration::from_secs(4));
stale_observation.scanner_epoch = Some(4);
stale_observation.scanner_cycle = Some(12);
stale_observation.usage_snapshot_converged = Some(false);
stale_observation.usage_snapshot_authoritative_baseline = Some(backup.snapshot_identity());
assert!(observed_data_usage_is_newer(&stale_observation, &backup));
assert!(!observed_data_usage_is_newer(&stale_observation, &current));
save_config(
store.clone(),
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
serde_json::to_vec(&stale_observation).expect("encode observation of an older namespace"),
)
.await
.expect("persist an observation anchored to the old backup");
for (payload, expected) in [
(b"not a usage snapshot".to_vec(), &current),
(serde_json::to_vec(&after_delete).expect("encode post-delete usage"), &after_delete),
] {
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), payload)
.await
.expect("replace primary usage snapshot");
for cache in [data_usage_snapshot_cache(), admin_data_usage_snapshot_cache()] {
cache
.write()
.await
.as_mut()
.expect("usage cache should be populated")
.loaded_at = tokio::time::Instant::now() - Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
}
for loaded in [
load_data_usage_from_backend_cached(store.clone()).await,
load_admin_data_usage_from_backend_cached(store.clone()).await,
] {
assert_eq!(loaded.expect("refresh an expired usage cache"), *expected);
}
assert!(
!data_usage_snapshot_cache()
.read()
.await
.as_ref()
.expect("quota cache should remain populated")
.degraded_baseline
.contains_key("retired-incarnation"),
"a newer observation anchored to an older namespace must not backfill ghost usage"
);
}
invalidate_data_usage_snapshot_cache().await;
}
#[tokio::test]
#[serial]
async fn expired_admin_usage_refresh_orders_observations_by_generation_before_time() {
use crate::bucket::metadata_sys::test_support::isolated_store_over_temp_disks;
use crate::config::com::save_config;
let (_dirs, store) = isolated_store_over_temp_disks().await;
let mut authoritative = data_usage_info_for_test("bucket", 1, 10, SystemTime::UNIX_EPOCH + Duration::from_secs(100));
authoritative.scanner_epoch = Some(7);
authoritative.scanner_cycle = Some(10);
authoritative.usage_snapshot_converged = Some(true);
save_config(
store.clone(),
DATA_USAGE_OBJ_NAME_PATH.as_str(),
serde_json::to_vec(&authoritative).expect("encode authoritative usage"),
)
.await
.expect("persist the common authoritative baseline");
let observation = |epoch, cycle, seconds, objects, size| {
let mut info =
data_usage_info_for_test("bucket", objects, size, SystemTime::UNIX_EPOCH + Duration::from_secs(seconds));
info.scanner_epoch = Some(epoch);
info.scanner_cycle = Some(cycle);
info.usage_snapshot_converged = Some(false);
info.usage_snapshot_authoritative_baseline = Some(authoritative.snapshot_identity());
info
};
let first = observation(7, 11, 200, 11, 1_100);
let newer_cycle = observation(7, 12, 150, 12, 1_200);
let older_cycle = observation(7, 11, 300, 99, 9_900);
let newer_epoch = observation(8, 1, 50, 1, 100);
invalidate_data_usage_snapshot_cache().await;
for (observed, expected) in [
(&first, &first),
(&newer_cycle, &newer_cycle),
(&older_cycle, &newer_cycle),
(&newer_epoch, &newer_epoch),
] {
assert!(
observed_data_usage_is_newer(observed, &authoritative),
"the persisted observation must pass the real loader's selection gate"
);
save_config(
store.clone(),
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
serde_json::to_vec(observed).expect("encode scanner observation"),
)
.await
.expect("persist scanner observation");
if let Some(cached) = admin_data_usage_snapshot_cache().write().await.as_mut() {
cached.loaded_at = tokio::time::Instant::now() - Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
}
let loaded = load_admin_data_usage_from_backend_cached(store.clone())
.await
.expect("refresh admin usage through the real backend loader");
assert_eq!(
&loaded,
expected,
"same-baseline observation ordering must follow epoch/cycle before wall clock: incoming={:?}",
observed.snapshot_identity()
);
}
invalidate_data_usage_snapshot_cache().await;
}
#[test]
fn expired_usage_snapshot_refresh_accepts_new_authority_over_a_later_old_observation() {
let loaded_at = tokio::time::Instant::now();
let baseline = data_usage_info_for_test("bucket", 3, 42, SystemTime::UNIX_EPOCH + Duration::from_secs(1));
let mut observed = data_usage_info_for_test("bucket", 5, 60, SystemTime::UNIX_EPOCH + Duration::from_secs(4));
observed.usage_snapshot_converged = Some(false);
observed.usage_snapshot_authoritative_baseline = Some(baseline.snapshot_identity());
let mut cache = Some(CachedDataUsageSnapshot {
info: Some(observed),
last_load_failed: false,
loaded_at,
degraded_baseline: HashMap::new(),
});
let after_delete = data_usage_info_for_test("bucket", 1, 7, SystemTime::UNIX_EPOCH + Duration::from_secs(3));
let refreshed = cache_data_usage_snapshot_result(
&mut cache,
Ok(loaded_usage_for_test(after_delete.clone())),
loaded_at,
1,
1,
DataUsageInfo::clone,
)
.expect("matching refresh generation")
.expect("a newer authority must retire observations of its predecessor");
assert_eq!(refreshed, after_delete);
}
#[test]
fn expired_usage_snapshot_refresh_orders_lineage_by_epoch_cycle_then_time() {
let loaded_at = tokio::time::Instant::now();
for (cached_position, loaded_position, loaded_time, accept) in [
((Some(2), Some(10)), (Some(1), Some(99)), 110, false),
((Some(2), Some(10)), (Some(2), Some(9)), 110, false),
((Some(2), Some(10)), (Some(3), Some(1)), 90, true),
((Some(2), Some(10)), (Some(2), Some(11)), 90, true),
((Some(2), Some(10)), (Some(2), Some(10)), 110, true),
((Some(3), None), (Some(2), Some(10)), 110, false),
((Some(2), Some(10)), (Some(3), None), 90, true),
((None, None), (None, None), 110, true),
((None, None), (None, None), 90, false),
] {
let mut previous = data_usage_info_for_test("bucket", 3, 42, SystemTime::UNIX_EPOCH + Duration::from_secs(100));
(previous.scanner_epoch, previous.scanner_cycle) = cached_position;
let mut loaded = data_usage_info_for_test("bucket", 1, 7, SystemTime::UNIX_EPOCH + Duration::from_secs(loaded_time));
(loaded.scanner_epoch, loaded.scanner_cycle) = loaded_position;
let expected = if accept { &loaded } else { &previous };
let mut cache = Some(CachedDataUsageSnapshot {
info: Some(previous.clone()),
last_load_failed: false,
loaded_at,
degraded_baseline: HashMap::new(),
});
let refreshed = cache_data_usage_snapshot_result(
&mut cache,
Ok(loaded_usage_for_test(loaded.clone())),
loaded_at,
1,
1,
DataUsageInfo::clone,
)
.expect("matching refresh generation")
.expect("a valid usage snapshot remains available");
assert_eq!(
&refreshed, expected,
"lineage ordering must match scanner publication: cached={cached_position:?}, loaded={loaded_position:?}, loaded_time={loaded_time}"
);
}
}
#[test]
fn expired_usage_snapshot_refresh_preserves_rejection_backoff_and_baseline() {
let loaded_at = tokio::time::Instant::now();
let expected = data_usage_info_for_test("bucket", 3, 42, SystemTime::UNIX_EPOCH + Duration::from_secs(2));
let mut cache = Some(CachedDataUsageSnapshot {
info: Some(expected.clone()),
last_load_failed: false,
loaded_at,
degraded_baseline: HashMap::from([("bucket".to_string(), 42)]),
});
let refreshed_at = loaded_at + Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
let refreshed = cache_data_usage_snapshot_result(
&mut cache,
Ok(loaded_usage_for_test(data_usage_info_for_test("bucket", 1, 7, SystemTime::UNIX_EPOCH))),
refreshed_at,
1,
1,
DataUsageInfo::clone,
)
.expect("matching refresh generation")
.expect("the newer cached snapshot remains available");
assert_eq!(refreshed, expected);
let retained = cache.as_ref().expect("cache should remain populated");
assert_eq!(retained.degraded_baseline.get("bucket"), Some(&42));
assert_eq!(retained.loaded_at, refreshed_at, "an older fallback must retain the TTL backoff");
let result =
cache_data_usage_snapshot_result(&mut cache, Err(Error::ErasureReadQuorum), refreshed_at, 1, 1, DataUsageInfo::clone)
.expect("matching refresh generation");
assert!(matches!(result, Err(Error::ErasureReadQuorum)));
let failed = cache.as_ref().expect("negative cache should be populated");
assert!(failed.last_load_failed);
assert_eq!(failed.info.as_ref(), Some(&expected), "failure must retain the last good ordering proof");
}
#[test]
fn expired_usage_snapshot_refresh_accepts_clock_recovery() {
let loaded_at = tokio::time::Instant::now();
let now = SystemTime::now();
let future = now + rustfs_data_usage::USAGE_LAST_UPDATE_FUTURE_TOLERANCE + Duration::from_secs(60);
let mut cache = Some(CachedDataUsageSnapshot {
info: Some(data_usage_info_for_test("bucket", 3, 42, future)),
last_load_failed: false,
loaded_at,
degraded_baseline: HashMap::new(),
});
let recovered = data_usage_info_for_test("bucket", 1, 7, now);
let refreshed = cache_data_usage_snapshot_result(
&mut cache,
Ok(loaded_usage_for_test(recovered.clone())),
loaded_at,
1,
1,
DataUsageInfo::clone,
)
.expect("matching refresh generation")
.expect("a clock recovery snapshot must not be frozen behind an untrusted timestamp");
assert_eq!(refreshed, recovered);
}
#[test]
fn expired_usage_snapshot_refresh_keeps_last_good_lineage_across_a_failed_load() {
let loaded_at = tokio::time::Instant::now();
let current = data_usage_info_for_test("bucket", 3, 42, SystemTime::UNIX_EPOCH + Duration::from_secs(2));
let mut cache = None;
let first = cache_data_usage_snapshot_result(
&mut cache,
Ok(loaded_usage_for_test(current.clone())),
loaded_at,
1,
1,
DataUsageInfo::clone,
)
.expect("matching refresh generation")
.expect("prime the last successful snapshot");
assert_eq!(first, current);
let failed_at = loaded_at + Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
let failure =
cache_data_usage_snapshot_result(&mut cache, Err(Error::ErasureReadQuorum), failed_at, 1, 1, DataUsageInfo::clone)
.expect("matching refresh generation");
assert!(matches!(failure, Err(Error::ErasureReadQuorum)));
assert!(
fresh_cached_data_usage_snapshot(
&cache,
failed_at + Duration::from_secs(DATA_USAGE_UNKNOWN_BUCKET_REFRESH_SECS),
Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS),
|_| panic!("a failed load must not serve its retained snapshot as a success"),
)
.expect("the failed load remains inside its negative TTL")
.is_err()
);
assert!(
!unknown_bucket_reload_due(
&cache,
"unknown",
failed_at + Duration::from_secs(DATA_USAGE_UNKNOWN_BUCKET_REFRESH_SECS),
),
"an unknown bucket must not bypass the failed load's TTL"
);
let recovered_at = failed_at + Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
let recovered = cache_data_usage_snapshot_result(
&mut cache,
Ok(loaded_usage_for_test(data_usage_info_for_test("bucket", 1, 7, SystemTime::UNIX_EPOCH))),
recovered_at,
1,
1,
DataUsageInfo::clone,
)
.expect("matching refresh generation")
.expect("a readable older backup must preserve the last good lineage");
assert_eq!(recovered, current);
assert_eq!(
cache.as_ref().expect("recovered cache").degraded_baseline.get("bucket"),
Some(&42),
"a failed load must not erase the baseline's ordering proof"
);
assert_eq!(
fresh_cached_data_usage_snapshot(&cache, recovered_at, Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS), |info| info
.objects_total_size,)
.expect("the recovered load starts a fresh TTL")
.expect("the last successful snapshot is available again"),
42
);
}
#[test]
#[serial]
fn unavailable_observed_read_keeps_previous_baseline_coverage() {
@@ -5094,7 +4707,6 @@ mod tests {
let refresh_generation = data_usage_snapshot_generation();
let mut cache = Some(CachedDataUsageSnapshot {
info: Some(data_usage_info_for_test("bucket", 1, 42, SystemTime::UNIX_EPOCH)),
last_load_failed: false,
loaded_at,
degraded_baseline: HashMap::from([("observed-only".to_string(), 7_u64), ("covered".to_string(), 1)]),
});
@@ -5128,7 +4740,6 @@ mod tests {
let refresh_generation = data_usage_snapshot_generation();
let mut cache = Some(CachedDataUsageSnapshot {
info: Some(data_usage_info_for_test("stale", 1, 42, SystemTime::UNIX_EPOCH)),
last_load_failed: false,
loaded_at,
degraded_baseline: HashMap::new(),
});
@@ -6317,7 +5928,6 @@ mod tests {
let mut snapshot_cache = data_usage_snapshot_cache().write().await;
*snapshot_cache = Some(CachedDataUsageSnapshot {
info: Some(successor),
last_load_failed: false,
loaded_at: tokio::time::Instant::now(),
degraded_baseline: HashMap::new(),
});
@@ -6389,7 +5999,6 @@ mod tests {
let mut snapshot_cache = data_usage_snapshot_cache().write().await;
*snapshot_cache = Some(CachedDataUsageSnapshot {
info: Some(successor),
last_load_failed: false,
loaded_at: tokio::time::Instant::now(),
degraded_baseline: HashMap::new(),
});
@@ -6442,7 +6051,6 @@ mod tests {
let stale = data_usage_info_for_test(BUCKET, 7, 294, SystemTime::now());
*data_usage_snapshot_cache().write().await = Some(CachedDataUsageSnapshot {
info: Some(stale),
last_load_failed: false,
loaded_at: tokio::time::Instant::now(),
degraded_baseline: HashMap::new(),
});
@@ -21,7 +21,6 @@ fn scalar_projection_borrows_snapshot_and_preserves_unknown_and_zero() {
info.buckets_usage.insert("empty".to_string(), BucketUsageInfo::default());
let cache = Some(CachedDataUsageSnapshot {
info: Some(info),
last_load_failed: false,
loaded_at,
degraded_baseline: HashMap::new(),
});
@@ -67,7 +66,6 @@ fn projection_is_skipped_for_missing_failed_expired_or_invalidated_snapshots() {
cache = Some(CachedDataUsageSnapshot {
info: Some(data_usage_info_for_test("known", 1, 42, SystemTime::UNIX_EPOCH)),
last_load_failed: false,
loaded_at,
degraded_baseline: HashMap::new(),
});
@@ -129,33 +127,6 @@ fn projected_publication_moves_bucket_maps_into_cache() {
cached.buckets_usage.get_key_value("known").expect("cached bucket").0.as_ptr(),
original_key
);
let failure = cache_data_usage_snapshot_result(
&mut cache,
Err(Error::ErasureReadQuorum),
tokio::time::Instant::now(),
1,
1,
|_: &DataUsageInfo| -> u64 { panic!("a failed load must not project its retained snapshot") },
)
.expect("matching refresh generation");
assert!(matches!(failure, Err(Error::ErasureReadQuorum)));
let retained = cache
.as_ref()
.expect("negative cache")
.info
.as_ref()
.expect("last good ordering proof");
assert_eq!(
retained
.buckets_usage
.get_key_value("known")
.expect("retained bucket")
.0
.as_ptr(),
original_key,
"a failed load must retain the snapshot without copying its bucket maps"
);
}
#[test]
@@ -682,11 +682,15 @@ async fn get_pools_info(all_disks: &[Disk]) -> Result<HashMap<i32, HashMap<i32,
return Err(Error::other("ServerNotInitialized"));
};
collect_pools_info(all_disks, |pool_index, set_index| {
let store = store.clone();
async move {
let mut pools_info: HashMap<i32, HashMap<i32, ErasureSetInfo>> = HashMap::new();
for d in all_disks {
let pool_info = pools_info.entry(d.pool_index).or_default();
let erasure_set = pool_info.entry(d.set_index).or_default();
if erasure_set.id == 0 {
erasure_set.id = d.set_index;
let load = load_data_usage_cache(
store.pools[pool_index as usize].disk_set[set_index as usize].as_ref(),
store.pools[d.pool_index as usize].disk_set[d.set_index as usize].as_ref(),
DATA_USAGE_CACHE_NAME,
);
#[cfg(feature = "diagnostic-read-version-caller")]
@@ -694,34 +698,11 @@ async fn get_pools_info(all_disks: &[Disk]) -> Result<HashMap<i32, HashMap<i32,
rustfs_io_metrics::internode_metrics::ReadVersionEntrySource::ServerInfoPools,
load,
);
load.await
}
})
.await
}
async fn collect_pools_info<F, Fut>(all_disks: &[Disk], mut load_usage: F) -> Result<HashMap<i32, HashMap<i32, ErasureSetInfo>>>
where
F: FnMut(i32, i32) -> Fut,
Fut: std::future::Future<Output = Result<rustfs_data_usage::DataUsageCache>>,
{
let mut pools_info: HashMap<i32, HashMap<i32, ErasureSetInfo>> = HashMap::new();
for d in all_disks {
let pool_info = pools_info.entry(d.pool_index).or_default();
let erasure_set = match pool_info.entry(d.set_index) {
std::collections::hash_map::Entry::Occupied(entry) => entry.into_mut(),
std::collections::hash_map::Entry::Vacant(entry) => {
let erasure_set = entry.insert(ErasureSetInfo {
id: d.set_index,
..Default::default()
});
match load_usage(d.pool_index, d.set_index).await {
Ok(cache) => apply_erasure_set_usage(&cache, erasure_set),
Err(_) => erasure_set.usage_error = Some(DATA_USAGE_UNAVAILABLE_ERROR.to_string()),
}
erasure_set
match load.await {
Ok(cache) => apply_erasure_set_usage(&cache, erasure_set),
Err(_) => erasure_set.usage_error = Some(DATA_USAGE_UNAVAILABLE_ERROR.to_string()),
}
};
}
erasure_set.raw_capacity += d.total_space;
erasure_set.raw_usage += d.used_space;
@@ -1260,59 +1241,6 @@ mod tests {
assert_eq!(info.deployment_id, expected_deployment_id);
}
#[tokio::test]
async fn pool_usage_is_loaded_once_per_set_including_set_zero() {
let mut disks = Vec::new();
for (pool, set, count) in [(0, 0, 16), (1, 0, 16), (1, 1, 4)] {
disks.extend((0..count).map(|_| Disk {
pool_index: pool,
set_index: set,
used_space: 10,
total_space: 100,
..Default::default()
}));
}
let mut loads = Vec::new();
let info = super::collect_pools_info(&disks, |pool, set| {
loads.push((pool, set));
async move {
if pool == 1 && set == 0 {
return Err(crate::error::Error::other("usage unavailable"));
}
let mut cache = rustfs_data_usage::DataUsageCache::default();
cache.info.name = DATA_USAGE_ROOT.to_owned();
cache.info.last_update = Some(std::time::SystemTime::UNIX_EPOCH);
cache.info.snapshot_complete = true;
cache.replace(
DATA_USAGE_ROOT,
"",
rustfs_data_usage::DataUsageEntry {
size: 25,
objects: 2,
..Default::default()
},
);
Ok(cache)
}
})
.await
.expect("collect mixed successful and unavailable usage");
assert_eq!(loads, [(0, 0), (1, 0), (1, 1)]);
for (pool, set, count) in [(0, 0, 16), (1, 0, 16), (1, 1, 4)] {
let set_info = &info[&pool][&set];
assert_eq!(set_info.id, set);
assert_eq!(set_info.raw_usage, count * 10);
assert_eq!(set_info.raw_capacity, count * 100);
if pool == 1 && set == 0 {
assert_eq!(set_info.usage_error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
} else {
assert_eq!(set_info.usage, 25, "logical usage must not multiply by drive count");
assert_eq!(set_info.objects_count, 2);
assert!(set_info.usage_error.is_none());
}
}
}
#[serial]
#[tokio::test]
async fn local_server_property_includes_runtime_stats_without_endpoint_pools() {
@@ -167,16 +167,6 @@ impl ObjectLockAttempt {
}
}
}
/// Correlate a raw guard's acquisition with its peer identities without
/// claiming ownership of the guard's lifetime. Peer release events retain
/// the identities needed to reconstruct when the actual owner releases it.
pub(crate) fn observe_acquisition(&mut self, result: &Result<NamespaceLockGuard, LockError>) {
self.observe(result);
if let Some(context) = self.0.as_mut() {
context.finished = true;
}
}
}
fn duration_ms(duration: Duration) -> u64 {
@@ -510,45 +500,6 @@ pub(crate) mod tests {
)
}
#[test]
#[serial_test::serial]
fn raw_guard_acquisition_diagnostic_does_not_claim_guard_lifetime() {
temp_env::with_vars([(rustfs_config::ENV_OBJECT_LOCK_DIAG_ENABLE, Some("true"))], || {
let capture = CapturedLockEvents::default();
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("raw guard diagnostic runtime");
tracing::subscriber::with_default(capture.subscriber(), || {
runtime.block_on(async {
let lock = namespace();
let mut attempt = ObjectLockAttempt::start(
"bucket_incarnation_disk_read",
"bucket",
"transaction.lock",
None,
&lock,
"read",
Duration::from_secs(1),
);
let result = lock.get_read_lock(Duration::from_secs(1)).await;
attempt.observe_acquisition(&result);
drop(attempt);
let guard = result.expect("retain the raw guard after diagnostic completion");
assert!(!guard.is_lock_lost());
let states = capture
.rows()
.iter()
.map(|row| row["fields"]["state"].as_str().expect("diagnostic state string").to_owned())
.collect::<Vec<_>>();
assert_eq!(states, ["acquiring", "acquired"]);
drop(guard);
assert_eq!(capture.rows().len(), 2, "the acquisition observer must not invent a guard release");
});
});
});
}
#[test]
#[serial_test::serial]
fn object_lock_diagnostics_disabled_does_not_allocate_context_or_emit() {
+3 -205
View File
@@ -55,20 +55,6 @@ use tokio_util::sync::CancellationToken;
use tracing::{info, warn};
use uuid::Uuid;
/// Join the local disk's actual bootstrap worker for deterministic coverage tests.
/// Completion may mean legacy fallback after a failed scan; callers must assert
/// the expected index result separately. Remote disks have no local worker.
#[cfg(any(test, feature = "test-util"))]
pub async fn wait_for_multipart_index_bootstrap(disk: &crate::disk::DiskStore) -> Result<()> {
match disk.as_ref() {
crate::disk::Disk::Local(local) => {
local.disk.wait_for_multipart_index_bootstrap().await;
Ok(())
}
crate::disk::Disk::Remote(_) => Err(DiskError::MethodNotAllowed),
}
}
/// Disk health status constants
const DISK_HEALTH_OK: u32 = 0;
const DISK_HEALTH_FAULTY: u32 = 1;
@@ -867,8 +853,6 @@ pub(crate) struct DiskHealthMetricEpoch {
fallback_operation_metrics: StdRwLock<HashMap<&'static str, Arc<DiskOperationMetrics>>>,
#[cfg(test)]
multipart_cleanup_observer: std::sync::Mutex<Option<std::sync::Weak<ScopedMultipartCleanupCompletions>>>,
#[cfg(test)]
operation_timeout_override: std::sync::Mutex<Option<Duration>>,
/// Caller API operations currently executing through the disk health wrapper.
api_waiting: AtomicU32,
/// Operations rejected because the disk was unavailable for the caller.
@@ -896,8 +880,6 @@ impl Default for DiskHealthMetricEpoch {
fallback_operation_metrics: StdRwLock::new(HashMap::new()),
#[cfg(test)]
multipart_cleanup_observer: std::sync::Mutex::new(None),
#[cfg(test)]
operation_timeout_override: std::sync::Mutex::new(None),
api_waiting: AtomicU32::new(0),
total_errors_availability: AtomicU64::new(0),
total_errors_timeout: AtomicU64::new(0),
@@ -1591,20 +1573,6 @@ impl Default for DiskHealthTracker {
}
}
#[cfg(test)]
#[derive(Debug)]
pub(crate) struct ScopedDiskOperationTimeout {
metrics: Arc<DiskHealthMetricEpoch>,
previous: Option<Duration>,
}
#[cfg(test)]
impl Drop for ScopedDiskOperationTimeout {
fn drop(&mut self) {
*self.metrics.operation_timeout_override.lock().expect("test timeout lock") = self.previous;
}
}
/// LocalDiskWrapper wraps a DiskStore with health tracking capabilities.
/// This is similar to Go's xlStorageDiskIDCheck.
#[derive(Debug, Clone)]
@@ -1946,23 +1914,6 @@ impl LocalDiskWrapper {
self.health_check
}
#[cfg(test)]
pub(crate) fn override_operation_timeout_for_test(&self, timeout: Duration) -> ScopedDiskOperationTimeout {
assert!(!timeout.is_zero(), "fault injection needs a positive timeout");
// The metrics epoch follows this drive across clones and reconnects.
// Its scoped override cannot shorten unrelated healthy drives' I/O.
let previous = self
.metrics
.operation_timeout_override
.lock()
.expect("test timeout lock")
.replace(timeout);
ScopedDiskOperationTimeout {
metrics: Arc::clone(&self.metrics),
previous,
}
}
/// Enable health monitoring after disk creation.
/// Used to defer health checks until after startup format loading completes.
pub fn enable_health_check(&self) {
@@ -2447,17 +2398,12 @@ impl LocalDiskWrapper {
.await
}
pub(crate) fn has_clean_active_multipart_source(&self, upload_or_part_path: &str, part_path: bool) -> bool {
self.disk.has_clean_active_multipart_source(upload_or_part_path, part_path)
}
pub(crate) async fn try_publish_prepared_delete_local(
&self,
volume: &str,
object: &str,
transaction: uuid::Uuid,
namespace_owner: Option<Arc<dyn Send + Sync>>,
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
) -> Result<Option<(crate::disk::DeleteRecoveryJournal, bool, bool)>> {
self.track_disk_health_mutation(
"settle_delete_transaction",
@@ -2465,7 +2411,7 @@ impl LocalDiskWrapper {
|| {
Box::pin(async {
self.disk
.try_publish_prepared_delete_local_with_budget(volume, object, transaction, namespace_owner, work_budget)
.try_publish_prepared_delete_local(volume, object, transaction, namespace_owner)
.await
})
},
@@ -2481,7 +2427,6 @@ impl LocalDiskWrapper {
transaction: uuid::Uuid,
journal: &crate::disk::DeleteRecoveryJournal,
namespace_owner: Option<Arc<dyn Send + Sync>>,
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
) -> Result<bool> {
self.track_disk_health_mutation(
"settle_delete_transaction",
@@ -2489,14 +2434,7 @@ impl LocalDiskWrapper {
|| {
Box::pin(async {
self.disk
.try_finish_and_finalize_prepared_delete_local_with_budget(
volume,
object,
transaction,
journal,
namespace_owner,
work_budget,
)
.try_finish_and_finalize_prepared_delete_local(volume, object, transaction, journal, namespace_owner)
.await
})
},
@@ -2696,13 +2634,6 @@ impl LocalDiskWrapper {
}
return result;
}
#[cfg(test)]
let timeout_duration = self
.metrics
.operation_timeout_override
.lock()
.expect("test timeout lock")
.map_or(timeout_duration, |injected| timeout_duration.min(injected));
// Execute the operation with timeout
let result = tokio::time::timeout(timeout_duration, operation).await;
@@ -2751,18 +2682,6 @@ impl LocalDiskWrapper {
#[async_trait::async_trait]
impl DiskAPI for LocalDiskWrapper {
async fn list_multipart_index_roots(
&self,
bucket: &str,
marker: Option<&str>,
) -> Result<crate::disk::multipart_index::MultipartIndexPage> {
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
}
self.check_disk_stale().await?;
self.disk.list_multipart_index_roots(bucket, marker).await
}
async fn supports_guarded_delete_versions(&self) -> Result<()> {
// The local implementation shares this binary's capability. Availability
// and disk identity remain mutation checks and write-quorum inputs.
@@ -2816,62 +2735,6 @@ impl DiskAPI for LocalDiskWrapper {
)
.await
}
async fn discover_delete_transactions_bounded(
&self,
volume: &str,
object: &str,
max_entries: u32,
) -> Result<crate::disk::BoundedDeleteRecoveryDiscovery> {
self.track_disk_health_with_op(
"discover_delete_transactions",
|| async {
self.disk
.discover_delete_transactions_bounded(volume, object, max_entries)
.await
},
get_max_timeout_duration(),
)
.await
}
async fn inspect_delete_transaction_bounded(
&self,
volume: &str,
object: &str,
transaction: uuid::Uuid,
max_entries: u32,
) -> Result<crate::disk::BoundedDeleteRecoveryInspection> {
self.track_disk_health_with_op(
"inspect_delete_transaction",
|| async {
self.disk
.inspect_delete_transaction_bounded(volume, object, transaction, max_entries)
.await
},
get_max_timeout_duration(),
)
.await
}
async fn settle_delete_transaction_bounded(
&self,
volume: &str,
request: crate::disk::BoundedDeleteRecoverySettlement,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> Result<crate::disk::BoundedDeleteRecoverySettlementReply> {
self.track_disk_health_mutation(
"settle_delete_transaction",
DiskMetricMutation::Delete,
|| async {
self.disk
.settle_delete_transaction_bounded(volume, request, namespace_owner)
.await
},
crate::disk::get_delete_recovery_timeout()?,
)
.await
}
async fn discover_delete_transactions(
&self,
volume: &str,
@@ -3840,11 +3703,7 @@ mod tests {
assert!(futures::poll!(&mut operation).is_pending());
time::advance(SLOW_DISK_OPERATION_THRESHOLD - Duration::from_millis(1)).await;
assert!(futures::poll!(&mut operation).is_pending());
assert!(
!logs.contents().contains("disk_operation_slow") && !logs.contents().contains("WARN"),
"normal disk latency should not warn: {}",
logs.contents()
);
assert!(logs.contents().is_empty(), "normal disk latency should not warn");
time::advance(Duration::from_millis(1)).await;
assert!(futures::poll!(&mut operation).is_pending());
let warning = logs.contents();
@@ -6181,67 +6040,6 @@ mod tests {
assert!(health.offline_duration().is_none());
}
#[tokio::test]
async fn scoped_operation_timeout_is_per_disk_and_preserves_caller_policy() {
let directories = [
tempfile::tempdir().expect("faulted drive"),
tempfile::tempdir().expect("healthy drive"),
];
let mut wrappers = Vec::new();
for directory in &directories {
let endpoint = Endpoint::try_from(directory.path().to_str().expect("UTF-8 drive path")).expect("drive endpoint");
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local drive"));
wrappers.push(LocalDiskWrapper::new(disk, false));
}
let reconnected = LocalDiskWrapper::new_with_reconnect_state(
Arc::clone(&wrappers[0].disk),
false,
Some(wrappers[0].health_tracker_epoch_for_reconnect()),
);
let scope = wrappers[0].override_operation_timeout_for_test(Duration::from_secs(1));
tokio::time::pause();
let operation = || async {
tokio::time::sleep(Duration::from_secs(2)).await;
Ok(())
};
let (faulted, healthy) = tokio::join!(
reconnected.track_disk_health_with_op_and_timeout_action(
"read_metadata",
operation,
Duration::from_secs(3),
TimeoutHealthAction::IgnoreFailure,
),
wrappers[1].track_disk_health_with_op_and_timeout_action(
"read_metadata",
operation,
Duration::from_secs(3),
TimeoutHealthAction::IgnoreFailure,
),
);
assert_eq!(faulted.expect_err("only the selected drive must time out"), DiskError::Timeout);
healthy.expect("healthy drive keeps its caller's deadline");
let shorter_started = tokio::time::Instant::now();
let shorter = wrappers[0]
.track_disk_health_with_op_and_timeout_action(
"read_metadata",
operation,
Duration::from_millis(500),
TimeoutHealthAction::IgnoreFailure,
)
.await;
assert_eq!(shorter.expect_err("fault injection cannot extend a caller deadline"), DiskError::Timeout);
assert!(shorter_started.elapsed() < Duration::from_secs(1));
wrappers[0]
.track_disk_health_with_op("read_metadata", operation, Duration::ZERO)
.await
.expect("explicit unbounded physical ownership cannot acquire a test deadline");
drop(scope);
reconnected
.track_disk_health_with_op("read_metadata", operation, Duration::from_secs(3))
.await
.expect("dropping the scope restores reconnected callers' deadlines");
}
#[tokio::test]
async fn ignored_timeout_does_not_mark_drive_failure() {
let dir = tempfile::tempdir().expect("temp dir should be created");
-6
View File
@@ -172,8 +172,6 @@ pub enum DiskError {
#[error("guarded batch deletion is not supported by this disk")]
GuardedDeleteUnsupported,
#[error("foreground delete recovery work limit reached")]
DeleteRecoveryWorkLimit,
#[error("erasure write quorum")]
ErasureWriteQuorum,
@@ -772,7 +770,6 @@ impl Clone for DiskError {
DiskError::MethodNotAllowed => DiskError::MethodNotAllowed,
DiskError::ConditionalDeleteConflict => DiskError::ConditionalDeleteConflict,
DiskError::GuardedDeleteUnsupported => DiskError::GuardedDeleteUnsupported,
DiskError::DeleteRecoveryWorkLimit => DiskError::DeleteRecoveryWorkLimit,
DiskError::ErasureWriteQuorum => DiskError::ErasureWriteQuorum,
DiskError::ErasureReadQuorum => DiskError::ErasureReadQuorum,
DiskError::ShortWrite => DiskError::ShortWrite,
@@ -832,7 +829,6 @@ impl DiskError {
DiskError::RemoteClientUnavailable(_) => 0x2B,
DiskError::ConditionalDeleteConflict => 0x2C,
DiskError::GuardedDeleteUnsupported => 0x2D,
DiskError::DeleteRecoveryWorkLimit => 0x2E,
}
}
@@ -883,7 +879,6 @@ impl DiskError {
0x2B => Some(DiskError::RemoteClientUnavailable(String::new())),
0x2C => Some(DiskError::ConditionalDeleteConflict),
0x2D => Some(DiskError::GuardedDeleteUnsupported),
0x2E => Some(DiskError::DeleteRecoveryWorkLimit),
_ => None,
}
}
@@ -1667,7 +1662,6 @@ mod tests {
for (error, code) in [
(DiskError::ConditionalDeleteConflict, 0x2C),
(DiskError::GuardedDeleteUnsupported, 0x2D),
(DiskError::DeleteRecoveryWorkLimit, 0x2E),
] {
assert_eq!(error.to_u32(), code);
assert_eq!(DiskError::from_u32(code), Some(error.clone()));

Some files were not shown because too many files have changed in this diff Show More