mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-08 03:23:23 +00:00
Compare commits
89 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 0f125bb4ec | |||
| bda712df56 | |||
| c5b15b7279 | |||
| 00102f5737 | |||
| 55246fa932 | |||
| 2a6da8208b | |||
| 93b704420f | |||
| dfd4d3e213 | |||
| 1c1baa6b08 | |||
| 0147b8671d | |||
| 8cd751f9d1 | |||
| 5ea2991850 | |||
| 1556e05b8a | |||
| 32238a179c | |||
| 6b70584dde | |||
| d6210ac17b | |||
| 158aa276e9 | |||
| 3362421240 | |||
| 57df9181d0 | |||
| 371f5e4ffc | |||
| 8a5a2c9c93 | |||
| babfeaed00 | |||
| 736bf777b4 | |||
| 7babc7d621 | |||
| 4d46bd25f8 | |||
| ff5f491c5a | |||
| a8be876fd9 | |||
| fe3ada0689 | |||
| 926355ae1f | |||
| 7f5fa4b8f6 | |||
| a97c65895c | |||
| 73e7afa01a | |||
| e0f26a540d | |||
| 3b3cc018aa | |||
| f4f5635aab | |||
| 36ad5a6b87 | |||
| 6c14128072 | |||
| addc258a24 | |||
| 73ea527f55 | |||
| 55da14ad82 | |||
| 116a5299b2 | |||
| fbc54f7212 | |||
| 3aa492f5ee | |||
| 00a81b4048 | |||
| 20ef9ebf15 | |||
| cc08e0d11b | |||
| 69294523d5 | |||
| cd4d86f28a | |||
| 2f55cf69e1 | |||
| fff6e94378 | |||
| 41b13aeb8b | |||
| 88587a8f94 | |||
| 8c8b5da63f | |||
| 00a238f838 | |||
| 965d805a4f | |||
| 9d66e04ae3 | |||
| 12c58d25b5 | |||
| 1ac1d883f4 | |||
| 45ba33291a | |||
| 636eded277 | |||
| ed4099ebe1 | |||
| 3a1fac16e3 | |||
| b00e85f1a4 | |||
| ecbddc188a | |||
| 2d63931bba | |||
| d429c6f792 | |||
| e494dc2b4b | |||
| 2b7c927ee8 | |||
| 647846956f | |||
| 04690f7da5 | |||
| 3c0829049d | |||
| 890a463218 | |||
| 25f45f6309 | |||
| e3700614a1 | |||
| d4ab7704b3 | |||
| 422e724051 | |||
| e0bf54197d | |||
| b4af43c884 | |||
| 1cb4c0ebe2 | |||
| e5563e6004 | |||
| 0a72b29a34 | |||
| 51275cb7d8 | |||
| 3bf270b49f | |||
| d8d064bd18 | |||
| e5653ca055 | |||
| 6413a11889 | |||
| 9e4ea9fdd1 | |||
| 1a841fb7c7 | |||
| b8ee0301e9 |
@@ -1,2 +1,2 @@
|
||||
sha256-linux=686dcc075e41a75dde3d37e5bd0393d016469f02f83aa3508f92bfc17f26a4d4
|
||||
sha256-darwin=686dcc075e41a75dde3d37e5bd0393d016469f02f83aa3508f92bfc17f26a4d4
|
||||
sha256-linux=348fa5eed9ae58962c06b33283ff4f15122cff97bce9b4eb1e3ffcb227264b0b
|
||||
sha256-darwin=348fa5eed9ae58962c06b33283ff4f15122cff97bce9b4eb1e3ffcb227264b0b
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=67251cb7d9978d38e899268cac4d7898297ff7e5b5c48c9c1546dc760b93568d
|
||||
sha256-linux=673da76c2e5710c4622fe2fa32d2b297730895769487fcf1152ee1c2366f32f4
|
||||
sha256-darwin=219ac11e720b3f54dfee4d5d5e95171785ba159b2fe01db3037a8f5061c1ad32
|
||||
sha256-linux=2d34698cc8ec52d5b6c37ac612919a1d21f36cd56a1af539a7522a4eea518e26
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=3814cdde8cacf4187e18188d342ed70931cc32e268e42e5bb438d565bb4d0f81
|
||||
sha256-linux=663025a80273579f95ae3f49af02c0afcbfce94df09701c360a13e607b7637db
|
||||
sha256-darwin=e84e04208f19dc0596b10d4d1b799ce43d6cc431415902f9c5458aa8232bc2d8
|
||||
sha256-linux=1f9c7add72c44d1c12ef4ac050bb7917431be657c0c90b29f70c3df56f8bbae3
|
||||
|
||||
@@ -1 +1 @@
|
||||
sha256=e9dc896774122a304e2b8b90f28c82d4d4e0af8ded36a82b147b30945ec994e3
|
||||
sha256=5bdd4c2eadbc41625ae3184fff34062e000d328108436259b2b7c4fe2531ecaf
|
||||
|
||||
@@ -28,6 +28,7 @@ script-tests: ## Run shell script tests
|
||||
./scripts/test_build_rustfs_options.sh
|
||||
./scripts/test_docker_runtime_timezone.sh
|
||||
./scripts/test_entrypoint_credentials.sh
|
||||
./scripts/test_entrypoint_directories.sh
|
||||
./scripts/test_internode_grpc_ab_bench.sh
|
||||
./scripts/test_object_batch_bench_enhanced.sh
|
||||
./scripts/test_package_service_scripts.sh
|
||||
|
||||
+10
-4
@@ -234,7 +234,7 @@ test-group = 'embedded-test-ports'
|
||||
# processes cannot exhaust those budgets. Keep deadlines and assertions intact.
|
||||
# The short HTTP fixtures are tracked in rustfs/backlog#2714.
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))'
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|cleanup_accepts_missing_object_and_still_deletes_the_bucket|cleanup_diagnostics_identify_each_disconnected_transport_stage|cleanup_diagnostics_identify_each_failed_http_stage_without_private_context|cleanup_diagnostics_identify_timeouts_without_resetting_the_shared_deadline|oversized_response_is_rejected_without_buffering_it)$/)))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# This cancellation probe hashes the serving executable before acquiring its
|
||||
@@ -548,7 +548,7 @@ filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnost
|
||||
test-group = 'embedded-test-ports'
|
||||
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))'
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|cleanup_accepts_missing_object_and_still_deletes_the_bucket|cleanup_diagnostics_identify_each_disconnected_transport_stage|cleanup_diagnostics_identify_each_failed_http_stage_without_private_context|cleanup_diagnostics_identify_timeouts_without_resetting_the_shared_deadline|oversized_response_is_rejected_without_buffering_it)$/)))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# Preserve the cancellation probe's startup budget and assertions in CI too.
|
||||
@@ -701,7 +701,7 @@ test-group = 'ecstore-serial-flaky'
|
||||
[profile.e2e-smoke]
|
||||
default-filter = """
|
||||
package(e2e_test) & (
|
||||
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat|data_usage)_test::|^fake_s3_target::/)
|
||||
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|multipart_upload_id|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat|data_usage)_test::|^fake_s3_target::/)
|
||||
| test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/)
|
||||
| test(/^reliant::lifecycle::/)
|
||||
| test(/^conditional_delete_objects_test::/)
|
||||
@@ -827,7 +827,7 @@ slow-timeout = { period = "120s", terminate-after = 12, grace-period = "10s" }
|
||||
# ---------------------------------------------------------------------------
|
||||
# e2e-distributed profile — 4-node 4-disk Actions suite
|
||||
# ---------------------------------------------------------------------------
|
||||
# Storage-sensitive PR / nightly / dispatch lane owned by
|
||||
# Nightly and manual-dispatch lane owned by
|
||||
# .github/workflows/e2e-distributed.yml.
|
||||
# Each case starts four rustfs processes (and for site replication, two
|
||||
# clusters). Upgrade cases also require RUSTFS_UPGRADE_SOURCE_BINARY.
|
||||
@@ -976,6 +976,12 @@ test-group = 'e2e-vault'
|
||||
filter = 'package(e2e_test) & (test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter)$/) | test(=reliant::tiering::test_hermetic_transition_restore_failure_expiry_and_retry))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# The mc fixture uploads hundreds of objects concurrently before measuring listing.
|
||||
# Reserve the runner so other server fixtures do not starve its disk operations.
|
||||
[[profile.e2e-full.overrides]]
|
||||
filter = 'package(e2e_test) & test(=mc_mirror_small_bucket_test::test_mc_mirror_small_bucket_completes_without_list_timeout)'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# Fixed, small contract for every executable server change. These existing
|
||||
# scenarios exercise write/read, anonymous denial and retained bytes on restart.
|
||||
[profile.core-contract]
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"plan_id": "rustfs-release-validation",
|
||||
"description": "Versioned certification plan for every public RustFS release exit (rustfs/backlog#2662, R01). preview, prerelease and stable all require this explicit plan. Scripts/release/release_gate.py fail-closes publication on any missing, duplicate, stale, substituted or non-PASS required item.",
|
||||
"release_types": ["preview", "prerelease", "stable"],
|
||||
"targets": [
|
||||
{"os": "linux", "arch": "x86_64", "abi": "musl"},
|
||||
{"os": "linux", "arch": "aarch64", "abi": "musl"},
|
||||
{"os": "linux", "arch": "x86_64", "abi": "gnu"},
|
||||
{"os": "linux", "arch": "aarch64", "abi": "gnu"},
|
||||
{"os": "macos", "arch": "aarch64", "abi": "none"},
|
||||
{"os": "windows", "arch": "x86_64", "abi": "msvc"}
|
||||
],
|
||||
"targets_basis": "Platform matrix of .github/workflows/build.yml (prepare-platform-matrix). gnu and macOS targets build with the pyroscope feature; musl and Windows targets build with no extra features.",
|
||||
"features": ["pyroscope"],
|
||||
"trusted_workflows": [
|
||||
{"repository": "rustfs/s3chaos", "workflow_id": "release-gate.yml"}
|
||||
],
|
||||
"trusted_workflows_basis": "The shared evidence envelope producer lives in rustfs/s3chaos (G01, fault/release_gate.rs) and its plan/live split workflow is .github/workflows/release-gate.yml there. Only results from these identities may enter a release certification; PR-uploaded JSON never gains release authority. R02 must extend this list when additional verifier workflows land, never widen it to untrusted sources.",
|
||||
"lanes": [
|
||||
{
|
||||
"lane": "identity",
|
||||
"harness_repository": "rustfs/s3chaos",
|
||||
"harness_sha": "9a48f55c1d6d58ce63a8fb1ea7317e024f32f11d",
|
||||
"harness_basis": "Fixed G01 evidence-envelope producer revision (rustfs/s3chaos PR #102 head recorded in rustfs/backlog#2658). R02 re-pins this when the live verifier ships.",
|
||||
"timeout_seconds": 1800,
|
||||
"environment_capabilities": ["process-exec"],
|
||||
"cases": ["artifact-identity", "startup"],
|
||||
"applies_to": {
|
||||
"release_types": ["preview", "prerelease", "stable"],
|
||||
"platforms": "all"
|
||||
},
|
||||
"applies_basis": "Artifact identity and startup checks are mandatory for every release type, preview included, on every shipped target (rustfs/backlog#2662 frozen design item 5)."
|
||||
},
|
||||
{
|
||||
"lane": "upgrade",
|
||||
"harness_repository": "rustfs/s3chaos",
|
||||
"harness_sha": "9a48f55c1d6d58ce63a8fb1ea7317e024f32f11d",
|
||||
"harness_basis": "Same fixed envelope producer as the identity lane; the workflow SHA and the product build SHA are bound separately and need not be equal.",
|
||||
"timeout_seconds": 5400,
|
||||
"environment_capabilities": ["process-exec", "previous-release-artifact"],
|
||||
"cases": [
|
||||
"direct-upgrade",
|
||||
"mixed-version-upgrade",
|
||||
"bucket-config-upgrade",
|
||||
"bucket-config-rollback",
|
||||
"multipart-layout-upgrade",
|
||||
"multipart-layout-baseline"
|
||||
],
|
||||
"cases_basis": "Case IDs of the pinned previous-release matrix in .github/workflows/e2e-upgrade.yml (source.version/asset/sha256 and the case list mapped there).",
|
||||
"applies_to": {
|
||||
"release_types": ["prerelease", "stable"],
|
||||
"platforms": [{"os": "linux", "arch": "x86_64", "abi": "gnu"}]
|
||||
},
|
||||
"applies_basis": "preview releases use the pre-declared smaller set (identity lane only). The upgrade lane is only exercised on linux-x86_64-gnu, the platform of the pinned source assets."
|
||||
}
|
||||
],
|
||||
"upgrade_sources": {
|
||||
"basis": "Exactly the two previous-release sources pinned in .github/workflows/e2e-upgrade.yml are exercised today (source.version / source.asset / source.sha256). No other N-1/N-2 compatibility is claimed, and these digests are not inferred from tags.",
|
||||
"sources": [
|
||||
{
|
||||
"version": "1.0.1",
|
||||
"role": "N-1",
|
||||
"artifact_name": "rustfs-linux-x86_64-gnu-v1.0.1.zip",
|
||||
"artifact_sha256": "9742e183cb0dfb0d48fa909797af572fe47b7f827569036baa4156e0d3958ee5"
|
||||
},
|
||||
{
|
||||
"version": "1.0.0-rc.5",
|
||||
"role": "N-2",
|
||||
"artifact_name": "rustfs-linux-x86_64-gnu-v1.0.0-rc.5.zip",
|
||||
"artifact_sha256": "3ee8df71e8edcfada533be452c4135868f697bc515460ae97b027313eade7a3d"
|
||||
}
|
||||
],
|
||||
"rollback_supported": false,
|
||||
"rollback_basis": "Only the bucket-config-rollback case (rollback_to_previous_release_reads_current_bucket_metadata) is exercised. Full data-plane rollback is not certified and is never a default claim."
|
||||
}
|
||||
}
|
||||
+33
-3
@@ -22,9 +22,9 @@ The following files in the project root are essential for Docker operations:
|
||||
| File | Description | Usage |
|
||||
| :--- | :--- | :--- |
|
||||
| **`docker-buildx.sh`** | **Multi-Arch Build Script**<br>Automates building and pushing Docker images for `amd64` and `arm64`. Supports release and dev channels. | `./docker-buildx.sh --push` |
|
||||
| **`Dockerfile`** | **Production Image (Alpine)**<br>Lightweight image using musl libc. Downloads pre-built binaries from GitHub Releases. | `docker build -t rustfs:latest .` |
|
||||
| **`Dockerfile.glibc`** | **Production Image (Ubuntu)**<br>Standard image using glibc. Useful if you need specific dynamic libraries. | `docker build -f Dockerfile.glibc .` |
|
||||
| **`Dockerfile.source`** | **Development Image**<br>Builds RustFS from source code. Includes build tools. Ideal for local development and CI. | `docker build -f Dockerfile.source .` |
|
||||
| **`Dockerfile`** | **Production Image (Alpine)**<br>Lightweight image using musl libc. Downloads verified RustFS and `rc` binaries from GitHub Releases and bundles `/etc/rustfs/bootstrap`. | `docker build -t rustfs:latest .` |
|
||||
| **`Dockerfile.glibc`** | **Production Image (Ubuntu)**<br>Standard image using glibc. Includes the bundled bootstrap configuration. | `docker build -f Dockerfile.glibc .` |
|
||||
| **`Dockerfile.source`** | **Development Image**<br>Builds RustFS from source code and bundles `rc` and the default bootstrap configuration. | `docker build -f Dockerfile.source .` |
|
||||
|
||||
### Docker Compose Configurations
|
||||
|
||||
@@ -110,6 +110,36 @@ docker compose up -d
|
||||
|
||||
---
|
||||
|
||||
## Docker Compose environment values
|
||||
|
||||
Use map syntax when quoting environment values in YAML:
|
||||
|
||||
```yaml
|
||||
services:
|
||||
rustfs:
|
||||
image: rustfs/rustfs:latest
|
||||
ports:
|
||||
- "9000:9000"
|
||||
- "9001:9001"
|
||||
volumes:
|
||||
- ./data:/data
|
||||
- ./logs:/var/log/rustfs
|
||||
environment:
|
||||
RUSTFS_ADDRESS: ":9000"
|
||||
RUSTFS_CONSOLE_ADDRESS: ":9001"
|
||||
RUSTFS_CONSOLE_ENABLE: "true"
|
||||
RUSTFS_OBS_LOGGER_LEVEL: error
|
||||
RUSTFS_OBS_LOG_DIRECTORY: "/var/log/rustfs/"
|
||||
RUSTFS_ACCESS_KEY: "${RUSTFS_ACCESS_KEY:?Set RUSTFS_ACCESS_KEY}"
|
||||
RUSTFS_SECRET_KEY: "${RUSTFS_SECRET_KEY:?Set RUSTFS_SECRET_KEY}"
|
||||
```
|
||||
|
||||
Create the host directories and grant the container user write access before starting this example; official images run as UID/GID `10001:10001`. Set both credential variables to your own values, then run `docker compose config` to inspect the resolved configuration locally before `docker compose up -d`. The resolved output includes credentials, so do not share it publicly.
|
||||
|
||||
With list syntax, use `- RUSTFS_OBS_LOG_DIRECTORY=/var/log/rustfs/` or quote the whole item as `- "RUSTFS_OBS_LOG_DIRECTORY=/var/log/rustfs/"`. Writing `- RUSTFS_OBS_LOG_DIRECTORY="/var/log/rustfs/"` passes the inner quotes literally, making this a relative path beginning with `"`, which can fail with `mkdir: cannot create directory '"': Permission denied`. This also applies to addresses and credentials: fix every affected list item, rather than only the log directory. The entrypoint preserves configured values, including literal quotes in valid paths and secrets.
|
||||
|
||||
---
|
||||
|
||||
## 🔧 Common Operations
|
||||
|
||||
### Cleaning Up
|
||||
|
||||
@@ -28,6 +28,15 @@ processors:
|
||||
check_interval: 1s
|
||||
limit_mib: 1024
|
||||
spike_limit_mib: 256
|
||||
transform/metrics:
|
||||
error_mode: ignore
|
||||
metric_statements:
|
||||
# RustFS process metrics already expose process_pid as a metric label.
|
||||
# Remove only the duplicate datapoint label and keep the shared resource
|
||||
# process.pid available to sibling metrics and lock histograms.
|
||||
- context: datapoint
|
||||
statements:
|
||||
- delete_key(attributes, "process_pid") where IsMatch(metric.name, "^rustfs_system_process_")
|
||||
transform/logs:
|
||||
error_mode: ignore
|
||||
log_statements:
|
||||
@@ -112,7 +121,7 @@ service:
|
||||
exporters: [ otlp/tempo, otlp/jaeger ]
|
||||
metrics:
|
||||
receivers: [ otlp ]
|
||||
processors: [ batch ]
|
||||
processors: [ transform/metrics, memory_limiter, batch ]
|
||||
exporters: [ prometheus ]
|
||||
logs:
|
||||
receivers: [ otlp ]
|
||||
|
||||
@@ -32,6 +32,8 @@ updates:
|
||||
- "overtrue"
|
||||
- "majinghe"
|
||||
ignore:
|
||||
- dependency-name: "pulsar"
|
||||
versions: [ "6.9.x" ]
|
||||
- dependency-name: "object_store"
|
||||
versions: [ "0.13.x" ]
|
||||
- dependency-name: "libunftp"
|
||||
|
||||
@@ -296,6 +296,32 @@
|
||||
"origin": "baseline",
|
||||
"alert_contract": "scheduled-validation"
|
||||
},
|
||||
{
|
||||
"workflow": ".github/workflows/rustfs-validation-sweep.yml",
|
||||
"name": "RustFS Validation Sweep",
|
||||
"roles": [
|
||||
"periodic",
|
||||
"manual"
|
||||
],
|
||||
"events": [
|
||||
"schedule",
|
||||
"workflow_dispatch"
|
||||
],
|
||||
"cadence": "nightly",
|
||||
"timezone": "UTC",
|
||||
"schedule": [
|
||||
"30 19 * * *"
|
||||
],
|
||||
"owner": "@overtrue",
|
||||
"policy": "active",
|
||||
"monitor": "workflow",
|
||||
"max_age_hours": 60,
|
||||
"result_owner": "rustfs-validation-sweep.yml",
|
||||
"disposition": "canonical validation cases executed in rotating batches",
|
||||
"failure_owner": "workflow",
|
||||
"origin": "baseline",
|
||||
"alert_contract": "scheduled-validation"
|
||||
},
|
||||
{
|
||||
"workflow": ".github/workflows/btf-prototype-reference.yml",
|
||||
"name": "BTF prototype reference",
|
||||
@@ -743,12 +769,10 @@
|
||||
"workflow": ".github/workflows/e2e-distributed.yml",
|
||||
"name": "e2e-distributed",
|
||||
"roles": [
|
||||
"pr",
|
||||
"periodic",
|
||||
"manual"
|
||||
],
|
||||
"events": [
|
||||
"pull_request",
|
||||
"workflow_dispatch",
|
||||
"schedule"
|
||||
],
|
||||
@@ -823,12 +847,11 @@
|
||||
"workflow": ".github/workflows/e2e-upgrade.yml",
|
||||
"name": "Upgrade Compatibility",
|
||||
"roles": [
|
||||
"pr",
|
||||
"periodic",
|
||||
"manual"
|
||||
"manual",
|
||||
"release"
|
||||
],
|
||||
"events": [
|
||||
"pull_request",
|
||||
"push",
|
||||
"schedule",
|
||||
"workflow_dispatch"
|
||||
|
||||
+311
-5
@@ -814,13 +814,18 @@ jobs:
|
||||
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
|
||||
export AWS_DEFAULT_REGION="auto"
|
||||
|
||||
# Determine upload path based on build type
|
||||
# Determine upload path based on build type. Release and prerelease
|
||||
# builds only stage their bytes here: the official
|
||||
# artifacts/rustfs/release/ path is written solely by the
|
||||
# certification-gated promote-release-artifacts job, from these same
|
||||
# staged objects. Development builds keep the uncertified dev channel
|
||||
# and never touch official or latest channels.
|
||||
if [[ "$BUILD_TYPE" == "development" ]]; then
|
||||
R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/dev/"
|
||||
echo "📤 Uploading development build to R2 dev directory"
|
||||
else
|
||||
R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/release/"
|
||||
echo "📤 Uploading release build to R2 release directory"
|
||||
R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
|
||||
echo "📤 Staging release build for certification-gated promotion"
|
||||
fi
|
||||
|
||||
# Upload all rustfs zip files to R2 using S3-compatible API
|
||||
@@ -869,6 +874,221 @@ jobs:
|
||||
echo "This run does not trigger Docker publishing."
|
||||
fi
|
||||
|
||||
# Aggregate the parent candidate batch. This job always collects results and
|
||||
# retains its evidence, including on failure, and every publication job
|
||||
# below requires its success. Missing, stale, substituted or non-PASS
|
||||
# evidence fails closed here: only the trusted verification sources
|
||||
# (rustfs/s3chaos release-gate envelopes, rustfs/backlog#2658) can satisfy
|
||||
# it, so a PR-uploaded JSON set never gains release authority.
|
||||
release-certification:
|
||||
name: Release Certification
|
||||
needs: [ build-check, build-rustfs ]
|
||||
if: ${{ !cancelled() && needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') }}
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download all build artifacts
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
path: ./artifacts
|
||||
pattern: rustfs-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Write candidate and collect trusted validation results
|
||||
env:
|
||||
# Cross-repo read of the trusted verification run. Falls back to the
|
||||
# workflow token, which cannot read rustfs/s3chaos; a missing token
|
||||
# therefore fail-closes instead of certifying.
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
PRODUCT_COMMIT: ${{ github.sha }}
|
||||
VERSION: ${{ needs.build-check.outputs.version }}
|
||||
BUILD_TYPE: ${{ needs.build-check.outputs.build_type }}
|
||||
FEATURES: ${{ needs.build-check.outputs.build_type == 'development' && '' || 'pyroscope' }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
mkdir -p results release-gate-output
|
||||
PLAN_SHA256=$(python3 -c "import hashlib,json,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('.config/release-validation-plan.json'))))")
|
||||
|
||||
python3 - "$PLAN_SHA256" > candidate.json <<'PY'
|
||||
import hashlib, json, os, sys, zipfile
|
||||
|
||||
plan_sha256 = sys.argv[1]
|
||||
profile = {"release": "stable", "prerelease": "prerelease", "preview": "preview"}.get(os.environ["BUILD_TYPE"])
|
||||
if profile is None:
|
||||
raise SystemExit("release certification only covers preview, prerelease and stable builds")
|
||||
|
||||
def sha256(path):
|
||||
digest = hashlib.sha256()
|
||||
with open(path, "rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
deliverables = []
|
||||
for name in sorted(os.listdir("artifacts")):
|
||||
if not name.startswith("rustfs-") or not name.endswith(".zip"):
|
||||
continue
|
||||
path = os.path.join("artifacts", name)
|
||||
members = zipfile.ZipFile(path).namelist()
|
||||
binary_name = "rustfs.exe" if "rustfs.exe" in members else "rustfs"
|
||||
with zipfile.ZipFile(path) as archive:
|
||||
binary_bytes = archive.read(binary_name)
|
||||
import tempfile
|
||||
with tempfile.NamedTemporaryFile() as handle:
|
||||
handle.write(binary_bytes)
|
||||
handle.flush()
|
||||
binary_sha = sha256(handle.name)
|
||||
deliverables.append({
|
||||
"kind": "zip",
|
||||
"name": name,
|
||||
"location": "https://github.com/rustfs/rustfs/releases/download/{}/{}".format(os.environ["VERSION"], name),
|
||||
"sha256": sha256(path),
|
||||
"size": os.path.getsize(path),
|
||||
"binary_sha256": binary_sha,
|
||||
})
|
||||
if not deliverables:
|
||||
raise SystemExit("no built deliverables found to certify")
|
||||
candidate = {
|
||||
"schema_version": 1,
|
||||
"product_commit": os.environ["PRODUCT_COMMIT"],
|
||||
"version": os.environ["VERSION"],
|
||||
"release_profile": profile,
|
||||
"features": [f for f in os.environ.get("FEATURES", "").split(",") if f],
|
||||
"plan_sha256": plan_sha256,
|
||||
"validation_batch": {"run_id": os.environ["GITHUB_RUN_ID"], "run_attempt": int(os.environ["GITHUB_RUN_ATTEMPT"])},
|
||||
"deliverables": deliverables,
|
||||
}
|
||||
json.dump(candidate, sys.stdout, indent=2, sort_keys=True)
|
||||
print()
|
||||
PY
|
||||
|
||||
# Trusted verification evidence only: the release-gate envelopes of
|
||||
# rustfs/s3chaos for this exact product commit. Absence blocks below.
|
||||
TRUSTED_REPO="rustfs/s3chaos"
|
||||
TRUSTED_WORKFLOW="release-gate.yml"
|
||||
TRUSTED_RUN=$(gh api "repos/${TRUSTED_REPO}/actions/workflows/${TRUSTED_WORKFLOW}/runs?head_sha=${PRODUCT_COMMIT}&status=success&per_page=1" --jq '.workflow_runs[0].id // empty' || true)
|
||||
if [[ -z "$TRUSTED_RUN" ]]; then
|
||||
echo "❌ No trusted verification run of ${TRUSTED_REPO}/${TRUSTED_WORKFLOW} for ${PRODUCT_COMMIT}; refusing to certify" >&2
|
||||
else
|
||||
gh run download "$TRUSTED_RUN" -R "$TRUSTED_REPO" -n release-validation-evidence -D results || \
|
||||
echo "❌ Trusted verification run ${TRUSTED_RUN} published no release-validation-evidence artifact" >&2
|
||||
fi
|
||||
|
||||
python3 scripts/release/release_gate.py validate \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate candidate.json \
|
||||
--results results \
|
||||
--output release-gate-output
|
||||
|
||||
- name: Upload certification evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: release-certification
|
||||
path: |
|
||||
candidate.json
|
||||
results/
|
||||
release-gate-output/
|
||||
if-no-files-found: warn
|
||||
retention-days: 30
|
||||
|
||||
# The only writer of the official R2 release path. Promotion re-checks the
|
||||
# exact bytes about to be published against the passed certification list
|
||||
# and copies the staged objects byte-for-byte: no rebuild-after-verify.
|
||||
promote-release-artifacts:
|
||||
name: Promote Release Artifacts
|
||||
needs: [ build-check, build-rustfs, release-certification ]
|
||||
# Target-channel lock for the official R2 release path.
|
||||
concurrency:
|
||||
group: release-channel-r2-release
|
||||
cancel-in-progress: false
|
||||
if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
env:
|
||||
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
|
||||
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
|
||||
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
|
||||
R2_BUCKET: ${{ secrets.R2_BUCKET }}
|
||||
AWS_EC2_METADATA_DISABLED: true
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download build and certification artifacts
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
path: ./artifacts
|
||||
pattern: rustfs-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Download certification evidence
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: release-certification
|
||||
path: ./certification
|
||||
|
||||
- name: Verify and promote staged release artifacts
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
if [[ -z "$R2_ACCESS_KEY_ID" || -z "$R2_SECRET_ACCESS_KEY" || -z "$R2_ENDPOINT" || -z "$R2_BUCKET" ]]; then
|
||||
echo "❌ R2 credentials missing; refusing to promote to the official channel" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for zip_file in artifacts/rustfs-*.zip; do
|
||||
[[ -f "$zip_file" ]] || continue
|
||||
name="$(basename "$zip_file")"
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$zip_file" \
|
||||
--artifact-name "$name"
|
||||
done
|
||||
|
||||
if ! command -v aws >/dev/null 2>&1; then
|
||||
sudo apt-get update && sudo apt-get install -y awscli
|
||||
fi
|
||||
export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
|
||||
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
|
||||
export AWS_DEFAULT_REGION="auto"
|
||||
|
||||
STAGING_PATH="s3://${R2_BUCKET}/artifacts/rustfs/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
|
||||
RELEASE_PATH="s3://${R2_BUCKET}/artifacts/rustfs/release/"
|
||||
for zip_file in artifacts/rustfs-*.zip; do
|
||||
[[ -f "$zip_file" ]] || continue
|
||||
name="$(basename "$zip_file")"
|
||||
staged="$(mktemp)"
|
||||
aws s3 cp "${STAGING_PATH}${name}" "$staged" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
staged_sha="$(sha256sum "$staged" | awk '{print $1}')"
|
||||
built_sha="$(sha256sum "$zip_file" | awk '{print $1}')"
|
||||
if [[ "$staged_sha" != "$built_sha" ]]; then
|
||||
echo "❌ Staged object ${name} does not match the verified bytes" >&2
|
||||
exit 1
|
||||
fi
|
||||
aws s3 cp "$staged" "${RELEASE_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
rm -f "$staged"
|
||||
echo "✅ Promoted ${name} (${built_sha})"
|
||||
done
|
||||
|
||||
# Create GitHub Release for every valid release tag, including previews
|
||||
create-release:
|
||||
name: Create GitHub Release
|
||||
@@ -1028,6 +1248,10 @@ jobs:
|
||||
update-latest-version:
|
||||
name: Update Latest Version
|
||||
needs: [ build-check, publish-release ]
|
||||
# Target-channel lock for the latest.json version pointer.
|
||||
concurrency:
|
||||
group: release-channel-oss-latest
|
||||
cancel-in-progress: false
|
||||
if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
@@ -1068,6 +1292,45 @@ jobs:
|
||||
rm -rf "$OSSUTIL_DIR" "$OSSUTIL_ZIP"
|
||||
chmod +x "$OSSUTIL_BIN"
|
||||
|
||||
# Candidate/pointer ordering: an older candidate must never overwrite
|
||||
# a newer pointer, even if its promotion job finishes later.
|
||||
CURRENT_LATEST="${RUNNER_TEMP}/current-latest.json"
|
||||
if "$OSSUTIL_BIN" cp oss://rustfs-version/latest.json "$CURRENT_LATEST" 2>/dev/null; then
|
||||
python3 - "$CURRENT_LATEST" "$VERSION" <<'PY'
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
|
||||
def key(version):
|
||||
match = re.match(r"^(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?$", version)
|
||||
if not match:
|
||||
raise SystemExit(3)
|
||||
core = tuple(int(part) for part in match.groups()[:3])
|
||||
prerelease = match.group(4)
|
||||
if prerelease is None:
|
||||
return core + (1,)
|
||||
return core + (0, tuple(int(p) if p.isdigit() else p for p in re.split(r"[.-]", prerelease)))
|
||||
|
||||
try:
|
||||
current = json.load(open(sys.argv[1], encoding="utf-8")).get("version", "")
|
||||
proposed = sys.argv[2]
|
||||
if current and key(current) > key(proposed):
|
||||
raise SystemExit(1)
|
||||
except SystemExit:
|
||||
raise
|
||||
except (OSError, ValueError, TypeError):
|
||||
raise SystemExit(3)
|
||||
PY
|
||||
guard_status=$?
|
||||
if [[ "$guard_status" -eq 1 ]]; then
|
||||
echo "Current latest.json points at a newer version; refusing to downgrade" >&2
|
||||
exit 0
|
||||
elif [[ "$guard_status" -ne 0 ]]; then
|
||||
echo "Existing latest.json is unparsable; refusing to overwrite it blindly" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Create latest.json
|
||||
cat > latest.json << EOF
|
||||
{
|
||||
@@ -1084,16 +1347,59 @@ jobs:
|
||||
|
||||
echo "✅ Updated latest.json for ${RELEASE_TYPE} release $VERSION"
|
||||
|
||||
# Publish release (remove draft status)
|
||||
# Publish release (remove draft status). Requires the release-certification
|
||||
# aggregate: without a complete legal success there is no public write.
|
||||
publish-release:
|
||||
name: Publish Release
|
||||
needs: [ build-check, create-release, upload-release-assets ]
|
||||
needs: [ build-check, create-release, upload-release-assets, release-certification ]
|
||||
# Target-channel lock: concurrent releases serialize on this channel so an
|
||||
# older candidate cannot interleave its publish with a newer one.
|
||||
concurrency:
|
||||
group: release-channel-github-release
|
||||
cancel-in-progress: false
|
||||
if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'preview' || needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: write
|
||||
actions: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download certification evidence
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: release-certification
|
||||
path: ./certification
|
||||
|
||||
- name: Verify release assets against certification
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
TAG="${{ needs.build-check.outputs.version }}"
|
||||
mkdir -p "$RUNNER_TEMP/promotion-assets"
|
||||
gh release download "$TAG" --dir "$RUNNER_TEMP/promotion-assets" --pattern '*.zip'
|
||||
found=0
|
||||
for zip_file in "$RUNNER_TEMP"/promotion-assets/*.zip; do
|
||||
[[ -f "$zip_file" ]] || continue
|
||||
found=1
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$zip_file" \
|
||||
--artifact-name "$(basename "$zip_file")"
|
||||
done
|
||||
if [[ "$found" -ne 1 ]]; then
|
||||
echo "No release zip assets to verify against the certification" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Publish release
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
|
||||
@@ -96,7 +96,7 @@ env:
|
||||
RUST_BACKTRACE: 1
|
||||
|
||||
jobs:
|
||||
# Readers: test-and-lint (workspace and ILM), build-rustfs-debug-binary,
|
||||
# Readers: workspace-lint-and-build, workspace-tests (workspace and ILM), build-rustfs-debug-binary,
|
||||
# e2e-tests, e2e-full.
|
||||
warm-ci-dev:
|
||||
name: Warm ci-dev
|
||||
@@ -146,7 +146,7 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
# --all-targets covers the test binaries nextest builds, including
|
||||
# e2e_test, which test-and-lint's own run excludes. The second build adds
|
||||
# e2e_test, which the workspace run excludes. The second build adds
|
||||
# the e2e-test-hooks feature resolution that build-rustfs-debug-binary uses
|
||||
# and that no lint lane enables.
|
||||
- name: Build ci-dev superset
|
||||
|
||||
+95
-34
@@ -113,7 +113,7 @@ jobs:
|
||||
printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
|
||||
echo "Base CI policy unavailable; running the full matrix."
|
||||
fi
|
||||
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^rehearse_shards=' "$GITHUB_OUTPUT"; then
|
||||
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=3$' "$GITHUB_OUTPUT"; then
|
||||
printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'rehearse_shards=false' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
|
||||
echo "Base policy does not cover this workflow contract; retaining the full matrix."
|
||||
fi
|
||||
@@ -160,7 +160,7 @@ jobs:
|
||||
- name: Run shared quick checks
|
||||
uses: ./.github/actions/quick-checks
|
||||
|
||||
test-and-lint:
|
||||
workspace-lint-and-build:
|
||||
name: Workspace Lint and Build
|
||||
if: contains(fromJSON('["full", "e2e"]'), needs.classify-changes.outputs.mode) && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
@@ -193,13 +193,13 @@ jobs:
|
||||
|
||||
- name: Prepare test evidence
|
||||
run: |
|
||||
mkdir -p artifacts/test-and-lint
|
||||
mkdir -p artifacts/workspace
|
||||
{
|
||||
echo "run_id=${GITHUB_RUN_ID}"
|
||||
echo "job=${GITHUB_JOB}"
|
||||
echo "runner=${RUNNER_NAME}"
|
||||
echo "started_at=$(date --utc --iso-8601=seconds)"
|
||||
} > artifacts/test-and-lint/run-metadata.txt
|
||||
} > artifacts/workspace/run-metadata.txt
|
||||
|
||||
# Clippy gates every event, pull requests included. When it ran only after
|
||||
# merge, a lint regression merged freely and then failed every main run
|
||||
@@ -233,11 +233,11 @@ jobs:
|
||||
|
||||
- name: Run documentation tests
|
||||
run: |
|
||||
mkdir -p artifacts/test-and-lint
|
||||
mkdir -p artifacts/workspace
|
||||
set +e
|
||||
timeout --verbose --signal=TERM --kill-after=30s 15m \
|
||||
python3 scripts/ci_test_artifact.py cargo doc --configuration workspace \
|
||||
2>&1 | tee artifacts/test-and-lint/doctest.log
|
||||
2>&1 | tee artifacts/workspace/doctest.log
|
||||
status=${PIPESTATUS[0]}
|
||||
{
|
||||
echo "command=cargo test --all --exclude e2e_test --doc"
|
||||
@@ -246,7 +246,7 @@ jobs:
|
||||
echo
|
||||
echo "Remaining test-related processes:"
|
||||
pgrep -af 'cargo|rustdoc|target/.*/deps/' || true
|
||||
} > artifacts/test-and-lint/doctest-diagnostics.txt
|
||||
} > artifacts/workspace/doctest-diagnostics.txt
|
||||
exit "${status}"
|
||||
|
||||
- name: Upload test reports and diagnostics
|
||||
@@ -255,7 +255,7 @@ jobs:
|
||||
with:
|
||||
name: junit-test-and-lint-${{ github.run_number }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
artifacts/test-and-lint
|
||||
artifacts/workspace
|
||||
retention-days: 3
|
||||
if-no-files-found: error
|
||||
|
||||
@@ -268,7 +268,7 @@ jobs:
|
||||
|
||||
workspace-tests:
|
||||
name: Workspace tests
|
||||
needs: [ test-and-lint, classify-changes ]
|
||||
needs: [ workspace-lint-and-build, classify-changes ]
|
||||
# The archive retains compile-time fixture paths and the large test set
|
||||
# requires the same runner image until the hosted-runtime pilot passes.
|
||||
runs-on: sm-standard-4
|
||||
@@ -293,14 +293,14 @@ jobs:
|
||||
|
||||
- name: Extract and list workspace tests
|
||||
run: |
|
||||
mkdir -p artifacts/test-and-lint
|
||||
mkdir -p artifacts/workspace
|
||||
cargo nextest list --profile ci --archive-file "$RUNNER_TEMP/workspace-tests.tar.zst" --workspace-remap "$GITHUB_WORKSPACE" --extract-to "$GITHUB_WORKSPACE" --extract-overwrite --message-format json \
|
||||
> artifacts/test-and-lint/core-test-listing.json
|
||||
python3 scripts/check_test_wiring.py --check-core artifacts/test-and-lint/core-test-listing.json
|
||||
> artifacts/workspace/core-test-listing.json
|
||||
python3 scripts/check_test_wiring.py --check-core artifacts/workspace/core-test-listing.json
|
||||
|
||||
- name: Run nextest tests
|
||||
run: |
|
||||
mkdir -p artifacts/test-and-lint
|
||||
mkdir -p artifacts/workspace
|
||||
rm -f target/nextest/ci/junit.xml
|
||||
./scripts/ci/resource_sampler.sh start nextest
|
||||
trap './scripts/ci/resource_sampler.sh stop' EXIT
|
||||
@@ -309,7 +309,7 @@ jobs:
|
||||
cargo nextest run --profile ci --cargo-metadata target/nextest/cargo-metadata.json --binaries-metadata target/nextest/binaries-metadata.json \
|
||||
--workspace-remap "$GITHUB_WORKSPACE" --target-dir-remap "$GITHUB_WORKSPACE/target" \
|
||||
--status-level all --final-status-level all \
|
||||
2>&1 | tee artifacts/test-and-lint/nextest.log
|
||||
2>&1 | tee artifacts/workspace/nextest.log
|
||||
status=${PIPESTATUS[0]}
|
||||
{
|
||||
echo "command=workspace archive execution"
|
||||
@@ -321,13 +321,13 @@ jobs:
|
||||
echo
|
||||
echo "Kernel OOM / kill events:"
|
||||
dmesg -T 2>/dev/null | grep -iE 'oom|out of memory|killed process' | tail -20 || true
|
||||
} > artifacts/test-and-lint/nextest-diagnostics.txt
|
||||
} > artifacts/workspace/nextest-diagnostics.txt
|
||||
exit "${status}"
|
||||
|
||||
- name: Verify workspace execution completeness
|
||||
run: |
|
||||
python3 scripts/ci_test_artifact.py evidence artifacts/test-and-lint/core-test-listing.json target/nextest/ci/junit.xml
|
||||
./scripts/check_migration_gate_count.sh evidence artifacts/test-and-lint/core-test-listing.json target/nextest/ci/junit.xml
|
||||
python3 scripts/ci_test_artifact.py evidence artifacts/workspace/core-test-listing.json target/nextest/ci/junit.xml
|
||||
./scripts/check_migration_gate_count.sh evidence artifacts/workspace/core-test-listing.json target/nextest/ci/junit.xml
|
||||
|
||||
- name: Run ignored ILM integration tests serially
|
||||
if: needs.classify-changes.outputs.mode == 'full'
|
||||
@@ -358,7 +358,7 @@ jobs:
|
||||
name: workspace-test-evidence-${{ github.run_attempt }}
|
||||
path: |
|
||||
target/nextest/ci/junit.xml
|
||||
artifacts/test-and-lint
|
||||
artifacts/workspace
|
||||
artifacts/ilm-integration
|
||||
retention-days: 3
|
||||
if-no-files-found: error
|
||||
@@ -368,7 +368,7 @@ jobs:
|
||||
if: needs.classify-changes.outputs.rehearse_shards == 'true'
|
||||
run: |
|
||||
python3 scripts/ci_test_shards.py prepare \
|
||||
--listing artifacts/test-and-lint/core-test-listing.json \
|
||||
--listing artifacts/workspace/core-test-listing.json \
|
||||
--junit target/nextest/ci/junit.xml --output artifacts/workspace-shard-plan
|
||||
- name: Upload same-run shard plan
|
||||
if: needs.classify-changes.outputs.rehearse_shards == 'true'
|
||||
@@ -449,7 +449,7 @@ jobs:
|
||||
|
||||
connect-feature-boundaries:
|
||||
name: Connect Feature Boundaries
|
||||
if: needs.classify-changes.outputs.mode == 'full' && needs.classify-changes.outputs.connect != 'false' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
if: needs.classify-changes.outputs.mode == 'full' && needs.classify-changes.outputs.connect != 'false' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
@@ -497,13 +497,72 @@ jobs:
|
||||
- name: Check offline enrollment E2E root boundary
|
||||
run: ./scripts/check_offline_enrollment_e2e.sh
|
||||
|
||||
fips-provider:
|
||||
name: FIPS provider boundaries
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
# This builds the RustFS library test target, which exceeds hosted-runner disk capacity.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
mode: [default, fips]
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
CARGO_BUILD_JOBS: "2"
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: ci-fips-provider
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
- name: Install FIPS build tools
|
||||
if: matrix.mode == 'fips'
|
||||
run: sudo apt-get install -y cmake golang clang libclang-dev
|
||||
|
||||
- name: Verify provider installation
|
||||
shell: bash
|
||||
run: |
|
||||
args=()
|
||||
if [[ '${{ matrix.mode }}' == 'fips' ]]; then
|
||||
args+=(--features fips)
|
||||
fi
|
||||
cargo nextest run --locked -p rustfs-tls-runtime --lib "${args[@]}" --no-tests fail -E 'test(provider_policy_isolated)'
|
||||
cargo nextest run --locked -p rustfs-s3-client --lib "${args[@]}" --no-tests fail -E 'test(overflowing_closed_range_is_rejected_before_network_io)'
|
||||
cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(validate_pulsar)'
|
||||
cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(webhook_provider_policy_isolated)'
|
||||
if [[ '${{ matrix.mode }}' == 'fips' ]]; then
|
||||
cargo nextest run --locked -p rustfs-protocols --lib --features ftps,webdav,fips --no-tests fail -E 'test(protocols_reject_incompatible_host_provider)'
|
||||
cargo nextest run --locked -p rustfs-targets --lib --features fips --no-tests fail -E 'test(fips_postgres_rejects_incompatible_host_provider)'
|
||||
cargo nextest run --locked -p rustfs-s3-client --lib --features fips --no-tests fail -E 'test(fips_transition_rejects_incompatible_host_provider)'
|
||||
fi
|
||||
tests=(provider_installation_isolated prepared_generation_shares_crypto_and_serves_sni_with_mtls)
|
||||
if [[ '${{ matrix.mode }}' == 'fips' ]]; then
|
||||
tests+=(fips_provider_builds_fips_tls_configs fips_provider_tls_handshake_accepts_and_rejects_algorithms)
|
||||
fi
|
||||
for test_name in "${tests[@]}"; do
|
||||
cargo nextest run --locked -p rustfs --lib "${args[@]}" --no-tests fail -E "test($test_name)"
|
||||
done
|
||||
cargo clippy --locked -p rustfs --lib "${args[@]}" -- -D warnings
|
||||
|
||||
test-and-lint-rio-v2:
|
||||
name: Test and Lint (rio-v2)
|
||||
# Alternate feature configurations run on the nightly schedule, merge
|
||||
# groups and manual dispatch. The default workspace remains the PR gate and
|
||||
# the per-push main baseline: repeating four feature builds on every merge
|
||||
# bought nothing the nightly run does not report within a day.
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["merge_group", "schedule", "workflow_dispatch"]'), github.event_name)
|
||||
# Specialized boundary, hardware and alternate-feature lanes run only on
|
||||
# the nightly schedule and manual dispatch. They do not gate a merge: the
|
||||
# nightly run reports a regression within a day, and repeating these builds
|
||||
# on every PR revision and merge group bought no earlier signal worth the
|
||||
# runner occupancy. Workspace lint and tests, the core contract, the debug
|
||||
# build, E2E smoke and the S3 lanes remain the pull-request gate.
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
@@ -540,12 +599,12 @@ jobs:
|
||||
# --profile ci so the quarantine list (and its junit flaky markers)
|
||||
# covers this leg too; the default profile is the local no-retry
|
||||
# profile and silently ignored quarantined flakes here (rustfs#6703).
|
||||
mkdir -p artifacts/test-and-lint
|
||||
mkdir -p artifacts/workspace
|
||||
rm -f target/nextest/ci/junit.xml
|
||||
./scripts/ci/resource_sampler.sh start nextest-rio-v2
|
||||
trap './scripts/ci/resource_sampler.sh stop' EXIT
|
||||
cargo nextest run --profile ci -p rustfs -p rustfs-ecstore --features rio-v2 \
|
||||
2>&1 | tee artifacts/test-and-lint/nextest.log
|
||||
2>&1 | tee artifacts/workspace/nextest.log
|
||||
cargo test -p rustfs --doc --features rio-v2
|
||||
|
||||
- name: Upload rio-v2 test reports and diagnostics
|
||||
@@ -555,14 +614,14 @@ jobs:
|
||||
name: junit-test-and-lint-rio-v2-${{ github.run_number }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
target/nextest/ci/junit.xml
|
||||
artifacts/test-and-lint
|
||||
artifacts/workspace
|
||||
retention-days: 3
|
||||
if-no-files-found: error
|
||||
|
||||
test-and-lint-protocols:
|
||||
name: "Test and Lint (${{ matrix.features.name }})"
|
||||
# Same cadence as test-and-lint-rio-v2 above.
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["merge_group", "schedule", "workflow_dispatch"]'), github.event_name)
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
@@ -631,7 +690,7 @@ jobs:
|
||||
path: |
|
||||
target/nextest/ci/junit.xml
|
||||
artifacts/protocol-tests
|
||||
artifacts/test-and-lint
|
||||
artifacts/workspace
|
||||
retention-days: 3
|
||||
if-no-files-found: warn
|
||||
|
||||
@@ -783,7 +842,7 @@ jobs:
|
||||
# job had neither, so each closed/merged PR really ran the whole io_uring
|
||||
# suite (measured 4m17s / 7m19s / 7m31s on runs 30678272341 / 30678117601 /
|
||||
# 30662728539) and kept the cancellation run in progress for minutes.
|
||||
if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
# GitHub-hosted ubuntu-latest runs a recent kernel with io_uring and, unlike
|
||||
# a container, applies no seccomp filter that would block io_uring_setup — so
|
||||
@@ -978,7 +1037,7 @@ jobs:
|
||||
path: |
|
||||
${{ runner.temp }}/rustfs-e2e-smoke-logs/
|
||||
${{ runner.temp }}/rustfs-e2e-smoke-list.json
|
||||
artifacts/test-and-lint
|
||||
artifacts/workspace
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Upload e2e smoke JUnit report
|
||||
@@ -1389,12 +1448,13 @@ jobs:
|
||||
- classify-changes
|
||||
- typos
|
||||
- quick-checks
|
||||
- test-and-lint
|
||||
- workspace-lint-and-build
|
||||
- workspace-tests
|
||||
- workspace-shards
|
||||
- workspace-shard-evidence
|
||||
- core-contract
|
||||
- connect-feature-boundaries
|
||||
- fips-provider
|
||||
- test-and-lint-rio-v2
|
||||
- test-and-lint-protocols
|
||||
- build-rustfs-debug-binary
|
||||
@@ -1422,10 +1482,11 @@ jobs:
|
||||
needs:
|
||||
- classify-changes
|
||||
- connect-feature-boundaries
|
||||
- fips-provider
|
||||
- required-checks
|
||||
- typos
|
||||
- quick-checks
|
||||
- test-and-lint
|
||||
- workspace-lint-and-build
|
||||
- workspace-tests
|
||||
- core-contract
|
||||
- test-and-lint-rio-v2
|
||||
|
||||
@@ -95,7 +95,7 @@ jobs:
|
||||
|
||||
- name: Run CLA Bot
|
||||
if: github.event_name != 'merge_group'
|
||||
uses: overtrue/cla-bot@5e5929846a60cfb42b16d0d38e135ef67d94fc88 # v0.0.9
|
||||
uses: overtrue/cla-bot@4bd1387c19691168c42fcf5edfa17b15819b2ab3 # v0.1.1
|
||||
with:
|
||||
github-token: ${{ github.token }}
|
||||
registry-token: ${{ steps.registry-token.outputs.token }}
|
||||
|
||||
@@ -329,13 +329,75 @@ jobs:
|
||||
echo " - Create latest: $create_latest"
|
||||
echo " - Source ref: $source_ref"
|
||||
|
||||
# Consume the trusted release certification for the exact version this
|
||||
# workflow wraps and verify the certified input digests. A manual dispatch
|
||||
# without that certification fails here and pushes no official tag; only
|
||||
# results from the trusted verification sources carry release authority.
|
||||
release-certification:
|
||||
name: Release Certification
|
||||
needs: build-check
|
||||
if: needs.build-check.outputs.should_build == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ needs.build-check.outputs.source_ref }}
|
||||
|
||||
- name: Download and verify release certification
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
VERSION: ${{ needs.build-check.outputs.version }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification
|
||||
run_id="$TRIGGERING_RUN_ID"
|
||||
if [[ -z "$run_id" ]]; then
|
||||
run_id="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
|
||||
-f branch="$VERSION" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
fi
|
||||
if [[ -z "$run_id" ]]; then
|
||||
echo "No certified Build and Release run found for ${VERSION}; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! gh run download "$run_id" -n release-certification -D certification; then
|
||||
echo "Build run ${run_id} published no release-certification artifact; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
|
||||
--artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
|
||||
|
||||
# Input digest binding: every zip this image wraps must be the
|
||||
# certified byte-for-byte content, never a mutable latest pointer.
|
||||
while IFS=$'\t' read -r name location expected; do
|
||||
echo "Verifying certified input ${name}"
|
||||
actual="$(curl -fsSL "$location" | sha256sum | awk '{print $1}')"
|
||||
if [[ "$actual" != "$expected" ]]; then
|
||||
echo "Certified input ${name} does not match its recorded digest" >&2
|
||||
exit 1
|
||||
fi
|
||||
done < <(jq -r '.deliverables[] | select(.name | contains("linux")) | [.name, .location, .sha256] | @tsv' certification/candidate.json)
|
||||
|
||||
# Build multi-arch Docker images
|
||||
# Strategy: Build images using pre-built binaries from dl.rustfs.com
|
||||
# Supports both release and dev channel binaries based on build context
|
||||
# Only runs when should_build is true (which includes workflow success check)
|
||||
build-docker:
|
||||
name: Build Docker Images
|
||||
needs: build-check
|
||||
needs: [ build-check, release-certification ]
|
||||
if: needs.build-check.outputs.should_build == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
@@ -443,6 +505,14 @@ jobs:
|
||||
# Output tags
|
||||
echo "tags=$TAGS" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Staging identity for this run's promotion: official tags are only
|
||||
# created after the staged image verifies against the certification,
|
||||
# and they point at this same digest (no rebuild-after-verify).
|
||||
STAGING_TAG="staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${VARIANT_SUFFIX}"
|
||||
STAGING_TAGS="${{ env.REGISTRY_DOCKERHUB }}:${STAGING_TAG},${{ env.REGISTRY_GHCR }}:${STAGING_TAG},${{ env.REGISTRY_QUAY }}:${STAGING_TAG}"
|
||||
echo "staging_tags=$STAGING_TAGS" >> "$GITHUB_OUTPUT"
|
||||
echo "staging_tag=$STAGING_TAG" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Generate labels
|
||||
LABELS="org.opencontainers.image.title=RustFS"
|
||||
LABELS="$LABELS,org.opencontainers.image.description=RustFS distributed object storage system"
|
||||
@@ -467,7 +537,9 @@ jobs:
|
||||
file: ${{ matrix.file }}
|
||||
platforms: ${{ env.DOCKER_PLATFORMS }}
|
||||
push: ${{ needs.build-check.outputs.should_push == 'true' }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
# Push to staging tags only: official tags are created by the
|
||||
# certification-gated promotion step below from this same digest.
|
||||
tags: ${{ steps.meta.outputs.staging_tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
# No layer cache. This build compiles nothing — it downloads a
|
||||
# release zip and runs apk/apt — so the cache could only save the
|
||||
@@ -493,6 +565,114 @@ jobs:
|
||||
no-cache: false
|
||||
pull: true
|
||||
|
||||
# A later-built wrapper: the staged image must complete this platform's
|
||||
# own binary identity verification before any official tag is written.
|
||||
# It may not inherit an unverified package's conclusion.
|
||||
- name: Verify staged image against certification
|
||||
if: needs.build-check.outputs.should_push == 'true'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
VERSION: ${{ needs.build-check.outputs.version }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification
|
||||
run_id="$TRIGGERING_RUN_ID"
|
||||
if [[ -z "$run_id" ]]; then
|
||||
run_id="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
|
||||
-f branch="$VERSION" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
fi
|
||||
gh run download "$run_id" -n release-certification -D certification
|
||||
|
||||
STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
|
||||
case "${{ matrix.variant }}" in
|
||||
musl) libc="musl" ;;
|
||||
glibc) libc="gnu" ;;
|
||||
esac
|
||||
for platform in linux/amd64 linux/arm64; do
|
||||
case "$platform" in
|
||||
linux/amd64) arch_sub="x86_64" ;;
|
||||
linux/arm64) arch_sub="aarch64" ;;
|
||||
esac
|
||||
docker pull --platform "$platform" "$STAGING_REF"
|
||||
container="$(docker create --platform "$platform" "$STAGING_REF")"
|
||||
docker cp "$container:/usr/bin/rustfs" "$RUNNER_TEMP/staged-rustfs"
|
||||
docker rm "$container" >/dev/null
|
||||
actual="$(sha256sum "$RUNNER_TEMP/staged-rustfs" | awk '{print $1}')"
|
||||
expected="$(jq -r --arg needle "linux-${arch_sub}-${libc}" \
|
||||
'[.deliverables[] | select(.name | contains($needle)) | .binary_sha256][0] // empty' \
|
||||
certification/candidate.json)"
|
||||
if [[ -z "$expected" || "$actual" != "$expected" ]]; then
|
||||
echo "Staged image binary for ${platform} does not match the certified binary digest" >&2
|
||||
exit 1
|
||||
fi
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$RUNNER_TEMP/staged-rustfs" \
|
||||
--artifact-role binary
|
||||
done
|
||||
|
||||
- name: Promote staged image to official tags
|
||||
if: needs.build-check.outputs.should_push == 'true'
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
|
||||
IFS=',' read -r -a official_tags <<< "${{ steps.meta.outputs.tags }}"
|
||||
for tag in "${official_tags[@]}"; do
|
||||
docker buildx imagetools create -t "$tag" "$STAGING_REF"
|
||||
echo "Promoted $tag -> $STAGING_REF"
|
||||
done
|
||||
|
||||
# Downstream consumers (Helm) verify this record instead of inheriting
|
||||
# an unverified wrapper's conclusion.
|
||||
- name: Record image promotion
|
||||
if: needs.build-check.outputs.should_push == 'true'
|
||||
shell: bash
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
VERSION: ${{ needs.build-check.outputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification
|
||||
run_id="$TRIGGERING_RUN_ID"
|
||||
if [[ -z "$run_id" ]]; then
|
||||
run_id="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
|
||||
-f branch="$VERSION" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
fi
|
||||
gh run download "$run_id" -n release-certification -D certification
|
||||
STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
|
||||
digest="$(docker buildx imagetools inspect "$STAGING_REF" --format '{{json .Manifest}}' | jq -r '.digest')"
|
||||
candidate_sha="$(python3 -c "import json,hashlib,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('certification/candidate.json'))))")"
|
||||
jq -n \
|
||||
--arg version "$VERSION" \
|
||||
--arg digest "$digest" \
|
||||
--arg staging_ref "$STAGING_REF" \
|
||||
--arg candidate_sha256 "$candidate_sha" \
|
||||
--arg tags "${{ steps.meta.outputs.tags }}" \
|
||||
--slurpfile candidate certification/candidate.json \
|
||||
'{schema_version: 1, version: $version, image_digest: $digest, staging_ref: $staging_ref,
|
||||
promoted_tags: ($tags | split(",")), candidate_sha256: $candidate_sha256,
|
||||
binary_sha256s: ($candidate[0].deliverables | map(.binary_sha256) | unique)}' \
|
||||
> image-promotion-record.json
|
||||
|
||||
- name: Upload image promotion record
|
||||
if: needs.build-check.outputs.should_push == 'true'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: image-promotion-record${{ matrix.suffix }}
|
||||
path: image-promotion-record.json
|
||||
if-no-files-found: error
|
||||
retention-days: 30
|
||||
|
||||
# Note: Manifest creation is no longer needed as we only build one variant
|
||||
# Multi-arch manifests are automatically created by docker/build-push-action
|
||||
|
||||
|
||||
@@ -17,8 +17,8 @@
|
||||
# Each selected test starts a real localhost cluster via
|
||||
# `RustFSTestClusterEnvironment` (4 processes; 4 drives per node unless the
|
||||
# case is a two-site 4-node 1-drive pair or a 4-node upgrade). Membership is
|
||||
# `[profile.e2e-distributed]` in `.config/nextest.toml`. Storage-sensitive PRs,
|
||||
# nightly runs, and manual dispatches all execute the same fail-closed suite.
|
||||
# `[profile.e2e-distributed]` in `.config/nextest.toml`. The nightly schedule
|
||||
# and manual dispatches execute the same fail-closed suite.
|
||||
# Upgrade cases download the same pinned previous release as e2e-upgrade.yml.
|
||||
#
|
||||
# Isolated pool filesystems: expand/decommission/rebalance cases require
|
||||
@@ -33,32 +33,6 @@
|
||||
name: e2e-distributed
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- "Cargo.lock"
|
||||
- "Cargo.toml"
|
||||
- ".config/nextest.toml"
|
||||
- ".config/e2e-distributed-selection.txt"
|
||||
- ".github/workflows/e2e-distributed.yml"
|
||||
- "crates/audit/**"
|
||||
- "crates/common/**"
|
||||
- "crates/config/**"
|
||||
- "crates/e2e_test/**"
|
||||
- "crates/ecstore/**"
|
||||
- "crates/filemeta/**"
|
||||
- "crates/heal/**"
|
||||
- "crates/iam/**"
|
||||
- "crates/lock/**"
|
||||
- "crates/madmin/**"
|
||||
- "crates/notify/**"
|
||||
- "crates/replication/**"
|
||||
- "crates/s3-client/**"
|
||||
- "crates/s3-ops/**"
|
||||
- "crates/s3-types/**"
|
||||
- "crates/scanner/**"
|
||||
- "crates/storage-api/**"
|
||||
- "crates/utils/**"
|
||||
- "rustfs/**"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
filter:
|
||||
|
||||
@@ -12,30 +12,19 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Single-node upgrade and rollback lane.
|
||||
#
|
||||
# The weekly Monday schedule, version tags, and workflow_dispatch run this
|
||||
# suite. Distributed upgrade cases stay on the nightly e2e-distributed lane.
|
||||
|
||||
name: Upgrade Compatibility
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- ".github/workflows/e2e-upgrade.yml"
|
||||
- "crates/e2e_test/src/common.rs"
|
||||
- "crates/e2e_test/src/distributed/**"
|
||||
- "crates/e2e_test/src/fake_s3_target/**"
|
||||
- "crates/e2e_test/src/lib.rs"
|
||||
- "crates/e2e_test/src/replication_extension_test.rs"
|
||||
- "crates/e2e_test/src/rpc_fault_proxy.rs"
|
||||
- "crates/e2e_test/src/upgrade_compatibility_test.rs"
|
||||
- "crates/ecstore/**"
|
||||
- "crates/filemeta/**"
|
||||
- "crates/heal/**"
|
||||
- "crates/kms/**"
|
||||
- "crates/storage-api/**"
|
||||
- "rustfs/**"
|
||||
- "Cargo.lock"
|
||||
push:
|
||||
tags:
|
||||
- "[0-9]*.[0-9]*.[0-9]*"
|
||||
schedule:
|
||||
# 03:17 UTC every Monday.
|
||||
- cron: "17 3 * * 1"
|
||||
workflow_dispatch:
|
||||
|
||||
@@ -44,7 +33,7 @@ permissions:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
|
||||
@@ -30,9 +30,102 @@ permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
# Consume the same trusted release certification as every other publication
|
||||
# exit and verify the image promotion record of the Docker workflow. The
|
||||
# chart wraps no binary of its own, so it may only reference an image whose
|
||||
# platform verification already passed; it must not inherit an unverified
|
||||
# package's conclusion. A run without that evidence fails closed here and
|
||||
# never reaches the chart repository.
|
||||
release-certification:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
outputs:
|
||||
version: ${{ steps.version.outputs.raw_tag }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Normalize release version
|
||||
id: version
|
||||
env:
|
||||
RAW_INPUT: ${{ github.event.inputs.version }}
|
||||
RAW_BRANCH: ${{ github.event.workflow_run.head_branch }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
RAW="$RAW_INPUT"
|
||||
else
|
||||
RAW="$RAW_BRANCH"
|
||||
fi
|
||||
case "$RAW" in
|
||||
refs/tags/*) RAW_TAG="${RAW#refs/tags/}" ;;
|
||||
*) RAW_TAG="$RAW" ;;
|
||||
esac
|
||||
echo "raw_tag=$RAW_TAG" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Download and verify release certification and image record
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
RAW_TAG: ${{ steps.version.outputs.raw_tag }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification image-record
|
||||
|
||||
build_run="$TRIGGERING_RUN_ID"
|
||||
if [[ -z "$build_run" ]]; then
|
||||
build_run="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
|
||||
-f branch="$RAW_TAG" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
fi
|
||||
if [[ -z "$build_run" ]]; then
|
||||
echo "No certified Build and Release run found for ${RAW_TAG}; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! gh run download "$build_run" -n release-certification -D certification; then
|
||||
echo "Build run ${build_run} published no release-certification artifact; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
|
||||
--artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
|
||||
|
||||
docker_run="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/docker.yml/runs" \
|
||||
-f branch="$RAW_TAG" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
if [[ -z "$docker_run" ]] || ! gh run download "$docker_run" -n image-promotion-record -D image-record; then
|
||||
echo "No image promotion record for ${RAW_TAG}; a chart may not wrap an unverified image" >&2
|
||||
exit 1
|
||||
fi
|
||||
candidate_sha="$(python3 -c "import json,hashlib,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('certification/candidate.json'))))")"
|
||||
record_sha="$(jq -r '.candidate_sha256 // empty' image-record/image-promotion-record.json)"
|
||||
if [[ "$record_sha" != "$candidate_sha" ]]; then
|
||||
echo "Image promotion record does not belong to this certification" >&2
|
||||
exit 1
|
||||
fi
|
||||
overlap="$(jq -r --slurpfile candidate certification/candidate.json \
|
||||
'[.binary_sha256s[]] - ($candidate[0].deliverables | map(.binary_sha256)) | length' \
|
||||
image-record/image-promotion-record.json)"
|
||||
if [[ "$overlap" -ne 0 ]]; then
|
||||
echo "Image promotion record carries binary digests outside the certification" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
build-helm-package:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
needs: [ release-certification ]
|
||||
if: |
|
||||
(github.event_name == 'workflow_dispatch' && !contains(github.event.inputs.version, '-preview')) ||
|
||||
(
|
||||
|
||||
@@ -87,18 +87,206 @@ jobs:
|
||||
install-build-packaging-tools: 'false'
|
||||
install-test-tools: 'false'
|
||||
|
||||
- name: Download static console assets
|
||||
shell: bash
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
# Packaging tools are off on this lane, so setup does not install unzip.
|
||||
if ! command -v unzip >/dev/null 2>&1; then
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq unzip
|
||||
fi
|
||||
if ! command -v unzip >/dev/null 2>&1; then
|
||||
echo "unzip is required to extract console assets" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p ./rustfs/static
|
||||
|
||||
verify_sha256() {
|
||||
local expected="$1"
|
||||
local file="$2"
|
||||
local actual=""
|
||||
local actual_line=""
|
||||
|
||||
if command -v sha256sum >/dev/null 2>&1; then
|
||||
actual_line=$(sha256sum -- "$file") || return 1
|
||||
elif command -v shasum >/dev/null 2>&1; then
|
||||
actual_line=$(shasum -a 256 -- "$file") || return 1
|
||||
else
|
||||
echo "No SHA-256 verification tool found" >&2
|
||||
return 1
|
||||
fi
|
||||
actual="${actual_line%%[[:space:]]*}"
|
||||
|
||||
if [ "$actual" = "$expected" ]; then
|
||||
echo "SHA256 verified OK: $actual"
|
||||
return 0
|
||||
else
|
||||
echo "SHA256 mismatch: expected=$expected actual=$actual" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
download_console_assets() {
|
||||
local curl_bin="curl"
|
||||
local python_bin="python3"
|
||||
local console_api="https://api.github.com/repos/rustfs/console/releases/latest"
|
||||
local console_json="console-release.json"
|
||||
local console_url
|
||||
local console_sha256
|
||||
local curl_auth_args=()
|
||||
console_tag=""
|
||||
|
||||
chmod +w ./rustfs/static/LICENSE || true
|
||||
|
||||
if ! command -v "$python_bin" >/dev/null 2>&1; then
|
||||
python_bin="python"
|
||||
fi
|
||||
if ! command -v "$python_bin" >/dev/null 2>&1; then
|
||||
echo "No Python interpreter found for release metadata parsing" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ -n "${GITHUB_TOKEN:-}" ]]; then
|
||||
curl_auth_args=(-H "Authorization: Bearer ${GITHUB_TOKEN}" -H "X-GitHub-Api-Version: 2022-11-28")
|
||||
fi
|
||||
|
||||
"$curl_bin" "${curl_auth_args[@]}" --fail -L "$console_api" \
|
||||
-o "$console_json" --retry 3 --retry-delay 5 --max-time 300 || return 1
|
||||
|
||||
read -r console_tag console_url console_sha256 < <("$python_bin" - "$console_json" <<'PY'
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
|
||||
with open(sys.argv[1], encoding="utf-8") as handle:
|
||||
release = json.load(handle)
|
||||
|
||||
tag = release.get("tag_name", "") or "unknown"
|
||||
|
||||
for asset in release.get("assets", []):
|
||||
name = asset.get("name", "")
|
||||
digest = asset.get("digest", "")
|
||||
url = asset.get("browser_download_url", "")
|
||||
if name.startswith("rustfs-console-") and name.endswith(".zip") and digest.startswith("sha256:") and url:
|
||||
sha256 = digest.split(":", 1)[1]
|
||||
if not re.fullmatch(r"[0-9a-fA-F]{64}", sha256):
|
||||
raise SystemExit(f"console zip asset has invalid sha256 digest: {sha256}")
|
||||
sys.stdout.buffer.write(f"{tag} {url} {sha256}\n".encode("utf-8"))
|
||||
break
|
||||
else:
|
||||
raise SystemExit("no console zip asset with sha256 digest found")
|
||||
PY
|
||||
) || return 1
|
||||
|
||||
if [[ -z "$console_url" || -z "$console_sha256" ]]; then
|
||||
echo "Console release metadata is missing URL or SHA-256 digest" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "Console release: ${console_tag}"
|
||||
echo "Downloading console asset: ${console_url}"
|
||||
"$curl_bin" --fail -L "$console_url" -o console.zip --retry 3 --retry-delay 5 --max-time 300 || return 1
|
||||
verify_sha256 "$console_sha256" console.zip || return 2
|
||||
unzip -o console.zip -d ./rustfs/static || return 2
|
||||
}
|
||||
|
||||
if download_console_assets; then
|
||||
rm -f console.zip console-release.json
|
||||
else
|
||||
status=$?
|
||||
rm -f console.zip console-release.json
|
||||
if [[ "$status" -eq 2 ]]; then
|
||||
echo "Console asset integrity verification failed" >&2
|
||||
fi
|
||||
echo "Failed to download verified console assets" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -s ./rustfs/static/index.html ]]; then
|
||||
echo "Console asset archive is missing static/index.html" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
asset_count=$(find ./rustfs/static -type f | wc -l | tr -d '[:space:]')
|
||||
echo "Console assets ready: version=${console_tag:-unknown}, ${asset_count} files extracted to ./rustfs/static"
|
||||
|
||||
- name: Build RustFS
|
||||
# Pin the glibc baseline (`<target>.<glibc>` for cargo-zigbuild) so
|
||||
# nightlies run on older distros such as RHEL/Rocky 9 (glibc 2.34); #8062.
|
||||
run: cargo zigbuild --release --locked --target x86_64-unknown-linux-gnu.2.31 -p rustfs --bins
|
||||
shell: bash
|
||||
run: |
|
||||
# Force rebuild by touching build.rs so a restored cache cannot
|
||||
# reuse a rustfs artifact compiled before these console assets existed.
|
||||
touch rustfs/build.rs
|
||||
cargo zigbuild --release --locked --target x86_64-unknown-linux-gnu.2.31 -p rustfs --bin rustfs
|
||||
|
||||
- name: Verify glibc symbol baseline
|
||||
run: |
|
||||
python3 scripts/release/check_glibc.py 2.31 \
|
||||
target/x86_64-unknown-linux-gnu/release/rustfs \
|
||||
target/x86_64-unknown-linux-gnu/release/rustfs-cli
|
||||
target/x86_64-unknown-linux-gnu/release/rustfs
|
||||
target/x86_64-unknown-linux-gnu/release/rustfs --version | grep -Fx 'build target : x86_64-unknown-linux-gnu'
|
||||
|
||||
- name: Verify console
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
find_free_port() {
|
||||
python3 - <<'PY'
|
||||
import socket
|
||||
|
||||
with socket.socket() as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
print(sock.getsockname()[1])
|
||||
PY
|
||||
}
|
||||
|
||||
binary="target/x86_64-unknown-linux-gnu/release/rustfs"
|
||||
data_dir="$(mktemp -d)"
|
||||
log_file="${RUNNER_TEMP}/rustfs-console-smoke.log"
|
||||
server_pid=""
|
||||
trap '
|
||||
if [[ -n "${server_pid:-}" ]]; then
|
||||
kill "$server_pid" 2>/dev/null || true
|
||||
wait "$server_pid" 2>/dev/null || true
|
||||
fi
|
||||
rm -rf "$data_dir"
|
||||
' EXIT
|
||||
|
||||
if [[ ! -x "$binary" ]]; then
|
||||
echo "rustfs binary not found: ${binary}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
api_port="$(find_free_port)"
|
||||
console_port="$(find_free_port)"
|
||||
console_url="http://127.0.0.1:${console_port}/rustfs/console/"
|
||||
|
||||
"$binary" server \
|
||||
--address "127.0.0.1:${api_port}" \
|
||||
--console-enable \
|
||||
--console-address "127.0.0.1:${console_port}" \
|
||||
--access-key console-smoke \
|
||||
--secret-key console-smoke-secret \
|
||||
"$data_dir" >"$log_file" 2>&1 &
|
||||
server_pid=$!
|
||||
|
||||
response=""
|
||||
for _ in {1..40}; do
|
||||
response="$(curl --silent --output /dev/null --write-out '%{http_code} %{content_type}' "$console_url" || true)"
|
||||
if [[ "$response" == 200\ text/html* ]]; then
|
||||
exit 0
|
||||
fi
|
||||
sleep 0.25
|
||||
done
|
||||
|
||||
echo "Console endpoint did not return 200 text/html: $response" >&2
|
||||
cat "$log_file"
|
||||
exit 1
|
||||
|
||||
- name: Build DEB package
|
||||
id: deb
|
||||
shell: bash
|
||||
@@ -346,12 +534,12 @@ jobs:
|
||||
path: ${{ steps.publish.outputs.candidate_file }}
|
||||
if-no-files-found: error
|
||||
|
||||
# Publish the deb/rpm pair to the auto-testing repo's `assets` branch so
|
||||
# engineers can download and install the nightly directly. The branch is
|
||||
# a single-commit orphan rewritten on every build, which keeps the repo
|
||||
# small while the latest files stay reachable at stable raw URLs.
|
||||
# Publish the deb/rpm pair as assets of the rolling `nightly` release on
|
||||
# rustfs/auto-testing (see scripts/release/publish_nightly_assets.sh).
|
||||
# The dispatch below runs only after this upload and the immutable R2
|
||||
# candidate both succeed. It does not replace rustfs-functional-chain.yml,
|
||||
# which still starts from this workflow's workflow_run completion.
|
||||
# kms-vault-lane and kms-vault-ha-failover stay parallel with this job.
|
||||
- name: Publish packages to auto-testing release assets
|
||||
env:
|
||||
ASSETS_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
@@ -360,6 +548,90 @@ jobs:
|
||||
DEB_DATE: ${{ steps.deb.outputs.deb_date }}
|
||||
BUILD_REF: ${{ env.NIGHTLY_BUILD_REF }}
|
||||
run: bash scripts/release/publish_nightly_assets.sh
|
||||
|
||||
# Cross-repo repository_dispatch cannot use GITHUB_TOKEN. The secret is a
|
||||
# PAT or GitHub App token for rustfs/auto-testing: classic `repo` scope,
|
||||
# or fine-grained Contents write (the permission GitHub checks for
|
||||
# POST /repos/{owner}/{repo}/dispatches). A dispatch failure fails this
|
||||
# step, and therefore this job, so the scheduled failure alert fires.
|
||||
# GitHub allows at most 10 top-level client_payload properties, so the
|
||||
# schema 2 candidate object is nested under `candidate` and the release
|
||||
# fields sit beside it.
|
||||
- name: Dispatch nightly candidate to auto-testing
|
||||
if: ${{ steps.publish.outputs.candidate_file != '' }}
|
||||
env:
|
||||
DISPATCH_TOKEN: ${{ secrets.AUTO_TESTING_DISPATCH_TOKEN }}
|
||||
CANDIDATE_FILE: ${{ steps.publish.outputs.candidate_file }}
|
||||
DEB_DATE: ${{ steps.deb.outputs.deb_date }}
|
||||
SOURCE_REPO: ${{ github.repository }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
: "${DISPATCH_TOKEN:?AUTO_TESTING_DISPATCH_TOKEN is required to dispatch rustfs/auto-testing}"
|
||||
: "${CANDIDATE_FILE:?nightly candidate manifest is required}"
|
||||
: "${DEB_DATE:?DEB_DATE is required}"
|
||||
[[ "${DEB_DATE}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]
|
||||
[[ "${SOURCE_REPO}" == "rustfs/rustfs" ]]
|
||||
|
||||
payload="${RUNNER_TEMP}/nightly-candidate-dispatch-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
|
||||
jq -n --slurpfile candidate "${CANDIDATE_FILE}" \
|
||||
--arg event_type "nightly-candidate-ready" \
|
||||
--arg source_repo "${SOURCE_REPO}" \
|
||||
--arg latest_alias_url "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb" \
|
||||
--arg auto_testing_release "https://github.com/rustfs/auto-testing/releases/tag/nightly" \
|
||||
--arg channel "nightly" \
|
||||
--arg deb_date "${DEB_DATE}" \
|
||||
--arg nightly_workflow ".github/workflows/nightly-gnu.yml" \
|
||||
'{
|
||||
event_type: $event_type,
|
||||
source_repo: $source_repo,
|
||||
latest_alias_url: $latest_alias_url,
|
||||
auto_testing_release: $auto_testing_release,
|
||||
channel: $channel,
|
||||
deb_date: $deb_date,
|
||||
nightly_workflow: $nightly_workflow,
|
||||
candidate: $candidate[0]
|
||||
}' > "${payload}"
|
||||
if ! jq -e '
|
||||
.event_type == "nightly-candidate-ready"
|
||||
and .source_repo == "rustfs/rustfs"
|
||||
and .channel == "nightly"
|
||||
and .deb_date == env.DEB_DATE
|
||||
and .nightly_workflow == ".github/workflows/nightly-gnu.yml"
|
||||
and .latest_alias_url == "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb"
|
||||
and .auto_testing_release == "https://github.com/rustfs/auto-testing/releases/tag/nightly"
|
||||
and .candidate.schema == 2
|
||||
and ((.candidate | keys | sort) == ["build_run_attempt","build_run_id","package_sha256","package_url","schema","source_ref","source_sha","workflow_sha"])
|
||||
and ((keys | sort) == ["auto_testing_release","candidate","channel","deb_date","event_type","latest_alias_url","nightly_workflow","source_repo"])
|
||||
' "${payload}" >/dev/null; then
|
||||
echo "nightly dispatch payload does not match the nightly-candidate-ready contract" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
body="${RUNNER_TEMP}/nightly-candidate-dispatch-body-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
|
||||
jq -n --slurpfile client_payload "${payload}" \
|
||||
'{event_type: "nightly-candidate-ready", client_payload: $client_payload[0]}' > "${body}"
|
||||
response="${RUNNER_TEMP}/nightly-candidate-dispatch-response-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.txt"
|
||||
status="$(curl -sS --retry 3 --connect-timeout 15 --max-time 60 \
|
||||
-o "${response}" -w '%{http_code}' \
|
||||
-X POST \
|
||||
-H "Authorization: Bearer ${DISPATCH_TOKEN}" \
|
||||
-H "Accept: application/vnd.github+json" \
|
||||
-H "X-GitHub-Api-Version: 2022-11-28" \
|
||||
-H "Content-Type: application/json" \
|
||||
--data-binary @"${body}" \
|
||||
https://api.github.com/repos/rustfs/auto-testing/dispatches)" || {
|
||||
echo "repository_dispatch request to rustfs/auto-testing failed before an HTTP status" >&2
|
||||
exit 1
|
||||
}
|
||||
if [[ "${status}" != "204" ]]; then
|
||||
echo "repository_dispatch to rustfs/auto-testing failed: HTTP ${status}" >&2
|
||||
if [[ -s "${response}" ]]; then
|
||||
cat "${response}" >&2
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
echo "dispatched nightly-candidate-ready to rustfs/auto-testing (deb_date=${DEB_DATE}, run=${GITHUB_RUN_ID}, attempt=${GITHUB_RUN_ATTEMPT})"
|
||||
# Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
|
||||
#
|
||||
# RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and
|
||||
|
||||
+210
-14
@@ -225,10 +225,57 @@ jobs:
|
||||
echo " Build run ID: $BUILD_RUN_ID"
|
||||
echo " Build run number: $RUN_NUMBER"
|
||||
|
||||
# Consume the trusted release certification of the resolved build run and
|
||||
# verify the certified input digests before any package is wrapped. A run
|
||||
# without that certification fails closed here; packages built later must
|
||||
# complete their own verification and cannot inherit an unverified
|
||||
# conclusion.
|
||||
release-certification:
|
||||
name: Release Certification
|
||||
needs: resolve
|
||||
if: needs.resolve.result == 'success'
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download and verify release certification
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
BUILD_RUN_ID: ${{ needs.resolve.outputs.build_run_id }}
|
||||
BUILD_TYPE: ${{ needs.resolve.outputs.build_type }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification
|
||||
if [[ "$BUILD_TYPE" == "development" ]]; then
|
||||
# Development artifacts are explicitly uncertified and never touch
|
||||
# official or latest channels; only the staging/dev paths remain.
|
||||
jq -n '{schema_version: 1, certified: false, reason: "development artifacts are explicitly uncertified"}' \
|
||||
> certification/development-uncertified.json
|
||||
exit 0
|
||||
fi
|
||||
if ! gh run download "$BUILD_RUN_ID" -n release-certification -D certification; then
|
||||
echo "Build run ${BUILD_RUN_ID} published no release-certification artifact; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
|
||||
--artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
|
||||
|
||||
# Build DEB and RPM packages for each architecture and libc variant
|
||||
package:
|
||||
name: Package (${{ matrix.arch }}-${{ matrix.libc }})
|
||||
needs: resolve
|
||||
needs: [ resolve, release-certification ]
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
@@ -494,6 +541,55 @@ jobs:
|
||||
echo "rpm_file=$RPM_FILE" >> "$GITHUB_OUTPUT"
|
||||
echo "✅ RPM built: $RPM_FILE"
|
||||
|
||||
- name: Download certification evidence
|
||||
if: needs.resolve.outputs.build_type != 'development'
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: release-certification
|
||||
path: ./certification
|
||||
|
||||
# Later-built wrapper verification: the binary inside each package must
|
||||
# match the certified binary digest for this platform before anything is
|
||||
# uploaded, and the package itself stays on staging until promotion.
|
||||
- name: Verify wrapped binary against certification
|
||||
if: needs.resolve.outputs.build_type != 'development'
|
||||
shell: bash
|
||||
env:
|
||||
DEB_FILE: ${{ steps.deb.outputs.deb_file }}
|
||||
RPM_FILE: ${{ steps.rpm.outputs.rpm_file }}
|
||||
ARTIFACT_NAME: ${{ matrix.artifact_name }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
expected="$(jq -r --arg needle "$ARTIFACT_NAME" \
|
||||
'[.deliverables[] | select(.name | contains($needle)) | .binary_sha256][0] // empty' \
|
||||
certification/candidate.json)"
|
||||
if [[ -z "$expected" ]]; then
|
||||
echo "No certified binary digest recorded for ${ARTIFACT_NAME}" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! command -v rpm2cpio >/dev/null 2>&1; then
|
||||
sudo apt-get update && sudo apt-get install -y rpm2cpio
|
||||
fi
|
||||
if ! command -v cpio >/dev/null 2>&1; then
|
||||
sudo apt-get update && sudo apt-get install -y cpio
|
||||
fi
|
||||
for package_file in "$DEB_FILE" "$RPM_FILE"; do
|
||||
case "$package_file" in
|
||||
*.deb)
|
||||
dpkg-deb --fsys-tarfile "$package_file" | tar -xOf - ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
|
||||
;;
|
||||
*.rpm)
|
||||
rpm2cpio "$package_file" | cpio -i --to-stdout ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
|
||||
;;
|
||||
esac
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$RUNNER_TEMP/wrapped-rustfs" \
|
||||
--artifact-role binary
|
||||
done
|
||||
|
||||
- name: Upload packages to artifacts
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
@@ -531,10 +627,14 @@ jobs:
|
||||
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
|
||||
export AWS_DEFAULT_REGION="auto"
|
||||
|
||||
# Build-phase writes stay on staging for release types: the official
|
||||
# packages/release and packages/latest paths are written only by the
|
||||
# certification-gated promote-packages job, from these same staged
|
||||
# objects. Development packages keep the uncertified dev channel.
|
||||
if [[ "$BUILD_TYPE" == "development" ]]; then
|
||||
R2_PREFIX="artifacts/rustfs/packages/dev"
|
||||
else
|
||||
R2_PREFIX="artifacts/rustfs/packages/release"
|
||||
R2_PREFIX="artifacts/rustfs/packages/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}"
|
||||
fi
|
||||
R2_PATH="s3://${R2_BUCKET}/${R2_PREFIX}/"
|
||||
|
||||
@@ -549,18 +649,6 @@ jobs:
|
||||
|
||||
echo "✅ Upload complete"
|
||||
|
||||
# Also upload as latest for release/prerelease
|
||||
if [[ "$BUILD_TYPE" == "release" || "$BUILD_TYPE" == "prerelease" ]]; then
|
||||
LATEST_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/latest/"
|
||||
for f in "$DEB_FILE" "$RPM_FILE"; do
|
||||
if [[ -n "$f" && -f "$f" ]]; then
|
||||
echo "Uploading latest: $(basename "$f")"
|
||||
aws s3 cp "$f" "$LATEST_PATH" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
fi
|
||||
done
|
||||
echo "✅ Latest packages updated"
|
||||
fi
|
||||
|
||||
- name: Upload packages to GitHub Release
|
||||
if: needs.resolve.outputs.tag != ''
|
||||
env:
|
||||
@@ -614,6 +702,114 @@ jobs:
|
||||
|
||||
echo "✅ GitHub release assets updated"
|
||||
|
||||
# The only writer of the official packages/release and packages/latest R2
|
||||
# paths. Promotion re-checks each package's wrapped binary against the
|
||||
# passed certification list and copies the staged objects byte-for-byte:
|
||||
# no rebuild-after-verify, no mutable latest resolution downstream.
|
||||
promote-packages:
|
||||
name: Promote Packages
|
||||
needs: [ resolve, package, release-certification ]
|
||||
# Target-channel lock for the official package paths.
|
||||
concurrency:
|
||||
group: release-channel-r2-packages
|
||||
cancel-in-progress: false
|
||||
if: needs.resolve.outputs.tag != '' && (needs.resolve.outputs.build_type == 'release' || needs.resolve.outputs.build_type == 'prerelease')
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
env:
|
||||
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
|
||||
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
|
||||
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
|
||||
R2_BUCKET: ${{ secrets.R2_BUCKET }}
|
||||
AWS_EC2_METADATA_DISABLED: true
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download packages and certification
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
path: ./packages
|
||||
pattern: packages-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Download certification evidence
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: release-certification
|
||||
path: ./certification
|
||||
|
||||
- name: Verify and promote staged packages
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
if [[ -z "$R2_ACCESS_KEY_ID" || -z "$R2_SECRET_ACCESS_KEY" || -z "$R2_ENDPOINT" || -z "$R2_BUCKET" ]]; then
|
||||
echo "❌ R2 credentials missing; refusing to promote to the official channel" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
STAGING_PREFIX="s3://${R2_BUCKET}/artifacts/rustfs/packages/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
|
||||
RELEASE_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/release/"
|
||||
LATEST_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/latest/"
|
||||
|
||||
if ! command -v aws >/dev/null 2>&1; then
|
||||
sudo apt-get update && sudo apt-get install -y awscli
|
||||
fi
|
||||
export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
|
||||
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
|
||||
export AWS_DEFAULT_REGION="auto"
|
||||
|
||||
if ! command -v rpm2cpio >/dev/null 2>&1; then
|
||||
sudo apt-get install -y rpm2cpio
|
||||
fi
|
||||
if ! command -v cpio >/dev/null 2>&1; then
|
||||
sudo apt-get install -y cpio
|
||||
fi
|
||||
|
||||
promoted=0
|
||||
for package_file in packages/*.deb packages/*.rpm; do
|
||||
[[ -f "$package_file" ]] || continue
|
||||
name="$(basename "$package_file")"
|
||||
case "$package_file" in
|
||||
*.deb)
|
||||
dpkg-deb --fsys-tarfile "$package_file" | tar -xOf - ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
|
||||
;;
|
||||
*.rpm)
|
||||
rpm2cpio "$package_file" | cpio -i --to-stdout ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
|
||||
;;
|
||||
esac
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$RUNNER_TEMP/wrapped-rustfs" \
|
||||
--artifact-role binary
|
||||
|
||||
staged="$(mktemp)"
|
||||
aws s3 cp "${STAGING_PREFIX}${name}" "$staged" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
staged_sha="$(sha256sum "$staged" | awk '{print $1}')"
|
||||
built_sha="$(sha256sum "$package_file" | awk '{print $1}')"
|
||||
if [[ "$staged_sha" != "$built_sha" ]]; then
|
||||
echo "❌ Staged package ${name} does not match the verified bytes" >&2
|
||||
exit 1
|
||||
fi
|
||||
aws s3 cp "$staged" "${RELEASE_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
aws s3 cp "$staged" "${LATEST_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
rm -f "$staged"
|
||||
promoted=$((promoted + 1))
|
||||
echo "✅ Promoted ${name} (${built_sha})"
|
||||
done
|
||||
if [[ "$promoted" -eq 0 ]]; then
|
||||
echo "No staged packages found to promote" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Summary
|
||||
summary:
|
||||
name: Summary
|
||||
|
||||
@@ -0,0 +1,457 @@
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: RustFS Validation Sweep
|
||||
|
||||
# Executes the migrated canonical validation cases (rustfs/auto-testing#113)
|
||||
# on the shared functional fleet. Everything with a lane executor is swept in
|
||||
# rotating nightly batches so the whole set is covered roughly weekly; a
|
||||
# single case or one module can be dispatched manually. That is the
|
||||
# implemented catalog (66 today) plus the eight planned cases whose dedicated
|
||||
# executors already exist and have live PASS evidence - fresh nightly
|
||||
# evidence feeds their acceptance review. heal.* cases stay on the
|
||||
# interactive validation flow: they require reviewed native/tool collectors
|
||||
# built by separate validation-tools runs before fleet execution.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# After the nightly functional chain settles (chain starts ~16:48 UTC and
|
||||
# runs for about two hours).
|
||||
- cron: "30 19 * * *"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
case:
|
||||
description: 'Single canonical validation case id (overrides module and rotation), e.g. pool.decommission'
|
||||
type: string
|
||||
required: false
|
||||
default: ''
|
||||
module:
|
||||
description: 'Restrict the sweep to one catalog module, e.g. bucket-replication'
|
||||
type: string
|
||||
required: false
|
||||
default: ''
|
||||
limit:
|
||||
description: 'Maximum cases per sweep (0 = run everything selected)'
|
||||
type: number
|
||||
required: false
|
||||
default: 12
|
||||
build_run_id:
|
||||
description: 'Successful nightly package run; empty resolves the latest'
|
||||
type: string
|
||||
required: false
|
||||
default: ''
|
||||
build_run_attempt:
|
||||
description: 'Attempt of that package run'
|
||||
type: string
|
||||
required: false
|
||||
default: '1'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: rustfs-validation-sweep
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES || 'vm000 vm001 vm002 vm003' }}
|
||||
RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER || 'azureuser' }}
|
||||
RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
|
||||
RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
|
||||
RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
|
||||
PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
jobs:
|
||||
sweep:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 150
|
||||
steps:
|
||||
- name: Checkout chain tooling
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Resolve nightly candidate
|
||||
id: candidate
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
BUILD_RUN_ID: ${{ inputs.build_run_id }}
|
||||
BUILD_RUN_ATTEMPT: ${{ inputs.build_run_attempt }}
|
||||
CHAIN_OUTPUT: ${{ runner.temp }}/validation-sweep-candidate.json
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Scheduled runs carry no inputs and the resolver only derives a
|
||||
# build from a workflow_run payload (which a schedule cannot
|
||||
# provide), so resolve the newest successful nightly build
|
||||
# explicitly before invoking it.
|
||||
if [ -z "${BUILD_RUN_ID:-}" ]; then
|
||||
LATEST="$(gh api "repos/rustfs/rustfs/actions/workflows/nightly-gnu.yml/runs?branch=main&status=success&per_page=1" --jq '.workflow_runs[0]')"
|
||||
BUILD_RUN_ID="$(printf '%s' "${LATEST}" | jq -r '.id')"
|
||||
BUILD_RUN_ATTEMPT="$(printf '%s' "${LATEST}" | jq -r '.run_attempt')"
|
||||
export BUILD_RUN_ID BUILD_RUN_ATTEMPT
|
||||
echo "scheduled sweep pinned to nightly ${BUILD_RUN_ID} attempt ${BUILD_RUN_ATTEMPT}"
|
||||
fi
|
||||
python3 scripts/resolve_functional_candidate.py
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
root = Path(os.environ["RUNNER_TEMP"]) / "validation-sweep"
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
chain = json.loads((Path(os.environ["RUNNER_TEMP"]) / "validation-sweep-candidate.json").read_text())
|
||||
manifest = chain["candidate"]["manifest"]
|
||||
(root / "candidate.json").write_text(json.dumps({
|
||||
"package_url": manifest["package_url"],
|
||||
"package_sha256": manifest["package_sha256"],
|
||||
"source_sha": manifest["source_sha"],
|
||||
}, sort_keys=True) + "\n")
|
||||
with open(os.environ["GITHUB_OUTPUT"], "a") as output:
|
||||
output.write("testing_sha=" + chain["testing_sha"] + "\n")
|
||||
output.write("package_url=" + manifest["package_url"] + "\n")
|
||||
output.write("source_sha=" + manifest["source_sha"] + "\n")
|
||||
PY
|
||||
|
||||
- name: Checkout auto-testing scripts
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
repository: rustfs/auto-testing
|
||||
ref: ${{ steps.candidate.outputs.testing_sha }}
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
path: auto-testing
|
||||
persist-credentials: false
|
||||
|
||||
- name: Prepare validation Python runtime
|
||||
# The pool / ilm / replication validation runners require a managed
|
||||
# Python, exactly like the chain's validation job provides.
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install validation Python requirements
|
||||
# Case scripts call python3 and import boto3 at startup. Install the
|
||||
# functional chain's client into a per-run env so that interpreter
|
||||
# sees it without changing the runner's shared Python.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
validation_python="${RUNNER_TEMP}/rustfs-validation-python-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
test ! -e "${validation_python}"
|
||||
python3 -m venv "${validation_python}"
|
||||
"${validation_python}/bin/python" -m pip install \
|
||||
-r auto-testing/scripts/validation_ilm/requirements.txt
|
||||
printf '%s\n' "${validation_python}/bin" >> "${GITHUB_PATH}"
|
||||
|
||||
- name: Select sweep cases
|
||||
id: select
|
||||
env:
|
||||
SELECTED_CASE: ${{ inputs.case }}
|
||||
SELECTED_MODULE: ${{ inputs.module }}
|
||||
CASE_LIMIT: ${{ inputs.limit }}
|
||||
SWEEP_EVENT: ${{ github.event_name }}
|
||||
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 - <<'PY'
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
sweep_root = Path(os.environ["SWEEP_ROOT"])
|
||||
|
||||
catalog = json.loads(Path("auto-testing/validation-cases.json").read_text())
|
||||
def sweepable(case):
|
||||
status = case.get("source_status")
|
||||
if status == "implemented":
|
||||
return True
|
||||
# The remaining planned non-heal cases route to the lane
|
||||
# executors and already have live PASS evidence; sweeping them
|
||||
# nightly produces fresh evidence for their acceptance review.
|
||||
return status == "planned" and not case["id"].startswith("heal.")
|
||||
|
||||
selected = [(case["id"], module["id"]) for module in catalog["modules"]
|
||||
for case in module["cases"]
|
||||
if sweepable(case)]
|
||||
if not selected:
|
||||
raise SystemExit("catalog has no sweepable cases")
|
||||
|
||||
one_case = os.environ.get("SELECTED_CASE", "").strip()
|
||||
module_filter = os.environ.get("SELECTED_MODULE", "").strip()
|
||||
known = {case_id for case_id, _ in selected}
|
||||
modules = {module_id for _, module_id in selected}
|
||||
if one_case:
|
||||
if one_case not in known:
|
||||
raise SystemExit(f"unknown case or no lane executor for it: {one_case}")
|
||||
selected = [(one_case, dict(selected)[one_case])]
|
||||
elif module_filter:
|
||||
if module_filter not in modules:
|
||||
raise SystemExit(f"unknown module or no implemented cases: {module_filter}")
|
||||
selected = [pair for pair in selected if pair[1] == module_filter]
|
||||
elif os.environ.get("SWEEP_EVENT") == "schedule":
|
||||
# Stateless rotation: every night take one window of the catalog
|
||||
# so a full sweep needs len(selected)/limit nights and no state
|
||||
# has to be carried between runs.
|
||||
limit = max(1, int(os.environ.get("CASE_LIMIT") or 12))
|
||||
day = datetime.datetime.now(datetime.timezone.utc).timetuple().tm_yday
|
||||
start = (day * limit) % len(selected)
|
||||
selected = (selected[start:] + selected[:start])[:limit]
|
||||
limit = int(os.environ.get("CASE_LIMIT") or 0)
|
||||
if one_case or module_filter or (not os.environ.get("SWEEP_EVENT") == "schedule"):
|
||||
if limit > 0:
|
||||
selected = selected[:limit]
|
||||
|
||||
(sweep_root / "sweep-cases.txt").write_text("".join(f"{case_id}\n" for case_id, _ in selected))
|
||||
with open(os.environ["GITHUB_OUTPUT"], "a") as output:
|
||||
output.write(f"count={len(selected)}\n")
|
||||
print(f"selected {len(selected)} case(s):", ", ".join(c for c, _ in selected))
|
||||
PY
|
||||
|
||||
- name: Admit clean shared environment
|
||||
id: environment
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-validation-sweep.json
|
||||
run: python3 scripts/functional_environment.py enter --suite heal
|
||||
|
||||
- name: Run validation cases
|
||||
id: run_cases
|
||||
timeout-minutes: 120
|
||||
env:
|
||||
AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
|
||||
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
|
||||
run: |
|
||||
set -uo pipefail
|
||||
mkdir -p "${SWEEP_ROOT}/results"
|
||||
: > "${SWEEP_ROOT}/results.txt"
|
||||
while read -r CASE_ID; do
|
||||
[ -n "${CASE_ID}" ] || continue
|
||||
echo "::group::case ${CASE_ID}"
|
||||
rm -rf "${SWEEP_ROOT}/run"
|
||||
if ./auto-testing/scripts/run_validation_case.sh \
|
||||
"${CASE_ID}" "${SWEEP_ROOT}/candidate.json" \
|
||||
"${AUTO_TESTING_SHA}" "${SWEEP_ROOT}/run" \
|
||||
> "${SWEEP_ROOT}/results/${CASE_ID//\//_}.log" 2>&1; then
|
||||
printf '%s\tPASS\n' "${CASE_ID}" >> "${SWEEP_ROOT}/results.txt"
|
||||
echo "case ${CASE_ID}: PASS"
|
||||
else
|
||||
printf '%s\tFAIL\n' "${CASE_ID}" >> "${SWEEP_ROOT}/results.txt"
|
||||
echo "case ${CASE_ID}: FAIL (log retained in the sweep artifact)"
|
||||
fi
|
||||
if [ -f "${SWEEP_ROOT}/run/report.md" ]; then
|
||||
cp "${SWEEP_ROOT}/run/report.md" "${SWEEP_ROOT}/results/${CASE_ID//\//_}.report.md" 2>/dev/null || true
|
||||
fi
|
||||
echo "::endgroup::"
|
||||
# Fleet hygiene between cases: the case runners set up their own
|
||||
# topology, this purge guarantees the next case starts clean even
|
||||
# when the previous one died mid-run.
|
||||
for node in ${RUSTFS_NODES}; do
|
||||
# -n keeps ssh from consuming the case list this loop reads from stdin
|
||||
ssh -n -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
${SUDO} dpkg -P rustfs
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -f /etc/default/rustfs
|
||||
' || echo "inter-case cleanup failed on ${node} (continuing)"
|
||||
done
|
||||
done < "${SWEEP_ROOT}/sweep-cases.txt"
|
||||
echo "sweep finished: $(wc -l < "${SWEEP_ROOT}/results.txt") case(s) executed"
|
||||
|
||||
- name: Generate sweep report
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
env:
|
||||
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
|
||||
PACKAGE_URL: ${{ steps.candidate.outputs.package_url }}
|
||||
SOURCE_SHA: ${{ steps.candidate.outputs.source_sha }}
|
||||
AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
root = Path(os.environ["SWEEP_ROOT"])
|
||||
catalog = json.loads(Path("auto-testing/validation-cases.json").read_text())
|
||||
module_of = {case["id"]: module["id"] for module in catalog["modules"] for case in module["cases"]}
|
||||
rows = []
|
||||
for line in (root / "results.txt").read_text().splitlines():
|
||||
if not line:
|
||||
continue
|
||||
case_id, status = line.split("\t")
|
||||
rows.append((case_id, module_of.get(case_id, "?"), status))
|
||||
passed = sum(1 for _, _, s in rows if s == "PASS")
|
||||
failed = len(rows) - passed
|
||||
run_url = (f"{os.environ.get('GITHUB_SERVER_URL', 'https://github.com')}/"
|
||||
f"{os.environ.get('GITHUB_REPOSITORY', 'rustfs/rustfs')}/actions/runs/"
|
||||
f"{os.environ.get('GITHUB_RUN_ID', '')}")
|
||||
|
||||
report = [
|
||||
"# RustFS validation sweep report", "",
|
||||
f"- Run: {run_url}",
|
||||
f"- Attempt: {os.environ.get('GITHUB_RUN_ATTEMPT', '')}",
|
||||
f"- Trigger: {os.environ.get('GITHUB_EVENT_NAME', '')}",
|
||||
f"- Package: {os.environ.get('PACKAGE_URL', 'nightly (R2 latest)')}",
|
||||
f"- Source SHA: {os.environ.get('SOURCE_SHA', '')}",
|
||||
f"- auto-testing: {os.environ.get('AUTO_TESTING_SHA', '')}", "",
|
||||
"Canonical validation cases (rustfs/auto-testing#113) executed by the sweep, grouped by catalog module.", "",
|
||||
]
|
||||
# Group the executed cases by catalog module (the major category:
|
||||
# pool lifecycle, heal, ILM expiration, bucket/site replication, kms,
|
||||
# data services, identity, s3 compatibility). Per-module subtotals
|
||||
# deliberately use "- Subtotal:" so the dashboard overview regexes
|
||||
# (which anchor on "- Total:") keep reading the grand totals below.
|
||||
by_module = {}
|
||||
module_order = []
|
||||
for case_id, module, status in rows:
|
||||
if module not in by_module:
|
||||
by_module[module] = []
|
||||
module_order.append(module)
|
||||
by_module[module].append((case_id, status))
|
||||
report += ["## Case Summary", ""]
|
||||
for module in module_order:
|
||||
cases = by_module[module]
|
||||
mod_pass = sum(1 for _, st in cases if st == "PASS")
|
||||
mod_fail = len(cases) - mod_pass
|
||||
report += [f"### {module} ({len(cases)})", "",
|
||||
"| Case | Status |", "| --- | --- |"]
|
||||
report += [f"| {case_id} | {status} |" for case_id, status in cases]
|
||||
report += ["", f"- Subtotal: {mod_pass} passed, {mod_fail} failed", ""]
|
||||
report += [
|
||||
"## Totals", "",
|
||||
f"- Total: {len(rows)}",
|
||||
f"- PASS: {passed}",
|
||||
f"- FAIL: {failed}",
|
||||
"- UNSUPPORTED: 0", "",
|
||||
"Failing cases keep this workflow red only as a report; the sweep artifact",
|
||||
"retains every case log and per-case report for diagnosis.", "",
|
||||
]
|
||||
(root / "report.md").write_text("\n".join(report))
|
||||
(root / "cases.md").write_text("\n".join(f"| {c} | {m} | {s} |" for c, m, s in rows) + "\n")
|
||||
PY
|
||||
cat "${SWEEP_ROOT}/report.md" >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 2
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
|
||||
REPORT_FILE: ${{ runner.temp }}/validation-sweep/report.md
|
||||
SUITE: validation
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
|
||||
exit 0
|
||||
fi
|
||||
DATE="$(date -u +%Y-%m-%d)"
|
||||
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
|
||||
B64_FILE="$(mktemp)"
|
||||
python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
|
||||
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${SHA}" ]; then
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
|
||||
'{message:$msg, content:($content|rtrimstr("\n"))}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
fi
|
||||
echo "dashboard updated: ${REPORT_PATH}"
|
||||
|
||||
- name: Retain sweep evidence
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: rustfs-validation-sweep-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
${{ runner.temp }}/validation-sweep/report.md
|
||||
${{ runner.temp }}/validation-sweep/results.txt
|
||||
${{ runner.temp }}/validation-sweep/sweep-cases.txt
|
||||
${{ runner.temp }}/validation-sweep/candidate.json
|
||||
${{ runner.temp }}/validation-sweep/results/
|
||||
${{ runner.temp }}/validation-sweep/run/
|
||||
# warn (not error): an early infra failure legitimately leaves no
|
||||
# per-case evidence, and the failing step is already red.
|
||||
if-no-files-found: warn
|
||||
retention-days: 30
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
id: environment_cleanup
|
||||
timeout-minutes: 5
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
cleanup_status=0
|
||||
for node in ${RUSTFS_NODES}; do
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
|
||||
set -euo pipefail
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
${SUDO} dpkg -P rustfs
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -f /etc/default/rustfs
|
||||
' || cleanup_status=$?
|
||||
done
|
||||
exit "${cleanup_status}"
|
||||
|
||||
- name: Release environment only after successful cleanup
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-validation-sweep.json
|
||||
CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
|
||||
run: python3 scripts/functional_environment.py finish --suite heal --cleanup-outcome "$CLEANUP_OUTCOME"
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
run: echo "RustFS validation sweep failed (harness/environment breakdown); see the sweep artifact for case logs."
|
||||
|
||||
- name: Enforce sweep result
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
env:
|
||||
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if grep -q $'\tFAIL$' "${SWEEP_ROOT}/results.txt"; then
|
||||
echo "validation sweep has failing cases:"
|
||||
grep $'\tFAIL$' "${SWEEP_ROOT}/results.txt"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
# A scheduled sweep failure usually means a canonical validation case
|
||||
# regressed or the fleet environment broke; surface it as a tracking
|
||||
# issue instead of leaving it in the Actions tab.
|
||||
needs: [sweep]
|
||||
if: github.repository == 'rustfs/rustfs' && (always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure'))
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Open or update failure-tracking issue
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}
|
||||
@@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- IAM snapshot/deletion replay, target-assigned delete-marker purges, timestamp ordering, and best-effort peer broadcast now close the control-plane gaps found by the R6 review ([#7195](https://github.com/rustfs/rustfs/pull/7195)).
|
||||
- A `set-remote-target?update=true&creds=true` request on a site-replication peer's bucket target is now rejected with `400 InvalidRequest`, naming the target and why it counts as peer-owned. It used to return success while keeping the stored endpoint, credentials, and TLS settings, so an endpoint edit from the Console's bucket replication form appeared to apply but did not. Change a peer's address with a site-replication edit instead; see "Changing a site's address" in `docs/operations/site-replication-operations.md`. The `sync`, `bandwidth`, `proxy`, and `path` update groups are unchanged.
|
||||
- Site-replication edits that omit the site name, such as `mc admin replicate update --deployment-id <ID> --endpoint <URL>`, now keep the stored name on the editing site and on every receiver instead of renaming the site after the new endpoint host. No persisted format changes.
|
||||
- The persisted replication MRF journal is now replayed every 5 minutes while the server runs, not only at startup. Delete-marker purges whose targets were unreachable, and replications that overflowed the worker queues, now converge within one replay interval after the target returns instead of waiting for a restart. A target that stays down is re-attempted once per retained entry per interval across the cluster: the start time of the last pass is shared through a small `config/replication/mrf.bin.replay` object, so nodes do not each replay once per interval. Older versions ignore that object; existing persisted formats are unchanged.
|
||||
- Upgrade and rollback: upgrade every node in one site consecutively and verify reconciliation before moving to the next site; do not intentionally run a site mixed-version. Target-version ledger keys are harmless on rollback, although old code cannot use their routing. Before rolling back past [#7307](https://github.com/rustfs/rustfs/pull/7307), drain or repair every pending version purge: older code can free a retained version's data directory before its remote purge is acknowledged. See `docs/operations/site-replication-operations.md`.
|
||||
|
||||
### Security
|
||||
|
||||
Generated
+444
-232
File diff suppressed because it is too large
Load Diff
+20
-22
@@ -147,7 +147,7 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.2" }
|
||||
async-channel = "2.5.0"
|
||||
async_zip = { default-features = false, version = "0.0.19" }
|
||||
mysql_async = { default-features = false, version = "0.37.1" }
|
||||
async-compression = { version = "0.4.51" }
|
||||
async-compression = { version = "0.4.52" }
|
||||
async-trait = "0.1.92"
|
||||
async-nats = { version = "0.51.0", default-features = false }
|
||||
axum = "0.8.9"
|
||||
@@ -168,16 +168,16 @@ http-body-util = "0.1.5"
|
||||
memchr = "2.8.3"
|
||||
minlz = "1.2.3"
|
||||
reqwest = "0.13.5"
|
||||
rustfs-kafka-async = { version = "1.4.0" }
|
||||
rustfs-kafka-async = { version = "1.4.1" }
|
||||
socket2 = { version = "0.6.5" }
|
||||
tokio = { version = "1.54.1" }
|
||||
tokio-rustls = { default-features = false, version = "0.26.6" }
|
||||
tokio-stream = { version = "0.1.19" }
|
||||
tokio-test = "0.4.6"
|
||||
tokio-util = { version = "0.7.20" }
|
||||
tonic = { version = "0.14.8" }
|
||||
tonic-prost = { version = "0.14.8" }
|
||||
tonic-prost-build = { version = "0.14.8" }
|
||||
tonic = { version = "0.14.9" }
|
||||
tonic-prost = { version = "0.14.9" }
|
||||
tonic-prost-build = { version = "0.14.9" }
|
||||
tower = { version = "0.5.3" }
|
||||
tower-http = { version = "0.7.1" }
|
||||
h3 = "0.0.8"
|
||||
@@ -191,6 +191,7 @@ quinn = { version = "0.11.12", default-features = false, features = [
|
||||
# Serialization and Data Formats
|
||||
apache-avro = { version = "0.22.0", features = ["snappy", "zstandard"] }
|
||||
bytes = { version = "1.12.1" }
|
||||
csv-core = "0.1.13"
|
||||
bytesize = "2.7.0"
|
||||
byteorder = "1.5.0"
|
||||
flatbuffers = "25.12.19"
|
||||
@@ -212,12 +213,13 @@ serde_urlencoded = "0.7.1"
|
||||
# releases.
|
||||
aes-gcm = { version = "0.11.1" }
|
||||
argon2 = { version = "0.6.0" }
|
||||
aws-lc-rs = { version = "1.19.0", default-features = false }
|
||||
blake2 = "0.11.0"
|
||||
chacha20poly1305 = { version = "0.11.0" }
|
||||
crc-fast = "1.10.0"
|
||||
hmac = { version = "0.13.0" }
|
||||
jsonwebtoken = { version = "11.1.0" }
|
||||
openidconnect = { default-features = false, version = "4.0" }
|
||||
openidconnect = { default-features = false, version = "4.1.0" }
|
||||
pbkdf2 = "0.13.0"
|
||||
p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
|
||||
rsa = { version = "=0.10.0-rc.18" }
|
||||
@@ -234,7 +236,7 @@ proptest = "1"
|
||||
# Time and Date
|
||||
chrono = { version = "0.4.45" }
|
||||
humantime = "2.4.0"
|
||||
jiff = { version = "0.2.38" }
|
||||
jiff = { version = "0.2.39" }
|
||||
time = { version = "0.3.55" }
|
||||
|
||||
# Database
|
||||
@@ -253,9 +255,9 @@ atoi = "3.1.1"
|
||||
atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.13.0" }
|
||||
aws-credential-types = { version = "1.3.1" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.128.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.156.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.123.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.129.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.157.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.124.0" }
|
||||
aws-smithy-async = { version = "1.3.1" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.5.2" }
|
||||
aws-smithy-runtime-api = { version = "1.19.2" }
|
||||
@@ -319,7 +321,7 @@ rustfs-erasure-codec = { version = "9.0.0" }
|
||||
reed-solomon-simd = "3.1.0"
|
||||
regex = { version = "1.13.1" }
|
||||
rumqttc = { package = "rumqttc-next", version = "0.34.0" }
|
||||
redis = { version = "1.8.0" }
|
||||
redis = { version = "1.8.1" }
|
||||
rustify = { version = "0.7", default-features = false }
|
||||
rustix = { version = "1.1.6" }
|
||||
rust-embed = { version = "8.13.0" }
|
||||
@@ -334,10 +336,10 @@ compact_str = "0.10.0"
|
||||
snap = "1.1.2"
|
||||
starshard = { version = "2.4.0" }
|
||||
strum = { version = "0.28.0" }
|
||||
sysinfo = "0.40.0"
|
||||
sysinfo = "0.40.1"
|
||||
temp-env = "0.3.6"
|
||||
tempfile = "3.27.0"
|
||||
test-case = "3.4.0"
|
||||
test-case = "3.4.1"
|
||||
thiserror = "2.0.21"
|
||||
tracing = { version = "0.1.44" }
|
||||
tracing-appender = "0.2.5"
|
||||
@@ -356,13 +358,13 @@ winapi-util = "0.1.11"
|
||||
windows = { version = "0.62.2" }
|
||||
windows-sys = "0.61.2"
|
||||
xxhash-rust = { version = "0.8.19" }
|
||||
zip = "9.1.3"
|
||||
zip = "9.3.0"
|
||||
zstd = "0.14.3"
|
||||
|
||||
# Observability and Metrics
|
||||
metrics = "0.24.6"
|
||||
metrics-util = "0.20"
|
||||
dial9 = { version = "0.5.6", default-features = false }
|
||||
dial9 = { version = "0.5.7", default-features = false }
|
||||
opentelemetry = { version = "0.33.1" }
|
||||
opentelemetry-appender-tracing = { version = "0.33.1" }
|
||||
opentelemetry-otlp = { version = "0.33.1" }
|
||||
@@ -377,16 +379,16 @@ libunftp = { version = "0.24.0" }
|
||||
unftp-core = "0.1.0"
|
||||
suppaftp = { version = "12.2.1" }
|
||||
rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
russh = { version = "0.65.0" }
|
||||
russh = { version = "0.65.1" }
|
||||
russh-sftp = "3.0.1"
|
||||
|
||||
# WebDAV
|
||||
dav-server = "0.11.0"
|
||||
|
||||
# Performance Analysis and Memory Profiling
|
||||
rustfs-mimalloc = { version = "0.7.2" }
|
||||
rustfs-mimalloc = { version = "0.7.3" }
|
||||
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
|
||||
hotpath = { version = "0.30.1", default-features = false }
|
||||
hotpath = { version = "0.30.3", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.49" }
|
||||
|
||||
@@ -427,7 +429,3 @@ strip = "none"
|
||||
# Pin upstream fixes until compatible crates.io releases are available.
|
||||
[patch.crates-io]
|
||||
zstd = { path = "crates/zstd-compat" }
|
||||
# Preserve RustFS OIDC flows while migrating the old elliptic-curve dependency chain.
|
||||
# Upstream: openidconnect-rs#244 and oauth2-rs#352.
|
||||
openidconnect = { git = "https://github.com/ramosbugs/openidconnect-rs", rev = "2f77ee73bc4554e1cb6279adb386005429fb4e41" }
|
||||
oauth2 = { git = "https://github.com/ramosbugs/oauth2-rs", rev = "89585f4a280afd7b285eaf1c9be0068874e03d22" }
|
||||
|
||||
+11
-3
@@ -16,6 +16,7 @@ FROM alpine:3.24.2 AS build
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RELEASE=latest
|
||||
ARG RC_RELEASE=latest
|
||||
|
||||
RUN apk add --no-cache ca-certificates curl jq unzip
|
||||
WORKDIR /build
|
||||
@@ -69,6 +70,9 @@ RUN set -eux; \
|
||||
chmod +x /build/rustfs; \
|
||||
rm -rf rustfs.zip /build/.tmp || true
|
||||
|
||||
COPY scripts/download_rc.sh /usr/local/bin/download_rc.sh
|
||||
RUN sh /usr/local/bin/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /build/rc
|
||||
|
||||
|
||||
FROM alpine:3.24.2
|
||||
|
||||
@@ -95,14 +99,18 @@ RUN apk upgrade --no-cache && \
|
||||
ca-certificates \
|
||||
coreutils \
|
||||
curl \
|
||||
jq \
|
||||
tzdata \
|
||||
&& test "$(TZ=Asia/Kolkata date +%z)" = "+0530"
|
||||
|
||||
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
|
||||
COPY --from=build /build/rustfs /usr/bin/rustfs
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
COPY --from=build /build/rc /usr/bin/rc
|
||||
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
|
||||
COPY bootstrap/ /etc/rustfs/bootstrap/
|
||||
|
||||
RUN chmod +x /usr/bin/rustfs /entrypoint.sh
|
||||
RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
|
||||
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
|
||||
|
||||
RUN addgroup -g 10001 -S rustfs && \
|
||||
adduser -u 10001 -G rustfs -S rustfs -D && \
|
||||
@@ -124,6 +132,6 @@ VOLUME ["/data"]
|
||||
|
||||
USER rustfs
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
|
||||
|
||||
CMD ["rustfs"]
|
||||
|
||||
@@ -12,7 +12,10 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
FROM rust:1.97.1-trixie
|
||||
FROM rust:1.99.1-trixie
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RC_RELEASE=latest
|
||||
|
||||
RUN set -eux; \
|
||||
export DEBIAN_FRONTEND=noninteractive; \
|
||||
@@ -21,6 +24,7 @@ RUN set -eux; \
|
||||
build-essential \
|
||||
ca-certificates \
|
||||
curl \
|
||||
jq \
|
||||
git \
|
||||
pkg-config \
|
||||
libssl-dev \
|
||||
@@ -36,6 +40,7 @@ COPY . .
|
||||
RUN ./scripts/static.sh
|
||||
RUN cargo run --bin gproto
|
||||
RUN cargo build --release --locked --bin rustfs
|
||||
RUN sh /usr/src/rustfs/scripts/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /usr/local/bin/rc
|
||||
|
||||
RUN set -eux; \
|
||||
groupadd -g 10001 rustfs; \
|
||||
@@ -50,8 +55,12 @@ RUN set -eux; \
|
||||
chown -R rustfs:rustfs /data /logs /app; \
|
||||
chmod 0750 /data /logs
|
||||
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
RUN install -m 0755 /usr/src/rustfs/target/release/rustfs /usr/bin/rustfs && chmod +x /entrypoint.sh
|
||||
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
|
||||
COPY bootstrap/ /etc/rustfs/bootstrap/
|
||||
RUN install -m 0755 /usr/src/rustfs/target/release/rustfs /usr/bin/rustfs && \
|
||||
install -m 0755 /usr/local/bin/rc /usr/bin/rc && \
|
||||
chmod +x /etc/rustfs/entrypoint.sh && \
|
||||
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
|
||||
|
||||
ENV RUSTFS_VOLUMES="/data" \
|
||||
RUST_LOG="warn" \
|
||||
@@ -63,5 +72,5 @@ ENV RUSTFS_VOLUMES="/data" \
|
||||
|
||||
EXPOSE 9000 9001
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
|
||||
CMD ["/usr/bin/rustfs"]
|
||||
|
||||
+11
-3
@@ -16,6 +16,7 @@ FROM ubuntu:26.04 AS build
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RELEASE=latest
|
||||
ARG RC_RELEASE=latest
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
@@ -76,6 +77,9 @@ RUN set -eux; \
|
||||
chmod +x /build/rustfs; \
|
||||
rm -rf rustfs.zip /build/.tmp || true
|
||||
|
||||
COPY scripts/download_rc.sh /usr/local/bin/download_rc.sh
|
||||
RUN sh /usr/local/bin/download_rc.sh "$TARGETARCH" gnu "$RC_RELEASE" /build/rc
|
||||
|
||||
FROM ubuntu:26.04
|
||||
|
||||
ARG RELEASE=latest
|
||||
@@ -99,14 +103,18 @@ RUN apt-get update && apt-get upgrade -y \
|
||||
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
curl \
|
||||
jq \
|
||||
tzdata \
|
||||
&& test "$(TZ=Asia/Kolkata date +%z)" = "+0530" \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY --from=build /build/rustfs /usr/bin/rustfs
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
COPY --from=build /build/rc /usr/bin/rc
|
||||
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
|
||||
COPY bootstrap/ /etc/rustfs/bootstrap/
|
||||
|
||||
RUN chmod +x /usr/bin/rustfs /entrypoint.sh
|
||||
RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
|
||||
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
|
||||
|
||||
RUN groupadd -g 10001 rustfs && \
|
||||
useradd -u 10001 -g rustfs -m -s /sbin/nologin rustfs && \
|
||||
@@ -131,6 +139,6 @@ VOLUME ["/data"]
|
||||
|
||||
USER rustfs
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
|
||||
|
||||
CMD ["rustfs"]
|
||||
|
||||
+19
-7
@@ -28,17 +28,19 @@ ARG TARGETPLATFORM
|
||||
ARG BUILDPLATFORM
|
||||
ARG TARGETARCH
|
||||
ARG RUSTFS_BUILD_FEATURES=""
|
||||
ARG RC_RELEASE=latest
|
||||
|
||||
# -----------------------------
|
||||
# Build stage
|
||||
# -----------------------------
|
||||
FROM rust:1.97.1-trixie AS builder
|
||||
FROM rust:1.99.1-trixie AS builder
|
||||
|
||||
# Re-declare args after FROM
|
||||
ARG TARGETPLATFORM
|
||||
ARG BUILDPLATFORM
|
||||
ARG TARGETARCH
|
||||
ARG RUSTFS_BUILD_FEATURES
|
||||
ARG RC_RELEASE
|
||||
|
||||
# Debug: print platforms
|
||||
RUN echo "Build info -> BUILDPLATFORM=${BUILDPLATFORM}, TARGETPLATFORM=${TARGETPLATFORM}"
|
||||
@@ -52,6 +54,7 @@ RUN set -eux; \
|
||||
build-essential \
|
||||
ca-certificates \
|
||||
curl \
|
||||
jq \
|
||||
git \
|
||||
pkg-config \
|
||||
libssl-dev \
|
||||
@@ -166,6 +169,8 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
||||
;; \
|
||||
esac
|
||||
|
||||
RUN sh /usr/src/rustfs/scripts/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /usr/local/bin/rc
|
||||
|
||||
# -----------------------------
|
||||
# Development stage (keeps toolchain)
|
||||
# -----------------------------
|
||||
@@ -199,10 +204,13 @@ ENV RUSTFS_VOLUMES="/data" \
|
||||
# We rely on cargo run to build and run
|
||||
EXPOSE 9000 9001
|
||||
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
RUN chmod +x /entrypoint.sh
|
||||
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
|
||||
COPY bootstrap/ /etc/rustfs/bootstrap/
|
||||
RUN install -m 0755 /usr/local/bin/rc /usr/bin/rc && \
|
||||
chmod +x /etc/rustfs/entrypoint.sh && \
|
||||
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
|
||||
CMD ["cargo", "run", "--bin", "rustfs", "--"]
|
||||
|
||||
# -----------------------------
|
||||
@@ -227,6 +235,7 @@ RUN set -eux; \
|
||||
apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
curl \
|
||||
jq \
|
||||
tzdata \
|
||||
coreutils; \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
@@ -248,8 +257,11 @@ RUN set -eux; \
|
||||
|
||||
# Copy the freshly built binary and the entrypoint
|
||||
COPY --from=builder /usr/local/bin/rustfs /usr/bin/rustfs
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
RUN chmod +x /usr/bin/rustfs /entrypoint.sh
|
||||
COPY --from=builder /usr/local/bin/rc /usr/bin/rc
|
||||
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
|
||||
COPY bootstrap/ /etc/rustfs/bootstrap/
|
||||
RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
|
||||
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
|
||||
|
||||
# Default environment (override in docker run/compose as needed)
|
||||
ENV RUSTFS_VOLUMES="/data" \
|
||||
@@ -263,5 +275,5 @@ EXPOSE 9000
|
||||
VOLUME ["/data"]
|
||||
|
||||
# Keep root here; entrypoint will drop privileges using chroot --userspec
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
|
||||
CMD ["/usr/bin/rustfs"]
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
# RustFS image bootstrap
|
||||
|
||||
The official image bundles this directory at `/etc/rustfs/bootstrap`. The default `buckets.json` is an empty JSON array, so the image starts RustFS directly and does not require a bootstrap endpoint or bootstrap credentials.
|
||||
|
||||
To create buckets, replace `buckets.json` in a derived image with a JSON array of bucket names. The entrypoint creates each bucket if it does not exist and applies `public-access-block.json`, whose default blocks all four public access paths. Edit that file in the derived image only when public access is an explicit requirement.
|
||||
|
||||
Optional per-bucket files use the bucket name as the filename without an extension:
|
||||
|
||||
| Directory | File format | Applied operation |
|
||||
| --- | --- | --- |
|
||||
| `policies/` | S3 bucket policy JSON | `rc bucket anonymous set-json` |
|
||||
| `versioning/` | JSON boolean (`true` or `false`) | Enable or suspend bucket versioning |
|
||||
| `lifecycle/` | `rc bucket lifecycle rule import` JSON | Replace lifecycle rules |
|
||||
| `cors/` | JSON or XML accepted by `rc bucket cors set` | Replace CORS rules |
|
||||
| `notifications/` | JSON array of `{ "arn": "...", "events": ["..."] }` | Add or replace event rules by ARN (prefix/suffix filters are not currently accepted) |
|
||||
|
||||
Every per-bucket file must correspond to a bucket listed in `buckets.json`. Applying these files is repeatable: bucket creation ignores existing buckets, while each configuration operation replaces or reconciles that bucket's requested state.
|
||||
|
||||
When `buckets.json` is non-empty, set `RUSTFS_BOOTSTRAP_ENDPOINT` to the RustFS S3 endpoint reachable from the container, for example `http://rustfs:9000` for a Compose service or the internal service DNS name in Kubernetes. The entrypoint waits on that endpoint's `/health/ready`; it never assumes `localhost`. `RUSTFS_BOOTSTRAP_TIMEOUT` controls the readiness wait in seconds (default `120`, maximum `3600`). `RUSTFS_BOOTSTRAP_DIR` can point to a different in-image configuration directory; it defaults to `/etc/rustfs/bootstrap`. Set `RUSTFS_ACCESS_KEY` and `RUSTFS_SECRET_KEY`, or their `_FILE` variants, for bootstrap authentication. The endpoint and credentials are read at runtime and are not baked into the image.
|
||||
|
||||
RustFS built-in IAM policies remain unchanged; the default bootstrap installs no custom bucket policies, lifecycle rules, versioning, CORS rules, or notification rules. The bundled RustFS CLI is resolved from a published release and its release checksum is verified. Set the Docker build argument `RC_RELEASE` to a version to pin it. Build the RustFS image after a CLI release containing the `bucket public-access-block` commands is available. For Kubernetes, the configured service must route traffic to the starting pod before its readiness probe succeeds (for example, with `publishNotReadyAddresses: true`); otherwise the service endpoint cannot reach the pod while the entrypoint waits for readiness.
|
||||
@@ -0,0 +1 @@
|
||||
[]
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"block_public_acls": true,
|
||||
"ignore_public_acls": true,
|
||||
"block_public_policy": true,
|
||||
"restrict_public_buckets": true
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -30,10 +30,10 @@ pub const ENV_TRUSTED_PROXY_VALIDATION_MODE: &str = "RUSTFS_TRUSTED_PROXY_VALIDA
|
||||
/// Default validation mode is "hop_by_hop".
|
||||
pub const DEFAULT_TRUSTED_PROXY_VALIDATION_MODE: &str = "hop_by_hop";
|
||||
|
||||
/// Environment variable to enable RFC 7239 "Forwarded" header support.
|
||||
/// Select RFC 7239 "Forwarded" instead of "X-Forwarded-For" in legacy mode.
|
||||
pub const ENV_TRUSTED_PROXY_ENABLE_RFC7239: &str = "RUSTFS_TRUSTED_PROXY_ENABLE_RFC7239";
|
||||
/// RFC 7239 support is enabled by default.
|
||||
pub const DEFAULT_TRUSTED_PROXY_ENABLE_RFC7239: bool = true;
|
||||
/// Legacy mode uses X-Forwarded-For unless RFC 7239 is explicitly selected.
|
||||
pub const DEFAULT_TRUSTED_PROXY_ENABLE_RFC7239: bool = false;
|
||||
|
||||
/// Environment variable for the maximum allowed proxy hops.
|
||||
pub const ENV_TRUSTED_PROXY_MAX_HOPS: &str = "RUSTFS_TRUSTED_PROXY_MAX_HOPS";
|
||||
|
||||
@@ -146,3 +146,9 @@ pub const ENV_TARGET_STORE_COMPRESS: &str = "RUSTFS_TARGET_STORE_COMPRESS";
|
||||
|
||||
/// Queue-store compression is enabled by default to reduce disk footprint.
|
||||
pub const DEFAULT_TARGET_STORE_COMPRESS: bool = true;
|
||||
|
||||
/// Maximum on-disk bytes in each target queue, including failed entries and pending writes.
|
||||
pub const ENV_TARGET_STORE_MAX_BYTES: &str = "RUSTFS_TARGET_STORE_MAX_BYTES";
|
||||
|
||||
/// Queue writes fail at one GiB unless the operator sets another positive byte budget.
|
||||
pub const DEFAULT_TARGET_STORE_MAX_BYTES: u64 = 1024 * 1024 * 1024;
|
||||
|
||||
@@ -175,12 +175,17 @@ pub const ENV_HTTP_SOCKET_RECV_BUFFER_BYTES: &str = "RUSTFS_HTTP_SOCKET_RECV_BUF
|
||||
pub const DEFAULT_HTTP_SOCKET_RECV_BUFFER_BYTES: usize = 0;
|
||||
|
||||
/// Environment variable for the S3 request-body inter-chunk read timeout
|
||||
/// (seconds). Default: 300. Set to 0 to disable.
|
||||
/// (seconds). Default: 300. Set to 0 to disable the inactivity timeout.
|
||||
///
|
||||
/// This bounds how long a `PutObject`/upload may wait for the *next* body chunk
|
||||
/// while more bytes are still expected. It resets on every chunk, so it does not
|
||||
/// penalize slow-but-progressing uploads — it only fires when a peer sends a
|
||||
/// partial body and then goes silent *without* closing the connection (no EOF).
|
||||
/// Independently of this knob, `PutObject` and `UploadPart` require at least
|
||||
/// 64 KiB of body progress per 60 seconds of server wait (the minimum-progress
|
||||
/// floor, logged as `upload_body_progress_stalled`); it bounds trickle uploads
|
||||
/// that would otherwise pin upload admission indefinitely, and it stays in
|
||||
/// force when this timeout is set to 0.
|
||||
/// A reverse proxy or CDN that forwards a truncated body this way would
|
||||
/// otherwise make RustFS wait forever for bytes that never arrive; with this
|
||||
/// timeout the stalled read is aborted and logged with the received/expected
|
||||
|
||||
@@ -26,7 +26,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
|
||||
| **protocols** | [`src/protocols/`](src/protocols) | FTPS, WebDAV, SFTP compliance. Fixed ports, own guide: [`src/protocols/README.md`](src/protocols/README.md) |
|
||||
| **reliant** | [`src/reliant/`](src/reliant) | Tests that reuse an **externally started** server (SQL/select, conditional writes, lifecycle, deleted-object reads, node-interact). Run via [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh); see [`src/reliant/README.md`](src/reliant/README.md) |
|
||||
| **cluster** | `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `namespace_lock_quorum_test`, `admin_timeout_regression_test`, `object_lambda_test`, `replication_extension_test`, `tier_stats_cluster_test` | Multi-node scenarios via `RustFSTestClusterEnvironment` |
|
||||
| **distributed 4×4** | [`src/distributed/`](src/distributed) | Storage-sensitive PR and nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
|
||||
| **distributed 4×4** | [`src/distributed/`](src/distributed) | Nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
|
||||
| **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
|
||||
| **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
|
||||
|
||||
@@ -194,9 +194,9 @@ the same profile for membership and execution with one nightly worker.
|
||||
| `s3s-e2e` black-box | `e2e-tests` + `e2e-tests-rio-v2` jobs | **Active** (external conformance tool) |
|
||||
| ILM / lifecycle (ignored) | `test-ilm-integration-serial` lane, `-j1` | **Active** (backlog#1148 ilm-1) |
|
||||
| KMS suite | `e2e-full` job, merge queue + main | **Active** |
|
||||
| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
|
||||
| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, release tags + weekly schedule + manual dispatch | **Active** |
|
||||
| Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
|
||||
| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (storage-sensitive PR / nightly / dispatch) |
|
||||
| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (nightly / dispatch) |
|
||||
| Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
|
||||
| Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
|
||||
| Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) |
|
||||
|
||||
@@ -502,10 +502,163 @@ mod tests {
|
||||
assert!(response.contains(if expected == reqwest::StatusCode::FORBIDDEN {
|
||||
"<Code>AccessDenied</Code>"
|
||||
} else {
|
||||
"<Code>BucketAlreadyExists</Code>"
|
||||
"<Code>BucketAlreadyOwnedByYou</Code>"
|
||||
}));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A deployment owns a single bucket namespace, so an IAM user that recreates
|
||||
/// an existing bucket is the owner, not another account. In the default region
|
||||
/// every such request returns the legacy idempotent `200 OK` (AWS CreateBucket
|
||||
/// semantics), including buckets created by root and by another IAM user.
|
||||
#[tokio::test]
|
||||
async fn test_create_bucket_duplicate_in_default_region_is_idempotent_for_iam_user()
|
||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec!["--region", "us-east-1"]).await?;
|
||||
let root = env.create_s3_client();
|
||||
|
||||
admin_create_user(&env, "duplicate-iam", "duplicate-iam-secret").await?;
|
||||
admin_attach_user_policy_via(
|
||||
AdminTransport::Signed,
|
||||
&env.url,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
"readwrite",
|
||||
"duplicate-iam",
|
||||
)
|
||||
.await?;
|
||||
let iam = Client::from_conf(build_test_s3_config(
|
||||
&env.url,
|
||||
"duplicate-iam",
|
||||
"duplicate-iam-secret",
|
||||
None,
|
||||
"duplicate-create",
|
||||
));
|
||||
|
||||
root.create_bucket()
|
||||
.bucket("duplicate-root-owned")
|
||||
.send()
|
||||
.await
|
||||
.expect("root creates the first bucket");
|
||||
iam.create_bucket()
|
||||
.bucket("duplicate-root-owned")
|
||||
.send()
|
||||
.await
|
||||
.expect("an IAM user recreating a root-owned bucket in us-east-1 gets 200 OK");
|
||||
|
||||
iam.create_bucket()
|
||||
.bucket("duplicate-iam-owned")
|
||||
.send()
|
||||
.await
|
||||
.expect("the IAM user creates its own bucket");
|
||||
iam.create_bucket()
|
||||
.bucket("duplicate-iam-owned")
|
||||
.send()
|
||||
.await
|
||||
.expect("the same IAM user recreating its bucket gets 200 OK");
|
||||
root.create_bucket()
|
||||
.bucket("duplicate-iam-owned")
|
||||
.send()
|
||||
.await
|
||||
.expect("root recreating an IAM-created bucket gets 200 OK");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Outside the default region the same ownership answer becomes
|
||||
/// `409 BucketAlreadyOwnedByYou` for every authorized requester. Requests that
|
||||
/// lack `s3:CreateBucket` still fail authorization before the duplicate check.
|
||||
#[tokio::test]
|
||||
async fn test_create_bucket_duplicate_outside_default_region_reports_owned_by_you() -> Result<(), Box<dyn Error + Send + Sync>>
|
||||
{
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec!["--region", "eu-west-1"]).await?;
|
||||
let region = "eu-west-1";
|
||||
let root = Client::from_conf(
|
||||
env.create_s3_client()
|
||||
.config()
|
||||
.to_builder()
|
||||
.region(Region::new(region))
|
||||
.build(),
|
||||
);
|
||||
|
||||
admin_create_user(&env, "duplicate-eu-iam", "duplicate-eu-iam-secret").await?;
|
||||
admin_attach_user_policy_via(
|
||||
AdminTransport::Signed,
|
||||
&env.url,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
"readwrite",
|
||||
"duplicate-eu-iam",
|
||||
)
|
||||
.await?;
|
||||
let iam = Client::from_conf(
|
||||
build_test_s3_config(&env.url, "duplicate-eu-iam", "duplicate-eu-iam-secret", None, "duplicate-create")
|
||||
.to_builder()
|
||||
.region(Region::new(region))
|
||||
.build(),
|
||||
);
|
||||
|
||||
admin_create_user(&env, "duplicate-eu-readonly", "duplicate-eu-readonly-secret").await?;
|
||||
admin_attach_user_policy_via(
|
||||
AdminTransport::Signed,
|
||||
&env.url,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
"readonly",
|
||||
"duplicate-eu-readonly",
|
||||
)
|
||||
.await?;
|
||||
let readonly = Client::from_conf(
|
||||
build_test_s3_config(
|
||||
&env.url,
|
||||
"duplicate-eu-readonly",
|
||||
"duplicate-eu-readonly-secret",
|
||||
None,
|
||||
"duplicate-create",
|
||||
)
|
||||
.to_builder()
|
||||
.region(Region::new(region))
|
||||
.build(),
|
||||
);
|
||||
|
||||
for (client, bucket) in [(&root, "duplicate-eu-root"), (&iam, "duplicate-eu-iam")] {
|
||||
client
|
||||
.create_bucket()
|
||||
.bucket(bucket)
|
||||
.send()
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("{bucket} must be created: {error:?}"));
|
||||
// Both the creator and the other authenticated identity own the namespace.
|
||||
for creator in [&root, &iam] {
|
||||
let error = creator
|
||||
.create_bucket()
|
||||
.bucket(bucket)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a duplicate create outside us-east-1 must report ownership");
|
||||
assert_eq!(
|
||||
error.as_service_error().and_then(ProvideErrorMetadata::code),
|
||||
Some("BucketAlreadyOwnedByYou"),
|
||||
"{bucket}"
|
||||
);
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(409), "{bucket}");
|
||||
}
|
||||
}
|
||||
|
||||
let denied = readonly
|
||||
.create_bucket()
|
||||
.bucket("duplicate-eu-root")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a requester without s3:CreateBucket must be denied before the duplicate check");
|
||||
assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,8 +16,193 @@
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::config::http::HttpResponse;
|
||||
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration};
|
||||
|
||||
fn assert_not_found<E: ProvideErrorMetadata + std::fmt::Debug>(error: &SdkError<E, HttpResponse>) {
|
||||
assert_eq!(error.raw_response().map(|r| r.status().as_u16()), Some(404), "{error:?}");
|
||||
assert_eq!(
|
||||
error.as_service_error().and_then(ProvideErrorMetadata::code),
|
||||
Some("NoSuchKey"),
|
||||
"{error:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn conditional_writes_cannot_replace_delete_marker_with_if_match() {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
|
||||
env.start_rustfs_server(vec![]).await.expect("start RustFS");
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "conditional-delete-marker-proof";
|
||||
create_versioned_bucket(&client, bucket).await;
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("source")
|
||||
.body(ByteStream::from_static(b"source"))
|
||||
.send()
|
||||
.await
|
||||
.expect("write copy source");
|
||||
|
||||
for operation in ["put", "copy", "multipart"] {
|
||||
let old = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.body(ByteStream::from_static(b"old"))
|
||||
.send()
|
||||
.await
|
||||
.expect("write historical version");
|
||||
let old_etag = old.e_tag().expect("historical ETag");
|
||||
let marker = client
|
||||
.delete_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.send()
|
||||
.await
|
||||
.expect("create delete marker");
|
||||
let marker_id = marker.version_id().expect("delete marker version");
|
||||
|
||||
let (upload_id, completed) = if operation == "multipart" {
|
||||
let upload = client
|
||||
.create_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.send()
|
||||
.await
|
||||
.expect("initiate multipart upload");
|
||||
let upload_id = upload.upload_id().expect("upload ID").to_owned();
|
||||
let part = client
|
||||
.upload_part()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.upload_id(&upload_id)
|
||||
.part_number(1)
|
||||
.body(ByteStream::from_static(b"new"))
|
||||
.send()
|
||||
.await
|
||||
.expect("upload part");
|
||||
let completed = CompletedMultipartUpload::builder()
|
||||
.parts(
|
||||
CompletedPart::builder()
|
||||
.part_number(1)
|
||||
.set_e_tag(part.e_tag().map(str::to_owned))
|
||||
.build(),
|
||||
)
|
||||
.build();
|
||||
(Some(upload_id), Some(completed))
|
||||
} else {
|
||||
(None, None)
|
||||
};
|
||||
|
||||
for etag in [old_etag, "\"00000000000000000000000000000000\""] {
|
||||
match operation {
|
||||
"put" => {
|
||||
let err = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.if_match(etag)
|
||||
.body(ByteStream::from_static(b"new"))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("If-Match must reject marker");
|
||||
assert_not_found(&err);
|
||||
}
|
||||
"copy" => {
|
||||
let err = client
|
||||
.copy_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.copy_source(format!("{bucket}/source"))
|
||||
.if_match(etag)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("If-Match copy must reject marker");
|
||||
assert_not_found(&err);
|
||||
}
|
||||
_ => {
|
||||
let err = client
|
||||
.complete_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.upload_id(upload_id.as_deref().expect("upload ID"))
|
||||
.multipart_upload(completed.clone().expect("completed parts"))
|
||||
.if_match(etag)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("If-Match completion must reject marker");
|
||||
assert_not_found(&err);
|
||||
}
|
||||
}
|
||||
let versions = client
|
||||
.list_object_versions()
|
||||
.bucket(bucket)
|
||||
.prefix(operation)
|
||||
.send()
|
||||
.await
|
||||
.expect("list versions after rejected write");
|
||||
assert_eq!(versions.versions().len(), 1, "{operation}: failed write created a version");
|
||||
assert_eq!(versions.delete_markers().len(), 1);
|
||||
assert_eq!(versions.delete_markers()[0].version_id(), Some(marker_id));
|
||||
assert_eq!(versions.delete_markers()[0].is_latest(), Some(true));
|
||||
}
|
||||
|
||||
match operation {
|
||||
"put" => {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.if_none_match("*")
|
||||
.body(ByteStream::from_static(b"new"))
|
||||
.send()
|
||||
.await
|
||||
.expect("If-None-Match PUT over marker");
|
||||
}
|
||||
"copy" => {
|
||||
client
|
||||
.copy_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.copy_source(format!("{bucket}/source"))
|
||||
.if_none_match("*")
|
||||
.send()
|
||||
.await
|
||||
.expect("If-None-Match copy over marker");
|
||||
}
|
||||
_ => {
|
||||
client
|
||||
.complete_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.upload_id(upload_id.as_deref().expect("upload ID"))
|
||||
.multipart_upload(completed.expect("completed parts"))
|
||||
.if_none_match("*")
|
||||
.send()
|
||||
.await
|
||||
.expect("If-None-Match completion over marker");
|
||||
}
|
||||
}
|
||||
let current = client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.send()
|
||||
.await
|
||||
.expect("read successful conditional write")
|
||||
.body
|
||||
.collect()
|
||||
.await
|
||||
.expect("read object body")
|
||||
.into_bytes();
|
||||
let expected: &[u8] = if operation == "copy" { b"source" } else { b"new" };
|
||||
assert_eq!(current.as_ref(), expected, "{operation}: unexpected current object");
|
||||
}
|
||||
}
|
||||
|
||||
async fn create_versioned_bucket(client: &Client, bucket: &str) {
|
||||
client
|
||||
|
||||
@@ -12,11 +12,249 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, unique_bucket};
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, payload_for, s3_probe_client, sha256_hex, unique_bucket};
|
||||
use crate::common::init_logging;
|
||||
use aws_sdk_s3::{
|
||||
Client,
|
||||
error::{ProvideErrorMetadata, SdkError},
|
||||
primitives::ByteStream,
|
||||
};
|
||||
use std::any::TypeId;
|
||||
use std::collections::BTreeSet;
|
||||
use std::fmt::Debug;
|
||||
use std::future::Future;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::Barrier;
|
||||
use tokio::time::Instant;
|
||||
|
||||
const LOCK_ADMISSION_BUDGET: Duration = Duration::from_secs(45);
|
||||
|
||||
async fn after_lock_contention<T, E, F, Fut>(operation: &str, mut call: F) -> TestResult<Result<T, SdkError<E>>>
|
||||
where
|
||||
E: Debug + ProvideErrorMetadata + 'static,
|
||||
F: FnMut() -> Fut,
|
||||
Fut: Future<Output = Result<T, SdkError<E>>>,
|
||||
{
|
||||
let started = Instant::now();
|
||||
let mut attempts = 0;
|
||||
// The workload uses unique unversioned keys. Repeat only explicitly
|
||||
// advertised lock backpressure; every step still needs its real result.
|
||||
let outcome = tokio::time::timeout(LOCK_ADMISSION_BUDGET, async {
|
||||
loop {
|
||||
attempts += 1;
|
||||
let result = call().await;
|
||||
let retry = result.as_ref().err().is_some_and(|error| {
|
||||
let Some(response) = error.raw_response() else {
|
||||
return false;
|
||||
};
|
||||
let Some(service_error) = error.as_service_error() else {
|
||||
return false;
|
||||
};
|
||||
// HEAD has no XML error body, so its SDK service error can
|
||||
// lack a code even when the server supplies retry headers.
|
||||
let namespace_code = service_error.code() == Some("ServiceUnavailable")
|
||||
|| (service_error.code().is_none()
|
||||
&& TypeId::of::<E>() == TypeId::of::<aws_sdk_s3::operation::head_object::HeadObjectError>());
|
||||
response.status().as_u16() == 503
|
||||
&& namespace_code
|
||||
&& response.headers().get("x-rustfs-error-cause") == Some("namespace_lock_timeout")
|
||||
&& response.headers().get("retry-after") == Some("1")
|
||||
});
|
||||
if !retry {
|
||||
return result;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
if attempts > 1 || !matches!(&outcome, Ok(Ok(_))) {
|
||||
eprintln!(
|
||||
"mixed workload request: operation={operation}, attempts={attempts}, elapsed={:?}",
|
||||
started.elapsed()
|
||||
);
|
||||
}
|
||||
outcome.map_err(|_| {
|
||||
format!("{operation} exceeded its {LOCK_ADMISSION_BUDGET:?} request budget after {attempts} attempts").into()
|
||||
})
|
||||
}
|
||||
|
||||
async fn assert_workload_bytes(client: &Client, bucket: &str, key: &str, expected: &[u8]) -> TestResult {
|
||||
let output = after_lock_contention(&format!("GET {key}"), || client.get_object().bucket(bucket).key(key).send()).await??;
|
||||
let actual = output.body.collect().await?.into_bytes();
|
||||
if actual.as_ref() != expected {
|
||||
return Err(format!(
|
||||
"object {key} bytes mismatch: expected {} bytes sha256={} got {} bytes sha256={}",
|
||||
expected.len(),
|
||||
sha256_hex(expected),
|
||||
actual.len(),
|
||||
sha256_hex(&actual)
|
||||
)
|
||||
.into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use aws_smithy_http_client::test_util::infallible_client_fn;
|
||||
use aws_smithy_types::body::SdkBody;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
type Reply = (u16, &'static str, Option<&'static str>, Option<&'static str>, &'static str);
|
||||
|
||||
fn client(respond: impl Fn(usize) -> Reply + Send + Sync + 'static) -> (Client, Arc<AtomicUsize>) {
|
||||
let requests = Arc::new(AtomicUsize::new(0));
|
||||
let seen = Arc::clone(&requests);
|
||||
let transport = infallible_client_fn(move |_| {
|
||||
let (status, code, cause, retry_after, body) = respond(seen.fetch_add(1, Ordering::SeqCst));
|
||||
let body = if code.is_empty() {
|
||||
body.to_string()
|
||||
} else {
|
||||
format!("<Error><Code>{code}</Code></Error>")
|
||||
};
|
||||
let mut response = http::Response::builder()
|
||||
.status(status)
|
||||
.header("content-type", "application/xml")
|
||||
.header("content-length", body.len());
|
||||
if let Some(cause) = cause {
|
||||
response = response.header("x-rustfs-error-cause", cause);
|
||||
}
|
||||
if let Some(retry_after) = retry_after {
|
||||
response = response.header("retry-after", retry_after);
|
||||
}
|
||||
response.body(SdkBody::from(body)).expect("scripted S3 response")
|
||||
});
|
||||
let config = crate::common::build_test_s3_config(
|
||||
"http://contention.example.invalid",
|
||||
"synthetic-access",
|
||||
"synthetic-secret",
|
||||
None,
|
||||
"contention-test",
|
||||
);
|
||||
let client = Client::from_conf(config.to_builder().http_client(transport).build());
|
||||
(s3_probe_client(&client), requests)
|
||||
}
|
||||
|
||||
fn contention() -> Reply {
|
||||
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("1"), "")
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn advertised_contention_recovers_after_standard_sdk_attempts_are_exhausted() -> TestResult {
|
||||
let (client, requests) = client(|attempt| {
|
||||
if attempt < 3 {
|
||||
contention()
|
||||
} else {
|
||||
(200, "", None, None, "")
|
||||
}
|
||||
});
|
||||
let standard = Client::from_conf(
|
||||
client
|
||||
.config()
|
||||
.to_builder()
|
||||
.retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(3))
|
||||
.build(),
|
||||
);
|
||||
standard
|
||||
.put_object()
|
||||
.bucket("bucket")
|
||||
.key("key")
|
||||
.body(ByteStream::from_static(b"body"))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("three advertised lock responses exhaust the standard SDK policy");
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 3);
|
||||
requests.store(0, Ordering::SeqCst);
|
||||
let started = Instant::now();
|
||||
after_lock_contention("PUT recovery", || {
|
||||
client
|
||||
.put_object()
|
||||
.bucket("bucket")
|
||||
.key("key")
|
||||
.body(ByteStream::from_static(b"body"))
|
||||
.send()
|
||||
})
|
||||
.await??;
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 4);
|
||||
assert!(started.elapsed() >= Duration::from_secs(3), "honor each Retry-After response");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn bodyless_head_contention_uses_explicit_retry_headers() -> TestResult {
|
||||
let (client, requests) = client(|attempt| {
|
||||
if attempt == 0 {
|
||||
(503, "", Some("namespace_lock_timeout"), Some("1"), "")
|
||||
} else {
|
||||
(200, "", None, None, "")
|
||||
}
|
||||
});
|
||||
after_lock_contention("HEAD recovery", || client.head_object().bucket("bucket").key("key").send()).await??;
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 2);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn unrelated_or_malformed_responses_are_not_retried() -> TestResult {
|
||||
for reply in [
|
||||
(500, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
|
||||
(403, "AccessDenied", None, None, ""),
|
||||
(404, "NoSuchKey", None, None, ""),
|
||||
(503, "ServiceUnavailable", Some("storage_write_quorum"), None, ""),
|
||||
(503, "ServiceUnavailable", Some("namespace_lock_unavailable"), Some("1"), ""),
|
||||
(503, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
|
||||
(503, "", Some("namespace_lock_timeout"), Some("1"), ""),
|
||||
(503, "ServiceUnavailable", None, Some("1"), ""),
|
||||
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), None, ""),
|
||||
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("2"), ""),
|
||||
] {
|
||||
let (client, requests) = client(move |_| reply);
|
||||
let result = after_lock_contention("PUT fatal response", || {
|
||||
client
|
||||
.put_object()
|
||||
.bucket("bucket")
|
||||
.key("key")
|
||||
.body(ByteStream::from_static(b"body"))
|
||||
.send()
|
||||
})
|
||||
.await?;
|
||||
assert!(result.is_err(), "fatal response cannot become success: {reply:?}");
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 1, "fatal response must remain visible: {reply:?}");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn persistent_lock_contention_exhausts_the_request_budget() {
|
||||
let (client, requests) = client(|_| contention());
|
||||
let started = Instant::now();
|
||||
let result = after_lock_contention("PUT unavailable", || {
|
||||
client
|
||||
.put_object()
|
||||
.bucket("bucket")
|
||||
.key("key")
|
||||
.body(ByteStream::from_static(b"body"))
|
||||
.send()
|
||||
})
|
||||
.await;
|
||||
assert!(result.is_err(), "persistent contention cannot acknowledge an operation");
|
||||
assert!(started.elapsed() >= LOCK_ADMISSION_BUDGET);
|
||||
assert!(requests.load(Ordering::SeqCst) > 1);
|
||||
assert!(requests.load(Ordering::SeqCst) <= LOCK_ADMISSION_BUDGET.as_secs() as usize + 1);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn acknowledged_wrong_bytes_fail_without_retry() {
|
||||
let (client, requests) = client(|_| (200, "", None, None, "wrong bytes"));
|
||||
let error = assert_workload_bytes(&client, "bucket", "key", b"expected bytes")
|
||||
.await
|
||||
.expect_err("a successful but incorrect read is a consistency failure");
|
||||
assert!(error.to_string().contains("bytes mismatch"));
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node() -> TestResult {
|
||||
@@ -24,7 +262,9 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
|
||||
let dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let bucket = unique_bucket("conc");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let clients = Arc::new(dist.clients()?);
|
||||
// This consistency probe owns its admission policy; unrelated responses
|
||||
// must not be hidden by the SDK's general-purpose retries.
|
||||
let clients = Arc::new(dist.clients()?.iter().map(s3_probe_client).collect::<Vec<_>>());
|
||||
let worker_count = 24;
|
||||
let rounds = 4;
|
||||
let barrier = Arc::new(Barrier::new(worker_count));
|
||||
@@ -44,31 +284,46 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
|
||||
let key = format!("source/worker-{idx:02}-round-{round}.bin");
|
||||
let copy_key = format!("retained/worker-{idx:02}-round-{round}.bin");
|
||||
let body = payload_for(&key, 64 * 1024);
|
||||
put_object(writer, &bucket, &key, body.clone()).await?;
|
||||
let context = format!("worker={idx} round={round}");
|
||||
after_lock_contention(&format!("{context} PUT source"), || {
|
||||
writer
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(&key)
|
||||
.body(ByteStream::from(body.clone()))
|
||||
.send()
|
||||
})
|
||||
.await??;
|
||||
|
||||
let head = reader.head_object().bucket(&bucket).key(&key).send().await?;
|
||||
let head = after_lock_contention(&format!("{context} HEAD source"), || {
|
||||
reader.head_object().bucket(&bucket).key(&key).send()
|
||||
})
|
||||
.await??;
|
||||
if head.content_length() != Some(body.len() as i64) {
|
||||
return Err(format!("HEAD returned the wrong size for {key}: {head:?}").into());
|
||||
}
|
||||
assert_object_bytes(reader, &bucket, &key, &body).await?;
|
||||
assert_workload_bytes(reader, &bucket, &key, &body).await?;
|
||||
|
||||
copier
|
||||
.copy_object()
|
||||
.bucket(&bucket)
|
||||
.key(©_key)
|
||||
.copy_source(format!("{bucket}/{key}"))
|
||||
.send()
|
||||
.await?;
|
||||
assert_object_bytes(writer, &bucket, ©_key, &body).await?;
|
||||
after_lock_contention(&format!("{context} COPY retained"), || {
|
||||
copier
|
||||
.copy_object()
|
||||
.bucket(&bucket)
|
||||
.key(©_key)
|
||||
.copy_source(format!("{bucket}/{key}"))
|
||||
.send()
|
||||
})
|
||||
.await??;
|
||||
assert_workload_bytes(writer, &bucket, ©_key, &body).await?;
|
||||
|
||||
writer.delete_object().bucket(&bucket).key(&key).send().await?;
|
||||
let missing = reader
|
||||
.head_object()
|
||||
.bucket(&bucket)
|
||||
.key(&key)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("deleted source key must not remain visible");
|
||||
after_lock_contention(&format!("{context} DELETE source"), || {
|
||||
writer.delete_object().bucket(&bucket).key(&key).send()
|
||||
})
|
||||
.await??;
|
||||
let missing = after_lock_contention(&format!("{context} HEAD deleted source"), || {
|
||||
reader.head_object().bucket(&bucket).key(&key).send()
|
||||
})
|
||||
.await?
|
||||
.expect_err("deleted source key must not remain visible");
|
||||
if missing.raw_response().map(|response| response.status().as_u16()) != Some(404) {
|
||||
return Err(format!("deleted source {key} returned an unexpected result: {missing:?}").into());
|
||||
}
|
||||
@@ -82,14 +337,22 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
|
||||
for handle in handles {
|
||||
inventory.extend(handle.await??);
|
||||
}
|
||||
assert_eq!(
|
||||
inventory.len(),
|
||||
worker_count * rounds,
|
||||
"every worker round must retain its acknowledged copy"
|
||||
);
|
||||
|
||||
let expected_keys: BTreeSet<_> = inventory.iter().map(|(key, _)| key.as_str()).collect();
|
||||
for (node_idx, client) in clients.iter().enumerate() {
|
||||
let listed = client.list_objects_v2().bucket(&bucket).prefix("retained/").send().await?;
|
||||
let listed = after_lock_contention(&format!("node={node_idx} LIST retained"), || {
|
||||
client.list_objects_v2().bucket(&bucket).prefix("retained/").send()
|
||||
})
|
||||
.await??;
|
||||
let listed_keys: BTreeSet<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
|
||||
assert_eq!(listed_keys, expected_keys, "node {node_idx} returned a divergent retained-key listing");
|
||||
for (key, body) in &inventory {
|
||||
assert_object_bytes(client, &bucket, key, body)
|
||||
assert_workload_bytes(client, &bucket, key, body)
|
||||
.await
|
||||
.map_err(|error| format!("node {node_idx} failed to read {key}: {error}"))?;
|
||||
}
|
||||
|
||||
@@ -25,7 +25,8 @@
|
||||
|
||||
use super::harness::{
|
||||
DistCluster, DistLayout, TestResult, cluster_admin, cluster_admin_ok, decommission_status_json, get_object_bytes,
|
||||
payload_for, put_object, s3_probe_client, s3_probe_error_code, sha256_hex, unique_bucket, wait_until,
|
||||
payload_for, put_object, rebalance_complete, rebalance_status_json, s3_probe_client, s3_probe_error_code, sha256_hex,
|
||||
unique_bucket, wait_until,
|
||||
};
|
||||
use crate::chaos::{VersionShardCensus, census_object_version_on_disk};
|
||||
use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, RustFSTestClusterEnvironment};
|
||||
@@ -47,6 +48,7 @@ use tokio::time::{Instant, sleep};
|
||||
use uuid::Uuid;
|
||||
|
||||
pub(crate) const MOVEMENT_BARRIER_DIR_ENV: &str = "RUSTFS_E2E_DATA_MOVEMENT_BARRIER_DIR";
|
||||
pub(crate) const STAGE_AFTER_ENTRY_ADMISSION: &str = "after_entry_admission";
|
||||
pub(crate) const STAGE_BEFORE_TARGET_COMMIT: &str = "before_target_commit";
|
||||
pub(crate) const STAGE_BEFORE_SOURCE_CLEANUP: &str = "before_source_cleanup";
|
||||
pub(crate) const STAGE_AFTER_RESPONSIBILITY_PERSISTED: &str = "after_responsibility_persisted";
|
||||
@@ -1166,8 +1168,21 @@ pub(crate) async fn clear_decommission(cluster: &RustFSTestClusterEnvironment, p
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn stop_rebalance(cluster: &RustFSTestClusterEnvironment) -> TestResult<String> {
|
||||
cluster_admin_ok(cluster, Method::POST, "/rustfs/admin/v3/rebalance/stop", None).await
|
||||
pub(crate) async fn stop_rebalance(cluster: &RustFSTestClusterEnvironment, expected_id: &str) -> TestResult<String> {
|
||||
let path = "/rustfs/admin/v3/rebalance/stop";
|
||||
let (status, response) = cluster_admin(cluster, Method::POST, path, None).await?;
|
||||
if status.is_success() {
|
||||
return Ok(response);
|
||||
}
|
||||
// The run can finish between the active observation and the stop request.
|
||||
// Only completion of that same run makes a not-started response acceptable.
|
||||
if status == StatusCode::NOT_FOUND
|
||||
&& response.contains("<Code>NoSuchResource</Code>")
|
||||
&& rebalance_complete(&rebalance_status_json(cluster).await?, expected_id)?
|
||||
{
|
||||
return Ok(response);
|
||||
}
|
||||
Err(format!("POST {path} failed: {status} {response}").into())
|
||||
}
|
||||
|
||||
pub(crate) fn decommission_pool_entry(status: &Value, pool_id: usize) -> TestResult<Value> {
|
||||
@@ -1585,6 +1600,129 @@ pub(crate) async fn assert_every_node_serves(dist: &DistCluster, bucket: &str, l
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const REBALANCE_NOT_STARTED: &str =
|
||||
"<Error><Code>NoSuchResource</Code><Message>pool rebalance is not started</Message></Error>";
|
||||
|
||||
async fn mock_rebalance_stop(
|
||||
stop_status: StatusCode,
|
||||
stop_body: &str,
|
||||
run_status: Value,
|
||||
) -> TestResult<(TestResult<String>, Vec<String>)> {
|
||||
use bytes::Bytes;
|
||||
use http_body_util::Full;
|
||||
use hyper::service::service_fn;
|
||||
use hyper_util::rt::TokioIo;
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio::task::JoinSet;
|
||||
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await?;
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(1).await?;
|
||||
cluster.nodes[0].url = format!("http://{}", listener.local_addr()?);
|
||||
let stop_body = stop_body.to_string();
|
||||
let (requests_tx, mut requests_rx) = mpsc::unbounded_channel();
|
||||
let mut server = JoinSet::new();
|
||||
server.spawn(async move {
|
||||
loop {
|
||||
let (stream, _) = match listener.accept().await {
|
||||
Ok(connection) => connection,
|
||||
Err(error) => return Err::<(), Box<dyn std::error::Error + Send + Sync>>(error.into()),
|
||||
};
|
||||
let stop_body = stop_body.clone();
|
||||
let run_status = run_status.clone();
|
||||
let requests_tx = requests_tx.clone();
|
||||
let handler = service_fn(move |request: hyper::Request<hyper::body::Incoming>| {
|
||||
let route = format!("{} {}", request.method(), request.uri().path());
|
||||
requests_tx.send(route.clone()).expect("record mock admin request");
|
||||
let (status, body) = match route.as_str() {
|
||||
"POST /rustfs/admin/v3/rebalance/stop" => (stop_status, stop_body.clone()),
|
||||
"GET /rustfs/admin/v3/rebalance/status" => (StatusCode::OK, run_status.to_string()),
|
||||
_ => (StatusCode::BAD_REQUEST, "unexpected admin request".to_string()),
|
||||
};
|
||||
async move { http::Response::builder().status(status).body(Full::new(Bytes::from(body))) }
|
||||
});
|
||||
hyper::server::conn::http1::Builder::new()
|
||||
.keep_alive(false)
|
||||
.serve_connection(TokioIo::new(stream), handler)
|
||||
.await?;
|
||||
}
|
||||
});
|
||||
let outcome = tokio::time::timeout(Duration::from_secs(5), stop_rebalance(&cluster, "run-1")).await?;
|
||||
server.abort_all();
|
||||
let requests = std::iter::from_fn(|| requests_rx.try_recv().ok()).collect();
|
||||
Ok((outcome, requests))
|
||||
}
|
||||
|
||||
fn completed_rebalance() -> Value {
|
||||
serde_json::json!({
|
||||
"id": "run-1",
|
||||
"pools": [
|
||||
{ "id": 0, "status": "Completed", "progress": { "versions": 113 } },
|
||||
{ "id": 1, "status": "None" }
|
||||
]
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rebalance_stop_success_does_not_query_completion() -> TestResult {
|
||||
let (outcome, requests) = mock_rebalance_stop(StatusCode::OK, "", Value::Null).await?;
|
||||
assert_eq!(outcome?, "");
|
||||
assert_eq!(requests, ["POST /rustfs/admin/v3/rebalance/stop"]);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rebalance_stop_accepts_completed_run_after_not_started_response() -> TestResult {
|
||||
let (outcome, requests) =
|
||||
mock_rebalance_stop(StatusCode::NOT_FOUND, REBALANCE_NOT_STARTED, completed_rebalance()).await?;
|
||||
outcome?;
|
||||
assert_eq!(
|
||||
requests,
|
||||
[
|
||||
"POST /rustfs/admin/v3/rebalance/stop",
|
||||
"GET /rustfs/admin/v3/rebalance/status"
|
||||
]
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rebalance_stop_rejects_unproven_completion_and_other_errors() -> TestResult {
|
||||
let complete = completed_rebalance();
|
||||
let mut invalid = vec![Value::Null];
|
||||
for (pointer, value) in [
|
||||
("/id", serde_json::json!("run-2")),
|
||||
("/pools/0/status", serde_json::json!("Started")),
|
||||
("/pools/0/status", serde_json::json!("Failed")),
|
||||
("/pools/0/status", serde_json::json!("Stopped")),
|
||||
("/pools/0/progress/versions", serde_json::json!(0)),
|
||||
] {
|
||||
let mut status = complete.clone();
|
||||
*status.pointer_mut(pointer).expect("completion fixture field") = value;
|
||||
invalid.push(status);
|
||||
}
|
||||
let mut pending = complete.clone();
|
||||
pending["stopPropagation"] = serde_json::json!({ "pendingTerminalReload": true });
|
||||
invalid.push(pending);
|
||||
let mut failed = complete.clone();
|
||||
failed["pools"][0]["lastError"] = serde_json::json!("migration failed");
|
||||
invalid.push(failed);
|
||||
for status in invalid {
|
||||
let (outcome, _) = mock_rebalance_stop(StatusCode::NOT_FOUND, REBALANCE_NOT_STARTED, status.clone()).await?;
|
||||
assert!(outcome.is_err(), "unproven completion accepted: {status}");
|
||||
}
|
||||
for (code, body) in [
|
||||
(StatusCode::FORBIDDEN, REBALANCE_NOT_STARTED),
|
||||
(StatusCode::INTERNAL_SERVER_ERROR, REBALANCE_NOT_STARTED),
|
||||
(StatusCode::NOT_FOUND, "<Error><Code>NoSuchBucket</Code></Error>"),
|
||||
] {
|
||||
let (outcome, requests) = mock_rebalance_stop(code, body, complete.clone()).await?;
|
||||
assert!(outcome.is_err(), "unrelated error accepted: {code} {body}");
|
||||
assert_eq!(requests, ["POST /rustfs/admin/v3/rebalance/stop"]);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn version_ids_normalize_null_spellings() {
|
||||
for spelling in [None, Some(""), Some("null"), Some("00000000-0000-0000-0000-000000000000")] {
|
||||
|
||||
@@ -28,7 +28,7 @@ use super::harness::{
|
||||
wait_for_rebalance_active, wait_for_rebalance_complete,
|
||||
};
|
||||
use super::scanner_heal_topology_fixture::{
|
||||
LiveWorkload, MOVEMENT_ACTIVE_TIMEOUT, MOVEMENT_COMPLETE_TIMEOUT, MovementOperation, SEED_POOL,
|
||||
LiveWorkload, MOVEMENT_ACTIVE_TIMEOUT, MOVEMENT_COMPLETE_TIMEOUT, MovementOperation, SEED_POOL, STAGE_AFTER_ENTRY_ADMISSION,
|
||||
STAGE_AFTER_RESPONSIBILITY_PERSISTED, STAGE_BEFORE_SOURCE_CLEANUP, STAGE_BEFORE_TARGET_COMMIT, TopologyManifest, TopologyRun,
|
||||
assert_bucket_durability_strict, assert_every_node_serves, assert_probe_objects, assert_process_durability_strict,
|
||||
assert_seed_pool_census_unchanged, assert_single_pool_placement, cancel_decommission, capture_seed_pool_census,
|
||||
@@ -274,6 +274,7 @@ async fn wait_for_rebalance_stopped_or_completed(run: &TopologyRun, rebalance_id
|
||||
#[tokio::test]
|
||||
async fn scanner_heal_topology_rebalance_survives_live_writes_stop_and_restart() -> TestResult {
|
||||
init_logging();
|
||||
require_test_hooks();
|
||||
let mut run = TopologyRun::start("rebalance").await?;
|
||||
let manifest = seed_manifest(&run).await?;
|
||||
assert_bucket_durability_strict(&run.dist.cluster, &manifest.buckets()).await?;
|
||||
@@ -288,23 +289,52 @@ async fn scanner_heal_topology_rebalance_survives_live_writes_stop_and_restart()
|
||||
.map(|entry| (entry.key.clone(), entry.sha256.clone()))
|
||||
.collect();
|
||||
|
||||
let first_id = start_rebalance(&run.dist.cluster).await?;
|
||||
wait_for_rebalance_active(&run.dist.cluster, &first_id, MOVEMENT_ACTIVE_TIMEOUT).await?;
|
||||
let during_rebalance = put_probe_objects(&run.dist, &scratch, "during-rebalance", 4).await?;
|
||||
let workload = tokio::spawn(run_live_workload(LiveWorkload {
|
||||
clients: run.dist.clients()?,
|
||||
bucket: scratch.clone(),
|
||||
prefix: "live".to_string(),
|
||||
writers: LIVE_WRITERS,
|
||||
rounds: LIVE_ROUNDS,
|
||||
baseline_bucket: manifest.plain_bucket.clone(),
|
||||
baseline: baseline.clone(),
|
||||
pause: Duration::from_millis(100),
|
||||
}));
|
||||
sleep(Duration::from_secs(2)).await;
|
||||
stop_rebalance(&run.dist.cluster).await?;
|
||||
let (first_outcome, first_status) = wait_for_rebalance_stopped_or_completed(&run, &first_id).await?;
|
||||
let ops = workload.await??;
|
||||
// Keep an admitted entry on the source until stop cancels admission. Start
|
||||
// propagation may await that entry, so observe and stop before awaiting ACK.
|
||||
let first_key = manifest.plain_key("external", 0)?;
|
||||
run.movement_barrier
|
||||
.arm_movement("rebalance", &manifest.plain_bucket, &first_key, STAGE_AFTER_ENTRY_ADMISSION)?;
|
||||
let observe_first = async {
|
||||
let (observed_id, reached) = tokio::time::timeout(MOVEMENT_ACTIVE_TIMEOUT, async {
|
||||
let reached = run.movement_barrier.wait_reached(MOVEMENT_ACTIVE_TIMEOUT).await?;
|
||||
let status = rebalance_status_json(&run.dist.cluster).await?;
|
||||
let observed_id = status
|
||||
.get("id")
|
||||
.and_then(Value::as_str)
|
||||
.ok_or("rebalance status omitted id")?;
|
||||
wait_for_rebalance_active(&run.dist.cluster, observed_id, MOVEMENT_ACTIVE_TIMEOUT).await?;
|
||||
Ok::<_, Box<dyn std::error::Error + Send + Sync>>((observed_id.to_string(), reached))
|
||||
})
|
||||
.await??;
|
||||
let probes = put_probe_objects(&run.dist, &scratch, "during-rebalance", 4).await?;
|
||||
let workload = run_live_workload(LiveWorkload {
|
||||
clients: run.dist.clients()?,
|
||||
bucket: scratch.clone(),
|
||||
prefix: "live".to_string(),
|
||||
writers: LIVE_WRITERS,
|
||||
rounds: LIVE_ROUNDS,
|
||||
baseline_bucket: manifest.plain_bucket.clone(),
|
||||
baseline: baseline.clone(),
|
||||
pause: Duration::from_millis(100),
|
||||
});
|
||||
let stop = async {
|
||||
sleep(Duration::from_secs(2)).await;
|
||||
stop_rebalance(&run.dist.cluster, &observed_id).await?;
|
||||
let (outcome, status) = wait_for_rebalance_stopped_or_completed(&run, &observed_id).await?;
|
||||
if outcome != "stopped" {
|
||||
return Err("rebalance completed before stop; mid-run stop was not exercised".into());
|
||||
}
|
||||
Ok::<_, Box<dyn std::error::Error + Send + Sync>>((outcome, status))
|
||||
};
|
||||
let (ops, (outcome, status)) = tokio::try_join!(workload, stop)?;
|
||||
Ok::<_, Box<dyn std::error::Error + Send + Sync>>((observed_id, reached, probes, ops, outcome, status))
|
||||
};
|
||||
let (first_id, (observed_id, first_reached, during_rebalance, ops, first_outcome, first_status)) =
|
||||
tokio::try_join!(start_rebalance(&run.dist.cluster), observe_first)?;
|
||||
if first_id != observed_id {
|
||||
return Err(format!("rebalance start acknowledged {first_id}, but the stopped run was {observed_id}").into());
|
||||
}
|
||||
run.movement_barrier.disarm()?;
|
||||
let live_state = expected_live_state(&ops);
|
||||
|
||||
for node_idx in 0..run.dist.cluster.nodes.len() {
|
||||
@@ -333,8 +363,40 @@ async fn scanner_heal_topology_rebalance_survives_live_writes_stop_and_restart()
|
||||
verify_from_every_node(&run.dist, &manifest, "after restart following the stop").await?;
|
||||
verify_live_state(&run.dist.client(0)?, &scratch, &live_state, "after restart following the stop").await?;
|
||||
|
||||
let second_id = start_rebalance(&run.dist.cluster).await?;
|
||||
wait_for_rebalance_active(&run.dist.cluster, &second_id, MOVEMENT_ACTIVE_TIMEOUT).await?;
|
||||
// A short second run can finish before the start response reaches us.
|
||||
// Hold one remaining source entry until its active state is observed.
|
||||
let mut second_key = None;
|
||||
for entry in manifest.plain_entries() {
|
||||
if pools_holding(&run.dist, &manifest.plain_bucket, &entry.key, None)?.contains(&SEED_POOL) {
|
||||
second_key = Some(entry.key.clone());
|
||||
break;
|
||||
}
|
||||
}
|
||||
let second_key = second_key.ok_or("no seed-pool object remains for the second rebalance")?;
|
||||
let second_barrier = run.movement_barrier.next_phase();
|
||||
second_barrier.arm_movement("rebalance", &manifest.plain_bucket, &second_key, STAGE_BEFORE_SOURCE_CLEANUP)?;
|
||||
let observe_second = async {
|
||||
let reached = second_barrier.wait_reached(MOVEMENT_ACTIVE_TIMEOUT).await?;
|
||||
let status = rebalance_status_json(&run.dist.cluster).await?;
|
||||
let observed_id = status
|
||||
.get("id")
|
||||
.and_then(Value::as_str)
|
||||
.ok_or("rebalance status omitted id")?;
|
||||
if observed_id == first_id {
|
||||
return Err("the second rebalance reused the stopped run id".into());
|
||||
}
|
||||
wait_for_rebalance_active(&run.dist.cluster, observed_id, MOVEMENT_ACTIVE_TIMEOUT).await?;
|
||||
second_barrier.disarm()?;
|
||||
second_barrier.release_movement()?;
|
||||
Ok::<_, Box<dyn std::error::Error + Send + Sync>>((observed_id.to_string(), reached))
|
||||
};
|
||||
// Worker propagation can hold the start response until movement drains.
|
||||
let (second_id, (observed_id, second_reached)) = tokio::try_join!(start_rebalance(&run.dist.cluster), async {
|
||||
tokio::time::timeout(MOVEMENT_ACTIVE_TIMEOUT, observe_second).await?
|
||||
})?;
|
||||
if second_id != observed_id {
|
||||
return Err(format!("rebalance start acknowledged {second_id}, but the active run was {observed_id}").into());
|
||||
}
|
||||
wait_for_rebalance_complete(&run.dist.cluster, &second_id, MOVEMENT_COMPLETE_TIMEOUT).await?;
|
||||
verify_from_every_node(&run.dist, &manifest, "after the second rebalance completed").await?;
|
||||
verify_live_state(&run.dist.client(3)?, &scratch, &live_state, "after the second rebalance completed").await?;
|
||||
@@ -351,8 +413,8 @@ async fn scanner_heal_topology_rebalance_survives_live_writes_stop_and_restart()
|
||||
&json!({
|
||||
"gate": "G-REB",
|
||||
"manifest": manifest_summary(&manifest),
|
||||
"first_run": { "id": first_id, "outcome": first_outcome, "status": first_status, "versions_held_by_two_pools_after_stop": retained_after_stop },
|
||||
"second_run": { "id": second_id, "versions_off_seed_pool": moved },
|
||||
"first_run": { "id": first_id, "outcome": first_outcome, "status": first_status, "versions_held_by_two_pools_after_stop": retained_after_stop, "barrier_object": first_key, "barrier": first_reached },
|
||||
"second_run": { "id": second_id, "versions_off_seed_pool": moved, "barrier_object": second_key, "barrier": second_reached },
|
||||
"live_ops": ops,
|
||||
"probe_placement_during_rebalance": during_placement,
|
||||
"process_durability_strict_nodes": strict_nodes,
|
||||
|
||||
@@ -1636,20 +1636,53 @@ async fn hot_window_with_old_followers(
|
||||
let mixed_initial_physical = if mixed { json!(converged_shards(&run)?) } else { Value::Null };
|
||||
let scheduler_config = run.assert_default_cadence().await?;
|
||||
let baseline = run.quiet_baseline().await?;
|
||||
let leader = run.leader().await?;
|
||||
let leader_pid = run.pid(leader)?;
|
||||
// A leader-lock refresh gap during old-binary replacement can make
|
||||
// `leader()` briefly observe a different node. Poll until the coordinator
|
||||
// is stable before comparing against the measured fleet. A persistent
|
||||
// change to an old-binary node is a genuine election outcome and fails.
|
||||
let (leader, leader_pid) = if mixed {
|
||||
let expected_leader = usize::try_from(
|
||||
mixed_fleet["leader"]
|
||||
.as_u64()
|
||||
.ok_or("mixed fleet omitted its measured leader")?,
|
||||
)?;
|
||||
let expected_pid = mixed_fleet["leader_pid"]
|
||||
.as_u64()
|
||||
.ok_or("mixed fleet omitted its measured leader pid")?;
|
||||
let mut resolved = None;
|
||||
for _ in 0..20 {
|
||||
let candidate = run.leader().await?;
|
||||
let candidate_pid = u64::from(run.pid(candidate)?);
|
||||
if Some(u64::try_from(candidate)?) == mixed_fleet["leader"].as_u64() && Some(candidate_pid) == Some(expected_pid) {
|
||||
resolved = Some((candidate, run.pid(candidate)?));
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
|
||||
}
|
||||
match resolved {
|
||||
Some(pair) => pair,
|
||||
None => {
|
||||
let current = match run.leader().await {
|
||||
Ok(node) => Some((node, run.pid(node).ok())),
|
||||
Err(_) => None,
|
||||
};
|
||||
return Err(format!(
|
||||
"the hot window did not retain its measured new coordinator after stabilization \
|
||||
(expected node {expected_leader} pid {expected_pid}, observed {current:?})"
|
||||
)
|
||||
.into());
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let leader = run.leader().await?;
|
||||
(leader, run.pid(leader)?)
|
||||
};
|
||||
let producer = (leader + 1) % 4;
|
||||
let producers = if fault.is_some() {
|
||||
vec![producer, (leader + 2) % 4]
|
||||
} else {
|
||||
vec![producer]
|
||||
};
|
||||
if mixed
|
||||
&& (mixed_fleet["leader"].as_u64() != Some(u64::try_from(leader)?)
|
||||
|| mixed_fleet["leader_pid"].as_u64() != Some(u64::from(leader_pid)))
|
||||
{
|
||||
return Err("the hot window did not retain its measured new coordinator".into());
|
||||
}
|
||||
let producer_pid = run.pid(producer)?;
|
||||
let producer_address = run.peer_address(producer);
|
||||
let [cold_a, cold_b, hot] = run.buckets.clone();
|
||||
|
||||
@@ -33,7 +33,7 @@ mod tests {
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashSet;
|
||||
use std::error::Error;
|
||||
use std::io::{Read, Write};
|
||||
use std::io::{Read, Seek, SeekFrom, Write};
|
||||
use std::net::SocketAddr;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
@@ -1654,6 +1654,89 @@ mod tests {
|
||||
TargetEndpointBlackhole,
|
||||
}
|
||||
|
||||
async fn release_heal_commit_barrier_after_shutdown(
|
||||
log: &Path,
|
||||
log_offset: u64,
|
||||
barrier: &Path,
|
||||
) -> Result<serde_json::Value, Box<dyn Error + Send + Sync>> {
|
||||
// Observe within the existing graceful process-stop budget.
|
||||
timeout(Duration::from_secs(45), async {
|
||||
loop {
|
||||
let mut file = std::fs::File::open(log)?;
|
||||
file.seek(SeekFrom::Start(log_offset))?;
|
||||
let mut tail = String::new();
|
||||
file.read_to_string(&mut tail)?;
|
||||
if let Some(event) = tail
|
||||
.lines()
|
||||
.filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
|
||||
.find(|event| {
|
||||
event["event"] == "server_shutdown_state"
|
||||
&& event["component"] == "main"
|
||||
&& event["subsystem"] == "startup"
|
||||
&& event["state"] == "stopping"
|
||||
})
|
||||
{
|
||||
// Local heal has stopped and HTTP shutdown has been signalled.
|
||||
// Let admitted peer RPCs finish; keeping this synthetic barrier
|
||||
// until process exit would make the fixture block its own drain.
|
||||
std::fs::remove_file(barrier)?;
|
||||
return Ok(event);
|
||||
}
|
||||
sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn heal_commit_barrier_waits_for_new_shutdown_event() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let log = directory.path().join("server.log");
|
||||
let barrier = directory.path().join("barrier");
|
||||
let event = serde_json::json!({
|
||||
"event": "server_shutdown_state", "component": "main", "subsystem": "startup", "state": "stopping"
|
||||
});
|
||||
std::fs::write(&log, format!("{event}\n"))?;
|
||||
std::fs::write(&barrier, "bucket/prefix/")?;
|
||||
let log_offset = log.metadata()?.len();
|
||||
let mut writer = std::fs::OpenOptions::new().append(true).open(&log)?;
|
||||
writeln!(writer, "{}", serde_json::json!({"event": "shutdown_signal_received"}))?;
|
||||
writeln!(writer, "{}", serde_json::json!({"event": "server_shutdown_state", "state": "stopping"}))?;
|
||||
|
||||
let release = release_heal_commit_barrier_after_shutdown(&log, log_offset, &barrier);
|
||||
tokio::pin!(release);
|
||||
assert!(
|
||||
futures::poll!(release.as_mut()).is_pending(),
|
||||
"old or unrelated events must not release the barrier"
|
||||
);
|
||||
assert!(barrier.is_file(), "the partial-rebuild barrier must remain armed before shutdown");
|
||||
tokio::time::advance(Duration::from_secs(6)).await;
|
||||
assert!(
|
||||
futures::poll!(release.as_mut()).is_pending(),
|
||||
"background shutdown may take longer than five seconds within the process-stop budget"
|
||||
);
|
||||
writeln!(writer, "{event}")?;
|
||||
tokio::time::advance(Duration::from_millis(10)).await;
|
||||
assert_eq!(release.await?, event, "retain the actual new shutdown witness");
|
||||
assert!(!barrier.exists(), "shutdown must release the synthetic hold");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn heal_commit_barrier_missing_shutdown_fails_without_releasing() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let log = directory.path().join("server.log");
|
||||
let barrier = directory.path().join("barrier");
|
||||
std::fs::write(&log, "")?;
|
||||
std::fs::write(&barrier, "bucket/prefix/")?;
|
||||
let error = release_heal_commit_barrier_after_shutdown(&log, 0, &barrier)
|
||||
.await
|
||||
.expect_err("missing shutdown evidence must fail the fixture");
|
||||
assert!(error.is::<tokio::time::error::Elapsed>());
|
||||
assert!(barrier.is_file(), "a missing shutdown witness must not release repairs");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn run_cluster_root_heal_interruption(scenario: InterruptionScenario) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let server_binary = rustfs_binary_path();
|
||||
let evidence_case = match scenario {
|
||||
@@ -1791,6 +1874,7 @@ mod tests {
|
||||
let mut created_online_objects = Vec::with_capacity(online_object_count);
|
||||
let mut expected_manifests = Vec::with_capacity(online_object_count);
|
||||
let mut unclean_shutdown_marker_observed = None;
|
||||
let mut shutdown_barrier_release = None;
|
||||
let mut attempt_count = 0usize;
|
||||
let max_online_attempts = online_object_count.saturating_mul(topology.total_drives().max(1));
|
||||
while expected_manifests.len() < online_object_count {
|
||||
@@ -2373,7 +2457,21 @@ mod tests {
|
||||
| InterruptionScenario::BackgroundTargetRestartEc84MultiSet
|
||||
| InterruptionScenario::BackgroundCoordinatorRestart
|
||||
) {
|
||||
cluster.stop_node_gracefully(interruption_node).await?;
|
||||
if interruption_node == 1
|
||||
&& let Some(barrier) = &commit_barrier
|
||||
{
|
||||
let target_log = PathBuf::from(format!("{log_dir}/node1.log"));
|
||||
let log_offset = target_log.metadata()?.len();
|
||||
let (stopped, released) = tokio::join!(
|
||||
cluster.stop_node_gracefully(interruption_node),
|
||||
release_heal_commit_barrier_after_shutdown(&target_log, log_offset, barrier),
|
||||
);
|
||||
// Await both branches even on error so the child is reaped.
|
||||
stopped?;
|
||||
shutdown_barrier_release = Some(released?);
|
||||
} else {
|
||||
cluster.stop_node_gracefully(interruption_node).await?;
|
||||
}
|
||||
} else {
|
||||
cluster.stop_node(interruption_node)?;
|
||||
}
|
||||
@@ -2415,7 +2513,9 @@ mod tests {
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(barrier) = &commit_barrier {
|
||||
if shutdown_barrier_release.is_none()
|
||||
&& let Some(barrier) = &commit_barrier
|
||||
{
|
||||
std::fs::remove_file(barrier)?;
|
||||
}
|
||||
cluster.start_node_from_binary(interruption_node, &server_binary).await?;
|
||||
@@ -2746,6 +2846,7 @@ mod tests {
|
||||
"all_peers_bound_to_generation_window": true,
|
||||
"pid_before": target_pid, "pid_after": restarted_pid,
|
||||
"unclean_shutdown_marker": unclean_shutdown_marker_observed.unwrap_or(false),
|
||||
"graceful_barrier_release": shutdown_barrier_release,
|
||||
"objects": evidence_objects, "node_listings": node_listings,
|
||||
});
|
||||
if let Some(outage_write) = outage_write_diagnostic {
|
||||
|
||||
@@ -283,6 +283,9 @@ mod ssec_copy_test;
|
||||
#[cfg(test)]
|
||||
mod multipart_storage_class_test;
|
||||
|
||||
#[cfg(test)]
|
||||
mod multipart_upload_id_test;
|
||||
|
||||
#[cfg(test)]
|
||||
mod storage_class_capability_test;
|
||||
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Multipart upload ID regression tests.
|
||||
//!
|
||||
//! Every operation that addresses an existing multipart upload must report an
|
||||
//! upload ID that does not name a live upload as `404 NoSuchUpload`, whether
|
||||
//! the ID is malformed or is a well-formed ID of an aborted upload.
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
||||
|
||||
type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
|
||||
|
||||
const BUCKET: &str = "multipart-upload-id";
|
||||
const KEY: &str = "target.bin";
|
||||
const COPY_SOURCE_KEY: &str = "copy-source.bin";
|
||||
// RustFS upload IDs are URL-safe base64; '.' is outside that alphabet, so
|
||||
// the object layer rejects this ID as malformed instead of looking it up.
|
||||
const MALFORMED_UPLOAD_ID: &str = "malformed.upload.id";
|
||||
const PART_NUMBER: i32 = 1;
|
||||
const PART_BODY: &[u8] = b"multipart upload id regression part";
|
||||
const PART_ETAG: &str = "\"00000000000000000000000000000000\"";
|
||||
const NO_SUCH_UPLOAD_STATUS: u16 = 404;
|
||||
const NO_SUCH_UPLOAD_CODE: &str = "NoSuchUpload";
|
||||
const OBJECT_NOT_FOUND_STATUS: u16 = 404;
|
||||
|
||||
/// Describes how `result` differs from `404 NoSuchUpload`, or returns `None` when it matches.
|
||||
fn no_such_upload_mismatch<T, E>(operation: &str, result: Result<T, SdkError<E>>) -> Option<String>
|
||||
where
|
||||
E: ProvideErrorMetadata + std::fmt::Debug,
|
||||
{
|
||||
let Err(error) = result else {
|
||||
return Some(format!("{operation}: succeeded"));
|
||||
};
|
||||
let status = error.raw_response().map(|response| response.status().as_u16());
|
||||
let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
|
||||
(status != Some(NO_SUCH_UPLOAD_STATUS) || code != Some(NO_SUCH_UPLOAD_CODE))
|
||||
.then(|| format!("{operation}: status {status:?}, code {code:?}: {error:?}"))
|
||||
}
|
||||
|
||||
async fn start_server_with_copy_source() -> Result<(RustFSTestEnvironment, Client), Box<dyn std::error::Error + Send + Sync>>
|
||||
{
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(Vec::new()).await?;
|
||||
env.create_test_bucket(BUCKET).await?;
|
||||
let client = env.create_s3_client();
|
||||
client
|
||||
.put_object()
|
||||
.bucket(BUCKET)
|
||||
.key(COPY_SOURCE_KEY)
|
||||
.body(ByteStream::from_static(PART_BODY))
|
||||
.send()
|
||||
.await?;
|
||||
Ok((env, client))
|
||||
}
|
||||
|
||||
/// Sends every operation that addresses an existing upload and asserts that all of them report `404 NoSuchUpload`.
|
||||
async fn assert_every_upload_operation_reports_no_such_upload(client: &Client, upload_id: &str) {
|
||||
let upload_part = client
|
||||
.upload_part()
|
||||
.bucket(BUCKET)
|
||||
.key(KEY)
|
||||
.upload_id(upload_id)
|
||||
.part_number(PART_NUMBER)
|
||||
.body(ByteStream::from_static(PART_BODY))
|
||||
.send()
|
||||
.await;
|
||||
let upload_part_copy = client
|
||||
.upload_part_copy()
|
||||
.bucket(BUCKET)
|
||||
.key(KEY)
|
||||
.upload_id(upload_id)
|
||||
.part_number(PART_NUMBER)
|
||||
.copy_source(format!("/{BUCKET}/{COPY_SOURCE_KEY}"))
|
||||
.send()
|
||||
.await;
|
||||
let list_parts = client.list_parts().bucket(BUCKET).key(KEY).upload_id(upload_id).send().await;
|
||||
let complete = client
|
||||
.complete_multipart_upload()
|
||||
.bucket(BUCKET)
|
||||
.key(KEY)
|
||||
.upload_id(upload_id)
|
||||
.multipart_upload(
|
||||
CompletedMultipartUpload::builder()
|
||||
.parts(CompletedPart::builder().part_number(PART_NUMBER).e_tag(PART_ETAG).build())
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await;
|
||||
let abort = client
|
||||
.abort_multipart_upload()
|
||||
.bucket(BUCKET)
|
||||
.key(KEY)
|
||||
.upload_id(upload_id)
|
||||
.send()
|
||||
.await;
|
||||
|
||||
let mismatches: Vec<String> = [
|
||||
no_such_upload_mismatch("UploadPart", upload_part),
|
||||
no_such_upload_mismatch("UploadPartCopy", upload_part_copy),
|
||||
no_such_upload_mismatch("ListParts", list_parts),
|
||||
no_such_upload_mismatch("CompleteMultipartUpload", complete),
|
||||
no_such_upload_mismatch("AbortMultipartUpload", abort),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect();
|
||||
assert!(
|
||||
mismatches.is_empty(),
|
||||
"upload ID {upload_id:?}: operations not reporting {NO_SUCH_UPLOAD_STATUS} {NO_SUCH_UPLOAD_CODE}:\n{}",
|
||||
mismatches.join("\n")
|
||||
);
|
||||
}
|
||||
|
||||
/// Asserts that the rejected operations left neither a multipart upload nor an object under `KEY`.
|
||||
async fn assert_no_upload_or_object_published(client: &Client) -> TestResult {
|
||||
let uploads = client.list_multipart_uploads().bucket(BUCKET).prefix(KEY).send().await?;
|
||||
assert!(uploads.uploads().is_empty(), "rejected operations must not leave an upload: {uploads:?}");
|
||||
let head = client.head_object().bucket(BUCKET).key(KEY).send().await;
|
||||
let status = head
|
||||
.as_ref()
|
||||
.err()
|
||||
.and_then(|error| error.raw_response())
|
||||
.map(|response| response.status().as_u16());
|
||||
assert_eq!(
|
||||
status,
|
||||
Some(OBJECT_NOT_FOUND_STATUS),
|
||||
"rejected operations must not publish {KEY}: {head:?}"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn malformed_upload_id_reports_no_such_upload_on_every_upload_operation() -> TestResult {
|
||||
let (_env, client) = start_server_with_copy_source().await?;
|
||||
|
||||
assert_every_upload_operation_reports_no_such_upload(&client, MALFORMED_UPLOAD_ID).await;
|
||||
assert_no_upload_or_object_published(&client).await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn aborted_upload_id_reports_no_such_upload_on_every_upload_operation() -> TestResult {
|
||||
let (_env, client) = start_server_with_copy_source().await?;
|
||||
let created = client.create_multipart_upload().bucket(BUCKET).key(KEY).send().await?;
|
||||
let upload_id = created.upload_id().ok_or("CreateMultipartUpload returned no upload ID")?;
|
||||
client
|
||||
.abort_multipart_upload()
|
||||
.bucket(BUCKET)
|
||||
.key(KEY)
|
||||
.upload_id(upload_id)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
assert_every_upload_operation_reports_no_such_upload(&client, upload_id).await;
|
||||
assert_no_upload_or_object_published(&client).await
|
||||
}
|
||||
}
|
||||
@@ -77,9 +77,7 @@ core protocol surface plus cross-protocol consistency:
|
||||
existing path returns an error, bad-password authentication is rejected
|
||||
- Cross-protocol via aws-sdk-s3: SFTP write then S3 read with SHA256 match,
|
||||
S3 write then SFTP read with SHA256 match
|
||||
- Cross-API directory visibility: SFTP-created sub-directory visible via S3
|
||||
ListObjectsV2, S3-created `__XLDIR__` marker visible via SFTP readdir as a
|
||||
directory entry
|
||||
- Cross-API directory visibility: SFTP-created sub-directory visible via S3 ListObjectsV2, and an S3 trailing-slash directory object visible via SFTP readdir as a directory entry
|
||||
|
||||
#### sftp_compliance (`test_sftp_compliance_suite`)
|
||||
|
||||
|
||||
@@ -394,9 +394,9 @@ pub async fn test_sftp_core_operations() -> Result<()> {
|
||||
info!("PASS: S3-written object matches via S3 GetObject and SFTP READ");
|
||||
|
||||
// --- Cross-API directory visibility: SFTP mkdir, S3 ListObjectsV2 ---
|
||||
// SFTP mkdir writes a __XLDIR__ marker. The rustfs S3 listing path
|
||||
// decodes that marker back to a trailing-slash key, so the asserted
|
||||
// pattern is "subdir_sftp/".
|
||||
// SFTP mkdir writes a zero-byte object at the trailing-slash key.
|
||||
// Storage encodes that key to the __XLDIR__ marker, and the S3
|
||||
// listing path decodes the marker back to a trailing-slash key.
|
||||
info!("Testing SFTP: SFTP-created sub-directory visible via S3 ListObjectsV2");
|
||||
let sftp_subdir_name = "subdir_sftp";
|
||||
let sftp_subdir_path = format!("/{bucket}/{sftp_subdir_name}");
|
||||
@@ -422,20 +422,21 @@ pub async fn test_sftp_core_operations() -> Result<()> {
|
||||
);
|
||||
info!("PASS: SFTP mkdir visible to S3 ListObjectsV2");
|
||||
|
||||
// --- Cross-API directory visibility: S3 marker, SFTP readdir ---
|
||||
// aws-sdk-s3 PutObject writes a zero-byte marker keyed with the
|
||||
// __XLDIR__ suffix. SFTP readdir must decode the marker back to a
|
||||
// bare directory entry whose file_type reports as a directory.
|
||||
info!("Testing SFTP: S3-created __XLDIR__ marker visible via SFTP readdir");
|
||||
// --- Cross-API directory visibility: S3 directory object, SFTP readdir ---
|
||||
// aws-sdk-s3 PutObject writes a zero-byte object at the trailing-slash
|
||||
// key. The __XLDIR__ suffix is reserved on the client-key boundary, so
|
||||
// a direct put of that suffix is rejected. Storage encodes the slash
|
||||
// key, and SFTP readdir must show a directory entry.
|
||||
info!("Testing SFTP: S3-created directory object visible via SFTP readdir");
|
||||
let s3_subdir_name = "subdir_s3";
|
||||
let s3_subdir_marker_key = format!("{s3_subdir_name}{XLDIR_SUFFIX}");
|
||||
let s3_subdir_key = format!("{s3_subdir_name}/");
|
||||
s3.put_object()
|
||||
.bucket(bucket)
|
||||
.key(&s3_subdir_marker_key)
|
||||
.key(&s3_subdir_key)
|
||||
.body(ByteStream::from_static(b""))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| anyhow!("S3 PutObject {}/{} failed: {:?}", bucket, s3_subdir_marker_key, e))?;
|
||||
.map_err(|e| anyhow!("S3 PutObject {}/{} failed: {:?}", bucket, s3_subdir_key, e))?;
|
||||
let bucket_entries: Vec<(String, bool)> = sftp
|
||||
.read_dir(&bucket_path)
|
||||
.await?
|
||||
@@ -444,9 +445,9 @@ pub async fn test_sftp_core_operations() -> Result<()> {
|
||||
let visible_via_sftp = bucket_entries.iter().any(|(name, is_dir)| name == s3_subdir_name && *is_dir);
|
||||
assert!(
|
||||
visible_via_sftp,
|
||||
"S3-created marker must appear as a directory in SFTP readdir: entries were {bucket_entries:?}"
|
||||
"S3-created directory object must appear as a directory in SFTP readdir: entries were {bucket_entries:?}"
|
||||
);
|
||||
info!("PASS: S3 marker visible to SFTP readdir as a directory");
|
||||
info!("PASS: S3 directory object visible to SFTP readdir as a directory");
|
||||
|
||||
// --- Pre-cleanup of cross-protocol fixtures ---
|
||||
// Removes the new files and sub-directories so the existing rmdir
|
||||
|
||||
@@ -42,6 +42,7 @@ fn lock_result_from_response(response: rustfs_lock::LockResponse) -> GenerallyLo
|
||||
success: response.success,
|
||||
error_info: response.error,
|
||||
lock_info: response.lock_info.and_then(|info| serde_json::to_string(&info).ok()),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -50,6 +51,7 @@ fn lock_result_from_error(error: impl Into<String>) -> GenerallyLockResult {
|
||||
success: false,
|
||||
error_info: Some(error.into()),
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -59,6 +61,7 @@ fn lock_result_from_release(lock_id: &rustfs_lock::LockId, success: bool) -> Gen
|
||||
success: true,
|
||||
error_info: None,
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
}
|
||||
} else {
|
||||
lock_result_from_error(format!("lock not found for release: {lock_id}"))
|
||||
@@ -164,6 +167,7 @@ impl NodeService for MinimalLockNodeService {
|
||||
success: false,
|
||||
error_info: Some(format!("can not decode args, err: {err}")),
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
}));
|
||||
}
|
||||
};
|
||||
@@ -175,6 +179,7 @@ impl NodeService for MinimalLockNodeService {
|
||||
success: result.success,
|
||||
error_info: result.error,
|
||||
lock_info: lock_info_json,
|
||||
..Default::default()
|
||||
}))
|
||||
}
|
||||
Err(err) => Ok(Response::new(GenerallyLockResponse {
|
||||
@@ -184,6 +189,7 @@ impl NodeService for MinimalLockNodeService {
|
||||
args.resource, args.owner, err
|
||||
)),
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
})),
|
||||
}
|
||||
}
|
||||
@@ -197,6 +203,7 @@ impl NodeService for MinimalLockNodeService {
|
||||
success: false,
|
||||
error_info: Some(format!("can not decode args, err: {err}")),
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
}));
|
||||
}
|
||||
};
|
||||
@@ -208,6 +215,7 @@ impl NodeService for MinimalLockNodeService {
|
||||
success: result.success,
|
||||
error_info: result.error_info,
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
}))
|
||||
}
|
||||
Err(err) => Ok(Response::new(GenerallyLockResponse {
|
||||
@@ -217,6 +225,7 @@ impl NodeService for MinimalLockNodeService {
|
||||
args.resource, args.owner, err
|
||||
)),
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
})),
|
||||
}
|
||||
}
|
||||
@@ -230,6 +239,7 @@ impl NodeService for MinimalLockNodeService {
|
||||
success: false,
|
||||
error_info: Some(format!("can not decode args, err: {err}")),
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
}));
|
||||
}
|
||||
};
|
||||
@@ -241,6 +251,7 @@ impl NodeService for MinimalLockNodeService {
|
||||
success: result.success,
|
||||
error_info: result.error_info,
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
}))
|
||||
}
|
||||
Err(err) => Ok(Response::new(GenerallyLockResponse {
|
||||
@@ -250,6 +261,7 @@ impl NodeService for MinimalLockNodeService {
|
||||
args.resource, args.owner, err
|
||||
)),
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
})),
|
||||
}
|
||||
}
|
||||
@@ -263,6 +275,7 @@ impl NodeService for MinimalLockNodeService {
|
||||
success: false,
|
||||
error_info: Some(format!("can not decode args, err: {err}")),
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
}));
|
||||
}
|
||||
};
|
||||
@@ -272,11 +285,13 @@ impl NodeService for MinimalLockNodeService {
|
||||
success,
|
||||
error_info: None,
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
})),
|
||||
Err(err) => Ok(Response::new(GenerallyLockResponse {
|
||||
success: false,
|
||||
error_info: Some(format!("can not refresh, err: {err}")),
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,8 +31,8 @@ use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::operation::list_object_versions::ListObjectVersionsOutput;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, DeleteMarkerEntry, ObjectVersion, ServerSideEncryption,
|
||||
VersioningConfiguration,
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Delete, DeleteMarkerEntry, ObjectIdentifier, ObjectVersion,
|
||||
ServerSideEncryption, VersioningConfiguration,
|
||||
};
|
||||
use base64_simd::STANDARD as BASE64_STANDARD;
|
||||
use bytes::Bytes;
|
||||
@@ -3729,6 +3729,8 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
|
||||
init_logging();
|
||||
|
||||
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||
let source_log = format!("{}/source.log", source_env.temp_dir);
|
||||
source_env.capture_log_path = Some(source_log.clone());
|
||||
let mut source_env_vars = replication_fast_env();
|
||||
source_env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
source_env.start_rustfs_server_with_env(vec![], &source_env_vars).await?;
|
||||
@@ -3791,19 +3793,66 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
|
||||
"target did not preserve the source delete-marker version ID"
|
||||
);
|
||||
|
||||
source_client
|
||||
.delete_object()
|
||||
let wait_for_source_writeback = || async {
|
||||
let metrics_url = format!("{}/{source_bucket}?replication-metrics", source_env.url);
|
||||
timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
let response = signed_request(
|
||||
http::Method::GET,
|
||||
&metrics_url,
|
||||
&source_env.access_key,
|
||||
&source_env.secret_key,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
let metrics: serde_json::Value = response.error_for_status()?.json().await?;
|
||||
let queued = metrics["queued"]["curr"]["count"]
|
||||
.as_f64()
|
||||
.ok_or("missing replication queue count")?;
|
||||
if metrics["provider_available"] == true && queued == 0.0 {
|
||||
return Ok::<(), Box<dyn Error + Send + Sync>>(());
|
||||
}
|
||||
sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
})
|
||||
.await?
|
||||
};
|
||||
wait_for_source_writeback().await?;
|
||||
|
||||
let purged = source_client
|
||||
.delete_objects()
|
||||
.bucket(source_bucket)
|
||||
.key(object_key)
|
||||
.version_id(&purged_version_id)
|
||||
.delete(
|
||||
Delete::builder()
|
||||
.objects(
|
||||
ObjectIdentifier::builder()
|
||||
.key(object_key)
|
||||
.version_id(&purged_version_id)
|
||||
.build()?,
|
||||
)
|
||||
.objects(
|
||||
ObjectIdentifier::builder()
|
||||
.key(object_key)
|
||||
.version_id(delete_marker_version_id)
|
||||
.build()?,
|
||||
)
|
||||
.build()?,
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
assert!(purged.errors().is_empty(), "batch version purge failed: {:?}", purged.errors());
|
||||
assert_eq!(purged.deleted().len(), 2, "both the data version and marker must be purged");
|
||||
assert_replication_converged(&source_client, source_bucket, &target_client, target_bucket).await?;
|
||||
let target_state = list_replication_state(&target_client, target_bucket).await?;
|
||||
assert!(
|
||||
target_state.iter().all(|entry| entry.version_id != purged_version_id),
|
||||
"target retained the explicitly purged object version"
|
||||
);
|
||||
assert!(
|
||||
target_state.iter().all(|entry| entry.version_id != delete_marker_version_id),
|
||||
"target retained the explicitly purged delete marker"
|
||||
);
|
||||
assert!(
|
||||
target_state
|
||||
.iter()
|
||||
@@ -3819,6 +3868,19 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
|
||||
.await?;
|
||||
assert_eq!(retained.body.collect().await?.into_bytes().as_ref(), b"versioned replication payload v2");
|
||||
|
||||
// Wait for the source workers to finish their final writeback, which runs
|
||||
// after the remote purge has already become visible (rustfs#9121).
|
||||
wait_for_source_writeback().await?;
|
||||
let logs = fs::read_to_string(source_log).await?;
|
||||
let writeback_failures: Vec<_> = logs
|
||||
.lines()
|
||||
.filter(|line| line.contains("apply_replication_delete_state"))
|
||||
.collect();
|
||||
assert!(
|
||||
writeback_failures.is_empty(),
|
||||
"a completed batch purge must not report a source writeback failure: {writeback_failures:?}"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -2146,3 +2146,4 @@ mod tests {
|
||||
);
|
||||
}
|
||||
}
|
||||
// event-drill: isolated harness-only change
|
||||
|
||||
@@ -94,3 +94,12 @@ pub(crate) mod scanner_peer_bucket {
|
||||
pub(crate) use ecstore_erasure::{ReedSolomonEncoder, calc_shard_size};
|
||||
pub(crate) use ecstore_rpc::PeerRestClient;
|
||||
}
|
||||
|
||||
/// Storage handles used to seed a pre-validation bucket record while the fixture server is stopped.
|
||||
#[cfg(test)]
|
||||
pub(crate) mod bucket_tag_fixture {
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata::load_bucket_metadata;
|
||||
pub(crate) use rustfs_ecstore::api::layout::{DisksLayout, EndpointServerPools};
|
||||
pub(crate) use rustfs_ecstore::api::runtime::InstanceContext;
|
||||
pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks_with_instance_ctx};
|
||||
}
|
||||
|
||||
@@ -360,12 +360,26 @@ impl OpaMock {
|
||||
|
||||
pub(crate) async fn expect_bucket_tags(&mut self, action: &str, bucket: &str, department: Option<&str>) -> TestResult {
|
||||
let payload = self.next_request().await?;
|
||||
Self::assert_bucket_tags(&payload, action, bucket, department);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn expect_versioned_tag_permission(&mut self, bucket: &str, object: &str, version: &str) -> TestResult {
|
||||
let payload = self.next_request().await?;
|
||||
Self::assert_bucket_tags(&payload, "s3:GetObjectVersionTagging", bucket, Some("finance"));
|
||||
assert_eq!(payload["input"]["resource"]["object"], object);
|
||||
assert_eq!(payload["input"]["context"]["conditions"]["versionid"], serde_json::json!([version]));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn assert_bucket_tags(payload: &Value, action: &str, bucket: &str, department: Option<&str>) {
|
||||
let input = &payload["input"];
|
||||
assert_eq!(input["identity"]["account"], "opabuckettags");
|
||||
assert_eq!(input["identity"]["is_owner"], false);
|
||||
assert_eq!(input["action"], action);
|
||||
assert_eq!(input["resource"]["bucket"], bucket);
|
||||
let expected = department.map(|value| serde_json::json!([value]));
|
||||
assert_eq!(input["context"]["conditions"].get("ExistingBucketTag/department"), expected.as_ref());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn wait_for_validation(&mut self) -> TestResult {
|
||||
@@ -678,22 +692,61 @@ async fn test_list_buckets_opa_contract() -> TestResult {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Stop and reap the directory's owning server before opening the fixture store.
|
||||
async fn seed_legacy_duplicate_bucket_tags(env: &mut RustFSTestEnvironment, bucket: &str) -> TestResult {
|
||||
use crate::storage_api::bucket_tag_fixture::{
|
||||
DisksLayout, ECStore, EndpointServerPools, InstanceContext, init_local_disks_with_instance_ctx, load_bucket_metadata,
|
||||
};
|
||||
|
||||
let server = env
|
||||
.process
|
||||
.as_mut()
|
||||
.ok_or("missing server process before offline fixture write")?;
|
||||
server.kill()?;
|
||||
server.wait()?;
|
||||
env.process = None;
|
||||
let layout = DisksLayout::from_volumes(std::slice::from_ref(&env.temp_dir))?;
|
||||
let (pools, _) = EndpointServerPools::create_server_endpoints(&env.address, &layout).await?;
|
||||
let context = std::sync::Arc::new(InstanceContext::new());
|
||||
init_local_disks_with_instance_ctx(&context, pools.clone()).await?;
|
||||
let store = ECStore::new_with_instance_ctx(env.address.parse()?, pools, Default::default(), context.clone()).await?;
|
||||
let _cancel = context
|
||||
.background_cancel_token()
|
||||
.expect("store binds its shutdown token")
|
||||
.drop_guard();
|
||||
let mut metadata = load_bucket_metadata(store.clone(), bucket).await?;
|
||||
let incarnation = metadata.bucket_incarnation_id;
|
||||
let policy = metadata.policy_config_json.clone();
|
||||
assert!(!incarnation.is_nil());
|
||||
assert!(!policy.is_empty(), "the bucket-policy Allow must survive legacy seeding");
|
||||
// Encode the duplicate tag set with the same bucket record and erasure writer used by the store.
|
||||
let xml = b"<Tagging><TagSet><Tag><Key>department</Key><Value>finance</Value></Tag><Tag><Key>department</Key><Value>engineering</Value></Tag></TagSet></Tagging>";
|
||||
metadata.tagging_config_xml = xml.to_vec();
|
||||
metadata.save_with_store(store.clone()).await?;
|
||||
let persisted = load_bucket_metadata(store, bucket).await?;
|
||||
assert_eq!(persisted.bucket_incarnation_id, incarnation);
|
||||
assert_eq!(persisted.policy_config_json, policy);
|
||||
assert_eq!(persisted.tagging_config_xml.as_slice(), xml);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Tag, Tagging, VersioningConfiguration,
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Tag, Tagging, TaggingDirective, VersioningConfiguration,
|
||||
};
|
||||
|
||||
init_logging();
|
||||
let mut opa = OpaMock::start().await?;
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_without_cleanup_with_env(&[
|
||||
("RUSTFS_POLICY_PLUGIN_URL", opa.url.as_str()),
|
||||
let opa_url = opa.url.clone();
|
||||
let server_env = [
|
||||
("RUSTFS_POLICY_PLUGIN_URL", opa_url.as_str()),
|
||||
("RUSTFS_POLICY_PLUGIN_AUTH_TOKEN", OPA_AUTH_TOKEN),
|
||||
("NO_PROXY", "127.0.0.1,localhost"),
|
||||
])
|
||||
.await?;
|
||||
];
|
||||
env.start_rustfs_server_without_cleanup_with_env(&server_env).await?;
|
||||
let secret = uuid::Uuid::new_v4().to_string();
|
||||
create_user(&env, "opabuckettags", &secret).await?;
|
||||
let client = aws_sdk_s3::Client::from_conf(build_test_s3_config(&env.url, "opabuckettags", &secret, None, "e2e-bucket-tags"));
|
||||
@@ -733,7 +786,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
.await?;
|
||||
assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
|
||||
client
|
||||
let head = client
|
||||
.head_object()
|
||||
.bucket(source)
|
||||
.key("report")
|
||||
@@ -741,6 +794,8 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
.send()
|
||||
.await?;
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
|
||||
assert_eq!(head.tag_count(), Some(1));
|
||||
opa.expect_versioned_tag_permission(source, "report", version).await?;
|
||||
client.list_objects_v2().bucket(source).send().await?;
|
||||
opa.expect_bucket_tags("s3:ListBucket", source, Some("finance")).await?;
|
||||
client.list_object_versions().bucket(source).send().await?;
|
||||
@@ -750,28 +805,65 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
admin.create_bucket().bucket(destination).send().await?;
|
||||
set_department(&admin, destination, "engineering").await?;
|
||||
let copy_source = format!("{source}/report?versionId={version}");
|
||||
let error = client
|
||||
.copy_object()
|
||||
.bucket(destination)
|
||||
.key("copied")
|
||||
.copy_source(©_source)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("source tags must not authorize a different destination");
|
||||
assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
|
||||
opa.expect_bucket_tags("s3:PutObject", destination, Some("engineering"))
|
||||
.await?;
|
||||
for directive in [TaggingDirective::Copy, TaggingDirective::Replace] {
|
||||
let error = client
|
||||
.copy_object()
|
||||
.bucket(destination)
|
||||
.key("copied")
|
||||
.copy_source(©_source)
|
||||
.set_tagging_directive((directive == TaggingDirective::Replace).then_some(TaggingDirective::Replace))
|
||||
.set_tagging((directive == TaggingDirective::Replace).then(|| "department=legal".to_owned()))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("source tags must not authorize a different destination");
|
||||
assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
|
||||
if directive == TaggingDirective::Copy {
|
||||
opa.expect_versioned_tag_permission(source, "report", version).await?;
|
||||
}
|
||||
opa.expect_bucket_tags("s3:PutObject", destination, Some("engineering"))
|
||||
.await?;
|
||||
let missing = admin
|
||||
.head_object()
|
||||
.bucket(destination)
|
||||
.key("copied")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("denied COPY and REPLACE must leave no destination object");
|
||||
assert_eq!(missing.raw_response().map(|response| response.status().as_u16()), Some(404));
|
||||
}
|
||||
set_department(&admin, destination, "finance").await?;
|
||||
client
|
||||
.copy_object()
|
||||
.bucket(destination)
|
||||
.key("copied")
|
||||
.copy_source(©_source)
|
||||
.send()
|
||||
.await?;
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
|
||||
opa.expect_bucket_tags("s3:PutObject", destination, Some("finance")).await?;
|
||||
for directive in [TaggingDirective::Copy, TaggingDirective::Replace] {
|
||||
client
|
||||
.copy_object()
|
||||
.bucket(destination)
|
||||
.key("copied")
|
||||
.copy_source(©_source)
|
||||
.set_tagging_directive((directive == TaggingDirective::Replace).then_some(TaggingDirective::Replace))
|
||||
.set_tagging((directive == TaggingDirective::Replace).then(|| "department=legal".to_owned()))
|
||||
.send()
|
||||
.await?;
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
|
||||
// The access hook checks preliminary tags, then the copy use case checks
|
||||
// the exact source reader's tags before publishing the destination.
|
||||
for _ in 0..2 {
|
||||
if directive == TaggingDirective::Copy {
|
||||
opa.expect_versioned_tag_permission(source, "report", version).await?;
|
||||
}
|
||||
opa.expect_bucket_tags("s3:PutObject", destination, Some("finance")).await?;
|
||||
}
|
||||
let tags = admin.get_object_tagging().bucket(destination).key("copied").send().await?;
|
||||
assert_eq!(tags.tag_set().len(), 1);
|
||||
assert_eq!(tags.tag_set()[0].key(), "department");
|
||||
assert_eq!(
|
||||
tags.tag_set()[0].value(),
|
||||
if directive == TaggingDirective::Copy {
|
||||
"engineering"
|
||||
} else {
|
||||
"legal"
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
let upload = client
|
||||
.create_multipart_upload()
|
||||
@@ -847,6 +939,14 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("engineering"))
|
||||
.await?;
|
||||
let missing = admin
|
||||
.head_object()
|
||||
.bucket(destination)
|
||||
.key("denied-source")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a denied source must leave no destination object");
|
||||
assert_eq!(missing.raw_response().map(|response| response.status().as_u16()), Some(404));
|
||||
set_department(&client, source, "finance")
|
||||
.await
|
||||
.expect_err("requested tags must not authorize their own mutation");
|
||||
@@ -885,7 +985,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
|
||||
opa.expect_bucket_tags("s3:GetObject", destination, Some("engineering"))
|
||||
.await?;
|
||||
admin
|
||||
let invalid_write = admin
|
||||
.put_bucket_tagging()
|
||||
.bucket(destination)
|
||||
.tagging(
|
||||
@@ -895,7 +995,25 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
.build()?,
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("new duplicate tags must be rejected before publication");
|
||||
assert_eq!(invalid_write.as_service_error().and_then(ProvideErrorMetadata::code), Some("InvalidTag"));
|
||||
assert_eq!(invalid_write.raw_response().map(|response| response.status().as_u16()), Some(400));
|
||||
|
||||
set_department(&admin, source, "finance").await?;
|
||||
seed_legacy_duplicate_bucket_tags(&mut env, destination).await?;
|
||||
env.start_rustfs_server_without_cleanup_with_env(&server_env).await?;
|
||||
// Prove this process can authorize the same caller and read a healthy bucket's persisted tags.
|
||||
let read = client
|
||||
.get_object()
|
||||
.bucket(source)
|
||||
.key("report")
|
||||
.version_id(version)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
|
||||
|
||||
let error = client
|
||||
.get_object()
|
||||
.bucket(destination)
|
||||
|
||||
@@ -48,6 +48,7 @@ const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY";
|
||||
const G09_EVIDENCE_DIR_ENV: &str = "RUSTFS_SCANNER_HEAL_G09_EVIDENCE_DIR";
|
||||
const RC5_COMMIT: &str = "40a2470feb567201165a5b809b7598bb4b1f68f5";
|
||||
const STABLE_1_0_1_COMMIT: &str = "6de965ae3c965a78ff819fbcd7acd4aa44177d92";
|
||||
const STABLE_1_0_4_COMMIT: &str = "46337942d55f3a8e6a97b147ca98d9ea8115555d";
|
||||
const SSE_MASTER_KEY_ENV: &str = "RUSTFS_SSE_S3_MASTER_KEY";
|
||||
const SSE_MASTER_KEY: &str = "QkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkI=";
|
||||
const PLAIN_BUCKET: &str = "upgrade-plain-data";
|
||||
@@ -954,7 +955,8 @@ async fn exercise_mixed_cluster(
|
||||
phase: &str,
|
||||
current_node: usize,
|
||||
previous_node: usize,
|
||||
) -> TestResult {
|
||||
previous_has_delete_recovery: bool,
|
||||
) -> Result<Option<String>, BoxError> {
|
||||
let clients = cluster.create_all_clients()?;
|
||||
wait_for_upgrade_write_readiness(&clients, MIXED_BUCKET, phase, LISTING_CONVERGENCE_TIMEOUT).await?;
|
||||
wait_for_drive_readmission(&clients, MIXED_BUCKET, phase, DRIVE_READMISSION_TIMEOUT).await?;
|
||||
@@ -1009,6 +1011,30 @@ async fn exercise_mixed_cluster(
|
||||
vec![u8::try_from(MULTIPART_WORKERS - 1)?; 64 * 1024]
|
||||
);
|
||||
|
||||
// 1.0.4 already implements the delete decision protocol. Use a separate
|
||||
// old-writer key so the original seed still spans the entire rollout.
|
||||
if previous_has_delete_recovery {
|
||||
let key = format!(".upgrade-delete/{phase}/previous-writer");
|
||||
previous_client
|
||||
.put_object()
|
||||
.bucket(MIXED_BUCKET)
|
||||
.key(&key)
|
||||
.body(ByteStream::from_static(UPGRADE_READINESS_BODY))
|
||||
.send()
|
||||
.await?;
|
||||
for client in &clients {
|
||||
assert_eq!(read_object(client, MIXED_BUCKET, &key, None).await?.1, UPGRADE_READINESS_BODY);
|
||||
}
|
||||
upgrade_probe_client(current_client)
|
||||
.delete_object()
|
||||
.bucket(MIXED_BUCKET)
|
||||
.key(&key)
|
||||
.send()
|
||||
.await?;
|
||||
assert_mixed_deletes_visible(&clients, std::slice::from_ref(&key)).await?;
|
||||
return Ok(Some(key));
|
||||
}
|
||||
|
||||
// Every online participant must understand the delete decision protocol.
|
||||
// An upgrade phase can serve reads/writes while intentionally refusing a
|
||||
// modern DELETE; rejecting it must leave the old writer's seed intact.
|
||||
@@ -1029,6 +1055,31 @@ async fn exercise_mixed_cluster(
|
||||
);
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn assert_mixed_deletes_visible(clients: &[Client], keys: &[String]) -> TestResult {
|
||||
for (node, client) in clients.iter().enumerate() {
|
||||
for key in keys {
|
||||
let error = upgrade_probe_client(client)
|
||||
.get_object()
|
||||
.bucket(MIXED_BUCKET)
|
||||
.key(key)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a successful mixed-version DELETE must remain visible through every reader");
|
||||
assert_eq!(
|
||||
error.raw_response().map(|response| response.status().as_u16()),
|
||||
Some(404),
|
||||
"node {node}: {key}"
|
||||
);
|
||||
assert_eq!(
|
||||
error.as_service_error().and_then(ProvideErrorMetadata::code),
|
||||
Some("NoSuchKey"),
|
||||
"node {node}: {key}"
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1112,8 +1163,9 @@ async fn wait_for_upgrade_writes(
|
||||
match response {
|
||||
Ok(Ok(_)) => break,
|
||||
Ok(Err(error)) => {
|
||||
let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
|
||||
if error.raw_response().map(|response| response.status().as_u16()) != Some(503)
|
||||
|| error.as_service_error().and_then(ProvideErrorMetadata::code) != Some("ServiceUnavailable")
|
||||
|| !matches!(code, Some("ServiceUnavailable" | "SlowDown" | "SlowDownRead"))
|
||||
{
|
||||
return Err(format!("{phase}: node {node} {probe} failed: {error:?}").into());
|
||||
}
|
||||
@@ -2239,6 +2291,9 @@ async fn direct_upgrade_from_previous_release_preserves_object_contracts() -> Te
|
||||
async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts() -> TestResult {
|
||||
init_logging();
|
||||
let previous_binary = source_binary()?;
|
||||
// Bind the expected protocol to the verified release source, never to the
|
||||
// DELETE result itself. The older CI source pins retain their 503 contract.
|
||||
let previous_has_delete_recovery = binary_source_revision(&previous_binary).await? == STABLE_1_0_4_COMMIT;
|
||||
let evidence_context = g09_evidence_context(&previous_binary).await?;
|
||||
let current_binary = rustfs_binary_path();
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(MIXED_NODE_COUNT).await?;
|
||||
@@ -2259,7 +2314,10 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
|
||||
cluster.stop_node(0)?;
|
||||
cluster.start_node_from_binary(0, ¤t_binary).await?;
|
||||
exercise_mixed_cluster(&cluster, "one-current-node", 0, 1).await?;
|
||||
let mut mixed_deleted_keys = Vec::new();
|
||||
if let Some(key) = exercise_mixed_cluster(&cluster, "one-current-node", 0, 1, previous_has_delete_recovery).await? {
|
||||
mixed_deleted_keys.push(key);
|
||||
}
|
||||
// One controlled out-of-policy attempt proves fail-closed behavior below
|
||||
// native write quorum; positive MPU writers remain paused while mixed.
|
||||
assert_mixed_pending_refusal(&cluster, &pending, &sources).await?;
|
||||
@@ -2267,7 +2325,9 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
for (node_idx, phase) in [(1, "two-current-nodes"), (2, "one-previous-node")] {
|
||||
cluster.stop_node(node_idx)?;
|
||||
cluster.start_node_from_binary(node_idx, ¤t_binary).await?;
|
||||
exercise_mixed_cluster(&cluster, phase, 0, 3).await?;
|
||||
if let Some(key) = exercise_mixed_cluster(&cluster, phase, 0, 3, previous_has_delete_recovery).await? {
|
||||
mixed_deleted_keys.push(key);
|
||||
}
|
||||
}
|
||||
|
||||
cluster.stop_node(3)?;
|
||||
@@ -2277,6 +2337,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
let completed_pending = finish_pending_multipart(&clients, &pending, pending_parts.concat()).await?;
|
||||
exercise_homogeneous_multipart(&cluster, "homogeneous-current").await?;
|
||||
verify_load_bodies(&clients, "homogeneous-previous", "multipart").await?;
|
||||
assert_mixed_deletes_visible(&clients, &mixed_deleted_keys).await?;
|
||||
|
||||
for (node_idx, client) in clients.iter().enumerate() {
|
||||
assert_eq!(
|
||||
@@ -2325,6 +2386,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
cluster.start_with_binary(&previous_binary).await?;
|
||||
prepare_release_baseline(&mut cluster, &previous_binary, "rollback").await?;
|
||||
let old_clients = cluster.create_all_clients()?;
|
||||
assert_mixed_deletes_visible(&old_clients, &mixed_deleted_keys).await?;
|
||||
assert_resumed_multipart(&old_clients, &completed_pending).await?;
|
||||
for phase in ["homogeneous-previous", "homogeneous-current"] {
|
||||
verify_load_bodies(&old_clients, phase, "multipart").await?;
|
||||
@@ -2389,6 +2451,8 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
"phases": phases,
|
||||
"objects_per_phase": objects_per_phase,
|
||||
"mixed_load": "plain-put",
|
||||
"mixed_delete_protocol_supported": previous_has_delete_recovery,
|
||||
"mixed_deleted_keys": mixed_deleted_keys,
|
||||
"homogeneous_multipart_phases": ["homogeneous-previous", "homogeneous-current", "homogeneous-rollback"],
|
||||
"multipart_objects_per_homogeneous_phase": MULTIPART_WORKERS * MULTIPART_UPLOADS_PER_WORKER,
|
||||
"assertions": [
|
||||
@@ -2421,7 +2485,11 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
"16x16 multipart load runs in homogeneous previous, current and rollback phases",
|
||||
"one-current-node Complete returns typed 503 and preserves actual source authority and parts",
|
||||
"the same legacy upload completes and replays once fully upgraded without adding a version",
|
||||
"modern DELETE fails closed and preserves every byte while old peers remain"
|
||||
if previous_has_delete_recovery {
|
||||
"mixed-version DELETE succeeds and stays absent through every reader after upgrade and rollback"
|
||||
} else {
|
||||
"modern DELETE fails closed and preserves every byte while old peers remain"
|
||||
}
|
||||
],
|
||||
}),
|
||||
)?;
|
||||
|
||||
@@ -31,6 +31,7 @@ workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
fips = ["rustfs-s3-client/fips", "rustfs-utils/fips"]
|
||||
diagnostic-read-version-caller = ["rustfs-io-metrics/diagnostic-read-version-caller"]
|
||||
# Sampled wall/CPU diagnostics only; does not change the read backend.
|
||||
diagnostic-local-batch = ["rustix/time"]
|
||||
|
||||
@@ -192,6 +192,7 @@ pub mod bucket {
|
||||
table_catalog_path_hash, unreadable_config_refusal,
|
||||
};
|
||||
pub use crate::bucket::metadata::{BUCKET_DURABILITY_CONFIG, BUCKET_ON_DEMAND_MIGRATION_CONFIG};
|
||||
pub use crate::bucket::metadata::{MAX_BUCKET_TAGS, validate_tag_set_entries};
|
||||
}
|
||||
|
||||
pub mod config_parse_mode {
|
||||
@@ -331,8 +332,8 @@ pub mod bucket {
|
||||
pub mod utils {
|
||||
pub use crate::bucket::utils::{
|
||||
check_bucket_and_object_names, check_list_objs_args, check_object_name_for_length_and_slash,
|
||||
check_valid_bucket_name_strict, deserialize, has_bad_path_component, is_meta_bucketname, is_valid_object_prefix,
|
||||
serialize,
|
||||
check_valid_bucket_name_strict, deserialize, has_bad_path_component, has_xml_invalid_control_characters,
|
||||
is_meta_bucketname, is_valid_object_prefix, serialize,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -451,9 +452,15 @@ pub mod disk {
|
||||
pub use crate::cluster::rpc::background_reader_stream::BackgroundReaderStream;
|
||||
pub use crate::disk::background_memory::BackgroundEncodeBuffer;
|
||||
pub use crate::disk::disk_store::get_object_disk_read_timeout;
|
||||
#[cfg(feature = "test-util")]
|
||||
pub use crate::disk::disk_store::wait_for_multipart_index_bootstrap;
|
||||
pub use crate::disk::local::{ReplacementExecutionLease, ScanGuard};
|
||||
#[cfg(feature = "test-util")]
|
||||
pub use crate::disk::local::{local_durability_modes, set_local_rename_data_fail_after_metadata_commit};
|
||||
pub use crate::disk::multipart_index::{
|
||||
MULTIPART_INDEX_RPC_MAX_BYTES, MULTIPART_INDEX_RPC_MAX_REQUEST_BYTES, MultipartIndexPage, MultipartIndexRpcReply,
|
||||
MultipartIndexRpcRequest,
|
||||
};
|
||||
pub use crate::disk::multipart_source::{
|
||||
MULTIPART_SOURCE_PROTOCOL_VERSION, MultipartAbortCertificate, MultipartAbortProof, MultipartAbortProofKind,
|
||||
MultipartAbortRound, MultipartSourceOperation, MultipartSourcePhase, MultipartSourceRequest, MultipartSourceResponse,
|
||||
@@ -466,13 +473,15 @@ pub mod disk {
|
||||
pub use crate::disk::os::{LocalPublicationPause, LocalPublicationStage};
|
||||
pub use crate::disk::{
|
||||
BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp,
|
||||
CheckPartsResp, ConditionalFileUpdate, CurrentDeleteGeneration, CurrentDeleteTransition,
|
||||
BoundedDeleteRecoveryDiscovery, BoundedDeleteRecoveryInspection, BoundedDeleteRecoverySettlement,
|
||||
BoundedDeleteRecoverySettlementReply, CheckPartsResp, ConditionalFileUpdate, CurrentDeleteGeneration,
|
||||
CurrentDeleteTransition, DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES, DELETE_RECOVERY_MAX_INSPECTION_ENTRIES,
|
||||
DELETE_RECOVERY_PROTOCOL_VERSION, DeleteOptions, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest,
|
||||
DeleteRecoveryInspection, DeleteRecoveryPage, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, DeleteRecoveryState,
|
||||
Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore, FileInfoVersions, FileReader, FileWriter,
|
||||
GuardedDeleteVersionsPayload, GuardedFileInfoVersions, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize,
|
||||
POOL_META_NATIVE_CAS_METADATA_SUFFIX, PartTransactionAction, PoolMetaNativeBackup, PoolMetaNativeInspection,
|
||||
PoolMetaNativeObject, PoolMetaNativeRecoveryRequest, PoolMetaNativeRecoveryResponse,
|
||||
GuardedDeleteVersionsPayload, GuardedFileInfoVersions, HEAL_REPLACEMENT_RECOVERY_DIR, HEALING_MARKER_PATH,
|
||||
NsScannerOpenRequest, OldCurrentSize, POOL_META_NATIVE_CAS_METADATA_SUFFIX, PartTransactionAction, PoolMetaNativeBackup,
|
||||
PoolMetaNativeInspection, PoolMetaNativeObject, PoolMetaNativeRecoveryRequest, PoolMetaNativeRecoveryResponse,
|
||||
QUOTA_MUTATION_FENCE_METADATA_SUFFIX, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_TMP_BUCKET,
|
||||
ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, SnapshotLeaseToken,
|
||||
UpdateMetadataOpts, VolumeInfo, WalkDirOptions, is_quota_mutation_fence_path, new_disk,
|
||||
@@ -617,7 +626,7 @@ pub mod rebalance {
|
||||
pub mod rio {
|
||||
pub use crate::io_support::rio::{
|
||||
DecryptReader, DynReader, EncryptReader, HardLimitReader, HashReader, ReadStream, Reader, WriteEncryption, WritePlan,
|
||||
boxed_reader, compression_metadata_value, wrap_reader,
|
||||
boxed_reader, compression_metadata_value, validate_compression_scheme, wrap_reader,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -654,7 +663,9 @@ pub mod set_disk {
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
pub mod test_util {
|
||||
pub use crate::bucket::quota::reservation::fail_next_quota_ledger_save_for_test;
|
||||
pub use crate::bucket::quota::reservation::{
|
||||
fail_next_quota_ledger_save_for_test, initialize_quota_ledger_for_test, retry_quota_reconciliation_for_test,
|
||||
};
|
||||
pub use crate::set_disk::ConditionalReadBenchmarkMetadataGuard;
|
||||
pub use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, PutObjectCommitPause};
|
||||
|
||||
|
||||
@@ -16,7 +16,8 @@ use crate::bucket::metadata::BucketMetadata;
|
||||
use crate::bucket::metadata_sys::get_bucket_targets_config;
|
||||
use crate::bucket::metadata_sys::get_replication_config;
|
||||
use crate::bucket::remote_s3_client::{
|
||||
PathStyle, REPLICATION_TARGET_RETRY_POLICY, RemoteCredentials, RemoteS3EndpointSpec, build_remote_s3_client,
|
||||
PathStyle, REPLICATION_TARGET_RETRY_POLICY, RemoteCredentials, RemoteS3EndpointSpec, RemoteS3RetryPolicy,
|
||||
UploadResponseDeadline, build_remote_s3_client,
|
||||
};
|
||||
use crate::bucket::replication::{ObjectLockIntegrity, object_lock_put_integrity, replication_etags_match};
|
||||
use crate::bucket::replication::{ReplicationStatusType, ReplicationTargetConfigBridge};
|
||||
@@ -119,6 +120,7 @@ impl From<&BucketTarget> for RemoteS3EndpointSpec {
|
||||
ca_cert_pem: (!target.ca_cert_pem.trim().is_empty()).then(|| target.ca_cert_pem.clone()),
|
||||
connect_timeout: None,
|
||||
read_timeout: None,
|
||||
operation_timeout: Some(TargetRequestTimeouts::from_env().request),
|
||||
// Replication has no retry budget of its own on the request path,
|
||||
// so it keeps the SDK's standard three attempts; stating it here
|
||||
// pins the behaviour to this line instead of an SDK default.
|
||||
@@ -1191,6 +1193,7 @@ impl BucketTargetSys {
|
||||
health_check_duration: target.health_check_duration,
|
||||
replicate_sync: target.replication_sync,
|
||||
client: Arc::new(client),
|
||||
timeouts: TargetRequestTimeouts::from_env(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1922,6 +1925,83 @@ impl std::fmt::Display for S3ClientError {
|
||||
}
|
||||
}
|
||||
|
||||
/// Replication request budgets, snapshotted when a target client is built.
|
||||
/// Streamed GET bodies are not subject to the upload-response budget.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct TargetRequestTimeouts {
|
||||
request: Duration,
|
||||
complete: Duration,
|
||||
minimum_transfer_rate: u64,
|
||||
}
|
||||
|
||||
impl Default for TargetRequestTimeouts {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
request: Duration::from_secs(60),
|
||||
complete: Duration::from_secs(15 * 60),
|
||||
minimum_transfer_rate: 1024 * 1024,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TargetRequestTimeouts {
|
||||
fn from_env() -> Self {
|
||||
let defaults = Self::default();
|
||||
let seconds = |name: &str, fallback: Duration| {
|
||||
let value = rustfs_utils::get_env_u64(name, fallback.as_secs());
|
||||
Duration::from_secs(if (1..=86400).contains(&value) {
|
||||
value
|
||||
} else {
|
||||
fallback.as_secs()
|
||||
})
|
||||
};
|
||||
let minimum_transfer_rate =
|
||||
rustfs_utils::get_env_u64("RUSTFS_REPLICATION_MIN_TRANSFER_RATE_BYTES_PER_SEC", defaults.minimum_transfer_rate);
|
||||
Self {
|
||||
request: seconds("RUSTFS_REPLICATION_REQUEST_TIMEOUT_SECS", defaults.request),
|
||||
complete: seconds("RUSTFS_REPLICATION_COMPLETE_TIMEOUT_SECS", defaults.complete),
|
||||
minimum_transfer_rate: if minimum_transfer_rate == 0 {
|
||||
defaults.minimum_transfer_rate
|
||||
} else {
|
||||
minimum_transfer_rate
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn upload_timeout(self, size: i64) -> Result<Duration, S3ClientError> {
|
||||
let size = u64::try_from(size).map_err(|_| S3ClientError::new("replication upload size must be non-negative"))?;
|
||||
// The SDK can make three attempts. Leave transmission headroom for
|
||||
// all three; the separate response deadline still bounds a trickling
|
||||
// target after each upload finishes.
|
||||
let attempts = match REPLICATION_TARGET_RETRY_POLICY {
|
||||
RemoteS3RetryPolicy::Disabled => 1,
|
||||
RemoteS3RetryPolicy::Standard { max_attempts } => u64::from(max_attempts.max(1)),
|
||||
};
|
||||
let seconds = size
|
||||
.div_ceil(self.minimum_transfer_rate)
|
||||
.checked_mul(attempts)
|
||||
.ok_or_else(|| S3ClientError::new("replication upload timeout exceeds supported duration"))?;
|
||||
let timeout = self
|
||||
.request
|
||||
.checked_add(Duration::from_secs(seconds))
|
||||
.ok_or_else(|| S3ClientError::new("replication upload timeout exceeds supported duration"))?;
|
||||
if Instant::now().checked_add(timeout).is_none() {
|
||||
return Err(S3ClientError::new("replication upload timeout exceeds supported duration"));
|
||||
}
|
||||
Ok(timeout)
|
||||
}
|
||||
|
||||
fn upload_override(self, size: i64) -> Result<aws_sdk_s3::config::Builder, S3ClientError> {
|
||||
Ok(aws_sdk_s3::config::Builder::new()
|
||||
.timeout_config(
|
||||
aws_smithy_types::timeout::TimeoutConfig::builder()
|
||||
.operation_timeout(self.upload_timeout(size)?)
|
||||
.build(),
|
||||
)
|
||||
.interceptor(UploadResponseDeadline(self.request)))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct TargetClient {
|
||||
pub endpoint: String,
|
||||
@@ -1935,6 +2015,7 @@ pub struct TargetClient {
|
||||
pub health_check_duration: Duration,
|
||||
pub replicate_sync: bool,
|
||||
pub client: Arc<S3Client>,
|
||||
pub timeouts: TargetRequestTimeouts,
|
||||
}
|
||||
|
||||
impl TargetClient {
|
||||
@@ -2360,12 +2441,11 @@ impl TargetClient {
|
||||
|
||||
Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
|
||||
});
|
||||
let mut config = self.timeouts.upload_override(size)?;
|
||||
if forwards_source_checksum {
|
||||
operation = operation.config_override(
|
||||
aws_sdk_s3::config::Builder::new()
|
||||
.request_checksum_calculation(aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired),
|
||||
);
|
||||
config = config.request_checksum_calculation(aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired);
|
||||
}
|
||||
operation = operation.config_override(config);
|
||||
match operation.send().await {
|
||||
Ok(output) => {
|
||||
// Under SSE-KMS/DSSE or SSE-C the target's ETag is not the MD5
|
||||
@@ -2501,6 +2581,7 @@ impl TargetClient {
|
||||
}
|
||||
Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
|
||||
})
|
||||
.config_override(self.timeouts.upload_override(size)?)
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
@@ -2538,6 +2619,13 @@ impl TargetClient {
|
||||
}
|
||||
Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
|
||||
})
|
||||
.config_override(
|
||||
aws_sdk_s3::config::Builder::new().timeout_config(
|
||||
aws_smithy_types::timeout::TimeoutConfig::builder()
|
||||
.operation_timeout(self.timeouts.complete)
|
||||
.build(),
|
||||
),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
@@ -2966,11 +3054,241 @@ mod tests {
|
||||
health_check_duration: Duration::from_secs(5),
|
||||
replicate_sync: false,
|
||||
client: Arc::new(client),
|
||||
timeouts: TargetRequestTimeouts::default(),
|
||||
},
|
||||
request_headers,
|
||||
)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct TimedResponseConnector {
|
||||
upload_delay: Duration,
|
||||
response_delay: Duration,
|
||||
prefix_bytes: usize,
|
||||
response: bytes::Bytes,
|
||||
frames: Arc<std::sync::atomic::AtomicUsize>,
|
||||
dropped: Arc<std::sync::atomic::AtomicUsize>,
|
||||
}
|
||||
|
||||
struct ResponseDrop(Arc<std::sync::atomic::AtomicUsize>);
|
||||
impl Drop for ResponseDrop {
|
||||
fn drop(&mut self) {
|
||||
self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
impl SmithyHttpConnector for TimedResponseConnector {
|
||||
fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
|
||||
let settings = self.clone();
|
||||
HttpConnectorFuture::new(async move {
|
||||
http_body_util::BodyExt::collect(request.into_body())
|
||||
.await
|
||||
.map_err(aws_smithy_runtime_api::client::result::ConnectorError::io)?;
|
||||
tokio::time::sleep(settings.upload_delay).await;
|
||||
let guard = ResponseDrop(Arc::clone(&settings.dropped));
|
||||
let stream = futures::stream::unfold(
|
||||
(settings.prefix_bytes, Some(settings.response), true, guard),
|
||||
move |(remaining, mut body, first, guard)| {
|
||||
let frames = Arc::clone(&settings.frames);
|
||||
async move {
|
||||
if remaining == 0 && body.is_none() {
|
||||
return None;
|
||||
}
|
||||
if !first {
|
||||
tokio::time::sleep(settings.response_delay).await;
|
||||
}
|
||||
let bytes = if remaining > 0 {
|
||||
bytes::Bytes::from_static(b" ")
|
||||
} else {
|
||||
body.take()?
|
||||
};
|
||||
frames.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
Some((
|
||||
Ok::<_, std::io::Error>(http_body::Frame::data(bytes)),
|
||||
(remaining.saturating_sub(1), body, false, guard),
|
||||
))
|
||||
}
|
||||
},
|
||||
)
|
||||
.fuse();
|
||||
let body = SdkBody::from_body_1_x(http_body_util::StreamBody::new(stream));
|
||||
let mut response =
|
||||
HttpResponse::new(aws_smithy_runtime_api::http::StatusCode::try_from(200u16).expect("status"), body);
|
||||
response.headers_mut().insert("etag", "\"test-etag\"");
|
||||
Ok(response)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn timed_response_target(
|
||||
upload_delay: Duration,
|
||||
response_delay: Duration,
|
||||
prefix_bytes: usize,
|
||||
response: &'static [u8],
|
||||
) -> (TargetClient, Arc<std::sync::atomic::AtomicUsize>, Arc<std::sync::atomic::AtomicUsize>) {
|
||||
let frames = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let dropped = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let connector = SharedHttpConnector::new(TimedResponseConnector {
|
||||
upload_delay,
|
||||
response_delay,
|
||||
prefix_bytes,
|
||||
response: bytes::Bytes::from_static(response),
|
||||
frames: Arc::clone(&frames),
|
||||
dropped: Arc::clone(&dropped),
|
||||
});
|
||||
let http_client = http_client_fn(move |_settings, _components| connector.clone());
|
||||
let (mut target, _) = header_recording_target_client(Vec::new());
|
||||
target.timeouts = TargetRequestTimeouts {
|
||||
request: Duration::from_secs(2),
|
||||
complete: Duration::from_secs(10),
|
||||
minimum_transfer_rate: 1024 * 1024,
|
||||
};
|
||||
target.client = Arc::new(S3Client::from_conf(
|
||||
target
|
||||
.client
|
||||
.config()
|
||||
.to_builder()
|
||||
.http_client(http_client)
|
||||
.retry_config(aws_sdk_s3::config::retry::RetryConfig::disabled())
|
||||
.timeout_config(
|
||||
aws_smithy_types::timeout::TimeoutConfig::builder()
|
||||
.operation_timeout(target.timeouts.request)
|
||||
.build(),
|
||||
)
|
||||
.build(),
|
||||
));
|
||||
(target, frames, dropped)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_upload_budgets_preserve_large_transfers_and_reject_invalid_sizes() {
|
||||
let policy = TargetRequestTimeouts::default();
|
||||
assert_eq!(policy.upload_timeout(0).expect("empty upload"), Duration::from_secs(60));
|
||||
assert_eq!(
|
||||
policy.upload_timeout(5 * 1024 * 1024 * 1024).expect("maximum single PUT"),
|
||||
Duration::from_secs(15420)
|
||||
);
|
||||
assert!(policy.upload_timeout(-1).is_err());
|
||||
let slow = TargetRequestTimeouts {
|
||||
minimum_transfer_rate: 1,
|
||||
..policy
|
||||
};
|
||||
assert!(slow.upload_timeout(i64::MAX).is_err());
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn replication_upload_response_deadline_is_independent_of_size_and_checksum_override() {
|
||||
for operation in ["put", "forwarded-checksum", "part"] {
|
||||
let (target, frames, dropped) = timed_response_target(Duration::from_secs(4), Duration::from_millis(500), 100, b"");
|
||||
let body = vec![b'x'; 2 * 1024 * 1024];
|
||||
let size = i64::try_from(body.len()).expect("size");
|
||||
let started = tokio::time::Instant::now();
|
||||
let error = if operation == "part" {
|
||||
target
|
||||
.put_object_part(
|
||||
"bucket",
|
||||
"key",
|
||||
"upload",
|
||||
1,
|
||||
size,
|
||||
ByteStream::from(body),
|
||||
&PutObjectPartOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect_err("response deadline")
|
||||
} else {
|
||||
let mut opts = PutObjectOptions::default();
|
||||
if operation == "forwarded-checksum" {
|
||||
opts.user_metadata.insert(
|
||||
"x-amz-checksum-sha256".to_owned(),
|
||||
"OoJ3yNhRwv3wwtZoGqEIPrPX9xwTnfLl+ka0wStN1g0=".to_owned(),
|
||||
);
|
||||
}
|
||||
target
|
||||
.put_object("bucket", "key", size, ByteStream::from(body), &opts)
|
||||
.await
|
||||
.expect_err("response deadline")
|
||||
};
|
||||
assert!(
|
||||
started.elapsed() >= Duration::from_secs(6),
|
||||
"upload gets its full independent window: {error}"
|
||||
);
|
||||
assert!(
|
||||
started.elapsed() < target.timeouts.upload_timeout(size).expect("budget"),
|
||||
"response must end before the larger total deadline"
|
||||
);
|
||||
assert!(
|
||||
frames.load(std::sync::atomic::Ordering::SeqCst) >= 2,
|
||||
"trickle delivered progress before failure"
|
||||
);
|
||||
assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1, "transport body released");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn replication_slow_upload_preserves_transfer_window_and_streaming_get_body() {
|
||||
let (target, _, _) = timed_response_target(Duration::from_secs(4), Duration::from_millis(100), 5, b"");
|
||||
let body = vec![b'x'; 2 * 1024 * 1024];
|
||||
let size = i64::try_from(body.len()).expect("size");
|
||||
target
|
||||
.put_object("bucket", "key", size, ByteStream::from(body), &PutObjectOptions::default())
|
||||
.await
|
||||
.expect("slow upload, prompt response");
|
||||
let (target, _, dropped) = timed_response_target(Duration::ZERO, Duration::from_secs(1), 5, b"payload");
|
||||
let response = target
|
||||
.get_object("bucket", "key", None, None, None, HeaderMap::new())
|
||||
.await
|
||||
.expect("GET headers");
|
||||
let started = tokio::time::Instant::now();
|
||||
let body = response
|
||||
.body
|
||||
.collect()
|
||||
.await
|
||||
.expect("streamed GET remains outside upload deadline")
|
||||
.into_bytes();
|
||||
assert!(started.elapsed() > target.timeouts.request);
|
||||
assert_eq!(&body[..], b" payload");
|
||||
assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn replication_complete_processing_whitespace_has_a_separate_bounded_window() {
|
||||
const RESULT: &[u8] = b"<CompleteMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><ETag>\"etag\"</ETag></CompleteMultipartUploadResult>";
|
||||
for prefix in [5, 100] {
|
||||
let (target, frames, dropped) = timed_response_target(Duration::ZERO, Duration::from_secs(1), prefix, RESULT);
|
||||
let started = tokio::time::Instant::now();
|
||||
let result = target
|
||||
.complete_multipart_upload(
|
||||
"bucket",
|
||||
"key",
|
||||
"upload",
|
||||
vec![CompletedPart::builder().part_number(1).e_tag("etag").build()],
|
||||
&PutObjectOptions::default(),
|
||||
)
|
||||
.await;
|
||||
if prefix == 5 {
|
||||
assert!(result.is_ok(), "legal processing whitespace: {result:?}");
|
||||
} else {
|
||||
assert!(result.is_err());
|
||||
assert!(started.elapsed() <= Duration::from_secs(11));
|
||||
}
|
||||
assert!(started.elapsed() > target.timeouts.request);
|
||||
assert!(frames.load(std::sync::atomic::Ordering::SeqCst) >= 2);
|
||||
assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn replication_control_operation_deadline_covers_response_trickle() {
|
||||
let (target, frames, dropped) =
|
||||
timed_response_target(Duration::ZERO, Duration::from_millis(500), 100, b"<VersioningConfiguration/>");
|
||||
let started = tokio::time::Instant::now();
|
||||
assert!(target.get_bucket_versioning("bucket").await.is_err());
|
||||
assert!(started.elapsed() <= Duration::from_secs(3));
|
||||
assert!(frames.load(std::sync::atomic::Ordering::SeqCst) >= 2);
|
||||
assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
fn streaming_test_body(payload: &'static [u8]) -> ByteStream {
|
||||
let stream = tokio_util::io::ReaderStream::new(std::io::Cursor::new(payload));
|
||||
let body = http_body_util::StreamBody::new(futures::StreamExt::map(stream, |r| r.map(http_body::Frame::data)));
|
||||
@@ -3279,6 +3597,7 @@ mod tests {
|
||||
health_check_duration: Duration::from_secs(5),
|
||||
replicate_sync: false,
|
||||
client: Arc::new(client),
|
||||
timeouts: TargetRequestTimeouts::default(),
|
||||
},
|
||||
request_uris,
|
||||
)
|
||||
@@ -3591,6 +3910,7 @@ mod tests {
|
||||
health_check_duration: Duration::from_secs(5),
|
||||
replicate_sync: false,
|
||||
client: Arc::new(client),
|
||||
timeouts: TargetRequestTimeouts::default(),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -5487,6 +5487,14 @@ pub async fn put_restore_opts(
|
||||
if !oi.user_tags.is_empty() {
|
||||
meta.insert(AMZ_OBJECT_TAGGING.to_string(), (*oi.user_tags).clone());
|
||||
}
|
||||
if let Some(expires) = oi.expires {
|
||||
meta.insert(
|
||||
"expires".to_string(),
|
||||
expires
|
||||
.format(&time::format_description::well_known::Rfc3339)
|
||||
.map_err(std::io::Error::other)?,
|
||||
);
|
||||
}
|
||||
let restore_expiry = lifecycle::expected_expiry_time(OffsetDateTime::now_utc(), rreq.days.unwrap_or(1));
|
||||
meta.insert(
|
||||
metadata_keys::RESTORE.to_string(),
|
||||
@@ -5506,7 +5514,6 @@ pub async fn put_restore_opts(
|
||||
// computed MD5 is not the object's public plaintext ETag.
|
||||
preserve_etag: oi.etag.clone(),
|
||||
shard_integrity_write_mode: Some(oi.shard_integrity_write_mode()),
|
||||
//expires: oi.expires,
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
@@ -7144,6 +7151,25 @@ mod tests {
|
||||
0,
|
||||
"tier generation lease should release after EOF"
|
||||
);
|
||||
let unread = get_transitioned_object_reader_with_tier_manager(
|
||||
&object_info.bucket,
|
||||
&object_info.name,
|
||||
&None,
|
||||
&HeaderMap::new(),
|
||||
&object_info,
|
||||
&ObjectOptions::default(),
|
||||
&manager,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("open an unread transitioned response");
|
||||
assert_eq!(TierConfigMgr::active_operation_lease_count(&manager, &tier).await, 1);
|
||||
drop(unread);
|
||||
assert_eq!(
|
||||
TierConfigMgr::active_operation_lease_count(&manager, &tier).await,
|
||||
0,
|
||||
"canceling an unread response must release the tier lease"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
|
||||
@@ -256,6 +256,38 @@ pub const BUCKET_POLICY_CONFIG: &str = "policy.json";
|
||||
pub const BUCKET_NOTIFICATION_CONFIG: &str = "notification.xml";
|
||||
pub const BUCKET_LIFECYCLE_CONFIG: &str = "lifecycle.xml";
|
||||
pub const BUCKET_SSECONFIG: &str = "bucket-encryption.xml";
|
||||
/// Shared tag-entry validation for object requests and persisted bucket tags.
|
||||
pub fn validate_tag_set_entries(tag_set: &[s3s::dto::Tag]) -> std::result::Result<(), &'static str> {
|
||||
fn is_valid_tag_text(value: &str) -> bool {
|
||||
static VALID_TAG_TEXT: std::sync::OnceLock<std::result::Result<regex::Regex, regex::Error>> = std::sync::OnceLock::new();
|
||||
VALID_TAG_TEXT
|
||||
.get_or_init(|| regex::Regex::new(r"^[\p{L}\p{N}\p{Z}+\-=._:/@]*$"))
|
||||
.as_ref()
|
||||
.is_ok_and(|pattern| pattern.is_match(value))
|
||||
}
|
||||
let mut keys = std::collections::HashSet::with_capacity(tag_set.len());
|
||||
for tag in tag_set {
|
||||
let key = tag
|
||||
.key
|
||||
.as_deref()
|
||||
.filter(|key| !key.is_empty())
|
||||
.ok_or("Tag key cannot be empty")?;
|
||||
if key.encode_utf16().count() > 128 || !is_valid_tag_text(key) {
|
||||
return Err("The TagKey you have provided is invalid");
|
||||
}
|
||||
let value = tag.value.as_deref().ok_or("Tag value cannot be null")?;
|
||||
if value.encode_utf16().count() > 256 || !is_valid_tag_text(value) {
|
||||
return Err("The TagValue you have provided is invalid");
|
||||
}
|
||||
if !keys.insert(key) {
|
||||
return Err("Cannot provide multiple Tags with the same key");
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub const MAX_BUCKET_TAGS: usize = 50;
|
||||
|
||||
pub const BUCKET_TAGGING_CONFIG: &str = "tagging.xml";
|
||||
pub const BUCKET_QUOTA_CONFIG_FILE: &str = "quota.json";
|
||||
pub const OBJECT_LOCK_CONFIG: &str = "object-lock.xml";
|
||||
@@ -990,6 +1022,20 @@ impl BucketMetadata {
|
||||
self.encryption_config_updated_at = updated;
|
||||
}
|
||||
BUCKET_TAGGING_CONFIG => {
|
||||
if !data.is_empty() {
|
||||
let tagging = deserialize::<Tagging>(&data).map_err(|_| {
|
||||
Error::InvalidArgument(self.name.clone(), String::new(), "Invalid bucket tagging XML".to_owned())
|
||||
})?;
|
||||
if tagging.tag_set.len() > MAX_BUCKET_TAGS {
|
||||
return Err(Error::InvalidArgument(
|
||||
self.name.clone(),
|
||||
String::new(),
|
||||
"Cannot have more than 50 tags per bucket".to_owned(),
|
||||
));
|
||||
}
|
||||
validate_tag_set_entries(&tagging.tag_set)
|
||||
.map_err(|message| Error::InvalidArgument(self.name.clone(), String::new(), message.to_owned()))?;
|
||||
}
|
||||
self.tagging_config_xml = data;
|
||||
// Drop the parsed form (like lifecycle above) so clearing the
|
||||
// payload can't leave stale parsed tags to be cached.
|
||||
@@ -1784,6 +1830,31 @@ mod test {
|
||||
/// backlog#2292: a replicated config is stamped with the source
|
||||
/// `updated_at` it was given, not the local clock, while the plain
|
||||
/// `update_config` entry keeps stamping the local clock.
|
||||
#[test]
|
||||
fn bucket_tag_updates_reject_unreadable_shapes_without_replacing_metadata() {
|
||||
let mut metadata = BucketMetadata::new("validated-tags");
|
||||
let valid = br#"<Tagging><TagSet><Tag><Key>team</Key><Value></Value></Tag></TagSet></Tagging>"#.to_vec();
|
||||
metadata
|
||||
.update_config(BUCKET_TAGGING_CONFIG, valid.clone())
|
||||
.expect("empty value is valid");
|
||||
let updated = metadata.tagging_config_updated_at;
|
||||
for invalid in [
|
||||
"<Tagging><TagSet><Tag><Key>team</Key></Tag></TagSet></Tagging>".to_owned(),
|
||||
"<Tagging><TagSet><Tag><Key></Key><Value>x</Value></Tag></TagSet></Tagging>".to_owned(),
|
||||
"<Tagging><TagSet><Tag><Key>team</Key><Value>x</Value></Tag><Tag><Key>team</Key><Value>y</Value></Tag></TagSet></Tagging>".to_owned(),
|
||||
format!("<Tagging><TagSet>{}</TagSet></Tagging>", (0..51).map(|i| format!("<Tag><Key>k{i}</Key><Value>v</Value></Tag>")).collect::<String>()),
|
||||
] {
|
||||
assert!(metadata.update_config_at(BUCKET_TAGGING_CONFIG, invalid.into_bytes(), updated + time::Duration::hours(1)).is_err());
|
||||
assert_eq!(metadata.tagging_config_xml, valid);
|
||||
assert_eq!(metadata.tagging_config_updated_at, updated);
|
||||
}
|
||||
metadata.tagging_config_xml = b"legacy corrupt tags".to_vec();
|
||||
metadata
|
||||
.update_config(BUCKET_TAGGING_CONFIG, Vec::new())
|
||||
.expect("deletion repairs legacy corruption");
|
||||
assert!(metadata.tagging_config_xml.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn update_config_at_stamps_the_given_time_and_update_config_stamps_now() {
|
||||
let source_time = OffsetDateTime::now_utc() - time::Duration::hours(3);
|
||||
|
||||
@@ -683,6 +683,7 @@ pub(crate) async fn update_validated_in<F>(
|
||||
bucket: &str,
|
||||
config_file: &str,
|
||||
data: Vec<u8>,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
validate: F,
|
||||
) -> Result<OffsetDateTime>
|
||||
where
|
||||
@@ -690,7 +691,7 @@ where
|
||||
{
|
||||
Box::pin(async {
|
||||
let sys = require_bucket_metadata_sys_in(ctx)?;
|
||||
let guard = acquire_config_write_guard(sys.clone(), bucket).await?;
|
||||
let guard = acquire_config_write_guard_for_incarnation(sys.clone(), bucket, expected_incarnation_id).await?;
|
||||
validate()?;
|
||||
update_under_config_write_guard(sys, &guard, config_file, data, None).await
|
||||
})
|
||||
@@ -1901,6 +1902,15 @@ impl BucketMetadataSys {
|
||||
self.api.upgrade()
|
||||
}
|
||||
|
||||
/// Whether the backing `ECStore` has been dropped.
|
||||
///
|
||||
/// Used by `InstanceContext::clear_bucket_metadata_sys_if_stale` during
|
||||
/// `ECStore::drop` so a shared bootstrap context does not retain a
|
||||
/// `BucketMetadataSys` whose weak handle is dangling.
|
||||
pub(crate) fn is_store_stale(&self) -> bool {
|
||||
self.api.upgrade().is_none()
|
||||
}
|
||||
|
||||
fn metadata_publish_lock(&self, bucket: &str) -> Arc<Mutex<MetadataPublishLockState>> {
|
||||
Self::bucket_lock_in(&self.metadata_publish_locks, bucket)
|
||||
}
|
||||
@@ -3418,7 +3428,7 @@ mod tests {
|
||||
),
|
||||
(
|
||||
BUCKET_TAGGING_CONFIG,
|
||||
r#"<Tagging><TagSet><Tag><Key>environment</Key><Value>测试-🦀</Value></Tag></TagSet></Tagging>"#.as_bytes(),
|
||||
r#"<Tagging><TagSet><Tag><Key>environment</Key><Value>测试-rust</Value></Tag></TagSet></Tagging>"#.as_bytes(),
|
||||
),
|
||||
(BUCKET_REPLICATION_CONFIG, NEW_WRITER_REPLICATION_XML),
|
||||
(
|
||||
@@ -4546,8 +4556,13 @@ mod tests {
|
||||
std::fs::create_dir_all(dir.path().join(bucket)).expect("bucket volume should be created");
|
||||
}
|
||||
let mut bm = BucketMetadata::new(bucket);
|
||||
bm.update_config(config_file, raw.to_vec())
|
||||
.expect("raw config should be stored");
|
||||
if config_file == crate::bucket::metadata::BUCKET_TAGGING_CONFIG {
|
||||
// Model legacy persisted corruption without passing the write-time validator.
|
||||
bm.tagging_config_xml = raw.to_vec();
|
||||
} else {
|
||||
bm.update_config(config_file, raw.to_vec())
|
||||
.expect("raw config should be stored");
|
||||
}
|
||||
sys.persist_new_and_set(bm).await.expect("initial metadata should persist");
|
||||
}
|
||||
|
||||
@@ -5200,6 +5215,61 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn validated_config_update_rejects_a_different_bucket_incarnation() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = "catalog-cleanup-incarnation";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let incarnation = store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("read bucket incarnation");
|
||||
let called = std::sync::atomic::AtomicBool::new(false);
|
||||
let result = store
|
||||
.update_bucket_metadata_config_for_incarnation_validated(
|
||||
bucket,
|
||||
"quota.json",
|
||||
br#"{"quota":1}"#.to_vec(),
|
||||
Uuid::new_v4(),
|
||||
|| {
|
||||
called.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(result, Err(Error::BucketNotFound(name)) if name == bucket));
|
||||
assert!(
|
||||
!called.load(std::sync::atomic::Ordering::SeqCst),
|
||||
"incarnation must be checked before the external fence"
|
||||
);
|
||||
let (metadata, _) = get_config_from_disk_with_presence_in(&store.ctx, bucket)
|
||||
.await
|
||||
.expect("read unchanged bucket metadata");
|
||||
assert!(
|
||||
metadata.quota_config_json.is_empty(),
|
||||
"a stale cleanup must not modify the replacement bucket"
|
||||
);
|
||||
store
|
||||
.update_bucket_metadata_config_for_incarnation_validated(
|
||||
bucket,
|
||||
"quota.json",
|
||||
br#"{"quota":1}"#.to_vec(),
|
||||
incarnation,
|
||||
|| Ok(()),
|
||||
)
|
||||
.await
|
||||
.expect("the current incarnation permits the validated update");
|
||||
let (metadata, _) = get_config_from_disk_with_presence_in(&store.ctx, bucket)
|
||||
.await
|
||||
.expect("read updated bucket metadata");
|
||||
assert_eq!(metadata.quota_config_json, br#"{"quota":1}"#);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_admission_rejects_stale_cached_no_quota() {
|
||||
@@ -5673,7 +5743,8 @@ mod tests {
|
||||
// update only needs one interleaving to show up.
|
||||
const ROUNDS: usize = 8;
|
||||
for round in 0..ROUNDS {
|
||||
let tagging = format!("<Tagging><Round>{round}</Round></Tagging>").into_bytes();
|
||||
let tagging =
|
||||
format!("<Tagging><TagSet><Tag><Key>round</Key><Value>{round}</Value></Tag></TagSet></Tagging>").into_bytes();
|
||||
let policy = format!(r#"{{"Version":"2012-10-17","Round":{round}}}"#).into_bytes();
|
||||
|
||||
let start = Arc::new(tokio::sync::Barrier::new(2));
|
||||
@@ -5746,7 +5817,7 @@ mod tests {
|
||||
|
||||
let blocked = tokio::spawn({
|
||||
let node_b = node_b.clone();
|
||||
async move { update_with_sys(node_b, bucket, BUCKET_TAGGING_CONFIG, b"<Tagging/>".to_vec()).await }
|
||||
async move { update_with_sys(node_b, bucket, BUCKET_TAGGING_CONFIG, b"<Tagging><TagSet/></Tagging>".to_vec()).await }
|
||||
});
|
||||
|
||||
// Long enough for the write to have finished had it not waited: the
|
||||
|
||||
@@ -119,6 +119,21 @@ async fn durable_quota_reporting_retires_degraded_floors_without_changing_ledger
|
||||
// Exercise the real storage entry point before testing the new
|
||||
// reporting semantics: the same ledger must admit and refuse
|
||||
// these writes even when a scanner floor claims a million bytes.
|
||||
if protocol == 1 {
|
||||
let mut bootstrap = PutObjReader::from_vec(vec![0x5a; 512]);
|
||||
assert!(
|
||||
matches!(
|
||||
store
|
||||
.put_object(&bucket, "bootstrap", &mut bootstrap, &ObjectOptions::default())
|
||||
.await,
|
||||
Err(StorageError::SlowDown)
|
||||
),
|
||||
"a missing authoritative ledger must not admit a write from scanner reporting"
|
||||
);
|
||||
crate::bucket::quota::reservation::retry_quota_reconciliation_for_test(store.clone(), &bucket)
|
||||
.await
|
||||
.expect("bootstrap authoritative usage through the production worker");
|
||||
}
|
||||
let mut too_large = PutObjReader::from_vec(vec![0x5a; 8192]);
|
||||
assert!(
|
||||
matches!(
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -40,7 +40,9 @@ use aws_smithy_runtime_api::client::http::{
|
||||
HttpConnector as SmithyHttpConnector, HttpConnectorFuture, SharedHttpConnector, http_client_fn,
|
||||
};
|
||||
use aws_smithy_runtime_api::client::interceptors::Intercept;
|
||||
use aws_smithy_runtime_api::client::interceptors::context::BeforeTransmitInterceptorContextMut;
|
||||
use aws_smithy_runtime_api::client::interceptors::context::{
|
||||
BeforeDeserializationInterceptorContextMut, BeforeTransmitInterceptorContextMut,
|
||||
};
|
||||
use aws_smithy_runtime_api::client::orchestrator::{HttpRequest, HttpResponse};
|
||||
use aws_smithy_runtime_api::client::result::ConnectorError;
|
||||
use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents;
|
||||
@@ -155,6 +157,8 @@ pub struct RemoteS3EndpointSpec {
|
||||
pub ca_cert_pem: Option<String>,
|
||||
pub connect_timeout: Option<Duration>,
|
||||
pub read_timeout: Option<Duration>,
|
||||
/// Entire SDK call, including non-streaming response bodies and retries.
|
||||
pub operation_timeout: Option<Duration>,
|
||||
/// How many wire requests one logical call may cost. Every consumer
|
||||
/// declares it; see [`RemoteS3RetryPolicy`].
|
||||
pub retry: RemoteS3RetryPolicy,
|
||||
@@ -252,6 +256,78 @@ pub(crate) fn remote_sdk_credentials(credentials: &RemoteCredentials, now: Syste
|
||||
Ok(builder.build())
|
||||
}
|
||||
|
||||
/// Only installed on non-streaming upload operations. The budget begins at
|
||||
/// response headers, independently of the potentially long request upload.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct UploadResponseDeadline(pub Duration);
|
||||
|
||||
impl Intercept for UploadResponseDeadline {
|
||||
fn name(&self) -> &'static str {
|
||||
"RustfsUploadResponseDeadline"
|
||||
}
|
||||
|
||||
fn modify_before_deserialization(
|
||||
&self,
|
||||
context: &mut BeforeDeserializationInterceptorContextMut<'_>,
|
||||
_runtime_components: &RuntimeComponents,
|
||||
_cfg: &mut ConfigBag,
|
||||
) -> Result<(), BoxError> {
|
||||
let response = context.response_mut();
|
||||
let inner = std::mem::replace(response.body_mut(), SdkBody::taken());
|
||||
*response.body_mut() = SdkBody::from_body_1_x(UploadResponseBody {
|
||||
inner: Some(inner),
|
||||
deadline: Box::pin(tokio::time::sleep(self.0)),
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
struct UploadResponseBody {
|
||||
inner: Option<SdkBody>,
|
||||
deadline: std::pin::Pin<Box<tokio::time::Sleep>>,
|
||||
}
|
||||
|
||||
impl http_body::Body for UploadResponseBody {
|
||||
type Data = bytes::Bytes;
|
||||
type Error = BoxError;
|
||||
|
||||
fn poll_frame(
|
||||
mut self: std::pin::Pin<&mut Self>,
|
||||
cx: &mut std::task::Context<'_>,
|
||||
) -> std::task::Poll<Option<Result<http_body::Frame<Self::Data>, Self::Error>>> {
|
||||
use std::future::Future as _;
|
||||
use std::task::Poll;
|
||||
if self.inner.is_none() {
|
||||
return Poll::Ready(None);
|
||||
}
|
||||
// The absolute response budget never renews on data, empty frames,
|
||||
// or trailers. A retry gets its own bounded response window.
|
||||
if self.deadline.as_mut().poll(cx).is_ready() {
|
||||
self.inner.take();
|
||||
return Poll::Ready(Some(Err(std::io::Error::new(
|
||||
std::io::ErrorKind::TimedOut,
|
||||
"replication response body deadline exceeded",
|
||||
)
|
||||
.into())));
|
||||
}
|
||||
let Some(inner) = self.inner.as_mut() else {
|
||||
return Poll::Ready(None);
|
||||
};
|
||||
let result = std::pin::Pin::new(inner).poll_frame(cx);
|
||||
if matches!(result, Poll::Ready(None) | Poll::Ready(Some(Err(_)))) {
|
||||
self.inner.take();
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn is_end_stream(&self) -> bool {
|
||||
self.inner.as_ref().is_none_or(http_body::Body::is_end_stream)
|
||||
}
|
||||
fn size_hint(&self) -> http_body::SizeHint {
|
||||
self.inner.as_ref().map(http_body::Body::size_hint).unwrap_or_default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Appends a caller-identifying token to the SDK `User-Agent`. Runs after
|
||||
/// signing: SigV4 excludes `user-agent` from the canonical request, so the
|
||||
/// signature stays valid.
|
||||
@@ -309,7 +385,7 @@ pub async fn build_remote_s3_config(spec: &RemoteS3EndpointSpec) -> Result<aws_s
|
||||
config_builder = config_builder.http_client(http_client);
|
||||
}
|
||||
|
||||
if spec.connect_timeout.is_some() || spec.read_timeout.is_some() {
|
||||
if spec.connect_timeout.is_some() || spec.read_timeout.is_some() || spec.operation_timeout.is_some() {
|
||||
let mut timeouts = TimeoutConfig::builder();
|
||||
if let Some(connect_timeout) = spec.connect_timeout {
|
||||
timeouts = timeouts.connect_timeout(connect_timeout);
|
||||
@@ -317,6 +393,9 @@ pub async fn build_remote_s3_config(spec: &RemoteS3EndpointSpec) -> Result<aws_s
|
||||
if let Some(read_timeout) = spec.read_timeout {
|
||||
timeouts = timeouts.read_timeout(read_timeout);
|
||||
}
|
||||
if let Some(operation_timeout) = spec.operation_timeout {
|
||||
timeouts = timeouts.operation_timeout(operation_timeout);
|
||||
}
|
||||
config_builder = config_builder.timeout_config(timeouts.build());
|
||||
}
|
||||
|
||||
@@ -674,6 +753,7 @@ mod tests {
|
||||
ca_cert_pem: None,
|
||||
connect_timeout: None,
|
||||
read_timeout: None,
|
||||
operation_timeout: None,
|
||||
retry: RemoteS3RetryPolicy::Disabled,
|
||||
user_agent_suffix: "",
|
||||
}
|
||||
@@ -1027,11 +1107,16 @@ mod tests {
|
||||
spec.user_agent_suffix = "RustFS-Test/0.0";
|
||||
spec.connect_timeout = Some(Duration::from_secs(5));
|
||||
spec.read_timeout = Some(Duration::from_secs(5));
|
||||
spec.operation_timeout = Some(Duration::from_secs(12));
|
||||
let config = build_remote_s3_config(&spec)
|
||||
.await
|
||||
.expect("spec should build")
|
||||
.http_client(http_client)
|
||||
.build();
|
||||
assert_eq!(
|
||||
config.timeout_config().and_then(TimeoutConfig::operation_timeout),
|
||||
Some(Duration::from_secs(12))
|
||||
);
|
||||
S3Client::from_conf(config)
|
||||
.head_bucket()
|
||||
.bucket("bucket")
|
||||
|
||||
@@ -33,6 +33,7 @@ pub(crate) struct ReplicationMetadataStore;
|
||||
impl ReplicationMetadataStore {
|
||||
pub(crate) const MRF_REPLICATION_FILE: &'static str = "config/replication/mrf.bin";
|
||||
pub(crate) const MRF_REPLICATION_RECOVERY_LOCK: &'static str = "config/replication/mrf.bin.recovery";
|
||||
pub(crate) const MRF_REPLICATION_REPLAY_STAMP: &'static str = "config/replication/mrf.bin.replay";
|
||||
pub(crate) const FORCE_DELETE_REPLICATION_FILE: &'static str = "config/replication/force-delete.bin";
|
||||
pub(crate) const FORCE_DELETE_REPLICATION_TRANSACTION_LOCK: &'static str = "config/replication/force-delete.bin.transaction";
|
||||
|
||||
@@ -116,6 +117,10 @@ mod tests {
|
||||
ReplicationMetadataStore::MRF_REPLICATION_RECOVERY_LOCK,
|
||||
"config/replication/mrf.bin.recovery"
|
||||
);
|
||||
assert_eq!(
|
||||
ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
|
||||
"config/replication/mrf.bin.replay"
|
||||
);
|
||||
assert_eq!(
|
||||
ReplicationMetadataStore::FORCE_DELETE_REPLICATION_FILE,
|
||||
"config/replication/force-delete.bin"
|
||||
|
||||
@@ -16,8 +16,8 @@ use super::replication_config_store::ReplicationConfigStore;
|
||||
use super::replication_error_boundary::{Error as EcstoreError, is_err_object_not_found, is_err_version_not_found};
|
||||
use super::replication_filemeta_boundary::{
|
||||
MrfOpKind, MrfReplicateEntry, REPLICATE_HEAL_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicatedTargetInfo,
|
||||
ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision, replicate_decision_for_admitted_targets,
|
||||
replication_statuses_map, version_purge_statuses_map,
|
||||
ReplicationGenerationSnapshot, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision,
|
||||
replicate_decision_for_admitted_targets, replication_statuses_map, version_purge_statuses_map,
|
||||
};
|
||||
use super::replication_lock_boundary::ReplicationLockTiming;
|
||||
use super::replication_logging::{EVENT_REPLICATION_CONFIG_LOOKUP_SKIPPED, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REPLICATION};
|
||||
@@ -86,6 +86,10 @@ const MRF_CORRUPT_FILE_PREFIX: &str = "config/replication/mrf.corrupt";
|
||||
const MRF_PENDING_CAP: usize = 200_000;
|
||||
const MRF_RETRY_INITIAL_DELAY: Duration = Duration::from_millis(100);
|
||||
const MRF_RETRY_MAX_DELAY: Duration = Duration::from_secs(5);
|
||||
/// Interval between runtime replays of the MRF journal, cluster-wide. A pass
|
||||
/// re-attempts every retained entry, so a target that stays down costs one
|
||||
/// attempt per entry per interval.
|
||||
const MRF_REPLAY_INTERVAL: Duration = Duration::from_secs(300);
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct DurableMrfBacklog {
|
||||
@@ -615,7 +619,7 @@ async fn read_mrf_entries<S: ReplicationObjectIO>(storage: Arc<S>) -> Result<Vec
|
||||
}
|
||||
|
||||
/// Acknowledge only the generation read by the recovery leader. Entries appended
|
||||
/// after that generation are retained as a suffix and replayed on the next startup.
|
||||
/// after that generation are retained as a suffix and replayed on the next pass.
|
||||
/// Lock order: recovery leader lock -> MRF journal object lock.
|
||||
async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
|
||||
storage: Arc<S>,
|
||||
@@ -667,7 +671,7 @@ async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
|
||||
Err(EcstoreError::PreconditionFailed)
|
||||
}
|
||||
|
||||
/// Acquires the MRF recovery leader lock for the startup replay.
|
||||
/// Acquires the MRF recovery leader lock for one journal replay pass.
|
||||
/// Returns `None` (after logging) when the lock cannot be created or another
|
||||
/// node is already processing the backlog.
|
||||
async fn acquire_mrf_recovery_guard<S: ReplicationStorage>(storage: &Arc<S>) -> Option<rustfs_lock::NamespaceLockGuard> {
|
||||
@@ -1018,7 +1022,7 @@ fn admitted_mrf_replicate_object(oi: ObjectInfo, entry: &MrfReplicateEntry, op_t
|
||||
}
|
||||
|
||||
/// Acknowledges the replayed MRF prefix and returns the retained backlog.
|
||||
/// On acknowledgement failure the backlog is preserved for the next startup and
|
||||
/// On acknowledgement failure the backlog is preserved for the next replay pass and
|
||||
/// re-read (falling back to the replayed snapshot) so the published summary stays accurate.
|
||||
async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
|
||||
storage: &Arc<S>,
|
||||
@@ -1033,7 +1037,7 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %error,
|
||||
"Failed to acknowledge the MRF recovery prefix; preserving it for the next startup"
|
||||
"Failed to acknowledge the MRF recovery prefix; preserving it for the next replay pass"
|
||||
);
|
||||
match read_mrf_entries(storage.clone()).await {
|
||||
Ok(current) => current,
|
||||
@@ -1051,6 +1055,120 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
|
||||
}
|
||||
}
|
||||
|
||||
fn unix_millis_now() -> i64 {
|
||||
i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000_000).unwrap_or(i64::MAX)
|
||||
}
|
||||
|
||||
/// Whether a periodic pass may run, given the start time of the last pass on
|
||||
/// any node. A stamp more than one interval ahead of this node's clock is skew
|
||||
/// or a bad value, so it does not suppress replay.
|
||||
fn mrf_replay_due(last_started_ms: Option<i64>, now_ms: i64, interval: Duration) -> bool {
|
||||
let Some(last_started_ms) = last_started_ms else {
|
||||
return true;
|
||||
};
|
||||
let interval_ms = i64::try_from(interval.as_millis()).unwrap_or(i64::MAX);
|
||||
now_ms.saturating_sub(last_started_ms) >= interval_ms || last_started_ms.saturating_sub(now_ms) >= interval_ms
|
||||
}
|
||||
|
||||
async fn read_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>) -> Option<i64> {
|
||||
let data = ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP)
|
||||
.await
|
||||
.ok()?;
|
||||
std::str::from_utf8(&data).ok()?.trim().parse().ok()
|
||||
}
|
||||
|
||||
async fn write_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>, started_ms: i64) {
|
||||
if let Err(error) = ReplicationConfigStore::save(
|
||||
storage.clone(),
|
||||
ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
|
||||
started_ms.to_string().into_bytes(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %error,
|
||||
"Failed to record the MRF replay start time"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// One MRF journal replay pass. The recovery leader reads the journal,
|
||||
/// re-drives every entry, and acknowledges the replayed prefix; entries that
|
||||
/// could not be admitted, and entries appended meanwhile, stay for the next pass.
|
||||
///
|
||||
/// Every node runs the periodic loop, and the leader lock only serializes
|
||||
/// passes. With `min_interval`, a pass is skipped when a pass on any node
|
||||
/// started within that interval, read from the shared replay stamp, so a
|
||||
/// cluster replays once per interval rather than once per node. The startup
|
||||
/// pass passes `None` and always runs. A pass that finds entries records its
|
||||
/// start time. Returns whether the journal was read.
|
||||
/// Lock order: recovery leader lock -> replay stamp object lock.
|
||||
async fn replay_mrf_journal<S: ReplicationStorage>(storage: &Arc<S>, min_interval: Option<Duration>) -> bool {
|
||||
let Some(recovery_guard) = acquire_mrf_recovery_guard(storage).await else {
|
||||
return false;
|
||||
};
|
||||
let started_ms = unix_millis_now();
|
||||
if let Some(interval) = min_interval
|
||||
&& !mrf_replay_due(read_mrf_replay_stamp(storage).await, started_ms, interval)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
let Some(entries) = load_mrf_recovery_entries(storage).await else {
|
||||
return true;
|
||||
};
|
||||
if !entries.is_empty() {
|
||||
write_mrf_replay_stamp(storage, started_ms).await;
|
||||
}
|
||||
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
|
||||
|
||||
let total = entries.len();
|
||||
let mut queued_count = 0usize;
|
||||
let mut retry_entries = Vec::new();
|
||||
|
||||
for entry in entries.iter() {
|
||||
let Some(admission) = replay_mrf_entry(entry, storage, &mut retry_entries).await else {
|
||||
continue;
|
||||
};
|
||||
|
||||
if admission == ReplicationQueueAdmission::Missed {
|
||||
retry_entries.push(entry.clone());
|
||||
} else if admission == ReplicationQueueAdmission::Queued {
|
||||
queued_count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
let retained = resolve_retained_mrf_entries(storage, &recovery_guard, &entries, &retry_entries).await;
|
||||
let retained_count = retained.len();
|
||||
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
|
||||
|
||||
if queued_count > 0 {
|
||||
info!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
recovered = queued_count,
|
||||
total,
|
||||
retained = retained_count,
|
||||
"Replayed MRF entries admitted for retry"
|
||||
);
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// Re-runs [`replay_mrf_journal`] every `period`. The recovery leader lock keeps
|
||||
/// one pass at a time across the cluster, including the startup pass, and the
|
||||
/// shared replay stamp spaces passes from different nodes by `period`.
|
||||
fn spawn_mrf_replay_loop<S: ReplicationStorage>(storage: Arc<S>, period: Duration) -> JoinHandle<()> {
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
tokio::time::sleep(period).await;
|
||||
replay_mrf_journal(&storage, Some(period)).await;
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("replication resync {active_resync_id} is already active for {bucket}/{arn}")]
|
||||
struct ResyncActiveConflictError {
|
||||
@@ -1171,6 +1289,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
// Start background tasks
|
||||
pool.start_mrf_persister().await;
|
||||
pool.start_mrf_processor().await;
|
||||
pool.start_mrf_replayer().await;
|
||||
pool.start_force_delete_processor().await;
|
||||
|
||||
pool
|
||||
@@ -1462,7 +1581,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
|
||||
/// Queues a replica task
|
||||
pub async fn queue_replica_task(&self, ri: ReplicateObjectInfo) -> ReplicationQueueAdmission {
|
||||
// A version that is already queued or being uploaded is not driven a
|
||||
// A source generation already queued or being uploaded is not driven a
|
||||
// second time: the scanner heal pass sees it as PENDING until the
|
||||
// first upload lands and would otherwise re-queue it every cycle
|
||||
// (backlog#2362). The key is released when the worker finishes, or
|
||||
@@ -1639,55 +1758,26 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
|
||||
/// Starts the MRF processor — one-shot at startup.
|
||||
///
|
||||
/// Reads the on-disk MRF file, re-injects admitted entries as Heal operations, and
|
||||
/// rewrites any entries that could not be admitted for a later startup retry.
|
||||
/// Replays the on-disk MRF journal once; [`Self::start_mrf_replayer`] repeats
|
||||
/// the pass while the server runs.
|
||||
async fn start_mrf_processor(&self) {
|
||||
let storage = self.storage.clone();
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
let Some(recovery_guard) = acquire_mrf_recovery_guard(&storage).await else {
|
||||
return;
|
||||
};
|
||||
|
||||
let Some(entries) = load_mrf_recovery_entries(&storage).await else {
|
||||
return;
|
||||
};
|
||||
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
|
||||
|
||||
let total = entries.len();
|
||||
let mut queued_count = 0usize;
|
||||
let mut retry_entries = Vec::new();
|
||||
|
||||
for entry in entries.iter() {
|
||||
let Some(admission) = replay_mrf_entry(entry, &storage, &mut retry_entries).await else {
|
||||
continue;
|
||||
};
|
||||
|
||||
if admission == ReplicationQueueAdmission::Missed {
|
||||
retry_entries.push(entry.clone());
|
||||
} else if admission == ReplicationQueueAdmission::Queued {
|
||||
queued_count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
let retained = resolve_retained_mrf_entries(&storage, &recovery_guard, &entries, &retry_entries).await;
|
||||
let retained_count = retained.len();
|
||||
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
|
||||
|
||||
if queued_count > 0 {
|
||||
info!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
recovered = queued_count,
|
||||
total,
|
||||
retained = retained_count,
|
||||
"Replayed MRF entries admitted for retry"
|
||||
);
|
||||
}
|
||||
replay_mrf_journal(&storage, None).await;
|
||||
});
|
||||
self.task_handles.lock().await.push(handle);
|
||||
}
|
||||
|
||||
/// Starts the runtime MRF replayer — ongoing background task.
|
||||
///
|
||||
/// Entries reach the journal while the server runs (worker-queue overflow,
|
||||
/// delete-marker purge intents whose watch window drained with targets still
|
||||
/// dirty). Replaying only at startup left them waiting for a restart.
|
||||
async fn start_mrf_replayer(&self) {
|
||||
let handle = spawn_mrf_replay_loop(self.storage.clone(), MRF_REPLAY_INTERVAL);
|
||||
self.task_handles.lock().await.push(handle);
|
||||
}
|
||||
|
||||
async fn start_force_delete_processor(&self) {
|
||||
let storage = self.storage.clone();
|
||||
let handle =
|
||||
@@ -1755,7 +1845,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
/// writes them to the on-disk MRF file every flush interval (default 10s,
|
||||
/// overridable via `RUSTFS_REPL_MRF_FLUSH_INTERVAL_MS`) or when 1 000 new
|
||||
/// entries accumulate. Runtime writers append under the journal lock; only the
|
||||
/// startup recovery leader may remove an acknowledged prefix.
|
||||
/// recovery leader of a replay pass may remove an acknowledged prefix.
|
||||
async fn start_mrf_persister(&self) {
|
||||
let Some(mut rx) = self.mrf_save_rx.lock().await.take() else {
|
||||
return;
|
||||
@@ -2314,28 +2404,47 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
}
|
||||
|
||||
/// Object versions currently queued or being uploaded, keyed by bucket,
|
||||
/// object name and version. `queue_replica_task` admits a version only once
|
||||
/// object name, version and source generation. `queue_replica_task` admits a generation only once
|
||||
/// while it is in flight; the scanner heal pass and MRF replays that arrive
|
||||
/// in the meantime are `Skipped` instead of driving a second complete upload
|
||||
/// (backlog#2362). Entries are removed when the worker finishes the task or
|
||||
/// when no worker accepted it.
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct ReplicationInFlight {
|
||||
keys: std::sync::Mutex<std::collections::HashSet<(String, String, Option<uuid::Uuid>)>>,
|
||||
keys: std::sync::Mutex<std::collections::HashSet<ReplicationInFlightKey>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
||||
struct ReplicationInFlightKey {
|
||||
bucket: String,
|
||||
object: String,
|
||||
version_id: Option<uuid::Uuid>,
|
||||
generation: ReplicationGenerationSnapshot,
|
||||
}
|
||||
|
||||
impl From<&ReplicateObjectInfo> for ReplicationInFlightKey {
|
||||
fn from(info: &ReplicateObjectInfo) -> Self {
|
||||
Self {
|
||||
bucket: info.bucket.clone(),
|
||||
object: info.name.clone(),
|
||||
version_id: info.version_id,
|
||||
generation: info.replication_generation.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ReplicationInFlight {
|
||||
fn lock(&self) -> std::sync::MutexGuard<'_, std::collections::HashSet<(String, String, Option<uuid::Uuid>)>> {
|
||||
fn lock(&self) -> std::sync::MutexGuard<'_, std::collections::HashSet<ReplicationInFlightKey>> {
|
||||
self.keys.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
}
|
||||
|
||||
/// Claim `ri`; `false` when the same version is already in flight.
|
||||
/// Claim `ri`; a metadata mutation is independent of its earlier generation.
|
||||
fn try_begin(&self, ri: &ReplicateObjectInfo) -> bool {
|
||||
self.lock().insert((ri.bucket.clone(), ri.name.clone(), ri.version_id))
|
||||
self.lock().insert(ReplicationInFlightKey::from(ri))
|
||||
}
|
||||
|
||||
fn finish(&self, ri: &ReplicateObjectInfo) {
|
||||
self.lock().remove(&(ri.bucket.clone(), ri.name.clone(), ri.version_id));
|
||||
self.lock().remove(&ReplicationInFlightKey::from(ri));
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -2348,26 +2457,21 @@ impl ReplicationInFlight {
|
||||
/// including when replication panics.
|
||||
struct ReplicationInFlightGuard {
|
||||
in_flight: Arc<ReplicationInFlight>,
|
||||
key: ReplicateObjectInfo,
|
||||
key: ReplicationInFlightKey,
|
||||
}
|
||||
|
||||
impl ReplicationInFlightGuard {
|
||||
fn new(in_flight: Arc<ReplicationInFlight>, ri: &ReplicateObjectInfo) -> Self {
|
||||
Self {
|
||||
in_flight,
|
||||
key: ReplicateObjectInfo {
|
||||
bucket: ri.bucket.clone(),
|
||||
name: ri.name.clone(),
|
||||
version_id: ri.version_id,
|
||||
..Default::default()
|
||||
},
|
||||
key: ReplicationInFlightKey::from(ri),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ReplicationInFlightGuard {
|
||||
fn drop(&mut self) {
|
||||
self.in_flight.finish(&self.key);
|
||||
self.in_flight.lock().remove(&self.key);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3421,6 +3525,8 @@ mod tests {
|
||||
allow_first_read: Notify,
|
||||
read_count: AtomicUsize,
|
||||
delete_storage_calls: AtomicUsize,
|
||||
delete_error: StdMutex<EcstoreError>,
|
||||
replay_stamp: StdMutex<Option<Vec<u8>>>,
|
||||
reported_size: StdMutex<Option<i64>>,
|
||||
stream_read_bytes: Arc<AtomicUsize>,
|
||||
write_count: AtomicUsize,
|
||||
@@ -3488,6 +3594,24 @@ mod tests {
|
||||
_h: Self::HeaderMap,
|
||||
_opts: &Self::ObjectOptions,
|
||||
) -> Result<Self::GetObjectReader, Self::Error> {
|
||||
if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
|
||||
let stamp = self
|
||||
.shared
|
||||
.replay_stamp
|
||||
.lock()
|
||||
.expect("test replay stamp lock should not be poisoned")
|
||||
.clone()
|
||||
.ok_or(EcstoreError::FileNotFound)?;
|
||||
return Ok(Self::GetObjectReader {
|
||||
object_info: ObjectInfo {
|
||||
size: i64::try_from(stamp.len()).expect("test stamp length should fit i64"),
|
||||
..Default::default()
|
||||
},
|
||||
stream: Box::new(Cursor::new(stamp)),
|
||||
buffered_body: None,
|
||||
body_source: Default::default(),
|
||||
});
|
||||
}
|
||||
if !object.ends_with("/.replication/resync.bin")
|
||||
&& !object.ends_with("config/replication/mrf.bin")
|
||||
&& !object.ends_with("config/replication/force-delete.bin")
|
||||
@@ -3551,6 +3675,16 @@ mod tests {
|
||||
let lock = self.new_ns_lock(bucket, object).await?;
|
||||
Some(lock.get_write_lock(Duration::from_secs(10)).await?)
|
||||
};
|
||||
if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
|
||||
let mut stamp = Vec::new();
|
||||
data.stream.read_to_end(&mut stamp).await.map_err(EcstoreError::from)?;
|
||||
*self
|
||||
.shared
|
||||
.replay_stamp
|
||||
.lock()
|
||||
.expect("test replay stamp lock should not be poisoned") = Some(stamp);
|
||||
return Ok(ObjectInfo::default());
|
||||
}
|
||||
if opts.http_preconditions.is_some()
|
||||
&& let Some(replacement) = self
|
||||
.shared
|
||||
@@ -3672,7 +3806,12 @@ mod tests {
|
||||
_opts: Self::ObjectOptions,
|
||||
) -> Result<Self::ObjectInfo, Self::Error> {
|
||||
self.shared.delete_storage_calls.fetch_add(1, Ordering::SeqCst);
|
||||
Err(EcstoreError::NotImplemented)
|
||||
Err(self
|
||||
.shared
|
||||
.delete_error
|
||||
.lock()
|
||||
.expect("delete error lock should not be poisoned")
|
||||
.clone())
|
||||
}
|
||||
|
||||
async fn delete_objects(
|
||||
@@ -3929,6 +4068,48 @@ mod tests {
|
||||
assert_eq!(current_queue(&pool, "in-flight-bucket").await, (3, 3 * 4096));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn queue_replica_task_admits_metadata_changes_during_an_initial_upload() {
|
||||
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
|
||||
let (tx, mut rx) = mpsc::channel(4);
|
||||
pool.workers.write().await.push(tx);
|
||||
let initial = ReplicateObjectInfo {
|
||||
bucket: "pending-tags-bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
version_id: Some(uuid::Uuid::new_v4()),
|
||||
size: 4096,
|
||||
replication_generation: ReplicationGenerationSnapshot {
|
||||
mutation_id: Some(uuid::Uuid::new_v4().to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
let metadata = ReplicateObjectInfo {
|
||||
op_type: ReplicationType::Metadata,
|
||||
user_tags: String::new(),
|
||||
replication_generation: ReplicationGenerationSnapshot {
|
||||
mutation_id: Some(uuid::Uuid::new_v4().to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
..initial.clone()
|
||||
};
|
||||
assert_eq!(pool.queue_replica_task(initial.clone()).await, ReplicationQueueAdmission::Queued);
|
||||
assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Queued);
|
||||
assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Skipped);
|
||||
assert_eq!(pool.in_flight.len(), 2);
|
||||
drop(ReplicationInFlightGuard::new(pool.in_flight.clone(), &initial));
|
||||
assert_eq!(pool.in_flight.len(), 1);
|
||||
assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Skipped);
|
||||
for expected in [&initial, &metadata] {
|
||||
let ReplicationOperation::Object(queued) = rx.recv().await.expect("queued operation") else {
|
||||
panic!("expected object replication");
|
||||
};
|
||||
assert_eq!(queued.replication_generation, expected.replication_generation);
|
||||
}
|
||||
drop(ReplicationInFlightGuard::new(pool.in_flight.clone(), &metadata));
|
||||
assert_eq!(pool.in_flight.len(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn queue_replica_task_releases_the_version_when_no_worker_accepts_it() {
|
||||
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
|
||||
@@ -4171,6 +4352,8 @@ mod tests {
|
||||
allow_first_read: Notify::new(),
|
||||
read_count: AtomicUsize::new(0),
|
||||
delete_storage_calls: AtomicUsize::new(0),
|
||||
delete_error: StdMutex::new(EcstoreError::NotImplemented),
|
||||
replay_stamp: StdMutex::new(None),
|
||||
reported_size: StdMutex::new(None),
|
||||
stream_read_bytes: Arc::new(AtomicUsize::new(0)),
|
||||
write_count: AtomicUsize::new(0),
|
||||
@@ -4221,6 +4404,103 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_worker_already_purged_source_completes_without_retry() {
|
||||
let arn = "arn:test";
|
||||
for delete_marker_version in [false, true] {
|
||||
for version_id in [Uuid::new_v4(), Uuid::nil()] {
|
||||
for error in [
|
||||
EcstoreError::VersionNotFound("source".to_string(), "purged".to_string(), version_id.to_string()),
|
||||
EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
|
||||
EcstoreError::FileVersionNotFound,
|
||||
EcstoreError::FileNotFound,
|
||||
] {
|
||||
let shared = empty_resync_shared_state();
|
||||
*shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
|
||||
let storage = Arc::new(LoadResyncNodeStore::new("purged-source", shared.clone()));
|
||||
let delete = DeletedObjectReplicationInfo {
|
||||
bucket: "source".to_string(),
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: "purged".to_string(),
|
||||
version_id: (!delete_marker_version).then_some(version_id),
|
||||
delete_marker_version_id: delete_marker_version.then_some(version_id),
|
||||
replication_state: Some(ReplicationState {
|
||||
replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
|
||||
purge_targets: HashMap::from([(arn.to_string(), VersionPurgeStatusType::Complete)]),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
op_type: ReplicationType::Delete,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(
|
||||
replicate_delete_with_outcome(delete, storage).await,
|
||||
"a confirmed target purge must complete when the source is already absent (marker={delete_marker_version}, version={version_id})"
|
||||
);
|
||||
assert_eq!(
|
||||
shared.delete_storage_calls.load(Ordering::SeqCst),
|
||||
2 + usize::from(!delete_marker_version && !version_id.is_nil()),
|
||||
"the worker must lock and attempt writeback, reading a non-null data version first"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_worker_missing_source_does_not_hide_unfinished_replication() {
|
||||
let arn = "arn:test";
|
||||
for (version_purge, target_status, error) in [
|
||||
(
|
||||
true,
|
||||
VersionPurgeStatusType::Pending,
|
||||
EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
|
||||
),
|
||||
(true, VersionPurgeStatusType::Complete, EcstoreError::ErasureWriteQuorum),
|
||||
(
|
||||
false,
|
||||
VersionPurgeStatusType::Empty,
|
||||
EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
|
||||
),
|
||||
] {
|
||||
let shared = empty_resync_shared_state();
|
||||
*shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
|
||||
let storage = Arc::new(LoadResyncNodeStore::new("unfinished-delete", shared));
|
||||
let delete = DeletedObjectReplicationInfo {
|
||||
bucket: "source".to_string(),
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: "purged".to_string(),
|
||||
version_id: version_purge.then(Uuid::new_v4),
|
||||
delete_marker: !version_purge,
|
||||
replication_state: Some(ReplicationState {
|
||||
replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
|
||||
targets: if version_purge {
|
||||
HashMap::new()
|
||||
} else {
|
||||
HashMap::from([(arn.to_string(), ReplicationStatusType::Completed)])
|
||||
},
|
||||
purge_targets: if version_purge {
|
||||
HashMap::from([(arn.to_string(), target_status)])
|
||||
} else {
|
||||
HashMap::new()
|
||||
},
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
op_type: ReplicationType::Delete,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(
|
||||
!replicate_delete_with_outcome(delete, storage).await,
|
||||
"unfinished target work, source I/O failures, and marker creation must remain retryable"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_worker_preserves_persisted_admission_without_live_config() {
|
||||
let arn = "arn:test";
|
||||
@@ -5304,6 +5584,8 @@ mod tests {
|
||||
allow_first_read: Notify::new(),
|
||||
read_count: AtomicUsize::new(0),
|
||||
delete_storage_calls: AtomicUsize::new(0),
|
||||
delete_error: StdMutex::new(EcstoreError::NotImplemented),
|
||||
replay_stamp: StdMutex::new(None),
|
||||
reported_size: StdMutex::new(None),
|
||||
stream_read_bytes: Arc::new(AtomicUsize::new(0)),
|
||||
write_count: AtomicUsize::new(0),
|
||||
@@ -6268,6 +6550,108 @@ mod tests {
|
||||
assert_eq!(retained.target_arns, entry.target_arns);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mrf_replay_loop_drains_entries_persisted_after_startup() {
|
||||
let shared = empty_resync_shared_state();
|
||||
let storage = Arc::new(LoadResyncNodeStore::new("mrf-runtime-replay", shared.clone()));
|
||||
// A force-delete intent from a superseded generation replays as
|
||||
// Skipped, so a pass that reaches it acknowledges it off the journal.
|
||||
let stale_intent = |object: &str| MrfReplicateEntry {
|
||||
bucket: "mrf-runtime-replay".to_string(),
|
||||
object: object.to_string(),
|
||||
op: MrfOpKind::Delete,
|
||||
force_delete_id: Some(uuid::Uuid::new_v4()),
|
||||
force_delete_local_commit: false,
|
||||
target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
|
||||
..Default::default()
|
||||
};
|
||||
let journal_drained = || async {
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while !shared.data.lock().expect("test data lock should not be poisoned").is_empty() {
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.is_ok()
|
||||
};
|
||||
|
||||
let handle = spawn_mrf_replay_loop(storage, Duration::from_millis(20));
|
||||
for object in ["first", "second"] {
|
||||
*shared.data.lock().expect("test data lock should not be poisoned") =
|
||||
encode_mrf_file(&[stale_intent(object)]).expect("MRF entry should encode");
|
||||
assert!(
|
||||
journal_drained().await,
|
||||
"an MRF entry persisted while the server runs must be replayed without a restart ({object})"
|
||||
);
|
||||
}
|
||||
handle.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mrf_periodic_replay_runs_once_per_interval_across_nodes() {
|
||||
let shared = empty_resync_shared_state();
|
||||
let node_a = Arc::new(LoadResyncNodeStore::new("mrf-node-a", shared.clone()));
|
||||
let node_b = Arc::new(LoadResyncNodeStore::new("mrf-node-b", shared.clone()));
|
||||
let interval = Duration::from_secs(1);
|
||||
let persist_backlog = |object: &str| {
|
||||
let entry = MrfReplicateEntry {
|
||||
bucket: "mrf-node-replay".to_string(),
|
||||
object: object.to_string(),
|
||||
op: MrfOpKind::Delete,
|
||||
force_delete_id: Some(uuid::Uuid::new_v4()),
|
||||
force_delete_local_commit: false,
|
||||
target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
|
||||
..Default::default()
|
||||
};
|
||||
*shared.data.lock().expect("test data lock should not be poisoned") =
|
||||
encode_mrf_file(&[entry]).expect("MRF entry should encode");
|
||||
};
|
||||
let journal_is_empty = || shared.data.lock().expect("test data lock should not be poisoned").is_empty();
|
||||
|
||||
persist_backlog("first");
|
||||
assert!(replay_mrf_journal(&node_a, Some(interval)).await, "node A's periodic pass should replay");
|
||||
assert!(journal_is_empty(), "node A's pass should acknowledge the replayed entry");
|
||||
|
||||
// Node B's timer fires within the same interval with a backlog present.
|
||||
persist_backlog("second");
|
||||
assert!(
|
||||
!replay_mrf_journal(&node_b, Some(interval)).await,
|
||||
"a node must not replay within one interval of another node's pass"
|
||||
);
|
||||
assert!(!journal_is_empty(), "the backlog should wait for the next interval");
|
||||
|
||||
assert!(
|
||||
replay_mrf_journal(&node_b, None).await,
|
||||
"the startup pass should replay regardless of the shared stamp"
|
||||
);
|
||||
assert!(journal_is_empty(), "the startup pass should acknowledge the replayed entry");
|
||||
|
||||
persist_backlog("third");
|
||||
tokio::time::sleep(interval).await;
|
||||
assert!(
|
||||
replay_mrf_journal(&node_a, Some(interval)).await,
|
||||
"a periodic pass should replay once the interval has elapsed"
|
||||
);
|
||||
assert!(journal_is_empty(), "the due pass should acknowledge the replayed entry");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mrf_replay_due_tolerates_missing_and_skewed_stamps() {
|
||||
let interval = Duration::from_secs(300);
|
||||
let now = 1_000_000_000;
|
||||
assert!(mrf_replay_due(None, now, interval));
|
||||
assert!(!mrf_replay_due(Some(now - 299_999), now, interval));
|
||||
assert!(mrf_replay_due(Some(now - 300_000), now, interval));
|
||||
assert!(
|
||||
!mrf_replay_due(Some(now + 1_000), now, interval),
|
||||
"small clock skew should still space passes"
|
||||
);
|
||||
assert!(
|
||||
mrf_replay_due(Some(now + 300_000), now, interval),
|
||||
"a stamp an interval ahead must not suppress replay"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mrf_entry_delete_marker_roundtrip() {
|
||||
let dm_vid = Uuid::new_v4();
|
||||
|
||||
@@ -2809,17 +2809,22 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
true
|
||||
}
|
||||
Err(e) => {
|
||||
error!(
|
||||
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
arn = %dobj.target_arn,
|
||||
object = %dobj.delete_object.object_name,
|
||||
operation = "apply_replication_delete_state",
|
||||
error = %e,
|
||||
"Replication target operation failed"
|
||||
);
|
||||
// A batch purge or another worker may have removed this version
|
||||
// already. Remote failures still control the final outcome below.
|
||||
let source_already_purged = is_version_purge && (is_err_object_not_found(&e) || is_err_version_not_found(&e));
|
||||
if !source_already_purged {
|
||||
error!(
|
||||
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
arn = %dobj.target_arn,
|
||||
object = %dobj.delete_object.object_name,
|
||||
operation = "apply_replication_delete_state",
|
||||
error = %e,
|
||||
"Replication target operation failed"
|
||||
);
|
||||
}
|
||||
send_event(EventArgs {
|
||||
event_name,
|
||||
bucket_name: bucket.clone(),
|
||||
@@ -2832,7 +2837,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
},
|
||||
..Default::default()
|
||||
});
|
||||
false
|
||||
source_already_purged
|
||||
}
|
||||
};
|
||||
|
||||
@@ -3010,7 +3015,7 @@ const DELETE_MARKER_PURGE_WATCH_INTERVAL: TokioDuration = TokioDuration::from_se
|
||||
/// If the marker disappears (deleted before or while the replica landed),
|
||||
/// purge the replicated marker from the targets, retrying failed targets on
|
||||
/// later rounds. When the window drains with targets still dirty, persist the
|
||||
/// purge intent as a durable MRF entry so the next startup replays it through
|
||||
/// purge intent as a durable MRF entry so the next MRF replay pass re-drives it through
|
||||
/// `purge_stale_delete_marker_targets`.
|
||||
async fn watch_and_purge_source_delete_marker<S: ReplicationStorage>(
|
||||
bucket: String,
|
||||
@@ -3101,7 +3106,7 @@ async fn enqueue_delete_marker_purge_mrf(dobj: &DeletedObjectReplicationInfo, fa
|
||||
/// Heal callers retry for the full watch window because the startup MRF
|
||||
/// processor runs before bucket metadata (and thus target clients) finishes
|
||||
/// initializing — the first pass can see `target_client_missing` and a later
|
||||
/// round resolves the client; the replay loop is serial and startup-only, so
|
||||
/// round resolves the client; replay passes are serial background work, so
|
||||
/// blocking it for up to the window per dirty entry is acceptable. Live
|
||||
/// callers run on replication workers where a down target would pin a worker
|
||||
/// for the whole window, so they attempt once and lean on the durable intent
|
||||
@@ -6604,6 +6609,7 @@ mod tests {
|
||||
health_check_duration: std::time::Duration::from_secs(5),
|
||||
replicate_sync: false,
|
||||
client: Arc::new(aws_sdk_s3::Client::from_conf(config)),
|
||||
timeouts: Default::default(),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -259,6 +259,10 @@ pub fn check_object_name_for_length_and_slash(bucket: &str, object: &str) -> Res
|
||||
pub fn check_new_object_args(bucket: &str, object: &str, data_movement: bool) -> Result<()> {
|
||||
check_put_object_args(bucket, object)?;
|
||||
|
||||
if !data_movement && !is_meta_bucketname(bucket) && has_xml_invalid_control_characters(object) {
|
||||
return Err(StorageError::ObjectNameInvalid(bucket.to_owned(), object.to_owned()));
|
||||
}
|
||||
|
||||
if cfg!(target_os = "windows") && !data_movement && !is_meta_bucketname(bucket) && object.contains('\\') {
|
||||
return Err(StorageError::InvalidArgument(
|
||||
bucket.to_owned(),
|
||||
@@ -270,6 +274,14 @@ pub fn check_new_object_args(bucket: &str, object: &str, data_movement: bool) ->
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// New public keys must remain representable in an unencoded XML listing.
|
||||
/// Existing keys still use the read/delete validators so they can be removed.
|
||||
pub fn has_xml_invalid_control_characters(object: &str) -> bool {
|
||||
object
|
||||
.chars()
|
||||
.any(|character| character < ' ' && !matches!(character, '\t' | '\n' | '\r'))
|
||||
}
|
||||
|
||||
pub fn check_copy_obj_args(bucket: &str, object: &str) -> Result<()> {
|
||||
check_bucket_and_object_names(bucket, object)
|
||||
}
|
||||
|
||||
@@ -34,7 +34,7 @@ use tokio::sync::Mutex as TokioMutex;
|
||||
use tokio::task::JoinSet;
|
||||
use tokio::time::timeout;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, error, warn};
|
||||
use tracing::{Instrument, debug, error, warn};
|
||||
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_METACACHE: &str = "metacache";
|
||||
@@ -224,6 +224,8 @@ pub struct ListPathRawOptions {
|
||||
pub recursive: bool,
|
||||
pub incl_deleted: bool,
|
||||
pub skip_hidden_prefix_check: bool,
|
||||
pub hidden_entry_budget: usize,
|
||||
pub pending_prefix: Option<String>,
|
||||
pub filter_prefix: Option<String>,
|
||||
pub forward_to: Option<String>,
|
||||
pub min_disks: usize,
|
||||
@@ -260,6 +262,8 @@ impl Clone for ListPathRawOptions {
|
||||
recursive: self.recursive,
|
||||
incl_deleted: self.incl_deleted,
|
||||
skip_hidden_prefix_check: self.skip_hidden_prefix_check,
|
||||
hidden_entry_budget: self.hidden_entry_budget,
|
||||
pending_prefix: self.pending_prefix.clone(),
|
||||
filter_prefix: self.filter_prefix.clone(),
|
||||
forward_to: self.forward_to.clone(),
|
||||
min_disks: self.min_disks,
|
||||
@@ -289,6 +293,8 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
|
||||
recursive: opts.recursive,
|
||||
incl_deleted: opts.incl_deleted,
|
||||
skip_hidden_prefix_check: opts.skip_hidden_prefix_check,
|
||||
hidden_entry_budget: opts.hidden_entry_budget,
|
||||
pending_prefix: opts.pending_prefix.clone(),
|
||||
report_notfound: opts.report_not_found,
|
||||
filter_prefix: opts.filter_prefix.clone(),
|
||||
forward_to: opts.forward_to.clone(),
|
||||
@@ -301,13 +307,15 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
|
||||
}
|
||||
|
||||
/// How a merged raw listing stream ended.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct ListPathRawOutcome {
|
||||
/// At least one producer stopped at its per-disk limit instead of at the
|
||||
/// end of its namespace, so the closed stream is not proof of exhaustion.
|
||||
/// Carried in-band by the metacache stream trailer, so remote producers
|
||||
/// report it as well.
|
||||
pub limit_reached: bool,
|
||||
/// Inclusive sorted boundary proved by every reader before a bounded stop.
|
||||
pub scan_cut: Option<String>,
|
||||
}
|
||||
|
||||
pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> disk::error::Result<()> {
|
||||
@@ -363,10 +371,15 @@ async fn list_path_raw_inner(
|
||||
let producer_errs: Arc<[OnceLock<DiskError>]> = (0..opts.disks.len()).map(|_| OnceLock::new()).collect::<Vec<_>>().into();
|
||||
|
||||
let cancel_rx = CancellationToken::new();
|
||||
let walk_options = Arc::new(walk_dir_options(&opts));
|
||||
|
||||
for (disk_idx, disk) in opts.disks.iter().enumerate() {
|
||||
let opdisk = disk.clone();
|
||||
let opts_clone = opts.clone();
|
||||
let walk_options = walk_options.clone();
|
||||
#[cfg(test)]
|
||||
let test_reader_behavior = opts.test_reader_behaviors.get(disk_idx).cloned();
|
||||
#[cfg(test)]
|
||||
let test_disk_io_context_observed = opts.test_disk_io_context_observed.clone();
|
||||
let fallback_claim_tracker = fallback_claim_tracker.clone();
|
||||
let fds_clone = fds.clone();
|
||||
#[cfg(test)]
|
||||
@@ -376,17 +389,17 @@ async fn list_path_raw_inner(
|
||||
let disk_io_context_clone = disk_io_context.clone();
|
||||
let (rd, wr) = tokio::io::duplex(64 * 1024);
|
||||
readers.push(MetacacheReader::new(rd));
|
||||
jobs.spawn(with_optional_disk_io_context(disk_io_context_clone, async move {
|
||||
let producer = with_optional_disk_io_context(disk_io_context_clone, async move {
|
||||
let mut wr = PublishedBytesWriter::new(wr);
|
||||
#[cfg(test)]
|
||||
if let Some(observer) = &opts_clone.test_disk_io_context_observed {
|
||||
if let Some(observer) = &test_disk_io_context_observed {
|
||||
observer.store(
|
||||
crate::disk::current_disk_io_context().is_some_and(|context| context.class == DiskIoClass::Background),
|
||||
std::sync::atomic::Ordering::Release,
|
||||
);
|
||||
}
|
||||
#[cfg(test)]
|
||||
let test_primary_error = if let Some(behavior) = opts_clone.test_reader_behaviors.get(disk_idx).cloned() {
|
||||
let test_primary_error = if let Some(behavior) = test_reader_behavior {
|
||||
match behavior {
|
||||
TestReaderBehavior::Eof => return Ok(()),
|
||||
TestReaderBehavior::Entries(entries) => {
|
||||
@@ -443,8 +456,6 @@ async fn list_path_raw_inner(
|
||||
None
|
||||
};
|
||||
|
||||
let wakl_opts = walk_dir_options(&opts_clone);
|
||||
|
||||
let mut need_fallback = false;
|
||||
let mut last_err = None;
|
||||
#[cfg(test)]
|
||||
@@ -459,7 +470,7 @@ async fn list_path_raw_inner(
|
||||
|
||||
if !need_fallback && let Some(disk) = opdisk {
|
||||
let primary_walk_started = std::time::Instant::now();
|
||||
match disk.walk_dir(wakl_opts, &mut wr).await {
|
||||
match disk.walk_dir((*walk_options).clone(), &mut wr).await {
|
||||
Ok(_res) => {
|
||||
rustfs_io_metrics::record_stage_duration(
|
||||
"metacache_walk_dir_primary",
|
||||
@@ -483,8 +494,8 @@ async fn list_path_raw_inner(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
bucket = %opts_clone.bucket,
|
||||
path = %opts_clone.path,
|
||||
bucket = %walk_options.bucket,
|
||||
path = %walk_options.base_dir,
|
||||
disk_index = disk_idx,
|
||||
state = "walk_dir_missing_path",
|
||||
error = ?err,
|
||||
@@ -495,8 +506,8 @@ async fn list_path_raw_inner(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
bucket = %opts_clone.bucket,
|
||||
path = %opts_clone.path,
|
||||
bucket = %walk_options.bucket,
|
||||
path = %walk_options.base_dir,
|
||||
disk_index = disk_idx,
|
||||
state = "walk_dir_failed",
|
||||
error = ?err,
|
||||
@@ -598,8 +609,8 @@ async fn list_path_raw_inner(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
bucket = %opts_clone.bucket,
|
||||
path = %opts_clone.path,
|
||||
bucket = %walk_options.bucket,
|
||||
path = %walk_options.base_dir,
|
||||
disk_index = disk_idx,
|
||||
state = "fallback_disk_missing_for_path",
|
||||
error = ?err,
|
||||
@@ -610,8 +621,8 @@ async fn list_path_raw_inner(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
bucket = %opts_clone.bucket,
|
||||
path = %opts_clone.path,
|
||||
bucket = %walk_options.bucket,
|
||||
path = %walk_options.base_dir,
|
||||
disk_index = disk_idx,
|
||||
state = "fallback_disk_missing",
|
||||
error = ?err,
|
||||
@@ -626,7 +637,7 @@ async fn list_path_raw_inner(
|
||||
}
|
||||
|
||||
let fallback_walk_started = std::time::Instant::now();
|
||||
match disk.as_ref().walk_dir(walk_dir_options(&opts_clone), &mut wr).await {
|
||||
match disk.as_ref().walk_dir((*walk_options).clone(), &mut wr).await {
|
||||
Ok(_r) => {
|
||||
rustfs_io_metrics::record_stage_duration(
|
||||
"metacache_walk_dir_fallback",
|
||||
@@ -652,8 +663,8 @@ async fn list_path_raw_inner(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
bucket = %opts_clone.bucket,
|
||||
path = %opts_clone.path,
|
||||
bucket = %walk_options.bucket,
|
||||
path = %walk_options.base_dir,
|
||||
disk_index = disk_idx,
|
||||
state = "fallback_walk_dir_missing_path",
|
||||
error = ?err,
|
||||
@@ -664,8 +675,8 @@ async fn list_path_raw_inner(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
bucket = %opts_clone.bucket,
|
||||
path = %opts_clone.path,
|
||||
bucket = %walk_options.bucket,
|
||||
path = %walk_options.base_dir,
|
||||
disk_index = disk_idx,
|
||||
state = "fallback_walk_dir_failed",
|
||||
error = ?err,
|
||||
@@ -687,12 +698,13 @@ async fn list_path_raw_inner(
|
||||
|
||||
// warn!("list_path_raw: while need_fallback done");
|
||||
Ok(())
|
||||
}));
|
||||
});
|
||||
jobs.spawn(producer.in_current_span());
|
||||
}
|
||||
|
||||
let revjob_rx = rx.clone();
|
||||
let mut revjobs = JoinSet::new();
|
||||
revjobs.spawn(async move {
|
||||
let merge = async move {
|
||||
// Consumer-side peek timeout, and a caveat worth understanding
|
||||
// (rustfs/backlog#1217).
|
||||
//
|
||||
@@ -722,6 +734,8 @@ async fn list_path_raw_inner(
|
||||
// Sticky across rounds: once any reader's trailer says its producer
|
||||
// stopped at the per-disk limit, the merged stream may hide more keys.
|
||||
let mut limit_reached = false;
|
||||
let mut scan_cut = None;
|
||||
let mut last_scanned_key: Option<String> = None;
|
||||
|
||||
loop {
|
||||
let mut current = MetaCacheEntry::default();
|
||||
@@ -733,7 +747,7 @@ async fn list_path_raw_inner(
|
||||
// );
|
||||
|
||||
if revjob_rx.is_cancelled() {
|
||||
return Ok(ListPathRawOutcome { limit_reached });
|
||||
return Ok(ListPathRawOutcome { limit_reached, scan_cut });
|
||||
}
|
||||
|
||||
let mut top_entries: Vec<Option<MetaCacheEntry>> = vec![None; readers.len()];
|
||||
@@ -747,6 +761,16 @@ async fn list_path_raw_inner(
|
||||
// Start every missing head read in the same round so one stalled
|
||||
// disk cannot multiply the wait budget by the erasure-set width.
|
||||
// Outcomes are still consumed below in stable disk-index order.
|
||||
if !revjob_rx.is_cancelled() {
|
||||
for (i, reader) in readers.iter_mut().enumerate() {
|
||||
if errs[i].is_none()
|
||||
&& pending_entries[i].is_none()
|
||||
&& let Some(Ok(Some(entry))) = reader.try_peek_buffered()
|
||||
{
|
||||
pending_entries[i] = Some(entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
let concurrent_peeks = readers.iter_mut().enumerate().filter_map(|(i, reader)| {
|
||||
if errs[i].is_some() || pending_entries[i].is_some() {
|
||||
return None;
|
||||
@@ -783,7 +807,12 @@ async fn list_path_raw_inner(
|
||||
continue;
|
||||
}
|
||||
// eof
|
||||
limit_reached |= r.limit_reached() == Some(true);
|
||||
if r.limit_reached() == Some(true) {
|
||||
limit_reached = true;
|
||||
scan_cut = Some(last_scanned_key.clone().ok_or_else(|| {
|
||||
DiskError::other("bounded listing reader ended without a scan boundary")
|
||||
})?);
|
||||
}
|
||||
at_eof += 1;
|
||||
// warn!("list_path_raw: peek eof, disk: {}", i);
|
||||
continue;
|
||||
@@ -857,7 +886,7 @@ async fn list_path_raw_inner(
|
||||
*r = MetacacheReader::new(detached_rd);
|
||||
continue;
|
||||
}
|
||||
PeekOutcome::Cancelled => return Ok(ListPathRawOutcome { limit_reached }),
|
||||
PeekOutcome::Cancelled => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
|
||||
}
|
||||
};
|
||||
|
||||
@@ -878,7 +907,7 @@ async fn list_path_raw_inner(
|
||||
}
|
||||
if entry.name == current.name {
|
||||
// If exact match, we agree.
|
||||
if current.matches_strict(&entry) {
|
||||
if current.matches_strict_borrowed(&entry) {
|
||||
top_entries[i] = Some(entry);
|
||||
agree += 1;
|
||||
} else {
|
||||
@@ -916,7 +945,7 @@ async fn list_path_raw_inner(
|
||||
if let Some(finished_fn) = opts.finished.as_ref() {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
|
||||
_ = finished_fn(&errs) => {}
|
||||
}
|
||||
}
|
||||
@@ -966,6 +995,14 @@ async fn list_path_raw_inner(
|
||||
return Err(classify_listing_quorum_failure(&failures));
|
||||
}
|
||||
|
||||
// The next head from a bounded reader is unknown. The previous
|
||||
// globally consumed group is a safe cut; consuming another reader's
|
||||
// later head would turn this absence of evidence into a skipped key.
|
||||
// Check quorum failures above before accepting the bounded result.
|
||||
if limit_reached {
|
||||
break;
|
||||
}
|
||||
|
||||
// Break if all at EOF or error.
|
||||
if at_eof + has_err == readers.len() {
|
||||
if has_err > 0
|
||||
@@ -973,7 +1010,7 @@ async fn list_path_raw_inner(
|
||||
{
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
|
||||
_ = finished_fn(&errs) => {}
|
||||
}
|
||||
}
|
||||
@@ -984,16 +1021,19 @@ async fn list_path_raw_inner(
|
||||
break;
|
||||
}
|
||||
|
||||
last_scanned_key = Some(current.name.clone());
|
||||
if agree == readers.len() && !opts.preserve_replica_metadata {
|
||||
for r in readers.iter_mut() {
|
||||
let _ = r.skip(1).await;
|
||||
if !r.skip_current() {
|
||||
let _ = r.skip(1).await;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(agreed_fn) = opts.agreed.as_ref() {
|
||||
// warn!("list_path_raw: agreed_fn start, current: {:?}", ¤t.name);
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
|
||||
result = agreed_fn(current) => result?
|
||||
}
|
||||
// warn!("list_path_raw: agreed_fn done");
|
||||
@@ -1009,7 +1049,7 @@ async fn list_path_raw_inner(
|
||||
}
|
||||
|
||||
for (i, r) in readers.iter_mut().enumerate() {
|
||||
if top_entries[i].is_some() {
|
||||
if top_entries[i].is_some() && !r.skip_current() {
|
||||
let _ = r.skip(1).await;
|
||||
}
|
||||
}
|
||||
@@ -1017,13 +1057,14 @@ async fn list_path_raw_inner(
|
||||
if let Some(partial_fn) = opts.partial.as_ref() {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
|
||||
_ = partial_fn(MetaCacheEntries(top_entries), &errs) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(ListPathRawOutcome { limit_reached })
|
||||
});
|
||||
Ok(ListPathRawOutcome { limit_reached, scan_cut })
|
||||
};
|
||||
revjobs.spawn(merge.in_current_span());
|
||||
|
||||
let merge_started = std::time::Instant::now();
|
||||
let Some(revjob) = revjobs.join_next().await else {
|
||||
@@ -1187,6 +1228,24 @@ mod tests {
|
||||
assert!(!options.skip_total_timeout);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cloned_reader_options_preserve_hidden_scan_budget_and_pending_prefix() {
|
||||
let request = ListPathRawOptions {
|
||||
bucket: "versioned".to_owned(),
|
||||
hidden_entry_budget: 100,
|
||||
pending_prefix: Some("pending/".to_owned()),
|
||||
forward_to: Some("pending/0099".to_owned()),
|
||||
..Default::default()
|
||||
};
|
||||
// Both walks are built once from the shared options; the per-job clone
|
||||
// must carry every field that the walk derives from the request.
|
||||
let job = request.clone();
|
||||
let walk = walk_dir_options(&job);
|
||||
assert_eq!(walk.hidden_entry_budget, 100);
|
||||
assert_eq!(walk.pending_prefix.as_deref(), request.pending_prefix.as_deref());
|
||||
assert_eq!(walk.forward_to.as_deref(), Some("pending/0099"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_path_raw_empty_disks_returns_read_quorum() {
|
||||
let err = list_path_raw(CancellationToken::new(), ListPathRawOptions::default())
|
||||
@@ -1237,6 +1296,163 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_path_raw_fanout_retains_only_linear_disk_references() {
|
||||
let root = tempfile::tempdir().expect("create disk root");
|
||||
let endpoint = crate::disk::endpoint::Endpoint::try_from(root.path().to_str().expect("UTF-8 disk path"))
|
||||
.expect("parse disk endpoint");
|
||||
let local = Arc::new(
|
||||
crate::disk::local::LocalDisk::new(&endpoint, false)
|
||||
.await
|
||||
.expect("open local disk"),
|
||||
);
|
||||
let wrapper = crate::disk::disk_store::LocalDiskWrapper::new(local, false);
|
||||
let disk = Arc::new(crate::disk::Disk::Local(Box::new(wrapper)));
|
||||
let baseline = Arc::strong_count(&disk);
|
||||
|
||||
for primary_count in [1, 4, 16] {
|
||||
for fallback_count in [0, 2] {
|
||||
let entered: Vec<_> = (0..primary_count).map(|_| Arc::new(tokio::sync::Notify::new())).collect();
|
||||
let stopped: Vec<_> = (0..primary_count).map(|_| Arc::new(tokio::sync::Notify::new())).collect();
|
||||
let behaviors = entered
|
||||
.iter()
|
||||
.zip(&stopped)
|
||||
.map(|(entered, stopped)| TestReaderBehavior::StallWithSignals {
|
||||
entered: entered.clone(),
|
||||
stopped: stopped.clone(),
|
||||
})
|
||||
.collect();
|
||||
let cancel = CancellationToken::new();
|
||||
let listing = tokio::spawn(list_path_raw(
|
||||
cancel.clone(),
|
||||
ListPathRawOptions {
|
||||
disks: vec![Some(disk.clone()); primary_count],
|
||||
fallback_disks: vec![Some(disk.clone()); fallback_count],
|
||||
min_disks: primary_count,
|
||||
test_reader_behaviors: behaviors,
|
||||
peek_timeout: Some(Duration::from_secs(60)),
|
||||
..Default::default()
|
||||
},
|
||||
));
|
||||
for signal in &entered {
|
||||
timeout(Duration::from_secs(2), signal.notified())
|
||||
.await
|
||||
.expect("every producer must enter the real fanout before counting references");
|
||||
}
|
||||
|
||||
// The merge owns the configured primary/fallback slots, each producer
|
||||
// owns its primary, and the shared fallback queue owns each fallback.
|
||||
let expected = baseline + 2 * (primary_count + fallback_count);
|
||||
let retained = Arc::strong_count(&disk);
|
||||
cancel.cancel();
|
||||
timeout(Duration::from_secs(2), listing)
|
||||
.await
|
||||
.expect("cancellation must stop the listing")
|
||||
.expect("listing task must join")
|
||||
.expect("external cancellation must finish cleanly");
|
||||
for signal in &stopped {
|
||||
timeout(Duration::from_secs(2), signal.notified())
|
||||
.await
|
||||
.expect("cancellation must drop every producer");
|
||||
}
|
||||
assert_eq!(Arc::strong_count(&disk), baseline, "cancellation must release every fanout reference");
|
||||
assert_eq!(
|
||||
retained, expected,
|
||||
"fanout must retain linear references for {primary_count} primary and {fallback_count} fallback slots"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn list_path_raw_producer_and_merge_preserve_request_span() {
|
||||
use tracing_subscriber::{Layer, layer::SubscriberExt};
|
||||
|
||||
struct ProducerSpan(Arc<AtomicBool>);
|
||||
#[derive(Default)]
|
||||
struct StateField(bool);
|
||||
impl tracing::field::Visit for StateField {
|
||||
fn record_debug(&mut self, _: &tracing::field::Field, _: &dyn std::fmt::Debug) {}
|
||||
|
||||
fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
|
||||
self.0 |= field.name() == "state" && value == "walk_dir_missing_path";
|
||||
}
|
||||
}
|
||||
impl<S> Layer<S> for ProducerSpan
|
||||
where
|
||||
S: tracing::Subscriber + for<'a> tracing_subscriber::registry::LookupSpan<'a>,
|
||||
{
|
||||
fn on_event(&self, event: &tracing::Event<'_>, context: tracing_subscriber::layer::Context<'_, S>) {
|
||||
let mut state = StateField::default();
|
||||
event.record(&mut state);
|
||||
if state.0
|
||||
&& context
|
||||
.lookup_current()
|
||||
.is_some_and(|span| span.name() == "flink_list_request")
|
||||
{
|
||||
self.0.store(true, std::sync::atomic::Ordering::Release);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The subscriber is thread-local; keep the spawned tasks on this runtime thread.
|
||||
let producer_observed = Arc::new(AtomicBool::new(false));
|
||||
let subscriber = tracing_subscriber::registry().with(ProducerSpan(producer_observed.clone()));
|
||||
let _subscriber = tracing::subscriber::set_default(subscriber);
|
||||
let request = tracing::info_span!("flink_list_request", request_id = "list-span-regression");
|
||||
let root = tempfile::tempdir().expect("create disk root");
|
||||
let endpoint = crate::disk::endpoint::Endpoint::try_from(root.path().to_str().expect("UTF-8 disk path"))
|
||||
.expect("parse disk endpoint");
|
||||
let local = Arc::new(
|
||||
crate::disk::local::LocalDisk::new(&endpoint, false)
|
||||
.await
|
||||
.expect("open local disk"),
|
||||
);
|
||||
let wrapper = crate::disk::disk_store::LocalDiskWrapper::new(local, false);
|
||||
let disk = Arc::new(crate::disk::Disk::Local(Box::new(wrapper)));
|
||||
let error = list_path_raw(
|
||||
CancellationToken::new(),
|
||||
ListPathRawOptions {
|
||||
disks: vec![Some(disk)],
|
||||
bucket: "missing-bucket".to_owned(),
|
||||
min_disks: 1,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.instrument(request.clone())
|
||||
.await
|
||||
.expect_err("missing bucket must fail the real disk producer");
|
||||
assert_eq!(error, DiskError::VolumeNotFound);
|
||||
assert!(
|
||||
producer_observed.load(std::sync::atomic::Ordering::Acquire),
|
||||
"disk error must retain the request span"
|
||||
);
|
||||
|
||||
let merge_observed = Arc::new(AtomicBool::new(false));
|
||||
let merge_sink = merge_observed.clone();
|
||||
let request_id = request.id();
|
||||
list_path_raw(
|
||||
CancellationToken::new(),
|
||||
ListPathRawOptions {
|
||||
disks: vec![None],
|
||||
min_disks: 1,
|
||||
test_reader_behaviors: vec![TestReaderBehavior::Entries(vec![fallback_test_entry()])],
|
||||
agreed: Some(Box::new(move |_| {
|
||||
merge_sink.store(tracing::Span::current().id() == request_id, std::sync::atomic::Ordering::Release);
|
||||
Box::pin(async { Ok(()) })
|
||||
})),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.instrument(request)
|
||||
.await
|
||||
.expect("merge should deliver the test entry");
|
||||
assert!(
|
||||
merge_observed.load(std::sync::atomic::Ordering::Acquire),
|
||||
"merge callback must retain the request span"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_joinset_walk_uses_the_local_disk_background_gate() {
|
||||
let root = tempfile::tempdir().expect("local disk root should be created");
|
||||
@@ -1304,6 +1520,7 @@ mod tests {
|
||||
outcome.limit_reached,
|
||||
"a reader whose trailer reports a limit stop must mark the merged stream as possibly incomplete"
|
||||
);
|
||||
assert_eq!(outcome.scan_cut.as_deref(), Some("bucket/object"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -32,7 +32,6 @@ use crate::storage_api_contracts::internode::{
|
||||
PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_CAPABILITY_VERSION,
|
||||
};
|
||||
|
||||
use hmac::{Hmac, KeyInit, Mac};
|
||||
use http::uri::Authority;
|
||||
use http::{HeaderMap, HeaderValue, Method, Uri};
|
||||
#[cfg(test)]
|
||||
@@ -47,6 +46,7 @@ use rustfs_io_metrics::internode_metrics::{
|
||||
global_internode_metrics,
|
||||
};
|
||||
use rustfs_object_data_cache::{MemoryBasis, resolve_effective_memory};
|
||||
use rustfs_utils::crypto::{HmacSha256, KeyInit, Mac};
|
||||
use rustfs_utils::get_env_bool;
|
||||
use sha2::Digest as _;
|
||||
use sha2::Sha256;
|
||||
@@ -58,8 +58,6 @@ use time::OffsetDateTime;
|
||||
use tracing::{error, info, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
type HmacSha256 = Hmac<Sha256>;
|
||||
|
||||
const SIGNATURE_HEADER: &str = "x-rustfs-signature";
|
||||
pub const RPC_BACKGROUND_ADMISSION_HEADER: &str = "x-rustfs-rpc-background-admission";
|
||||
const RPC_BACKGROUND_ADMISSION_V1: &str = "v1";
|
||||
@@ -1590,7 +1588,13 @@ fn verify_tonic_replay_scope_signature(audience: &str, path: &str, headers: &Hea
|
||||
if boot_epoch != tonic_rpc_boot_epoch() {
|
||||
return Err(std::io::Error::other("RPC boot epoch is stale"));
|
||||
}
|
||||
check_and_record_tonic_nonce(nonce, signed_at, path)
|
||||
check_and_record_tonic_nonce(nonce, signed_at, path)?;
|
||||
// A body-bound request also carries a usable v2 signature. Consume both
|
||||
// nonces so stripping the replay-scope headers cannot authorize it again.
|
||||
if content_sha256 != UNSIGNED_PAYLOAD {
|
||||
verify_tonic_rpc_signature_with_strictness(audience, path, headers, true)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Verify gRPC authentication, preferring v2 without downgrade on malformed v2 metadata.
|
||||
@@ -2581,6 +2585,43 @@ mod tests {
|
||||
assert_eq!(path_error.to_string(), "Invalid RPC replay scope signature");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_scope_consumes_the_body_bound_v2_nonce() {
|
||||
ensure_test_rpc_secret();
|
||||
let audience = "node-a:9000";
|
||||
let path = "/node_service.NodeService/RecoverDeleteTransaction";
|
||||
let digest = "a".repeat(64);
|
||||
for v3_first in [true, false] {
|
||||
let mut headers =
|
||||
gen_tonic_signature_headers(audience, "node_service.NodeService", "RecoverDeleteTransaction", Some(&digest))
|
||||
.expect("body-bound v2 headers should build");
|
||||
let timestamp = headers[TIMESTAMP_HEADER].to_str().expect("timestamp").to_string();
|
||||
headers.extend(
|
||||
gen_tonic_replay_scope_headers(audience, path, ×tamp, &digest, tonic_rpc_boot_epoch())
|
||||
.expect("replay-scope headers should build"),
|
||||
);
|
||||
let mut downgraded = headers.clone();
|
||||
for header in [
|
||||
RPC_REPLAY_SCOPE_VERSION_HEADER,
|
||||
RPC_REPLAY_SCOPE_SIGNATURE_HEADER,
|
||||
RPC_REPLAY_SCOPE_NONCE_HEADER,
|
||||
RPC_BOOT_EPOCH_HEADER,
|
||||
] {
|
||||
downgraded.remove(header);
|
||||
}
|
||||
let (first, replay) = if v3_first {
|
||||
(&headers, &downgraded)
|
||||
} else {
|
||||
(&downgraded, &headers)
|
||||
};
|
||||
verify_tonic_rpc_signature_with_policy(audience, path, first, true, false, false)
|
||||
.expect("the first delivery must be accepted");
|
||||
let error = verify_tonic_rpc_signature_with_policy(audience, path, replay, true, false, false)
|
||||
.expect_err("a mutation cannot replay through the other signature generation");
|
||||
assert_eq!(error.to_string(), "RPC request replay detected");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_scope_rejects_partial_metadata_and_stale_epoch_without_fallback() {
|
||||
ensure_test_rpc_secret();
|
||||
|
||||
@@ -2014,9 +2014,21 @@ impl PeerRestClient {
|
||||
}
|
||||
result
|
||||
})
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(Error::other(format!("load_bucket_metadata({bucket}) timed out"))));
|
||||
self.finalize_result(result).await
|
||||
.await;
|
||||
self.settle_bucket_metadata_reload(bucket, result).await
|
||||
}
|
||||
|
||||
async fn settle_bucket_metadata_reload(
|
||||
&self,
|
||||
bucket: &str,
|
||||
result: std::result::Result<Result<()>, tokio::time::error::Elapsed>,
|
||||
) -> Result<()> {
|
||||
match result {
|
||||
Ok(result) => self.finalize_result(result).await,
|
||||
// A local operation deadline does not establish that the peer is
|
||||
// unreachable. In particular, it must not gate IAM revocations.
|
||||
Err(_) => Err(Error::other(format!("load_bucket_metadata({bucket}) timed out"))),
|
||||
}
|
||||
}
|
||||
|
||||
async fn load_bucket_metadata_once(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
|
||||
@@ -4753,6 +4765,31 @@ mod tests {
|
||||
assert!(client.offline.load(Ordering::Acquire));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bucket_metadata_deadline_does_not_gate_healthy_peer_notifications() {
|
||||
let client = test_peer_client();
|
||||
let elapsed = tokio::time::timeout(std::time::Duration::ZERO, std::future::pending::<Result<()>>()).await;
|
||||
let error = client
|
||||
.settle_bucket_metadata_reload("slow-policy", elapsed)
|
||||
.await
|
||||
.expect_err("local budget expires");
|
||||
assert!(error.to_string().contains("timed out"));
|
||||
assert!(
|
||||
!client.offline.load(Ordering::Acquire),
|
||||
"local work must not suppress subsequent IAM RPCs"
|
||||
);
|
||||
client
|
||||
.settle_bucket_metadata_reload("healthy", Ok(Ok(())))
|
||||
.await
|
||||
.expect("peer reply");
|
||||
let error = client
|
||||
.settle_bucket_metadata_reload("unreachable", Ok(Err(Error::from(tonic::Status::unavailable("connection refused")))))
|
||||
.await
|
||||
.expect_err("real transport failure");
|
||||
assert!(error.to_string().contains("connection refused"));
|
||||
assert!(client.offline.load(Ordering::Acquire), "transport failures must still enter recovery");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn peer_rest_client_finalize_result_keeps_online_for_business_errors() {
|
||||
let client = test_peer_client();
|
||||
|
||||
@@ -996,21 +996,32 @@ fn internode_idempotent_read_retries() -> usize {
|
||||
static PREWARMED_PEERS: std::sync::LazyLock<std::sync::Mutex<std::collections::HashSet<String>>> =
|
||||
std::sync::LazyLock::new(|| std::sync::Mutex::new(std::collections::HashSet::new()));
|
||||
|
||||
fn claim_control_channel_prewarm(addr: &str) -> bool {
|
||||
let Ok(mut prewarmed) = PREWARMED_PEERS.lock() else {
|
||||
return false;
|
||||
};
|
||||
prewarmed.insert(addr.to_owned())
|
||||
}
|
||||
|
||||
fn release_control_channel_prewarm(addr: &str) {
|
||||
if let Ok(mut prewarmed) = PREWARMED_PEERS.lock() {
|
||||
prewarmed.remove(addr);
|
||||
}
|
||||
}
|
||||
|
||||
/// Best-effort background prewarm of a peer's control channel (grpc-optimization P3-1). Deduped per
|
||||
/// peer; a dial failure just falls through to the existing lazy connect and recovery monitor.
|
||||
/// peer; a dial failure releases the marker so a later disk construction can retry the prewarm.
|
||||
fn spawn_control_channel_prewarm(addr: String) {
|
||||
{
|
||||
let Ok(mut prewarmed) = PREWARMED_PEERS.lock() else {
|
||||
return;
|
||||
};
|
||||
if !prewarmed.insert(addr.clone()) {
|
||||
return;
|
||||
}
|
||||
if !claim_control_channel_prewarm(&addr) {
|
||||
return;
|
||||
}
|
||||
tokio::spawn(async move {
|
||||
match node_service_time_out_client_no_auth(&addr).await {
|
||||
Ok(_) => debug!(addr = %addr, "internode control channel prewarmed"),
|
||||
Err(err) => debug!(addr = %addr, error = %err, "internode control channel prewarm failed (best-effort)"),
|
||||
Err(err) => {
|
||||
release_control_channel_prewarm(&addr);
|
||||
debug!(addr = %addr, error = %err, "internode control channel prewarm failed (best-effort)");
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -1082,7 +1093,10 @@ impl RemoteDisk {
|
||||
if payload.len() > operation.max_wire_payload_bytes() {
|
||||
return Err(Error::other("delete recovery operation exceeds payload limit"));
|
||||
}
|
||||
let settle_timeout = if matches!(&operation, DeleteRecoveryRpcOperation::Settle { .. }) {
|
||||
let settle_timeout = if matches!(
|
||||
&operation,
|
||||
DeleteRecoveryRpcOperation::Settle { .. } | DeleteRecoveryRpcOperation::SettleBounded(_)
|
||||
) {
|
||||
Some(crate::disk::get_delete_recovery_timeout()?)
|
||||
} else {
|
||||
None
|
||||
@@ -1637,6 +1651,7 @@ impl RemoteDisk {
|
||||
let initial_probe_ok = Self::perform_connectivity_check(&addr).await.is_ok();
|
||||
if initial_probe_ok {
|
||||
health.record_operation_success(&endpoint, "connectivity_probe_success");
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
|
||||
} else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
|
||||
warn!(
|
||||
event = EVENT_REMOTE_DISK_HEALTH,
|
||||
@@ -1708,6 +1723,7 @@ impl RemoteDisk {
|
||||
// Perform basic connectivity check
|
||||
if Self::perform_connectivity_check(&addr).await.is_ok() {
|
||||
health.record_operation_success(&endpoint, "connectivity_probe_success");
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
|
||||
} else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
|
||||
warn!(
|
||||
event = EVENT_REMOTE_DISK_HEALTH,
|
||||
@@ -1778,6 +1794,7 @@ impl RemoteDisk {
|
||||
}
|
||||
_ = interval.tick() => {
|
||||
if Self::perform_recovery_probe(&addr, &endpoint).await.is_ok() {
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable(&addr);
|
||||
let became_online = health.mark_recovery_success(&endpoint, "disk_info_probe_success");
|
||||
debug!(
|
||||
event = EVENT_REMOTE_DISK_HEALTH,
|
||||
@@ -2009,6 +2026,7 @@ impl RemoteDisk {
|
||||
let operation_result = operation().await;
|
||||
if operation_result.is_ok() {
|
||||
self.health.record_operation_success(&self.endpoint, "operation_success");
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
|
||||
}
|
||||
self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
|
||||
.await;
|
||||
@@ -2023,6 +2041,7 @@ impl RemoteDisk {
|
||||
// Log success; the waiting guard balances every exit path.
|
||||
if operation_result.is_ok() {
|
||||
self.health.record_operation_success(&self.endpoint, "operation_success");
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
|
||||
}
|
||||
self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
|
||||
.await;
|
||||
@@ -2711,6 +2730,54 @@ impl RemoteDisk {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl DiskAPI for RemoteDisk {
|
||||
async fn list_multipart_index_roots(
|
||||
&self,
|
||||
bucket: &str,
|
||||
marker: Option<&str>,
|
||||
) -> Result<crate::disk::multipart_index::MultipartIndexPage> {
|
||||
use crate::disk::multipart_index::{MULTIPART_INDEX_RPC_MAX_BYTES, MultipartIndexRpcReply, MultipartIndexRpcRequest};
|
||||
let expected_disk = self
|
||||
.id
|
||||
.lock()
|
||||
.await
|
||||
.filter(|id| !id.is_nil())
|
||||
.ok_or(DiskError::DiskNotFound)?;
|
||||
let query = MultipartIndexRpcRequest::new(bucket, marker);
|
||||
let endpoint = self.endpoint.to_string();
|
||||
let canonical = query.canonical_request(&endpoint)?;
|
||||
let opts = serde_json::to_string(&query)?;
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(DiskInfoRequest {
|
||||
disk: endpoint.clone(),
|
||||
opts: opts.clone(),
|
||||
});
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
|
||||
let response = client.disk_info(request).await?.into_inner();
|
||||
if !response.success {
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
}
|
||||
if response.disk_info.len() > MULTIPART_INDEX_RPC_MAX_BYTES {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
// An old server ignores the additive query and returns ordinary DiskInfo.
|
||||
// It cannot satisfy the dedicated reply schema or the fresh challenge.
|
||||
let reply: MultipartIndexRpcReply =
|
||||
serde_json::from_str(&response.disk_info).map_err(|_| DiskError::MethodNotAllowed)?;
|
||||
reply.validate(&query, expected_disk)?;
|
||||
crate::cluster::rpc::verify_tonic_rpc_response_proof(
|
||||
&query.canonical_response(&endpoint, &reply.page)?,
|
||||
&reply.proof,
|
||||
)
|
||||
.map_err(Error::other)?;
|
||||
Ok(reply.page)
|
||||
},
|
||||
get_drive_disk_info_timeout(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn supports_guarded_delete_versions(&self) -> Result<()> {
|
||||
let response = self.guarded_delete_versions_rpc("", Vec::new(), true).await?;
|
||||
if !response.errors.is_empty() || !response.item_errors.is_empty() {
|
||||
@@ -3624,7 +3691,13 @@ impl DiskAPI for RemoteDisk {
|
||||
);
|
||||
}
|
||||
let rpc_started = read_version_stage_timer(batch_read_version_attribution_enabled);
|
||||
let response = match client.batch_read_version(request).await {
|
||||
let response_result = client.batch_read_version(request).await;
|
||||
if let Some(rpc_started) = rpc_started {
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_duration(
|
||||
rpc_started.elapsed(),
|
||||
);
|
||||
}
|
||||
let response = match response_result {
|
||||
Ok(response) => {
|
||||
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
|
||||
response.into_inner()
|
||||
@@ -4294,6 +4367,38 @@ impl DiskAPI for RemoteDisk {
|
||||
}
|
||||
}
|
||||
|
||||
async fn discover_delete_transactions_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
max_entries: u32,
|
||||
) -> Result<crate::disk::BoundedDeleteRecoveryDiscovery> {
|
||||
// Older peers reject this variant. Do not retry through the unbounded protocol;
|
||||
// they are reported as unsupported participants (rolling-upgrade contract).
|
||||
match self
|
||||
.recover_delete_transaction(
|
||||
volume,
|
||||
DeleteRecoveryRpcOperation::DiscoverBounded {
|
||||
object: object.to_owned(),
|
||||
max_entries,
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(DeleteRecoveryRpcReply::DiscoveredBounded(page)) => Ok(page),
|
||||
Ok(_) => Err(Error::FileCorrupt),
|
||||
Err(error)
|
||||
if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
|
||||
|| matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
|
||||
.is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
|
||||
{
|
||||
// No paginated fallback through the unbounded protocol; the peer
|
||||
// is an unsupported participant until it is upgraded.
|
||||
Err(Error::UnsupportedDisk)
|
||||
}
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
async fn discover_delete_transactions(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -4329,6 +4434,62 @@ impl DiskAPI for RemoteDisk {
|
||||
_ => Err(Error::other("unexpected delete discovery reply")),
|
||||
}
|
||||
}
|
||||
async fn settle_delete_transaction_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
request: crate::disk::BoundedDeleteRecoverySettlement,
|
||||
_namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<crate::disk::BoundedDeleteRecoverySettlementReply> {
|
||||
match self
|
||||
.recover_delete_transaction(volume, DeleteRecoveryRpcOperation::SettleBounded(request))
|
||||
.await
|
||||
{
|
||||
Ok(DeleteRecoveryRpcReply::SettledBounded(reply)) => Ok(reply),
|
||||
Ok(_) => Err(Error::FileCorrupt),
|
||||
Err(error)
|
||||
if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
|
||||
|| matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
|
||||
.is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
|
||||
{
|
||||
// No paginated fallback through the unbounded protocol; the peer
|
||||
// is an unsupported participant until it is upgraded.
|
||||
Err(Error::UnsupportedDisk)
|
||||
}
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
async fn inspect_delete_transaction_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
max_entries: u32,
|
||||
) -> Result<crate::disk::BoundedDeleteRecoveryInspection> {
|
||||
match self
|
||||
.recover_delete_transaction(
|
||||
volume,
|
||||
DeleteRecoveryRpcOperation::InspectBounded {
|
||||
object: object.to_owned(),
|
||||
transaction,
|
||||
max_entries,
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(DeleteRecoveryRpcReply::InspectedBounded(page)) => Ok(page),
|
||||
Ok(_) => Err(Error::FileCorrupt),
|
||||
Err(error)
|
||||
if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
|
||||
|| matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
|
||||
.is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
|
||||
{
|
||||
// No paginated fallback through the unbounded protocol; the peer
|
||||
// is an unsupported participant until it is upgraded.
|
||||
Err(Error::UnsupportedDisk)
|
||||
}
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
async fn inspect_delete_transaction(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -4551,7 +4712,10 @@ impl DiskAPI for RemoteDisk {
|
||||
read_multiple_req_bin: read_multiple_req_bin.into(),
|
||||
});
|
||||
|
||||
let response = client.read_multiple(request).await?.into_inner();
|
||||
let rpc_started = Instant::now();
|
||||
let response = client.read_multiple(request).await;
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_multiple_duration(rpc_started.elapsed());
|
||||
let response = response?.into_inner();
|
||||
|
||||
if !response.success {
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
@@ -4602,6 +4766,7 @@ impl DiskAPI for RemoteDisk {
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "write_all")?;
|
||||
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_request();
|
||||
let rpc_started = Instant::now();
|
||||
let response = match client.write_all(request).await {
|
||||
Ok(response) => response.into_inner(),
|
||||
Err(err) => {
|
||||
@@ -4610,6 +4775,7 @@ impl DiskAPI for RemoteDisk {
|
||||
}
|
||||
};
|
||||
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_duration(rpc_started.elapsed());
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_sent_bytes(data_len);
|
||||
|
||||
if !response.success {
|
||||
@@ -4707,6 +4873,7 @@ impl DiskAPI for RemoteDisk {
|
||||
});
|
||||
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_all_request();
|
||||
let rpc_started = Instant::now();
|
||||
let response = match client.read_all(request).await {
|
||||
Ok(response) => response.into_inner(),
|
||||
Err(err) => {
|
||||
@@ -4715,6 +4882,8 @@ impl DiskAPI for RemoteDisk {
|
||||
}
|
||||
};
|
||||
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_all_duration(rpc_started.elapsed());
|
||||
|
||||
if !response.success {
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_all_error();
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
@@ -4849,6 +5018,113 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bounded_delete_discovery_lost_remote_reply_leaves_no_cursor() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
|
||||
.await
|
||||
.expect("test peer must bind");
|
||||
let directory = tempfile::tempdir().expect("temporary disk");
|
||||
let endpoint = Endpoint::try_from(directory.path().to_str().expect("disk path")).expect("endpoint");
|
||||
let disk = Arc::new(
|
||||
crate::disk::local::LocalDisk::new(&endpoint, false)
|
||||
.await
|
||||
.expect("local disk"),
|
||||
);
|
||||
disk.make_volume("bucket").await.expect("bucket");
|
||||
let prefix = directory.path().join("bucket/prefix");
|
||||
std::fs::create_dir(&prefix).expect("prefix");
|
||||
for index in 0..64 {
|
||||
std::fs::create_dir(prefix.join(format!("child-{index}"))).expect("legitimate child");
|
||||
}
|
||||
let fixture = Arc::new(TestBoundedDiscovery {
|
||||
disk: Arc::clone(&disk),
|
||||
response_gate: TestFormatReadGate::default(),
|
||||
});
|
||||
*peer.peer.bounded_discovery.lock().expect("bounded fixture") = Some(Arc::clone(&fixture));
|
||||
let remote = remote_disk_for_addr(&peer.addr).await;
|
||||
let task = tokio::spawn(async move { remote.discover_delete_transactions_bounded("bucket", "prefix", 8).await });
|
||||
tokio::time::timeout(Duration::from_secs(5), fixture.response_gate.entered.notified())
|
||||
.await
|
||||
.expect("disk completed bounded scan");
|
||||
task.abort();
|
||||
assert!(task.await.expect_err("caller cancelled").is_cancelled());
|
||||
assert_eq!(disk.public_delete_discovery_sessions_for_test(), 0);
|
||||
fixture.response_gate.release.cancel();
|
||||
peer.stop().await;
|
||||
assert_eq!(disk.public_delete_discovery_sessions_for_test(), 0);
|
||||
assert!(prefix.join("child-63").is_dir());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_index_old_disk_info_handler_is_not_a_ready_empty_index() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
|
||||
.await
|
||||
.expect("old peer must bind");
|
||||
let remote = remote_disk_for_addr(&peer.addr).await;
|
||||
remote
|
||||
.set_disk_id(Some(Uuid::new_v4()))
|
||||
.await
|
||||
.expect("expected disk identity");
|
||||
assert!(matches!(
|
||||
remote.list_multipart_index_roots("bucket", None).await,
|
||||
Err(DiskError::MethodNotAllowed)
|
||||
));
|
||||
assert_eq!(
|
||||
peer.peer.disk_info_calls.load(Ordering::Acquire),
|
||||
1,
|
||||
"exercise the actual legacy JSON handler"
|
||||
);
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bounded_delete_discovery_old_peer_rejects_without_paginated_fallback() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
|
||||
.await
|
||||
.expect("old peer must bind");
|
||||
let remote = remote_disk_for_addr(&peer.addr).await;
|
||||
assert_eq!(
|
||||
remote
|
||||
.discover_delete_transactions_bounded("bucket", "prefix", 4096)
|
||||
.await
|
||||
.expect_err("old peer cannot prove a bounded inventory"),
|
||||
DiskError::UnsupportedDisk
|
||||
);
|
||||
let transaction = Uuid::new_v4();
|
||||
assert_eq!(
|
||||
remote
|
||||
.inspect_delete_transaction_bounded("bucket", "prefix", transaction, 65536)
|
||||
.await
|
||||
.expect_err("old peer cannot inspect with a budget"),
|
||||
DiskError::UnsupportedDisk
|
||||
);
|
||||
assert_eq!(
|
||||
remote
|
||||
.settle_delete_transaction_bounded(
|
||||
"bucket",
|
||||
crate::disk::BoundedDeleteRecoverySettlement {
|
||||
object: "prefix".to_owned(),
|
||||
transaction,
|
||||
expected_fingerprint: [0; 32],
|
||||
action: DeleteRecoveryAction::PublishCommit,
|
||||
max_entries: 65536,
|
||||
},
|
||||
None
|
||||
)
|
||||
.await
|
||||
.expect_err("old peer cannot settle with a budget"),
|
||||
DiskError::UnsupportedDisk
|
||||
);
|
||||
assert_eq!(
|
||||
*peer.peer.unsupported_methods.lock().expect("observed methods"),
|
||||
vec!["/node_service.NodeService/RecoverDeleteTransaction"; 3]
|
||||
);
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_source_recovery_old_peer_rejects_without_generic_fallback() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
@@ -5740,10 +6016,18 @@ mod tests {
|
||||
object_read_all_disks: Arc<StdMutex<Vec<String>>>,
|
||||
format_data: Bytes,
|
||||
read_all_data: Bytes,
|
||||
fail_format_reads: Arc<AtomicBool>,
|
||||
next_format_read_gate: Arc<StdMutex<Option<Arc<TestFormatReadGate>>>>,
|
||||
bounded_discovery: Arc<StdMutex<Option<Arc<TestBoundedDiscovery>>>>,
|
||||
unsupported_methods: Arc<StdMutex<Vec<String>>>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct TestBoundedDiscovery {
|
||||
disk: Arc<crate::disk::local::LocalDisk>,
|
||||
response_gate: TestFormatReadGate,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct TestFormatReadGate {
|
||||
entered: tokio::sync::Notify,
|
||||
@@ -5759,7 +6043,9 @@ mod tests {
|
||||
object_read_all_disks: Arc::default(),
|
||||
format_data,
|
||||
read_all_data,
|
||||
fail_format_reads: Arc::default(),
|
||||
next_format_read_gate: Arc::default(),
|
||||
bounded_discovery: Arc::default(),
|
||||
unsupported_methods: Arc::default(),
|
||||
}
|
||||
}
|
||||
@@ -5859,6 +6145,56 @@ mod tests {
|
||||
Ok(grpc.unary(method, request).await)
|
||||
})
|
||||
}
|
||||
"/node_service.NodeService/RecoverDeleteTransaction"
|
||||
if self.peer.bounded_discovery.lock().expect("bounded fixture").is_some() =>
|
||||
{
|
||||
#[derive(Clone)]
|
||||
struct BoundedSvc(Arc<TestBoundedDiscovery>);
|
||||
impl tonic::server::UnaryService<RecoverDeleteTransactionRequest> for BoundedSvc {
|
||||
type Response = rustfs_protos::proto_gen::node_service::RecoverDeleteTransactionResponse;
|
||||
type Future = Pin<Box<dyn Future<Output = std::result::Result<Response<Self::Response>, Status>> + Send>>;
|
||||
fn call(&mut self, request: Request<RecoverDeleteTransactionRequest>) -> Self::Future {
|
||||
let fixture = Arc::clone(&self.0);
|
||||
Box::pin(async move {
|
||||
let request = request.into_inner();
|
||||
let operation: DeleteRecoveryRpcOperation =
|
||||
rustfs_protos::decode_delete_recovery_payload(&request.operation)
|
||||
.map_err(|_| Status::invalid_argument("invalid bounded request"))?;
|
||||
let DeleteRecoveryRpcOperation::DiscoverBounded { object, max_entries } = operation else {
|
||||
return Err(Status::invalid_argument("unexpected fallback"));
|
||||
};
|
||||
let page = fixture
|
||||
.disk
|
||||
.discover_delete_transactions_bounded(&request.volume, &object, max_entries)
|
||||
.await
|
||||
.map_err(|_| Status::internal("bounded discovery failed"))?;
|
||||
// Lose the response only after the real disk operation has completed.
|
||||
fixture.response_gate.entered.notify_one();
|
||||
fixture.response_gate.release.cancelled().await;
|
||||
let result = rmp_serde::to_vec_named(&DeleteRecoveryRpcReply::DiscoveredBounded(page))
|
||||
.map_err(|_| Status::internal("encode bounded reply"))?;
|
||||
Ok(Response::new(rustfs_protos::proto_gen::node_service::RecoverDeleteTransactionResponse {
|
||||
success: true,
|
||||
protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION,
|
||||
result: result.into(),
|
||||
error: None,
|
||||
}))
|
||||
})
|
||||
}
|
||||
}
|
||||
let fixture = self
|
||||
.peer
|
||||
.bounded_discovery
|
||||
.lock()
|
||||
.expect("bounded fixture")
|
||||
.clone()
|
||||
.expect("configured fixture");
|
||||
Box::pin(async move {
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let mut grpc = tonic::server::Grpc::new(codec);
|
||||
Ok(grpc.unary(BoundedSvc(fixture), request).await)
|
||||
})
|
||||
}
|
||||
"/node_service.NodeService/ReadAll" => {
|
||||
#[derive(Clone)]
|
||||
struct ReadAllSvc(AuthenticatedReadPeer);
|
||||
@@ -5882,6 +6218,9 @@ mod tests {
|
||||
gate.entered.notify_one();
|
||||
gate.release.cancelled().await;
|
||||
}
|
||||
if peer.fail_format_reads.load(Ordering::Acquire) {
|
||||
return Err(Status::unavailable("format read temporarily unavailable"));
|
||||
}
|
||||
peer.format_data.clone()
|
||||
} else {
|
||||
peer.object_read_all_disks
|
||||
@@ -8847,6 +9186,124 @@ mod tests {
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(remote_disk_recovery_probe)]
|
||||
async fn endpoint_reconnect_failure_preserves_registered_remote_recovery() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(crate::disk::disk_store::ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, Some("true")),
|
||||
(rustfs_config::ENV_DRIVE_RETURNING_SUCCESS_THRESHOLD, Some("1")),
|
||||
],
|
||||
async {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let mut reference = crate::layout::format::FormatV3::new(1, 1);
|
||||
reference.erasure.this = reference.erasure.sets[0][0];
|
||||
let peer = TestGrpcPeer::spawn(
|
||||
Bytes::from(reference.to_json().expect("format should serialize")),
|
||||
Bytes::from_static(b"recovered-peer-data"),
|
||||
)
|
||||
.await
|
||||
.expect("authenticated regression peer is required");
|
||||
let set = missing_read_set(&peer, reference).await;
|
||||
let disk = crate::disk::new_disk(
|
||||
&set.set_endpoints[0],
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: true,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("registered remote disk should construct");
|
||||
let crate::disk::Disk::Remote(remote) = disk.as_ref() else {
|
||||
panic!("fixture must create a remote disk");
|
||||
};
|
||||
assert!(remote.health_check);
|
||||
remote.force_runtime_state_for_test(RuntimeDriveHealthState::Offline);
|
||||
set.disks.write().await[0] = Some(Arc::clone(&disk));
|
||||
|
||||
peer.peer.fail_format_reads.store(true, Ordering::Release);
|
||||
set.connect_disks().await;
|
||||
peer.peer.fail_format_reads.store(false, Ordering::Release);
|
||||
let cancelled_after_failure = remote.cancel_token.is_cancelled();
|
||||
assert!(peer.peer.read_all_calls() > 0, "reconnect must attempt the failing format RPC");
|
||||
assert!(
|
||||
Arc::ptr_eq(&disk, set.disks.read().await[0].as_ref().expect("existing slot must remain")),
|
||||
"a failed reconnect must not publish its unvalidated candidate"
|
||||
);
|
||||
remote.spawn_recovery_monitor_if_needed();
|
||||
let recovery = time::timeout(Duration::from_secs(5), async {
|
||||
while !disk.is_online().await {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
disk.read_all("bucket", "object").await
|
||||
})
|
||||
.await;
|
||||
disk.close().await.expect("stop the registered disk's monitoring");
|
||||
peer.stop().await;
|
||||
assert!(!cancelled_after_failure, "failed replacement must retain the registered monitor");
|
||||
assert_eq!(
|
||||
recovery
|
||||
.expect("failed reconnect must leave the registered disk able to recover without another sweep")
|
||||
.expect("the recovered registered disk must serve real I/O"),
|
||||
Bytes::from_static(b"recovered-peer-data")
|
||||
);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(remote_disk_recovery_probe)]
|
||||
async fn endpoint_reconnect_stops_old_remote_monitor_only_after_publication() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let mut reference = crate::layout::format::FormatV3::new(1, 1);
|
||||
reference.erasure.this = reference.erasure.sets[0][0];
|
||||
let peer = TestGrpcPeer::spawn(
|
||||
Bytes::from(reference.to_json().expect("format should serialize")),
|
||||
Bytes::from_static(b"replacement-peer-data"),
|
||||
)
|
||||
.await
|
||||
.expect("authenticated regression peer is required");
|
||||
let set = missing_read_set(&peer, reference).await;
|
||||
let disk = crate::disk::new_disk(
|
||||
&set.set_endpoints[0],
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: true,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("registered remote disk should construct");
|
||||
let crate::disk::Disk::Remote(remote) = disk.as_ref() else {
|
||||
panic!("fixture must create a remote disk");
|
||||
};
|
||||
remote.force_runtime_state_for_test(RuntimeDriveHealthState::Offline);
|
||||
set.disks.write().await[0] = Some(Arc::clone(&disk));
|
||||
let gate = peer.peer.pause_next_format_read();
|
||||
let pending_set = Arc::clone(&set);
|
||||
let pending = tokio::spawn(async move { pending_set.connect_disks().await });
|
||||
time::timeout(Duration::from_secs(5), gate.entered.notified())
|
||||
.await
|
||||
.expect("replacement must reach the format validation barrier");
|
||||
let cancelled_before_publication = remote.cancel_token.is_cancelled();
|
||||
gate.release.cancel();
|
||||
pending.await.expect("validated replacement must finish publication");
|
||||
let replacement = set.disks.read().await[0].clone().expect("replacement must occupy the slot");
|
||||
let readback = replacement.read_all("bucket", "object").await;
|
||||
replacement.close().await.expect("stop replacement monitoring");
|
||||
peer.stop().await;
|
||||
assert!(
|
||||
!cancelled_before_publication,
|
||||
"the registered monitor must survive pending format validation"
|
||||
);
|
||||
assert!(!Arc::ptr_eq(&disk, &replacement), "validated replacement must be published");
|
||||
assert!(remote.cancel_token.is_cancelled(), "the displaced monitor must stop after publication");
|
||||
assert_eq!(
|
||||
readback.expect("replacement must serve real I/O"),
|
||||
Bytes::from_static(b"replacement-peer-data")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(remote_disk_recovery_probe)]
|
||||
async fn endpoint_reconnect_admits_ready_drive_while_another_format_probe_is_blocked() {
|
||||
@@ -11543,4 +12000,15 @@ mod tests {
|
||||
|
||||
accept_task.abort();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prewarm_claim_is_deduplicated_and_released_after_failure() {
|
||||
let addr = "http://prewarm-retry-test.invalid:9000";
|
||||
release_control_channel_prewarm(addr);
|
||||
assert!(claim_control_channel_prewarm(addr));
|
||||
assert!(!claim_control_channel_prewarm(addr));
|
||||
release_control_channel_prewarm(addr);
|
||||
assert!(claim_control_channel_prewarm(addr));
|
||||
release_control_channel_prewarm(addr);
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -481,6 +481,14 @@ pub(crate) fn record_remote_disk_grpc_write_all_request() {
|
||||
.record_outgoing_request_for_operation_and_backend(INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC);
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_disk_grpc_write_all_duration(duration: Duration) {
|
||||
global_internode_metrics().record_duration_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_GRPC_WRITE_ALL,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
duration,
|
||||
);
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_disk_grpc_write_all_sent_bytes(bytes: usize) {
|
||||
global_internode_metrics().record_sent_bytes_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_GRPC_WRITE_ALL,
|
||||
@@ -521,6 +529,14 @@ pub(crate) fn record_remote_disk_grpc_read_all_request() {
|
||||
.record_outgoing_request_for_operation_and_backend(INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC);
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_disk_grpc_read_all_duration(duration: Duration) {
|
||||
global_internode_metrics().record_duration_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_GRPC_READ_ALL,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
duration,
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
fn read_version_request_mode(opts: &ReadOptions) -> ReadVersionRequestMode {
|
||||
match (opts.read_data, opts.healing, opts.incl_free_versions) {
|
||||
@@ -598,6 +614,17 @@ pub(crate) fn record_remote_disk_grpc_batch_read_version_stage(stage: &'static s
|
||||
);
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_disk_grpc_batch_read_version_duration(duration: Duration) {
|
||||
if !rustfs_io_metrics::get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
global_internode_metrics().record_duration_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
duration,
|
||||
);
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_disk_grpc_batch_read_version_error() {
|
||||
if !rustfs_io_metrics::get_stage_metrics_enabled() {
|
||||
return;
|
||||
@@ -690,6 +717,14 @@ pub(crate) fn record_remote_disk_grpc_read_multiple_recv_bytes(bytes: usize) {
|
||||
record_grpc_payload_size(INTERNODE_OPERATION_GRPC_READ_MULTIPLE, bytes);
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_disk_grpc_read_multiple_duration(duration: Duration) {
|
||||
global_internode_metrics().record_duration_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
duration,
|
||||
);
|
||||
}
|
||||
|
||||
/// Payload-size threshold (bytes) above which a unary internode gRPC response is counted as a
|
||||
/// "large payload" for alerting. Env-overridable via `RUSTFS_INTERNODE_RPC_LARGE_PAYLOAD_WARN_BYTES`.
|
||||
fn internode_rpc_large_payload_warn_bytes() -> usize {
|
||||
|
||||
@@ -3506,6 +3506,7 @@ mod tests {
|
||||
replication_decision: String::new(),
|
||||
checksum: None,
|
||||
multipart_completion_replayed: false,
|
||||
backfilled_old_current_size: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
use codec::{fixture_inline_pool_meta_replicas as replicas, fixture_split_inline_pool_meta_replicas as split};
|
||||
use std::path::PathBuf;
|
||||
use tokio::fs;
|
||||
use uuid::Uuid;
|
||||
|
||||
struct PhysicalFixture {
|
||||
_directories: Vec<tempfile::TempDir>,
|
||||
@@ -216,6 +217,264 @@ async fn pool_meta_native_physical_split_without_ordinary_commit_performs_no_mut
|
||||
"all current heads, identities and complete backups must remain unchanged"
|
||||
);
|
||||
assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
|
||||
state
|
||||
.ensure_write_safe("ordinary authority temporarily unavailable")
|
||||
.expect("missing ordinary authority alone must not install a permanent corruption latch");
|
||||
}
|
||||
|
||||
async fn interrupted_first_commit_fixture() -> (PhysicalFixture, PoolMetaCommittedCandidate) {
|
||||
let fixture = fixture(3).await;
|
||||
let PoolMetaReplica::Valid {
|
||||
previous: Some(previous),
|
||||
committed: false,
|
||||
..
|
||||
} = decode_pool_meta_replica(fixture.pending.clone())
|
||||
else {
|
||||
panic!("fixture must retain its acknowledged predecessor");
|
||||
};
|
||||
let physical = disks(&fixture.store).await;
|
||||
let prepared = replicas(&fixture.pending, OffsetDateTime::now_utc()).await;
|
||||
for (index, disk) in physical[0].iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &prepared[index])
|
||||
.await
|
||||
.expect("one pool must retain an ordinary-readable PREPARE");
|
||||
}
|
||||
(fixture, *previous)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_physical_interrupted_first_commit_restores_proven_predecessor() {
|
||||
let (mut fixture, previous) = interrupted_first_commit_fixture().await;
|
||||
let before = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(
|
||||
before[1..]
|
||||
.iter()
|
||||
.all(|read| matches!(read.replica, PoolMetaReplica::Unreadable(_)))
|
||||
);
|
||||
assert!(matches!(before[0].replica, PoolMetaReplica::Valid { committed: false, .. }));
|
||||
let mut state = clean_state(&fixture);
|
||||
let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
|
||||
assert!(
|
||||
repair(&fixture.store, &mut state, &fence)
|
||||
.await
|
||||
.expect("ordinary PREPARE must authorize restoring only its acknowledged predecessor")
|
||||
);
|
||||
state
|
||||
.ensure_write_safe("first commit rollback")
|
||||
.expect("rollback must finish its transaction");
|
||||
assert_ne!(
|
||||
previous.canonical, fixture.canonical,
|
||||
"the interrupted COMMIT must not become authoritative"
|
||||
);
|
||||
fixture.canonical = previous.canonical;
|
||||
fixture.revision = previous.revision;
|
||||
assert_committed(&fixture).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_physical_interrupted_rollback_retains_authority_and_resumes() {
|
||||
let (mut fixture, previous) = interrupted_first_commit_fixture().await;
|
||||
let pause = Arc::new(NativeApplyPause::default());
|
||||
let reached = pause.reached.notified();
|
||||
tokio::pin!(reached);
|
||||
reached.as_mut().enable();
|
||||
let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
|
||||
let mut state = clean_state(&fixture);
|
||||
let recovery = tokio::spawn({
|
||||
let store = Arc::clone(&fixture.store);
|
||||
let pause = Arc::clone(&pause);
|
||||
async move {
|
||||
AFTER_NATIVE_POOL_APPLY
|
||||
.scope(pause, async move { repair(&store, &mut state, &fence).await })
|
||||
.await
|
||||
}
|
||||
});
|
||||
tokio::time::timeout(std::time::Duration::from_secs(10), reached)
|
||||
.await
|
||||
.expect("real rollback should pause after one pool is durably restored");
|
||||
let partial = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(partial[0].replica, PoolMetaReplica::Valid { committed: false, .. }));
|
||||
assert!(matches!(&partial[1].replica, PoolMetaReplica::Valid { canonical, committed: true, .. }
|
||||
if canonical == &previous.canonical));
|
||||
assert!(matches!(partial[2].replica, PoolMetaReplica::Unreadable(_)));
|
||||
recovery.abort();
|
||||
assert!(
|
||||
recovery
|
||||
.await
|
||||
.expect_err("the paused repair must be cancelled")
|
||||
.is_cancelled()
|
||||
);
|
||||
let mut state = clean_state(&fixture);
|
||||
let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
|
||||
assert!(
|
||||
repair(&fixture.store, &mut state, &fence)
|
||||
.await
|
||||
.expect("restart must resume from the retained ordinary PREPARE")
|
||||
);
|
||||
fixture.canonical = previous.canonical;
|
||||
fixture.revision = previous.revision;
|
||||
assert_committed(&fixture).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_physical_rollback_resumes_after_last_prepare_loses_read_quorum() {
|
||||
let (mut fixture, previous) = interrupted_first_commit_fixture().await;
|
||||
let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
|
||||
let inspected = inspect(&fixture.store.pools, &fence)
|
||||
.await
|
||||
.expect("physical rollback inventory");
|
||||
let replacements = plan(
|
||||
&inspected,
|
||||
&fixture.canonical,
|
||||
fixture.revision,
|
||||
&fixture.identity,
|
||||
Some(&previous.canonical),
|
||||
)
|
||||
.await
|
||||
.expect("the ordinary PREPARE must authorize its predecessor");
|
||||
for (pool, count) in [(1, 4), (2, 4), (0, 2)] {
|
||||
for (index, replacement) in replacements[pool].iter().enumerate().take(count) {
|
||||
let snapshot = &inspected[pool].snapshots[index];
|
||||
assert_eq!(
|
||||
inspected[pool].disks[index]
|
||||
.pool_meta_native_recovery(Request::Apply {
|
||||
object: PoolMetaNativeObject::PoolBin,
|
||||
expected_disk_id: snapshot.disk_id,
|
||||
expected_current_sha256: snapshot.current.as_ref().map(|raw| Sha256::digest(raw).into()),
|
||||
expected_backup_inventory_sha256: snapshot.inventory_sha256().expect("original inventory hash"),
|
||||
replacement: replacement.clone(),
|
||||
})
|
||||
.await
|
||||
.expect("partial rollback must use the real conditional publication and sync path"),
|
||||
Response::Applied
|
||||
);
|
||||
}
|
||||
}
|
||||
let partial = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(partial[0].replica, PoolMetaReplica::Unreadable(_)));
|
||||
assert!(partial[1..].iter().all(|read| matches!(&read.replica,
|
||||
PoolMetaReplica::Valid { canonical, committed: true, .. } if canonical == &previous.canonical)));
|
||||
let mut state = clean_state(&fixture);
|
||||
assert!(
|
||||
repair(&fixture.store, &mut state, &fence)
|
||||
.await
|
||||
.expect("restored ordinary COMMIT must finish rollback after the last PREPARE becomes split")
|
||||
);
|
||||
fixture.canonical = previous.canonical;
|
||||
fixture.revision = previous.revision;
|
||||
assert_committed(&fixture).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_physical_rollback_backup_does_not_poison_reused_generation() {
|
||||
let (mut fixture, previous) = interrupted_first_commit_fixture().await;
|
||||
let mut state = clean_state(&fixture);
|
||||
let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
|
||||
assert!(repair(&fixture.store, &mut state, &fence).await.expect("initial rollback"));
|
||||
let revision = PoolMetaRevision {
|
||||
transaction_id: Some(Uuid::new_v4()),
|
||||
..fixture.revision
|
||||
};
|
||||
let mut meta = previous.meta.clone();
|
||||
meta.pools[0].last_update += Duration::seconds(1);
|
||||
let canonical = encode_pool_meta_v3_envelope(&meta, revision, true, None).expect("later COMMIT");
|
||||
let pending = encode_pool_meta_v3_envelope(&meta, revision, false, Some(&previous)).expect("later PREPARE");
|
||||
let modified = OffsetDateTime::now_utc() + Duration::seconds(1);
|
||||
let split = split(&pending, &canonical, modified).await;
|
||||
let commit = replicas(&canonical, modified).await;
|
||||
let prepared = replicas(&pending, modified).await;
|
||||
let directory = Uuid::new_v4();
|
||||
let retained = snapshots(&fixture.store).await;
|
||||
for (pool_index, pool) in disks(&fixture.store).await.into_iter().enumerate() {
|
||||
for (disk_index, disk) in pool.iter().enumerate() {
|
||||
assert!(
|
||||
!retained[pool_index][disk_index].backups.is_empty(),
|
||||
"the abandoned PREPARE must remain as real on-disk evidence"
|
||||
);
|
||||
let object = object_path(disk);
|
||||
fs::write(object.join("xl.meta"), &if pool_index < 2 { &split } else { &commit }[disk_index])
|
||||
.await
|
||||
.expect("install later partial COMMIT without clearing old backups");
|
||||
fs::create_dir(object.join(directory.to_string()))
|
||||
.await
|
||||
.expect("later backup directory");
|
||||
fs::write(object.join(directory.to_string()).join("xl.meta.bkp"), &prepared[disk_index])
|
||||
.await
|
||||
.expect("later PREPARE backup");
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
repair(&fixture.store, &mut state, &fence)
|
||||
.await
|
||||
.expect("an ordinary later COMMIT must survive the abandoned sibling PREPARE")
|
||||
);
|
||||
fixture.canonical = canonical;
|
||||
fixture.revision = revision;
|
||||
assert_committed(&fixture).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_physical_rollback_rejects_retained_commit_without_mutation() {
|
||||
let (fixture, _) = interrupted_first_commit_fixture().await;
|
||||
let physical = disks(&fixture.store).await;
|
||||
let commit = replicas(&fixture.canonical, OffsetDateTime::now_utc()).await;
|
||||
let directory = Uuid::new_v4();
|
||||
let object = object_path(&physical[0][0]);
|
||||
fs::create_dir(object.join(directory.to_string()))
|
||||
.await
|
||||
.expect("retained COMMIT directory");
|
||||
fs::write(object.join(directory.to_string()).join("xl.meta.bkp"), &commit[0])
|
||||
.await
|
||||
.expect("retain evidence that the interrupted transaction may already have committed");
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let mut state = clean_state(&fixture);
|
||||
let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
|
||||
assert!(repair(&fixture.store, &mut state, &fence).await.is_err());
|
||||
assert_eq!(snapshots(&fixture.store).await, before, "ambiguous COMMIT evidence must remain untouched");
|
||||
assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_physical_rollback_rejects_competing_prepares_without_mutation() {
|
||||
let (fixture, previous) = interrupted_first_commit_fixture().await;
|
||||
let pending = encode_pool_meta_v3_envelope(
|
||||
&previous.meta,
|
||||
PoolMetaRevision {
|
||||
transaction_id: Some(Uuid::new_v4()),
|
||||
..fixture.revision
|
||||
},
|
||||
false,
|
||||
Some(&previous),
|
||||
)
|
||||
.expect("competing PREPARE must remain structurally valid");
|
||||
let physical = disks(&fixture.store).await;
|
||||
let prepared = replicas(&pending, OffsetDateTime::now_utc()).await;
|
||||
for (index, disk) in physical[1].iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &prepared[index])
|
||||
.await
|
||||
.expect("second pool must expose the competing transaction through ordinary reads");
|
||||
}
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let mut state = clean_state(&fixture);
|
||||
let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
|
||||
let error = repair(&fixture.store, &mut state, &fence)
|
||||
.await
|
||||
.expect_err("two distinct prepared transactions must not authorize a guessed rollback");
|
||||
assert_eq!(
|
||||
error
|
||||
.pool_metadata_failure()
|
||||
.expect("transaction conflict must be typed")
|
||||
.kind,
|
||||
crate::error::PoolMetadataFailure::RecoveryRequired
|
||||
);
|
||||
assert_eq!(snapshots(&fixture.store).await, before, "competing transactions must remain untouched");
|
||||
assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
struct IdentityEpochFork {
|
||||
@@ -467,7 +726,7 @@ async fn pool_meta_native_physical_partial_apply_resumes_after_conditional_rejec
|
||||
let inspected = inspect(&fixture.store.pools, &fence)
|
||||
.await
|
||||
.expect("real native snapshots should be obtained");
|
||||
let replacements = plan(&inspected, &fixture.canonical, fixture.revision, &fixture.identity)
|
||||
let replacements = plan(&inspected, &fixture.canonical, fixture.revision, &fixture.identity, None)
|
||||
.await
|
||||
.expect("physical recovery plan should validate");
|
||||
let snapshot = &inspected[0].snapshots[0];
|
||||
@@ -628,3 +887,559 @@ async fn pool_meta_native_physical_three_plus_one_checkpoint_converges_all_shard
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async fn owner_repair_no_backups_fixture() -> PhysicalFixture {
|
||||
let fixture = fixture(0).await;
|
||||
for pool in disks(&fixture.store).await {
|
||||
for disk in pool {
|
||||
let object = object_path(&disk);
|
||||
let mut entries = fs::read_dir(&object).await.expect("fixture metadata directory");
|
||||
while let Some(entry) = entries.next_entry().await.expect("fixture directory entry") {
|
||||
if entry.file_type().await.expect("fixture entry type").is_dir() {
|
||||
fs::remove_dir_all(entry.path())
|
||||
.await
|
||||
.expect("remove task-owned fixture rollback directory");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
fixture
|
||||
}
|
||||
|
||||
async fn owner_repair_identity_heads(store: &ECStore) -> Vec<Vec<(Uuid, Bytes)>> {
|
||||
let mut all = Vec::new();
|
||||
for pool in &store.pools {
|
||||
let set = pool.get_disks_by_key(POOL_META_IDENTITY_NAME);
|
||||
let members = set
|
||||
.disks
|
||||
.read()
|
||||
.await
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(|disk| disk.expect("identity fixture disk must be online"))
|
||||
.collect::<Vec<_>>();
|
||||
let mut heads = Vec::new();
|
||||
for disk in members {
|
||||
match disk
|
||||
.pool_meta_native_recovery(Request::InspectIdentity {})
|
||||
.await
|
||||
.expect("physical identity inspection")
|
||||
{
|
||||
Response::IdentityInspected { disk_id, raw } => heads.push((disk_id, raw)),
|
||||
_ => panic!("identity inspection must return its physical head"),
|
||||
}
|
||||
}
|
||||
all.push(heads);
|
||||
}
|
||||
all
|
||||
}
|
||||
|
||||
async fn legacy_native_owner_fixture(version: u16, stale: bool) -> (PhysicalFixture, Vec<u8>) {
|
||||
let fixture = owner_repair_no_backups_fixture().await;
|
||||
let mut meta = match decode_pool_meta_replica(fixture.canonical.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
meta, committed: true, ..
|
||||
} => meta,
|
||||
_ => panic!("fixture must provide valid committed pool metadata"),
|
||||
};
|
||||
meta.version = version;
|
||||
let raw = meta
|
||||
.encode_config_data_for_v2_gate(version == POOL_META_VERSION)
|
||||
.expect("legacy fixture should encode in its original version");
|
||||
assert_eq!(u16::from_le_bytes([raw[2], raw[3]]), version);
|
||||
let all = disks(&fixture.store).await;
|
||||
let now = OffsetDateTime::now_utc();
|
||||
for pool in &all {
|
||||
let heads = replicas(&raw, now).await;
|
||||
for (index, disk) in pool.iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &heads[index])
|
||||
.await
|
||||
.expect("install exact legacy head");
|
||||
}
|
||||
}
|
||||
if stale {
|
||||
let old = replicas(&raw, now - Duration::seconds(10)).await;
|
||||
fs::write(object_path(&all[0][0]).join("xl.meta"), &old[0])
|
||||
.await
|
||||
.expect("install stale same-raw legacy head");
|
||||
}
|
||||
(fixture, raw)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_heal_v1_with_all_upgrade_gates_disabled_keeps_exact_raw() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
|
||||
],
|
||||
async {
|
||||
let (fixture, raw) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
fixture.store.heal_pool_meta_replica_under_fence(0, guard, &mut state),
|
||||
)
|
||||
.await
|
||||
.expect("legacy native heal must terminate");
|
||||
assert!(result.is_ok(), "fully online legacy native heal should succeed: {result:?}");
|
||||
drop(state);
|
||||
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
for read in &reads {
|
||||
assert!(
|
||||
matches!(&read.replica, PoolMetaReplica::Valid { raw: bytes, revision, .. }
|
||||
if bytes == &raw && revision.version == POOL_META_V1_VERSION),
|
||||
"native heal must not upgrade a V1 raw payload when both upgrade gates are disabled"
|
||||
);
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_heal_rejects_legacy_source_with_trailing_bytes_without_writes() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
|
||||
],
|
||||
async {
|
||||
for version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
|
||||
let (fixture, raw) = legacy_native_owner_fixture(version, false).await;
|
||||
let mut source_raw = raw.clone();
|
||||
// A second MessagePack value is readable by legacy decoding but
|
||||
// must not be copied as part of an unverified repair payload.
|
||||
source_raw.push(0xc0);
|
||||
assert!(
|
||||
matches!(decode_pool_meta_replica(source_raw.clone()), PoolMetaReplica::Valid { raw, .. } if raw == source_raw),
|
||||
"ordinary legacy reads must remain compatible with trailing bytes"
|
||||
);
|
||||
let all_disks = disks(&fixture.store).await;
|
||||
let source_heads = replicas(&source_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
|
||||
for (index, disk) in all_disks[0].iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
|
||||
.await
|
||||
.expect("install readable legacy source with trailing bytes");
|
||||
}
|
||||
let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
|
||||
.await
|
||||
.expect("legacy semantic selection must remain readable");
|
||||
assert_eq!(selection.legacy_raw.as_deref(), Some(source_raw.as_slice()));
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let identities = owner_repair_identity_heads(&fixture.store).await;
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
|
||||
)
|
||||
.await
|
||||
.expect("legacy native heal must terminate");
|
||||
drop(state);
|
||||
assert!(result.is_err(), "version {version}: unparsed trailing source bytes must not be republished");
|
||||
assert_eq!(snapshots(&fixture.store).await, before, "version {version}: no pool.bin head or backup may change");
|
||||
assert_eq!(
|
||||
owner_repair_identity_heads(&fixture.store).await,
|
||||
identities,
|
||||
"version {version}: no identity shard may change"
|
||||
);
|
||||
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &source_raw));
|
||||
assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw: target, .. } if target == &raw));
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_heal_mixed_v1_v2_preserves_observed_format_floor() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
|
||||
],
|
||||
async {
|
||||
for first_version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
|
||||
for target_pool in [0, 1] {
|
||||
let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
|
||||
let mut meta = match decode_pool_meta_replica(raw_v1.clone()) {
|
||||
PoolMetaReplica::Valid { meta, committed: true, .. } => meta,
|
||||
_ => panic!("initial V1 metadata must decode"),
|
||||
};
|
||||
meta.version = POOL_META_VERSION;
|
||||
let raw_v2 = meta.encode_config_data_for_v2_gate(true).expect("equivalent V2 payload must encode");
|
||||
assert_eq!(u16::from_le_bytes([raw_v1[2], raw_v1[3]]), POOL_META_V1_VERSION);
|
||||
assert_eq!(u16::from_le_bytes([raw_v2[2], raw_v2[3]]), POOL_META_VERSION);
|
||||
let semantic_v1 = match decode_pool_meta_replica(raw_v1.clone()) {
|
||||
PoolMetaReplica::Valid { canonical, .. } => canonical,
|
||||
_ => panic!("V1 payload must remain valid"),
|
||||
};
|
||||
let semantic_v2 = match decode_pool_meta_replica(raw_v2.clone()) {
|
||||
PoolMetaReplica::Valid { canonical, .. } => canonical,
|
||||
_ => panic!("V2 payload must remain valid"),
|
||||
};
|
||||
assert_eq!(semantic_v1, semantic_v2, "the two formats must carry equivalent metadata");
|
||||
let other_version = if first_version == POOL_META_V1_VERSION { POOL_META_VERSION } else { POOL_META_V1_VERSION };
|
||||
// Pool two follows pool zero; pools zero and one cover both
|
||||
// orderings and both target formats in separate runs.
|
||||
let versions = [first_version, other_version, first_version];
|
||||
let physical = disks(&fixture.store).await;
|
||||
let timestamp = OffsetDateTime::now_utc();
|
||||
for (pool_index, pool) in physical.iter().enumerate() {
|
||||
let raw = if versions[pool_index] == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
|
||||
let heads = replicas(raw, timestamp).await;
|
||||
for (index, disk) in pool.iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &heads[index]).await.expect("install mixed legacy pool head");
|
||||
}
|
||||
}
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
|
||||
.await.expect("equivalent legacy pools should be selectable");
|
||||
assert_eq!(selection.revision.version, POOL_META_VERSION, "the observed V2 format floor must be retained");
|
||||
assert_eq!(selection.canonical.as_deref(), Some(semantic_v2.as_slice()));
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
tokio::time::timeout(std::time::Duration::from_secs(30),
|
||||
fixture.store.heal_pool_meta_replica_under_fence(target_pool, guard, &mut state))
|
||||
.await.expect("mixed legacy heal must terminate")
|
||||
.expect("equivalent legacy metadata should heal without downgrading the format floor");
|
||||
drop(state);
|
||||
let after = snapshots(&fixture.store).await;
|
||||
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
for (pool_index, read) in reads.iter().enumerate() {
|
||||
let expected_version = if pool_index == target_pool { POOL_META_VERSION } else { versions[pool_index] };
|
||||
let expected_raw = if expected_version == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
|
||||
assert!(matches!(&read.replica,
|
||||
PoolMetaReplica::Valid { raw, canonical, revision, committed: true, .. }
|
||||
if raw == expected_raw && canonical == &semantic_v2 && revision.version == expected_version
|
||||
), "pool {pool_index}, first={first_version}, target={target_pool}: preserve equivalent payload and expected legacy version");
|
||||
for (index, snapshot) in after[pool_index].iter().enumerate() {
|
||||
codec::validate_inline_pool_meta_shard(snapshot.current.as_ref().expect("physical head"), index, expected_raw)
|
||||
.await.expect("every physical shard must match its selected legacy payload");
|
||||
}
|
||||
if pool_index != target_pool {
|
||||
assert_eq!(after[pool_index], before[pool_index], "healing one pool must not modify other pools");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_heal_rejects_v1_and_v3_prepare_in_either_pool_order() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
|
||||
],
|
||||
async {
|
||||
for prepare_pool in [0, 1] {
|
||||
let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
|
||||
let (legacy_meta, legacy_canonical) = match decode_pool_meta_replica(raw_v1.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
meta,
|
||||
canonical,
|
||||
revision,
|
||||
committed: true,
|
||||
..
|
||||
} if revision.version == POOL_META_V1_VERSION => (meta, canonical),
|
||||
_ => panic!("baseline V1 metadata must be committed and readable"),
|
||||
};
|
||||
let previous = PoolMetaCommittedCandidate {
|
||||
canonical: legacy_canonical,
|
||||
meta: legacy_meta.clone(),
|
||||
revision: PoolMetaRevision::legacy(POOL_META_V1_VERSION),
|
||||
};
|
||||
let mut next = legacy_meta;
|
||||
next.version = POOL_META_GENERATION_VERSION;
|
||||
next.pools[0].last_update += Duration::seconds(1);
|
||||
let revision = PoolMetaRevision {
|
||||
version: POOL_META_GENERATION_VERSION,
|
||||
cluster_id: fixture.revision.cluster_id,
|
||||
epoch: fixture.revision.epoch,
|
||||
generation: 1,
|
||||
transaction_id: Some(Uuid::new_v4()),
|
||||
};
|
||||
let pending = encode_pool_meta_v3_envelope(&next, revision, false, Some(&previous))
|
||||
.expect("V3 PREPARE with V1 predecessor must encode");
|
||||
assert!(matches!(decode_pool_meta_replica(pending.clone()),
|
||||
PoolMetaReplica::Valid { committed: false, revision: actual, .. } if actual == revision));
|
||||
let heads = replicas(&pending, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
|
||||
let physical = disks(&fixture.store).await;
|
||||
for (index, disk) in physical[prepare_pool].iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &heads[index])
|
||||
.await
|
||||
.expect("install task-owned V3 PREPARE evidence");
|
||||
}
|
||||
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(&reads[prepare_pool].replica,
|
||||
PoolMetaReplica::Valid { raw, committed: false, revision: actual, .. }
|
||||
if raw == &pending && *actual == revision));
|
||||
let other_pool = 1 - prepare_pool;
|
||||
assert!(matches!(&reads[other_pool].replica,
|
||||
PoolMetaReplica::Valid { raw, committed: true, revision: actual, .. }
|
||||
if raw == &raw_v1 && actual.version == POOL_META_V1_VERSION));
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let identities = owner_repair_identity_heads(&fixture.store).await;
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
fixture
|
||||
.store
|
||||
.heal_pool_meta_replica_under_fence(prepare_pool, guard, &mut state),
|
||||
)
|
||||
.await
|
||||
.expect("mixed V1/V3 PREPARE heal must terminate");
|
||||
drop(state);
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"pool {prepare_pool}: native heal must not overwrite a V3 PREPARE with legacy data"
|
||||
);
|
||||
assert_eq!(
|
||||
snapshots(&fixture.store).await,
|
||||
before,
|
||||
"pool {prepare_pool}: rejection must retain all physical head and backup evidence"
|
||||
);
|
||||
assert_eq!(
|
||||
owner_repair_identity_heads(&fixture.store).await,
|
||||
identities,
|
||||
"pool {prepare_pool}: rejection must not mutate identity evidence"
|
||||
);
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_heal_rejects_v1_legacy_dont_save_true_without_writes() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
|
||||
],
|
||||
async {
|
||||
let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
|
||||
let meta = match decode_pool_meta_replica(safe_v1.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
meta, committed: true, ..
|
||||
} => meta,
|
||||
_ => panic!("baseline V1 metadata must decode"),
|
||||
};
|
||||
let statuses = meta
|
||||
.pools
|
||||
.iter()
|
||||
.map(|pool| {
|
||||
assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
|
||||
LegacyPoolStatus {
|
||||
id: pool.id,
|
||||
cmd_line: pool.cmd_line.clone(),
|
||||
last_update: pool.last_update,
|
||||
decommission: None,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let legacy = LegacyPoolMeta {
|
||||
version: POOL_META_V1_VERSION,
|
||||
pools: statuses,
|
||||
dont_save: true,
|
||||
};
|
||||
let mut dangerous_raw = Vec::new();
|
||||
dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
|
||||
dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
|
||||
dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
|
||||
let old_reader: LegacyPoolMeta =
|
||||
rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode source");
|
||||
assert!(old_reader.dont_save, "the old reader would treat this source as permanently do-not-save");
|
||||
assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
|
||||
let source = match decode_pool_meta_replica(dangerous_raw.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
canonical,
|
||||
revision,
|
||||
committed: true,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(revision.version, POOL_META_V1_VERSION);
|
||||
canonical
|
||||
}
|
||||
_ => panic!("current reader must recognize the legacy source"),
|
||||
};
|
||||
assert!(matches!(decode_pool_meta_replica(safe_v1.clone()),
|
||||
PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
|
||||
if canonical == source && revision.version == POOL_META_V1_VERSION));
|
||||
let physical = disks(&fixture.store).await;
|
||||
let source_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
|
||||
for (index, disk) in physical[0].iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
|
||||
.await
|
||||
.expect("install legacy source on pool zero");
|
||||
}
|
||||
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
|
||||
assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let identities = owner_repair_identity_heads(&fixture.store).await;
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
|
||||
)
|
||||
.await
|
||||
.expect("unsafe legacy heal must terminate");
|
||||
drop(state);
|
||||
let error = result.expect_err("native heal must reject a source whose old-reader dont_save is true");
|
||||
assert!(
|
||||
error.to_string().contains("dont_save"),
|
||||
"rejection must identify the unsafe legacy field: {error}"
|
||||
);
|
||||
assert_eq!(
|
||||
snapshots(&fixture.store).await,
|
||||
before,
|
||||
"rejection must preserve every original head and backup"
|
||||
);
|
||||
assert_eq!(
|
||||
owner_repair_identity_heads(&fixture.store).await,
|
||||
identities,
|
||||
"rejection must not write identity shards"
|
||||
);
|
||||
let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw));
|
||||
assert!(
|
||||
matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1),
|
||||
"the dangerous source bytes must not be copied into the target pool"
|
||||
);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_heal_rejects_target_v1_dont_save_true_without_writes() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
|
||||
],
|
||||
async {
|
||||
let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
|
||||
let meta = match decode_pool_meta_replica(safe_v1.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
meta, committed: true, ..
|
||||
} => meta,
|
||||
_ => panic!("baseline V1 metadata must decode"),
|
||||
};
|
||||
let statuses = meta
|
||||
.pools
|
||||
.iter()
|
||||
.map(|pool| {
|
||||
assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
|
||||
LegacyPoolStatus {
|
||||
id: pool.id,
|
||||
cmd_line: pool.cmd_line.clone(),
|
||||
last_update: pool.last_update,
|
||||
decommission: None,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let legacy = LegacyPoolMeta {
|
||||
version: POOL_META_V1_VERSION,
|
||||
pools: statuses,
|
||||
dont_save: true,
|
||||
};
|
||||
let mut dangerous_raw = Vec::new();
|
||||
dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
|
||||
dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
|
||||
dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
|
||||
let old_reader: LegacyPoolMeta =
|
||||
rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode target");
|
||||
assert!(old_reader.dont_save, "the old reader interprets the target as do-not-save");
|
||||
assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
|
||||
let source_canonical = match decode_pool_meta_replica(safe_v1.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
canonical,
|
||||
revision,
|
||||
committed: true,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(revision.version, POOL_META_V1_VERSION);
|
||||
canonical
|
||||
}
|
||||
_ => panic!("safe pool-zero source must decode"),
|
||||
};
|
||||
assert!(
|
||||
matches!(decode_pool_meta_replica(dangerous_raw.clone()),
|
||||
PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
|
||||
if canonical == source_canonical && revision.version == POOL_META_V1_VERSION),
|
||||
"the dangerous target must remain semantically equivalent to the safe source"
|
||||
);
|
||||
let physical = disks(&fixture.store).await;
|
||||
let target_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
|
||||
for (index, disk) in physical[1].iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &target_heads[index])
|
||||
.await
|
||||
.expect("install legacy dangerous target head");
|
||||
}
|
||||
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
|
||||
assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let identities = owner_repair_identity_heads(&fixture.store).await;
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
|
||||
)
|
||||
.await
|
||||
.expect("unsafe legacy target heal must terminate");
|
||||
drop(state);
|
||||
let error = result.expect_err("native heal must reject a target whose old-reader dont_save is true");
|
||||
assert!(
|
||||
error.to_string().contains("dont_save"),
|
||||
"rejection must identify the unsafe legacy field: {error}"
|
||||
);
|
||||
assert_eq!(
|
||||
snapshots(&fixture.store).await,
|
||||
before,
|
||||
"rejection must preserve all physical heads and backups"
|
||||
);
|
||||
assert_eq!(
|
||||
owner_repair_identity_heads(&fixture.store).await,
|
||||
identities,
|
||||
"rejection must not write identity shards"
|
||||
);
|
||||
let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1));
|
||||
assert!(
|
||||
matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw),
|
||||
"unsafe target evidence must not be overwritten by the safe source"
|
||||
);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
@@ -30,6 +30,7 @@ pub(super) struct NativeApplyPause {
|
||||
#[cfg(test)]
|
||||
tokio::task_local! {
|
||||
pub(super) static AFTER_NATIVE_APPLY: Arc<NativeApplyPause>;
|
||||
pub(super) static AFTER_NATIVE_POOL_APPLY: Arc<NativeApplyPause>;
|
||||
}
|
||||
|
||||
struct InspectedPool {
|
||||
@@ -168,6 +169,7 @@ async fn plan(
|
||||
canonical: &[u8],
|
||||
revision: PoolMetaRevision,
|
||||
identity: &[u8],
|
||||
rollback: Option<&[u8]>,
|
||||
) -> Result<Vec<Vec<Bytes>>> {
|
||||
let mut pending = Vec::<Vec<u8>>::new();
|
||||
let mut predecessors = Vec::<Vec<u8>>::new();
|
||||
@@ -223,23 +225,83 @@ async fn plan(
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut obsolete = Vec::<Vec<u8>>::new();
|
||||
for payload in &decoded {
|
||||
match decode_pool_meta_replica(payload.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
// The last ordinary PREPARE can itself become split during rollback. In
|
||||
// that case another pool already exposes the restored ordinary COMMIT.
|
||||
// Its authority permits continuing the rollback, never promoting a child
|
||||
// transaction reconstructed from disk evidence.
|
||||
let mut continuation: Option<(Vec<u8>, PoolMetaRevision)> = None;
|
||||
if rollback.is_none() {
|
||||
let mut outstanding = Vec::new();
|
||||
for pool in inspected {
|
||||
for (index, snapshot) in pool.snapshots.iter().enumerate() {
|
||||
let raw = snapshot.current.as_ref().ok_or_else(invalid)?;
|
||||
if codec::validate_inline_pool_meta_shard(raw, index, canonical).await.is_err() {
|
||||
outstanding.push((index, raw));
|
||||
}
|
||||
}
|
||||
}
|
||||
for payload in &decoded {
|
||||
if let PoolMetaReplica::Valid {
|
||||
canonical: candidate,
|
||||
revision: candidate_revision,
|
||||
committed: false,
|
||||
previous: Some(previous),
|
||||
..
|
||||
} if candidate_revision == revision && candidate == canonical => {
|
||||
if !pending.contains(payload) {
|
||||
pending.push(payload.clone());
|
||||
} = decode_pool_meta_replica(payload.clone())
|
||||
&& previous.revision == revision
|
||||
&& previous.canonical == canonical
|
||||
{
|
||||
let mut matches = true;
|
||||
for (index, raw) in &outstanding {
|
||||
matches &= codec::validate_inline_pool_meta_shard(raw, *index, payload).await.is_ok()
|
||||
|| codec::validate_inline_pool_meta_shard(raw, *index, &candidate).await.is_ok();
|
||||
}
|
||||
if !predecessors.contains(&previous.canonical) {
|
||||
predecessors.push(previous.canonical);
|
||||
if matches {
|
||||
if !outstanding.is_empty()
|
||||
&& continuation.as_ref().is_some_and(|(selected, selected_revision)| {
|
||||
selected != &candidate || *selected_revision != candidate_revision
|
||||
})
|
||||
{
|
||||
return Err(known_validation(invalid(), "native_transaction_validation"));
|
||||
}
|
||||
continuation.get_or_insert((candidate, candidate_revision));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let (canonical, revision, rollback) = match &continuation {
|
||||
Some((transaction, next_revision)) => (transaction.as_slice(), *next_revision, Some(canonical)),
|
||||
None => (canonical, revision, rollback),
|
||||
};
|
||||
for payload in &decoded {
|
||||
if let PoolMetaReplica::Valid {
|
||||
canonical: candidate,
|
||||
revision: candidate_revision,
|
||||
committed: false,
|
||||
previous: Some(previous),
|
||||
..
|
||||
} = decode_pool_meta_replica(payload.clone())
|
||||
&& candidate_revision == revision
|
||||
&& candidate == canonical
|
||||
{
|
||||
pending.push(payload.clone());
|
||||
if !predecessors.contains(&previous.canonical) {
|
||||
predecessors.push(previous.canonical);
|
||||
}
|
||||
}
|
||||
}
|
||||
if pending.len() > 1 {
|
||||
return Err(known_validation(
|
||||
Error::other("native V3 prepare records disagree on the predecessor"),
|
||||
"native_transaction_validation",
|
||||
));
|
||||
}
|
||||
let mut obsolete = Vec::<Vec<u8>>::new();
|
||||
for payload in &decoded {
|
||||
if pending.contains(payload) {
|
||||
continue;
|
||||
}
|
||||
match decode_pool_meta_replica(payload.clone()) {
|
||||
PoolMetaReplica::Valid { revision: older, .. }
|
||||
if older.is_generation_protocol()
|
||||
&& older.cluster_id == revision.cluster_id
|
||||
@@ -251,6 +313,23 @@ async fn plan(
|
||||
// fully reconstructed and bound into the native CAS.
|
||||
obsolete.push(payload.clone());
|
||||
}
|
||||
PoolMetaReplica::Valid {
|
||||
revision: abandoned,
|
||||
committed: false,
|
||||
previous: Some(previous),
|
||||
..
|
||||
} if abandoned.is_generation_protocol()
|
||||
&& abandoned.cluster_id == revision.cluster_id
|
||||
&& abandoned.epoch == revision.epoch
|
||||
&& abandoned.generation == revision.generation
|
||||
&& abandoned.transaction_id != revision.transaction_id
|
||||
&& predecessors.contains(&previous.canonical) =>
|
||||
{
|
||||
// A rollback retains the failed PREPARE as evidence. A later
|
||||
// transaction may reuse that generation, but only a backup of
|
||||
// an uncommitted sibling with the same predecessor is obsolete.
|
||||
obsolete.push(payload.clone());
|
||||
}
|
||||
PoolMetaReplica::Valid {
|
||||
canonical: candidate,
|
||||
revision: candidate_revision,
|
||||
@@ -272,13 +351,7 @@ async fn plan(
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
if pending.len() > 1 {
|
||||
return Err(known_validation(
|
||||
Error::other("native V3 prepare records disagree on the predecessor"),
|
||||
"native_transaction_validation",
|
||||
));
|
||||
}
|
||||
if pending.is_empty() {
|
||||
if pending.is_empty() || rollback.is_some_and(|previous| !predecessors.iter().any(|candidate| candidate == previous)) {
|
||||
return Err(invalid());
|
||||
}
|
||||
for payload in &decoded {
|
||||
@@ -294,13 +367,25 @@ async fn plan(
|
||||
for (index, snapshot) in pool.snapshots.iter().enumerate() {
|
||||
let raw = snapshot.current.as_ref().ok_or_else(invalid)?;
|
||||
let mut matched = false;
|
||||
for payload in std::iter::once(canonical).chain(pending.iter().map(Vec::as_slice)) {
|
||||
for payload in std::iter::once(canonical)
|
||||
.chain(pending.iter().map(Vec::as_slice))
|
||||
.chain(rollback)
|
||||
{
|
||||
matched |= codec::validate_inline_pool_meta_shard(raw, index, payload).await.is_ok();
|
||||
}
|
||||
if !matched {
|
||||
return Err(invalid());
|
||||
}
|
||||
for backup in &snapshot.backups {
|
||||
if rollback.is_some()
|
||||
&& codec::validate_inline_pool_meta_shard(&backup.data, index, canonical)
|
||||
.await
|
||||
.is_ok()
|
||||
{
|
||||
// A retained COMMIT might have been authoritative before
|
||||
// this interruption. Never roll it back from PREPARE alone.
|
||||
return Err(invalid());
|
||||
}
|
||||
let mut matched = false;
|
||||
for payload in std::iter::once(canonical).chain(
|
||||
pending
|
||||
@@ -323,7 +408,7 @@ async fn plan(
|
||||
let mut replacements = Vec::with_capacity(inspected.len());
|
||||
for pool in inspected {
|
||||
replacements.push(
|
||||
codec::encode_inline_pool_meta_replicas(¤t(pool)?, canonical, modified)
|
||||
codec::encode_inline_pool_meta_replicas(¤t(pool)?, rollback.unwrap_or(canonical), modified)
|
||||
.await
|
||||
.map_err(|error| codec_validation(error, "native_metadata_validation"))?,
|
||||
);
|
||||
@@ -338,10 +423,12 @@ pub(super) async fn repair(
|
||||
) -> Result<bool> {
|
||||
fence.ensure_held()?;
|
||||
let reads = read_pool_meta_replicas(store.pools.clone(), true).await;
|
||||
if !reads
|
||||
let unreadable = reads
|
||||
.iter()
|
||||
.any(|read| matches!(read.replica, PoolMetaReplica::Unreadable(_)))
|
||||
{
|
||||
.enumerate()
|
||||
.filter_map(|(index, read)| matches!(read.replica, PoolMetaReplica::Unreadable(_)).then_some(index))
|
||||
.collect::<Vec<_>>();
|
||||
if unreadable.is_empty() {
|
||||
return Ok(false);
|
||||
}
|
||||
state.ensure_write_safe("native pool metadata recovery")?;
|
||||
@@ -360,8 +447,9 @@ pub(super) async fn repair(
|
||||
{
|
||||
return Err(invalid());
|
||||
}
|
||||
// Only an ordinary, successfully read COMMIT grants authority. Rebuilding
|
||||
// two erasure shards below proves integrity, never a commit decision.
|
||||
// Ordinary reads grant authority: a COMMIT selects its own state, while a
|
||||
// PREPARE can select only its acknowledged predecessor. Reconstruction
|
||||
// below proves integrity and never promotes an interrupted transaction.
|
||||
let ordinary_commits = reads
|
||||
.iter()
|
||||
.filter_map(|read| match &read.replica {
|
||||
@@ -375,7 +463,22 @@ pub(super) async fn repair(
|
||||
_ => None,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
if ordinary_commits.is_empty() {
|
||||
let ordinary_prepares = reads
|
||||
.iter()
|
||||
.filter_map(|read| match &read.replica {
|
||||
PoolMetaReplica::Valid {
|
||||
canonical,
|
||||
revision,
|
||||
committed: false,
|
||||
previous: Some(previous),
|
||||
..
|
||||
} if revision.is_generation_protocol() && previous.revision.is_generation_protocol() => {
|
||||
Some((canonical.clone(), *revision, previous.canonical.clone()))
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
if ordinary_commits.is_empty() && ordinary_prepares.is_empty() {
|
||||
return Err(invalid());
|
||||
}
|
||||
let selection = select_pool_meta_replica_reads(reads)
|
||||
@@ -395,10 +498,20 @@ pub(super) async fn repair(
|
||||
return Err(invalid());
|
||||
}
|
||||
let canonical = selection.canonical.as_ref().ok_or_else(invalid)?;
|
||||
let mut rollback = None;
|
||||
for (pending, revision, previous) in &ordinary_prepares {
|
||||
if previous == canonical {
|
||||
if rollback.is_some_and(|candidate| candidate != (pending, *revision)) {
|
||||
return Err(latch_validation(state, known_validation(invalid(), "native_transaction_validation")));
|
||||
}
|
||||
rollback = Some((pending, *revision));
|
||||
}
|
||||
}
|
||||
if !selection.revision.is_generation_protocol()
|
||||
|| !ordinary_commits
|
||||
.iter()
|
||||
.any(|(raw, revision)| raw == canonical && *revision == selection.revision)
|
||||
|| (rollback.is_none()
|
||||
&& !ordinary_commits
|
||||
.iter()
|
||||
.any(|(raw, revision)| raw == canonical && *revision == selection.revision))
|
||||
{
|
||||
return Err(invalid());
|
||||
}
|
||||
@@ -445,13 +558,18 @@ pub(super) async fn repair(
|
||||
.await
|
||||
.map_err(|error| latch_validation(state, error))?;
|
||||
let inspected = inspect(&store.pools, fence).await?;
|
||||
let replacements = plan(&inspected, canonical, selection.revision, &identity_bytes)
|
||||
let (transaction, revision) = rollback.unwrap_or((canonical, selection.revision));
|
||||
let replacements = plan(&inspected, transaction, revision, &identity_bytes, rollback.map(|_| canonical.as_slice()))
|
||||
.await
|
||||
.map_err(|error| latch_validation(state, error))?;
|
||||
fence.ensure_held()?;
|
||||
let mut arm = state.arm_transaction();
|
||||
arm.phase = Some("native_repair_apply");
|
||||
for (pool, replacements) in inspected.iter().zip(replacements) {
|
||||
let mut apply_order = inspected.iter().zip(replacements).enumerate().collect::<Vec<_>>();
|
||||
// Preserve ordinary authority until all unreadable pools are repaired.
|
||||
// Cancellation can then resume from a PREPARE or a restored COMMIT.
|
||||
apply_order.sort_by_key(|(index, _)| !unreadable.contains(index));
|
||||
for (_, (pool, replacements)) in apply_order {
|
||||
for ((disk, snapshot), replacement) in pool.disks.iter().zip(&pool.snapshots).zip(replacements) {
|
||||
fence.ensure_held()?;
|
||||
let request = Request::Apply {
|
||||
@@ -466,6 +584,11 @@ pub(super) async fn repair(
|
||||
_ => return Err(invalid()),
|
||||
}
|
||||
}
|
||||
#[cfg(test)]
|
||||
if let Ok(pause) = AFTER_NATIVE_POOL_APPLY.try_with(Arc::clone) {
|
||||
pause.reached.notify_one();
|
||||
pause.resume.notified().await;
|
||||
}
|
||||
}
|
||||
#[cfg(test)]
|
||||
if let Ok(pause) = AFTER_NATIVE_APPLY.try_with(Arc::clone) {
|
||||
@@ -627,7 +750,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn pool_meta_native_plan_restores_physical_split_with_fresh_common_identity() {
|
||||
let f = fixture().await;
|
||||
let repaired = plan(&f.pools, &f.commit, f.revision, &f.identity).await.unwrap();
|
||||
let repaired = plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.unwrap();
|
||||
assert_eq!(repaired.len(), 4);
|
||||
for pool in repaired {
|
||||
assert_eq!(
|
||||
@@ -648,7 +771,7 @@ mod tests {
|
||||
let mut f = fixture().await;
|
||||
let wrong = replicas(b"different identity", f.modified).await;
|
||||
f.pools[0].snapshots[0].identity = Some(wrong[0].clone());
|
||||
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
|
||||
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -671,10 +794,10 @@ mod tests {
|
||||
.unwrap();
|
||||
let wrong = replicas(&higher, f.modified + Duration::seconds(2)).await;
|
||||
f.pools[0].snapshots[0].current = Some(wrong[0].clone());
|
||||
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
|
||||
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
|
||||
f.pools[0].snapshots[0].current = Some(replicas(&f.pending, f.modified).await[0].clone());
|
||||
f.pools[0].snapshots[0].backups[0].data = wrong[0].clone();
|
||||
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
|
||||
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -717,17 +840,17 @@ mod tests {
|
||||
}
|
||||
}
|
||||
assert!(older_pending != f.pending);
|
||||
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_ok());
|
||||
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pool_meta_native_plan_rejects_missing_or_corrupt_evidence() {
|
||||
let mut f = fixture().await;
|
||||
let before = f.pools[1].snapshots[2].current.take();
|
||||
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
|
||||
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
|
||||
f.pools[1].snapshots[2].current = before;
|
||||
f.pools[1].snapshots[2].backups[0].data = Bytes::from_static(b"invalid metadata");
|
||||
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
|
||||
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -315,7 +315,7 @@ pub(super) async fn encode_inline_pool_meta_replicas(
|
||||
.encode_with_shard_integrity(
|
||||
Cursor::new(Bytes::copy_from_slice(canonical)),
|
||||
&mut writers,
|
||||
raw.len(),
|
||||
crate::erasure::coding::encode::ShardWritePolicy::object(raw.len()),
|
||||
1,
|
||||
IntegrityEncodeMode::Inline(canonical.len()),
|
||||
protected,
|
||||
|
||||
@@ -94,7 +94,6 @@ use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::fmt::Display;
|
||||
#[cfg(test)]
|
||||
use std::io::Cursor;
|
||||
use std::io::Write;
|
||||
use std::path::PathBuf;
|
||||
@@ -3720,33 +3719,45 @@ fn is_decommission_copy_quorum_deferral(err: &Error) -> bool {
|
||||
data_movement::data_movement_stage_source_as::<DecommissionCopyQuorumFailure>(err).is_some()
|
||||
}
|
||||
|
||||
/// The storage error of a version copy that kept timing out on a namespace
|
||||
/// lock through its in-place retries, kept typed so the entry failure policy
|
||||
/// defers it.
|
||||
/// A copy whose object namespace lock remained unavailable through its
|
||||
/// in-place retries, retained as a typed deferral.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("{0}")]
|
||||
struct DecommissionCopyLockTimeout(#[source] Error);
|
||||
struct DecommissionCopyLockUnavailable(#[source] Error);
|
||||
|
||||
fn is_decommission_copy_lock_timeout_error(err: &Error) -> bool {
|
||||
let err = data_movement::data_movement_stage_source(err).unwrap_or(err);
|
||||
matches!(err, Error::Lock(rustfs_lock::LockError::Timeout { .. }))
|
||||
fn is_decommission_copy_lock_unavailable_error(err: &Error, bucket: &str, object: &str) -> bool {
|
||||
let source = data_movement::data_movement_stage_source(err).unwrap_or(err);
|
||||
match source {
|
||||
Error::Lock(rustfs_lock::LockError::Timeout { .. }) => true,
|
||||
Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "read",
|
||||
bucket: lock_bucket,
|
||||
object: lock_object,
|
||||
..
|
||||
} => {
|
||||
data_movement::data_movement_error_is_stage(err, "prepare_put_object")
|
||||
&& lock_bucket == bucket
|
||||
&& lock_object == object
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// A version copy that timed out acquiring a namespace lock leaves the source
|
||||
/// A version copy that could not acquire its namespace lock leaves the source
|
||||
/// in place; the next attempt copies the version again or finds an equivalent
|
||||
/// committed target. The lock is held by another mutation of the key or, when
|
||||
/// this node restarted mid-copy, by its previous incarnation until that lock's
|
||||
/// lease expires on the peers. Either clears by waiting, so the entry is
|
||||
/// deferred instead of being counted as a permanent per-object failure.
|
||||
fn decommission_copy_lock_timeout_deferral(stage: &str, bucket: &str, object: &str, err: Error) -> Error {
|
||||
fn decommission_copy_lock_unavailable_deferral(stage: &str, bucket: &str, object: &str, err: Error) -> Error {
|
||||
data_movement::data_movement_context_error(
|
||||
format!("decommission {stage} timed out on a namespace lock for {bucket}/{object}: {err}"),
|
||||
DecommissionCopyLockTimeout(err),
|
||||
format!("decommission {stage} could not acquire a namespace lock for {bucket}/{object}: {err}"),
|
||||
DecommissionCopyLockUnavailable(err),
|
||||
)
|
||||
}
|
||||
|
||||
fn is_decommission_copy_lock_timeout_deferral(err: &Error) -> bool {
|
||||
data_movement::data_movement_stage_source_as::<DecommissionCopyLockTimeout>(err).is_some()
|
||||
fn is_decommission_copy_lock_unavailable_deferral(err: &Error) -> bool {
|
||||
data_movement::data_movement_stage_source_as::<DecommissionCopyLockUnavailable>(err).is_some()
|
||||
}
|
||||
|
||||
/// The deferral of a version copy whose in-place retries ended on `error`, if
|
||||
@@ -3756,8 +3767,8 @@ fn take_decommission_copy_deferral(stage: &str, bucket: &str, object: &str, erro
|
||||
return Some(decommission_copy_quorum_deferral(stage, bucket, object, err));
|
||||
}
|
||||
error
|
||||
.take_if(|err| is_decommission_copy_lock_timeout_error(err))
|
||||
.map(|err| decommission_copy_lock_timeout_deferral(stage, bucket, object, err))
|
||||
.take_if(|err| is_decommission_copy_lock_unavailable_error(err, bucket, object))
|
||||
.map(|err| decommission_copy_lock_unavailable_deferral(stage, bucket, object, err))
|
||||
}
|
||||
|
||||
fn resolve_decommission_entry_reload_result(result: Result<()>, bucket: &str, object_name: &str) -> Result<()> {
|
||||
@@ -5822,13 +5833,13 @@ enum DecommissionEntryOutcome {
|
||||
/// entry is done and must not be counted as a failure or escalated. A quorum
|
||||
/// failure of the source cleanup is deferred too: the copied source is still
|
||||
/// in place and the next attempt cleans it up again. So is a version copy that
|
||||
/// lost drive quorum or timed out on a namespace lock through its in-place
|
||||
/// lost drive quorum or could not acquire an object namespace lock through its in-place
|
||||
/// retries: the source still holds the version and the next attempt copies it
|
||||
/// again. Every other failure stays terminal.
|
||||
fn resolve_decommission_entry_failure(err: Error) -> Result<DecommissionEntryOutcome> {
|
||||
if is_decommission_source_cleanup_retryable_error(&err)
|
||||
|| is_decommission_copy_quorum_deferral(&err)
|
||||
|| is_decommission_copy_lock_timeout_deferral(&err)
|
||||
|| is_decommission_copy_lock_unavailable_deferral(&err)
|
||||
{
|
||||
return Ok(DecommissionEntryOutcome::Deferred(err));
|
||||
}
|
||||
@@ -6827,11 +6838,47 @@ impl Drop for PoolMetaTransactionArm {
|
||||
}
|
||||
}
|
||||
|
||||
fn ensure_legacy_pool_meta_republish_safe(data: &[u8]) -> Result<()> {
|
||||
if data.len() >= 4
|
||||
&& LittleEndian::read_u16(&data[2..4]) == POOL_META_V1_VERSION
|
||||
// The selection already validates the payload. The runtime-only
|
||||
// flag can only appear in the legacy three-field sequence or map.
|
||||
&& rmp::decode::read_array_len(&mut &data[4..])
|
||||
.or_else(|_| rmp::decode::read_map_len(&mut &data[4..]))
|
||||
.ok()
|
||||
== Some(3)
|
||||
&& let Ok(legacy) = rmp_serde::from_slice::<LegacyPoolMeta>(&data[4..])
|
||||
&& legacy.dont_save
|
||||
{
|
||||
return Err(Error::other("pool metadata legacy source contains dont_save=true"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Ordinary legacy reads retain their compatibility behavior, but bytes that
|
||||
// will be copied to another pool must not contain an unparsed extension.
|
||||
fn ensure_legacy_pool_meta_payload_complete(data: &[u8]) -> Result<()> {
|
||||
let payload = data
|
||||
.get(4..)
|
||||
.ok_or_else(|| Error::other("pool metadata legacy source is too short"))?;
|
||||
let mut decoder = rmp_serde::Deserializer::new(Cursor::new(payload));
|
||||
serde::de::IgnoredAny::deserialize(&mut decoder).map_err(|_| Error::other("pool metadata legacy source is invalid"))?;
|
||||
let payload_len = u64::try_from(payload.len()).map_err(|_| Error::other("pool metadata legacy source is too large"))?;
|
||||
if decoder.get_ref().position() != payload_len {
|
||||
return Err(Error::other("pool metadata legacy source has trailing bytes"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct PoolMetaSelection {
|
||||
meta: PoolMeta,
|
||||
revision: PoolMetaRevision,
|
||||
canonical: Option<Vec<u8>>,
|
||||
// Legacy semantic comparison canonicalizes to V2; a heal must instead
|
||||
// preserve verified raw bytes at the highest observed format floor.
|
||||
legacy_raw: Option<Vec<u8>>,
|
||||
legacy_has_unsafe_flag: bool,
|
||||
replica_state: PoolMetaReplicaState,
|
||||
cas_tokens: Vec<PoolMetaCasToken>,
|
||||
absent: bool,
|
||||
@@ -7222,6 +7269,8 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
|
||||
meta: PoolMeta::default(),
|
||||
revision: PoolMetaRevision::legacy(0),
|
||||
canonical: None,
|
||||
legacy_raw: None,
|
||||
legacy_has_unsafe_flag: false,
|
||||
replica_state: PoolMetaReplicaState {
|
||||
needs_repair: false,
|
||||
repair_write_safe: true,
|
||||
@@ -7276,6 +7325,8 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
|
||||
meta: selected.meta.clone(),
|
||||
revision: selected.revision,
|
||||
canonical: Some(selected.canonical.clone()),
|
||||
legacy_raw: None,
|
||||
legacy_has_unsafe_flag: false,
|
||||
replica_state: PoolMetaReplicaState {
|
||||
needs_repair,
|
||||
repair_write_safe,
|
||||
@@ -7289,6 +7340,10 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
|
||||
|
||||
// Legacy V1/V2 has no durable generation. Pool zero remains the commit
|
||||
// record; divergent backups without it are ambiguous and fail closed.
|
||||
let legacy_has_unsafe_flag = committed
|
||||
.iter()
|
||||
.any(|(_, raw, _)| ensure_legacy_pool_meta_republish_safe(raw).is_err());
|
||||
let mut floor_raw = None;
|
||||
let mut selected: Option<(usize, Vec<u8>, PoolMetaCommittedCandidate)> = None;
|
||||
for (idx, raw, candidate) in committed {
|
||||
if let Some((selected_idx, selected_raw, selected_candidate)) = selected.as_ref() {
|
||||
@@ -7302,19 +7357,36 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
|
||||
}
|
||||
} else {
|
||||
needs_repair |= selected_raw != &raw;
|
||||
if floor_raw.is_none()
|
||||
&& candidate.revision.version == observed_version
|
||||
&& raw
|
||||
.get(2..4)
|
||||
.is_some_and(|version| LittleEndian::read_u16(version) == observed_version)
|
||||
{
|
||||
floor_raw = Some(raw);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
selected = Some((idx, raw, candidate));
|
||||
}
|
||||
}
|
||||
let (_, _, mut selected) =
|
||||
let (_, selected_raw, mut selected) =
|
||||
selected.ok_or_else(|| Error::other("pool metadata recovery required: no valid legacy replica is available"))?;
|
||||
if selected.revision.version == observed_version
|
||||
&& selected_raw
|
||||
.get(2..4)
|
||||
.is_some_and(|version| LittleEndian::read_u16(version) == observed_version)
|
||||
{
|
||||
floor_raw = Some(selected_raw);
|
||||
}
|
||||
selected.meta.version = observed_version;
|
||||
selected.revision.version = observed_version;
|
||||
Ok(PoolMetaSelection {
|
||||
meta: selected.meta,
|
||||
revision: selected.revision,
|
||||
canonical: Some(selected.canonical),
|
||||
legacy_raw: floor_raw,
|
||||
legacy_has_unsafe_flag,
|
||||
replica_state: PoolMetaReplicaState {
|
||||
needs_repair,
|
||||
repair_write_safe,
|
||||
@@ -8308,8 +8380,9 @@ where
|
||||
})
|
||||
});
|
||||
let result = save_config_with_opts_and_metadata(pool, object, data, &opts).await;
|
||||
let may_have_mutated = !matches!(&result, Err(Error::PreconditionFailed));
|
||||
if !may_have_mutated {
|
||||
let write_not_started = matches!(&result, Err(Error::Io(inner)) if inner.get_ref().is_some_and(|source| source.is::<crate::set_disk::SystemMetadataWriteNotStarted>()));
|
||||
let may_have_mutated = !write_not_started && !matches!(&result, Err(Error::PreconditionFailed));
|
||||
if matches!(&result, Err(Error::PreconditionFailed)) {
|
||||
record_pool_meta_stale_write_rejection(phase);
|
||||
}
|
||||
let mut source = None;
|
||||
@@ -8320,6 +8393,8 @@ where
|
||||
source = Some(Arc::clone(&original));
|
||||
if matches!(original.as_ref(), Error::PreconditionFailed) {
|
||||
Err(Error::PreconditionFailed)
|
||||
} else if write_not_started {
|
||||
Err(Error::other(original))
|
||||
} else {
|
||||
Err(Error::other(pool_metadata_error(
|
||||
crate::error::PoolMetadataFailure::TransactionUnknown,
|
||||
@@ -14534,6 +14609,23 @@ impl ECStore {
|
||||
.canonical
|
||||
.as_ref()
|
||||
.ok_or_else(|| Error::other("pool metadata heal has no committed source"))?;
|
||||
if selection.generation_protocol_observed && !selection.revision.is_generation_protocol() {
|
||||
return Err(Error::other("pool metadata heal requires recovery of the observed generation protocol"));
|
||||
}
|
||||
if selection.legacy_has_unsafe_flag {
|
||||
return Err(Error::other("pool metadata legacy replica contains dont_save=true"));
|
||||
}
|
||||
let repair_data = if selection.revision.is_generation_protocol() {
|
||||
canonical.clone()
|
||||
} else {
|
||||
let raw = selection
|
||||
.legacy_raw
|
||||
.clone()
|
||||
.ok_or_else(|| Error::other("pool metadata heal cannot prove the legacy format floor"))?;
|
||||
ensure_legacy_pool_meta_payload_complete(&raw)?;
|
||||
raw
|
||||
};
|
||||
ensure_legacy_pool_meta_republish_safe(&repair_data)?;
|
||||
let pool = self
|
||||
.pools
|
||||
.get(pool_index)
|
||||
@@ -14544,10 +14636,10 @@ impl ECStore {
|
||||
.get(pool_index)
|
||||
.ok_or_else(|| Error::other("pool metadata heal has no target revision"))?;
|
||||
let mut arm = state.arm_transaction();
|
||||
save_pool_meta_object_cas(pool.clone(), POOL_META_NAME, canonical.clone(), token, &fence, "native_heal", &mut arm)
|
||||
save_pool_meta_object_cas(pool.clone(), POOL_META_NAME, repair_data.clone(), token, &fence, "native_heal", &mut arm)
|
||||
.await?;
|
||||
let (confirmed, _) = read_config_no_lock_preserve_empty_with_metadata(pool, POOL_META_NAME).await?;
|
||||
if confirmed != *canonical {
|
||||
if confirmed != repair_data {
|
||||
return Err(Error::other("pool metadata native heal readback diverged"));
|
||||
}
|
||||
fence.ensure_held()?;
|
||||
@@ -28353,7 +28445,7 @@ mod tests {
|
||||
"one version must commit before the next version times out"
|
||||
);
|
||||
assert!(
|
||||
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_timeout_deferral(err)),
|
||||
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_unavailable_deferral(err)),
|
||||
"{outcome:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -28387,6 +28479,88 @@ mod tests {
|
||||
assert_eq!(actual, b"written after versioning was enabled");
|
||||
}
|
||||
|
||||
/// A target lookup that loses its read-lock quorum must retain the source
|
||||
/// and resume an entry after an earlier version has already committed.
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn decommission_defers_target_lookup_read_lock_quorum_and_resumes() {
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
|
||||
let fixture = NullVersionDecommission::new(NullVersionTop::NewerVersion).await;
|
||||
let object = fixture.object;
|
||||
let attempted = Arc::new(AtomicUsize::new(0));
|
||||
let fault = DecommissionTestFaultGuard::install_with_error(
|
||||
Arc::new({
|
||||
let attempted = Arc::clone(&attempted);
|
||||
move |stage, _, faulted, _, success| {
|
||||
stage == "before_target_commit"
|
||||
&& faulted == object
|
||||
&& success
|
||||
&& attempted.fetch_add(1, Ordering::SeqCst) > 0
|
||||
}
|
||||
}),
|
||||
Some(Arc::new(|_, bucket, object| {
|
||||
data_movement::data_movement_stage_error_for_test(
|
||||
"decommission_object",
|
||||
"prepare_put_object",
|
||||
bucket,
|
||||
object,
|
||||
Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "read",
|
||||
bucket: bucket.to_owned(),
|
||||
object: object.to_owned(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
},
|
||||
)
|
||||
})),
|
||||
);
|
||||
let (outcome, _) = fixture.run_entry().await;
|
||||
assert!(
|
||||
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_unavailable_deferral(err)),
|
||||
"{outcome:?}"
|
||||
);
|
||||
fixture.assert_source_retained().await;
|
||||
assert_eq!(
|
||||
fixture
|
||||
.target_versions()
|
||||
.await
|
||||
.expect("first version committed")
|
||||
.versions
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
fixture.store.pool_meta.read().await.pools[0]
|
||||
.decommission
|
||||
.as_ref()
|
||||
.expect("active decommission")
|
||||
.items_decommission_failed,
|
||||
0
|
||||
);
|
||||
|
||||
drop(fault);
|
||||
let (outcome, _) = fixture.run_entry().await;
|
||||
assert!(matches!(outcome, DecommissionEntryOutcome::Complete), "{outcome:?}");
|
||||
fixture.assert_moved().await;
|
||||
let mut reader = fixture.store.pools[1]
|
||||
.get_object_reader(
|
||||
fixture.bucket,
|
||||
fixture.object,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
version_id: Some(fixture.seeded.versions[0].version_id.expect("newer version").to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("the newer version must remain readable");
|
||||
let mut actual = Vec::new();
|
||||
reader.stream.read_to_end(&mut actual).await.expect("read the newer version");
|
||||
assert_eq!(actual, b"written after versioning was enabled");
|
||||
}
|
||||
|
||||
/// A namespace lock still held on the target key, as by a copy whose node
|
||||
/// was killed mid-write, makes target selection fail with a typed
|
||||
/// object-lock timeout. A real held lock, not an injected error, must defer
|
||||
@@ -28429,7 +28603,7 @@ mod tests {
|
||||
|
||||
let (outcome, _) = fixture.run_entry().await;
|
||||
assert!(
|
||||
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_timeout_deferral(err)),
|
||||
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_unavailable_deferral(err)),
|
||||
"{outcome:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -28838,17 +29012,17 @@ mod tests {
|
||||
timeout(),
|
||||
);
|
||||
for err in [timeout(), prepare] {
|
||||
assert!(is_decommission_copy_lock_timeout_error(&err), "{err}");
|
||||
assert!(is_decommission_copy_lock_unavailable_error(&err, "bucket-a", "object-a"), "{err}");
|
||||
assert!(!is_decommission_copy_quorum_error(&err), "{err}");
|
||||
let mut error = Some(err);
|
||||
let deferred = take_decommission_copy_deferral("migrate_object", "bucket-a", "object-a", &mut error)
|
||||
.expect("a lock timeout copy must be deferred");
|
||||
assert!(error.is_none());
|
||||
assert!(is_decommission_copy_lock_timeout_deferral(&deferred));
|
||||
assert!(is_decommission_copy_lock_unavailable_deferral(&deferred));
|
||||
assert!(!is_decommission_copy_quorum_deferral(&deferred));
|
||||
let failure = data_movement::data_movement_stage_source_as::<DecommissionCopyLockTimeout>(&deferred)
|
||||
let failure = data_movement::data_movement_stage_source_as::<DecommissionCopyLockUnavailable>(&deferred)
|
||||
.expect("the deferred error must keep its lock-timeout origin");
|
||||
assert!(is_decommission_copy_lock_timeout_error(&failure.0));
|
||||
assert!(is_decommission_copy_lock_unavailable_error(&failure.0, "bucket-a", "object-a"));
|
||||
assert!(matches!(
|
||||
resolve_decommission_entry_failure(deferred),
|
||||
Ok(DecommissionEntryOutcome::Deferred(_))
|
||||
@@ -28873,6 +29047,43 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decommission_copy_defers_only_the_matching_object_read_lock_quorum() {
|
||||
let quorum = |mode, bucket: &str, object: &str| Error::NamespaceLockQuorumUnavailable {
|
||||
mode,
|
||||
bucket: bucket.to_owned(),
|
||||
object: object.to_owned(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
};
|
||||
let wrap = |stage, err| {
|
||||
data_movement::data_movement_stage_error_for_test("decommission_object", stage, "bucket-a", "object-a", err)
|
||||
};
|
||||
let mut error = Some(wrap("prepare_put_object", quorum("read", "bucket-a", "object-a")));
|
||||
let deferred = take_decommission_copy_deferral("migrate_object", "bucket-a", "object-a", &mut error)
|
||||
.expect("matching read-lock quorum must defer");
|
||||
assert!(error.is_none());
|
||||
assert!(matches!(
|
||||
resolve_decommission_entry_failure(deferred),
|
||||
Ok(DecommissionEntryOutcome::Deferred(_))
|
||||
));
|
||||
for err in [
|
||||
quorum("read", "bucket-a", "object-a"),
|
||||
wrap("source_recheck", quorum("read", "bucket-a", "object-a")),
|
||||
wrap("prepare_put_object", quorum("write", "bucket-a", "object-a")),
|
||||
wrap(
|
||||
"prepare_put_object",
|
||||
quorum("exact_delete_capacity_reconciliation", "bucket-a", "object-a"),
|
||||
),
|
||||
wrap("prepare_put_object", quorum("read", "other-bucket", "object-a")),
|
||||
wrap("prepare_put_object", quorum("read", "bucket-a", "other-object")),
|
||||
] {
|
||||
let mut error = Some(err);
|
||||
assert!(take_decommission_copy_deferral("migrate_object", "bucket-a", "object-a", &mut error).is_none());
|
||||
assert!(error.is_some(), "unrelated lock failures must stay terminal");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decommission_target_capacity_error_accepts_wrapped_capacity_errors() {
|
||||
let disk_full = Error::other(format!("decommission_object: put_object failed for bucket/object: {}", Error::DiskFull));
|
||||
@@ -29351,6 +29562,53 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_republish_guard_checks_only_three_field_sequence_or_map() {
|
||||
let frame = |payload: Vec<u8>| {
|
||||
let mut data = Vec::new();
|
||||
data.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
|
||||
data.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
|
||||
data.extend_from_slice(&payload);
|
||||
data
|
||||
};
|
||||
let two_fields = PersistedPoolMetaV1 {
|
||||
version: POOL_META_V1_VERSION,
|
||||
pools: Vec::new(),
|
||||
};
|
||||
for payload in [rmp_serde::to_vec(&two_fields), rmp_serde::to_vec_named(&two_fields)] {
|
||||
let data = frame(payload.expect("two-field V1 payload must serialize"));
|
||||
assert!(matches!(decode_pool_meta_replica(data.clone()), PoolMetaReplica::Valid { .. }));
|
||||
assert!(ensure_legacy_pool_meta_republish_safe(&data).is_ok());
|
||||
assert!(ensure_legacy_pool_meta_payload_complete(&data).is_ok());
|
||||
let mut trailing = data;
|
||||
trailing.push(0xc0);
|
||||
assert!(ensure_legacy_pool_meta_payload_complete(&trailing).is_err());
|
||||
}
|
||||
for dont_save in [false, true] {
|
||||
let legacy = LegacyPoolMeta {
|
||||
version: POOL_META_V1_VERSION,
|
||||
pools: Vec::new(),
|
||||
dont_save,
|
||||
};
|
||||
for payload in [rmp_serde::to_vec(&legacy), rmp_serde::to_vec_named(&legacy)] {
|
||||
let data = frame(payload.expect("three-field V1 payload must serialize"));
|
||||
assert!(matches!(decode_pool_meta_replica(data.clone()), PoolMetaReplica::Valid { .. }));
|
||||
assert_eq!(ensure_legacy_pool_meta_republish_safe(&data).is_err(), dont_save);
|
||||
assert!(ensure_legacy_pool_meta_payload_complete(&data).is_ok());
|
||||
let mut trailing = data;
|
||||
trailing.push(0xc0);
|
||||
assert!(ensure_legacy_pool_meta_payload_complete(&trailing).is_err());
|
||||
}
|
||||
}
|
||||
// This guard is only called for already-decoded replicas; malformed
|
||||
// containers must fail selection rather than becoming repair sources.
|
||||
let truncated = frame(vec![0x93, 0x01]);
|
||||
assert!(matches!(
|
||||
decode_pool_meta_replica(truncated),
|
||||
PoolMetaReplica::Corrupt(_) | PoolMetaReplica::Incompatible(_)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pool_meta_decode_supports_legacy_payload() {
|
||||
let start_time = OffsetDateTime::now_utc();
|
||||
@@ -30571,6 +30829,154 @@ mod pools_tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pool_meta_prewrite_quorum_rejection_allows_retry_after_disks_return() {
|
||||
let (dirs, disks, pool) = crate::set_disk::hermetic_set_disks_isolated(4).await;
|
||||
pool.disks.write().await[2..].fill(None);
|
||||
let state = super::PoolMetaWriteState::default();
|
||||
let data =
|
||||
super::initialized_pool_meta_identity_for_test(uuid::Uuid::new_v4(), 1).expect("encode the deployment identity");
|
||||
let mut transaction = state.arm_transaction();
|
||||
save_pool_meta_object_cas(
|
||||
pool.clone(),
|
||||
POOL_META_IDENTITY_NAME,
|
||||
data.clone(),
|
||||
&PoolMetaCasToken::Missing,
|
||||
&PoolMetaPersistenceFence::OfflineRecovery,
|
||||
"identity_cas",
|
||||
&mut transaction,
|
||||
)
|
||||
.await
|
||||
.expect_err("two of four disks cannot satisfy the identity write quorum");
|
||||
drop(transaction);
|
||||
for dir in &dirs {
|
||||
assert!(
|
||||
!dir.path()
|
||||
.join(crate::disk::RUSTFS_META_BUCKET)
|
||||
.join(POOL_META_IDENTITY_NAME)
|
||||
.exists(),
|
||||
"the rejected write must not create an identity replica"
|
||||
);
|
||||
}
|
||||
state
|
||||
.ensure_write_safe("retry after disks return")
|
||||
.expect("a rejected inline write must not poison the startup transaction");
|
||||
|
||||
let ordinary_error = crate::config::com::save_config_with_opts(
|
||||
pool.clone(),
|
||||
"ordinary-config",
|
||||
data.clone(),
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("ordinary writes must still fail without quorum");
|
||||
assert!(
|
||||
matches!(ordinary_error, Error::InsufficientWriteQuorum(_, _)),
|
||||
"ordinary callers must retain their existing quorum error"
|
||||
);
|
||||
|
||||
*pool.disks.write().await = disks.into_iter().map(Some).collect();
|
||||
let mut retry = state.arm_transaction();
|
||||
save_pool_meta_object_cas(
|
||||
pool.clone(),
|
||||
POOL_META_IDENTITY_NAME,
|
||||
data.clone(),
|
||||
&PoolMetaCasToken::Missing,
|
||||
&PoolMetaPersistenceFence::OfflineRecovery,
|
||||
"identity_cas",
|
||||
&mut retry,
|
||||
)
|
||||
.await
|
||||
.expect("the same transaction must succeed when the disks return");
|
||||
retry.disarm();
|
||||
drop(retry);
|
||||
assert_eq!(
|
||||
crate::config::com::read_config(pool, POOL_META_IDENTITY_NAME)
|
||||
.await
|
||||
.expect("read back the committed identity"),
|
||||
data
|
||||
);
|
||||
state
|
||||
.ensure_write_safe("after successful retry")
|
||||
.expect("retry must leave writes enabled");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pool_meta_prewrite_quorum_rejection_keeps_an_earlier_write_armed() {
|
||||
let (_dirs, disks, pool) = crate::set_disk::hermetic_set_disks_isolated(4).await;
|
||||
let state = super::PoolMetaWriteState::default();
|
||||
let data =
|
||||
super::initialized_pool_meta_identity_for_test(uuid::Uuid::new_v4(), 1).expect("encode the deployment identity");
|
||||
let mut transaction = state.arm_transaction();
|
||||
let written = save_pool_meta_object_cas(
|
||||
pool.clone(),
|
||||
POOL_META_IDENTITY_NAME,
|
||||
data.clone(),
|
||||
&PoolMetaCasToken::Missing,
|
||||
&PoolMetaPersistenceFence::OfflineRecovery,
|
||||
"prepare_cas",
|
||||
&mut transaction,
|
||||
)
|
||||
.await
|
||||
.expect("the earlier phase must really commit");
|
||||
pool.disks.write().await[2..].fill(None);
|
||||
save_pool_meta_object_cas(
|
||||
pool.clone(),
|
||||
POOL_META_IDENTITY_NAME,
|
||||
data.clone(),
|
||||
&PoolMetaCasToken::Existing(written.etag.expect("committed identity has an ETag")),
|
||||
&PoolMetaPersistenceFence::OfflineRecovery,
|
||||
"identity_cas",
|
||||
&mut transaction,
|
||||
)
|
||||
.await
|
||||
.expect_err("the next phase must fail without a write quorum");
|
||||
drop(transaction);
|
||||
*pool.disks.write().await = disks.into_iter().map(Some).collect();
|
||||
assert_eq!(
|
||||
crate::config::com::read_config(pool, POOL_META_IDENTITY_NAME)
|
||||
.await
|
||||
.expect("the earlier commit must survive the rejected write"),
|
||||
data
|
||||
);
|
||||
let error = state
|
||||
.ensure_write_safe("after a committed phase")
|
||||
.expect_err("a later rejection cannot discharge the earlier write");
|
||||
assert_eq!(
|
||||
error.pool_metadata_failure().expect("retain typed uncertainty").kind,
|
||||
crate::error::PoolMetadataFailure::TransactionUnknown
|
||||
);
|
||||
assert_eq!(error.pool_metadata_failure().expect("retain the earlier mutation").phase, "prepare_cas");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pool_meta_prewrite_noninline_failure_keeps_transaction_armed() {
|
||||
let (_dirs, _disks, pool) = crate::set_disk::hermetic_set_disks_isolated(4).await;
|
||||
pool.disks.write().await[2..].fill(None);
|
||||
let state = super::PoolMetaWriteState::default();
|
||||
let mut transaction = state.arm_transaction();
|
||||
save_pool_meta_object_cas(
|
||||
pool,
|
||||
POOL_META_IDENTITY_NAME,
|
||||
vec![0x51; 2 * 1024 * 1024],
|
||||
&PoolMetaCasToken::Missing,
|
||||
&PoolMetaPersistenceFence::OfflineRecovery,
|
||||
"identity_cas",
|
||||
&mut transaction,
|
||||
)
|
||||
.await
|
||||
.expect_err("the noninline write must fail without quorum");
|
||||
drop(transaction);
|
||||
assert!(
|
||||
state.ensure_write_safe("after writer setup").is_err(),
|
||||
"a failure after disk writers were created must remain fail-closed"
|
||||
);
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum IdentityRepairFault {
|
||||
Healthy,
|
||||
|
||||
@@ -317,8 +317,9 @@ impl Sets {
|
||||
&self.ctx
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn stop_endpoint_monitor_for_test(&self) {
|
||||
/// Stop endpoint reconnection and wait for any in-flight pass before a test owns disk probes.
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub async fn stop_endpoint_monitor_for_test(&self) {
|
||||
if let Some(exit_signal) = &self.exit_signal {
|
||||
let _ = exit_signal.send(());
|
||||
// Wait for an in-flight reconnect pass as well as its receiver to exit.
|
||||
|
||||
@@ -31,6 +31,8 @@ use tokio::fs;
|
||||
use tokio::time::Instant;
|
||||
|
||||
pub(crate) const BARRIER_DIR_ENV: &str = "RUSTFS_E2E_DATA_MOVEMENT_BARRIER_DIR";
|
||||
/// Rebalance holds its admission fence; no version of this entry has moved.
|
||||
pub(crate) const STAGE_AFTER_ENTRY_ADMISSION: &str = "after_entry_admission";
|
||||
/// Every target part is staged and the target version is not yet published.
|
||||
pub(crate) const STAGE_BEFORE_TARGET_COMMIT: &str = "before_target_commit";
|
||||
/// The target commit is durable and the source preflight matched; the source
|
||||
|
||||
@@ -605,6 +605,7 @@ fn resolve_data_movement_abort_result(
|
||||
#[derive(Debug)]
|
||||
struct DataMovementStageError {
|
||||
rendered: String,
|
||||
stage: Option<String>,
|
||||
source: Box<dyn std::error::Error + Send + Sync>,
|
||||
}
|
||||
|
||||
@@ -626,6 +627,7 @@ where
|
||||
{
|
||||
Error::other(DataMovementStageError {
|
||||
rendered,
|
||||
stage: None,
|
||||
source: Box::new(err),
|
||||
})
|
||||
}
|
||||
@@ -634,7 +636,21 @@ fn data_movement_stage_error<E>(op_label: &str, stage: &str, bucket: &str, objec
|
||||
where
|
||||
E: std::error::Error + Send + Sync + 'static,
|
||||
{
|
||||
data_movement_context_error(format!("{op_label}: {stage} failed for {bucket}/{object}: {err}"), err)
|
||||
Error::other(DataMovementStageError {
|
||||
rendered: format!("{op_label}: {stage} failed for {bucket}/{object}: {err}"),
|
||||
stage: Some(stage.to_owned()),
|
||||
source: Box::new(err),
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn data_movement_error_is_stage(err: &Error, stage: &str) -> bool {
|
||||
let Error::Io(io_err) = err else {
|
||||
return false;
|
||||
};
|
||||
io_err
|
||||
.get_ref()
|
||||
.and_then(|source| source.downcast_ref::<DataMovementStageError>())
|
||||
.is_some_and(|source| source.stage.as_deref() == Some(stage))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -55,6 +55,20 @@ use tokio_util::sync::CancellationToken;
|
||||
use tracing::{info, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Join the local disk's actual bootstrap worker for deterministic coverage tests.
|
||||
/// Completion may mean legacy fallback after a failed scan; callers must assert
|
||||
/// the expected index result separately. Remote disks have no local worker.
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub async fn wait_for_multipart_index_bootstrap(disk: &crate::disk::DiskStore) -> Result<()> {
|
||||
match disk.as_ref() {
|
||||
crate::disk::Disk::Local(local) => {
|
||||
local.disk.wait_for_multipart_index_bootstrap().await;
|
||||
Ok(())
|
||||
}
|
||||
crate::disk::Disk::Remote(_) => Err(DiskError::MethodNotAllowed),
|
||||
}
|
||||
}
|
||||
|
||||
/// Disk health status constants
|
||||
const DISK_HEALTH_OK: u32 = 0;
|
||||
const DISK_HEALTH_FAULTY: u32 = 1;
|
||||
@@ -853,6 +867,8 @@ pub(crate) struct DiskHealthMetricEpoch {
|
||||
fallback_operation_metrics: StdRwLock<HashMap<&'static str, Arc<DiskOperationMetrics>>>,
|
||||
#[cfg(test)]
|
||||
multipart_cleanup_observer: std::sync::Mutex<Option<std::sync::Weak<ScopedMultipartCleanupCompletions>>>,
|
||||
#[cfg(test)]
|
||||
operation_timeout_override: std::sync::Mutex<Option<Duration>>,
|
||||
/// Caller API operations currently executing through the disk health wrapper.
|
||||
api_waiting: AtomicU32,
|
||||
/// Operations rejected because the disk was unavailable for the caller.
|
||||
@@ -880,6 +896,8 @@ impl Default for DiskHealthMetricEpoch {
|
||||
fallback_operation_metrics: StdRwLock::new(HashMap::new()),
|
||||
#[cfg(test)]
|
||||
multipart_cleanup_observer: std::sync::Mutex::new(None),
|
||||
#[cfg(test)]
|
||||
operation_timeout_override: std::sync::Mutex::new(None),
|
||||
api_waiting: AtomicU32::new(0),
|
||||
total_errors_availability: AtomicU64::new(0),
|
||||
total_errors_timeout: AtomicU64::new(0),
|
||||
@@ -1573,6 +1591,20 @@ impl Default for DiskHealthTracker {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct ScopedDiskOperationTimeout {
|
||||
metrics: Arc<DiskHealthMetricEpoch>,
|
||||
previous: Option<Duration>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for ScopedDiskOperationTimeout {
|
||||
fn drop(&mut self) {
|
||||
*self.metrics.operation_timeout_override.lock().expect("test timeout lock") = self.previous;
|
||||
}
|
||||
}
|
||||
|
||||
/// LocalDiskWrapper wraps a DiskStore with health tracking capabilities.
|
||||
/// This is similar to Go's xlStorageDiskIDCheck.
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -1914,6 +1946,23 @@ impl LocalDiskWrapper {
|
||||
self.health_check
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn override_operation_timeout_for_test(&self, timeout: Duration) -> ScopedDiskOperationTimeout {
|
||||
assert!(!timeout.is_zero(), "fault injection needs a positive timeout");
|
||||
// The metrics epoch follows this drive across clones and reconnects.
|
||||
// Its scoped override cannot shorten unrelated healthy drives' I/O.
|
||||
let previous = self
|
||||
.metrics
|
||||
.operation_timeout_override
|
||||
.lock()
|
||||
.expect("test timeout lock")
|
||||
.replace(timeout);
|
||||
ScopedDiskOperationTimeout {
|
||||
metrics: Arc::clone(&self.metrics),
|
||||
previous,
|
||||
}
|
||||
}
|
||||
|
||||
/// Enable health monitoring after disk creation.
|
||||
/// Used to defer health checks until after startup format loading completes.
|
||||
pub fn enable_health_check(&self) {
|
||||
@@ -2398,12 +2447,17 @@ impl LocalDiskWrapper {
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) fn has_clean_active_multipart_source(&self, upload_or_part_path: &str, part_path: bool) -> bool {
|
||||
self.disk.has_clean_active_multipart_source(upload_or_part_path, part_path)
|
||||
}
|
||||
|
||||
pub(crate) async fn try_publish_prepared_delete_local(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: uuid::Uuid,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
|
||||
) -> Result<Option<(crate::disk::DeleteRecoveryJournal, bool, bool)>> {
|
||||
self.track_disk_health_mutation(
|
||||
"settle_delete_transaction",
|
||||
@@ -2411,7 +2465,7 @@ impl LocalDiskWrapper {
|
||||
|| {
|
||||
Box::pin(async {
|
||||
self.disk
|
||||
.try_publish_prepared_delete_local(volume, object, transaction, namespace_owner)
|
||||
.try_publish_prepared_delete_local_with_budget(volume, object, transaction, namespace_owner, work_budget)
|
||||
.await
|
||||
})
|
||||
},
|
||||
@@ -2427,6 +2481,7 @@ impl LocalDiskWrapper {
|
||||
transaction: uuid::Uuid,
|
||||
journal: &crate::disk::DeleteRecoveryJournal,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
|
||||
) -> Result<bool> {
|
||||
self.track_disk_health_mutation(
|
||||
"settle_delete_transaction",
|
||||
@@ -2434,7 +2489,14 @@ impl LocalDiskWrapper {
|
||||
|| {
|
||||
Box::pin(async {
|
||||
self.disk
|
||||
.try_finish_and_finalize_prepared_delete_local(volume, object, transaction, journal, namespace_owner)
|
||||
.try_finish_and_finalize_prepared_delete_local_with_budget(
|
||||
volume,
|
||||
object,
|
||||
transaction,
|
||||
journal,
|
||||
namespace_owner,
|
||||
work_budget,
|
||||
)
|
||||
.await
|
||||
})
|
||||
},
|
||||
@@ -2634,6 +2696,13 @@ impl LocalDiskWrapper {
|
||||
}
|
||||
return result;
|
||||
}
|
||||
#[cfg(test)]
|
||||
let timeout_duration = self
|
||||
.metrics
|
||||
.operation_timeout_override
|
||||
.lock()
|
||||
.expect("test timeout lock")
|
||||
.map_or(timeout_duration, |injected| timeout_duration.min(injected));
|
||||
// Execute the operation with timeout
|
||||
let result = tokio::time::timeout(timeout_duration, operation).await;
|
||||
|
||||
@@ -2682,6 +2751,18 @@ impl LocalDiskWrapper {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl DiskAPI for LocalDiskWrapper {
|
||||
async fn list_multipart_index_roots(
|
||||
&self,
|
||||
bucket: &str,
|
||||
marker: Option<&str>,
|
||||
) -> Result<crate::disk::multipart_index::MultipartIndexPage> {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
self.check_disk_stale().await?;
|
||||
self.disk.list_multipart_index_roots(bucket, marker).await
|
||||
}
|
||||
|
||||
async fn supports_guarded_delete_versions(&self) -> Result<()> {
|
||||
// The local implementation shares this binary's capability. Availability
|
||||
// and disk identity remain mutation checks and write-quorum inputs.
|
||||
@@ -2735,6 +2816,62 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
)
|
||||
.await
|
||||
}
|
||||
async fn discover_delete_transactions_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
max_entries: u32,
|
||||
) -> Result<crate::disk::BoundedDeleteRecoveryDiscovery> {
|
||||
self.track_disk_health_with_op(
|
||||
"discover_delete_transactions",
|
||||
|| async {
|
||||
self.disk
|
||||
.discover_delete_transactions_bounded(volume, object, max_entries)
|
||||
.await
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn inspect_delete_transaction_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: uuid::Uuid,
|
||||
max_entries: u32,
|
||||
) -> Result<crate::disk::BoundedDeleteRecoveryInspection> {
|
||||
self.track_disk_health_with_op(
|
||||
"inspect_delete_transaction",
|
||||
|| async {
|
||||
self.disk
|
||||
.inspect_delete_transaction_bounded(volume, object, transaction, max_entries)
|
||||
.await
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn settle_delete_transaction_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
request: crate::disk::BoundedDeleteRecoverySettlement,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<crate::disk::BoundedDeleteRecoverySettlementReply> {
|
||||
self.track_disk_health_mutation(
|
||||
"settle_delete_transaction",
|
||||
DiskMetricMutation::Delete,
|
||||
|| async {
|
||||
self.disk
|
||||
.settle_delete_transaction_bounded(volume, request, namespace_owner)
|
||||
.await
|
||||
},
|
||||
crate::disk::get_delete_recovery_timeout()?,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn discover_delete_transactions(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -3703,7 +3840,11 @@ mod tests {
|
||||
assert!(futures::poll!(&mut operation).is_pending());
|
||||
time::advance(SLOW_DISK_OPERATION_THRESHOLD - Duration::from_millis(1)).await;
|
||||
assert!(futures::poll!(&mut operation).is_pending());
|
||||
assert!(logs.contents().is_empty(), "normal disk latency should not warn");
|
||||
assert!(
|
||||
!logs.contents().contains("disk_operation_slow") && !logs.contents().contains("WARN"),
|
||||
"normal disk latency should not warn: {}",
|
||||
logs.contents()
|
||||
);
|
||||
time::advance(Duration::from_millis(1)).await;
|
||||
assert!(futures::poll!(&mut operation).is_pending());
|
||||
let warning = logs.contents();
|
||||
@@ -6040,6 +6181,67 @@ mod tests {
|
||||
assert!(health.offline_duration().is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scoped_operation_timeout_is_per_disk_and_preserves_caller_policy() {
|
||||
let directories = [
|
||||
tempfile::tempdir().expect("faulted drive"),
|
||||
tempfile::tempdir().expect("healthy drive"),
|
||||
];
|
||||
let mut wrappers = Vec::new();
|
||||
for directory in &directories {
|
||||
let endpoint = Endpoint::try_from(directory.path().to_str().expect("UTF-8 drive path")).expect("drive endpoint");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local drive"));
|
||||
wrappers.push(LocalDiskWrapper::new(disk, false));
|
||||
}
|
||||
let reconnected = LocalDiskWrapper::new_with_reconnect_state(
|
||||
Arc::clone(&wrappers[0].disk),
|
||||
false,
|
||||
Some(wrappers[0].health_tracker_epoch_for_reconnect()),
|
||||
);
|
||||
let scope = wrappers[0].override_operation_timeout_for_test(Duration::from_secs(1));
|
||||
tokio::time::pause();
|
||||
let operation = || async {
|
||||
tokio::time::sleep(Duration::from_secs(2)).await;
|
||||
Ok(())
|
||||
};
|
||||
let (faulted, healthy) = tokio::join!(
|
||||
reconnected.track_disk_health_with_op_and_timeout_action(
|
||||
"read_metadata",
|
||||
operation,
|
||||
Duration::from_secs(3),
|
||||
TimeoutHealthAction::IgnoreFailure,
|
||||
),
|
||||
wrappers[1].track_disk_health_with_op_and_timeout_action(
|
||||
"read_metadata",
|
||||
operation,
|
||||
Duration::from_secs(3),
|
||||
TimeoutHealthAction::IgnoreFailure,
|
||||
),
|
||||
);
|
||||
assert_eq!(faulted.expect_err("only the selected drive must time out"), DiskError::Timeout);
|
||||
healthy.expect("healthy drive keeps its caller's deadline");
|
||||
let shorter_started = tokio::time::Instant::now();
|
||||
let shorter = wrappers[0]
|
||||
.track_disk_health_with_op_and_timeout_action(
|
||||
"read_metadata",
|
||||
operation,
|
||||
Duration::from_millis(500),
|
||||
TimeoutHealthAction::IgnoreFailure,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(shorter.expect_err("fault injection cannot extend a caller deadline"), DiskError::Timeout);
|
||||
assert!(shorter_started.elapsed() < Duration::from_secs(1));
|
||||
wrappers[0]
|
||||
.track_disk_health_with_op("read_metadata", operation, Duration::ZERO)
|
||||
.await
|
||||
.expect("explicit unbounded physical ownership cannot acquire a test deadline");
|
||||
drop(scope);
|
||||
reconnected
|
||||
.track_disk_health_with_op("read_metadata", operation, Duration::from_secs(3))
|
||||
.await
|
||||
.expect("dropping the scope restores reconnected callers' deadlines");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ignored_timeout_does_not_mark_drive_failure() {
|
||||
let dir = tempfile::tempdir().expect("temp dir should be created");
|
||||
|
||||
@@ -172,6 +172,8 @@ pub enum DiskError {
|
||||
|
||||
#[error("guarded batch deletion is not supported by this disk")]
|
||||
GuardedDeleteUnsupported,
|
||||
#[error("foreground delete recovery work limit reached")]
|
||||
DeleteRecoveryWorkLimit,
|
||||
|
||||
#[error("erasure write quorum")]
|
||||
ErasureWriteQuorum,
|
||||
@@ -770,6 +772,7 @@ impl Clone for DiskError {
|
||||
DiskError::MethodNotAllowed => DiskError::MethodNotAllowed,
|
||||
DiskError::ConditionalDeleteConflict => DiskError::ConditionalDeleteConflict,
|
||||
DiskError::GuardedDeleteUnsupported => DiskError::GuardedDeleteUnsupported,
|
||||
DiskError::DeleteRecoveryWorkLimit => DiskError::DeleteRecoveryWorkLimit,
|
||||
DiskError::ErasureWriteQuorum => DiskError::ErasureWriteQuorum,
|
||||
DiskError::ErasureReadQuorum => DiskError::ErasureReadQuorum,
|
||||
DiskError::ShortWrite => DiskError::ShortWrite,
|
||||
@@ -829,6 +832,7 @@ impl DiskError {
|
||||
DiskError::RemoteClientUnavailable(_) => 0x2B,
|
||||
DiskError::ConditionalDeleteConflict => 0x2C,
|
||||
DiskError::GuardedDeleteUnsupported => 0x2D,
|
||||
DiskError::DeleteRecoveryWorkLimit => 0x2E,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -879,6 +883,7 @@ impl DiskError {
|
||||
0x2B => Some(DiskError::RemoteClientUnavailable(String::new())),
|
||||
0x2C => Some(DiskError::ConditionalDeleteConflict),
|
||||
0x2D => Some(DiskError::GuardedDeleteUnsupported),
|
||||
0x2E => Some(DiskError::DeleteRecoveryWorkLimit),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -1662,6 +1667,7 @@ mod tests {
|
||||
for (error, code) in [
|
||||
(DiskError::ConditionalDeleteConflict, 0x2C),
|
||||
(DiskError::GuardedDeleteUnsupported, 0x2D),
|
||||
(DiskError::DeleteRecoveryWorkLimit, 0x2E),
|
||||
] {
|
||||
assert_eq!(error.to_u32(), code);
|
||||
assert_eq!(DiskError::from_u32(code), Some(error.clone()));
|
||||
|
||||
@@ -5,6 +5,7 @@ use super::{FdCache, FdCacheEntry, FdKey};
|
||||
use std::fs::File;
|
||||
use std::os::unix::fs::FileExt;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
|
||||
fn key(volume: &str, path: &str, direct: bool) -> FdKey {
|
||||
FdKey {
|
||||
@@ -52,7 +53,11 @@ async fn uring_fd_cache_exact_invalidation_removes_both_modes_and_preserves_othe
|
||||
async fn uring_fd_cache_exact_invalidation_bumps_one_generation_and_rejects_old_epoch_inserts() {
|
||||
let (_directory, file) = fixture();
|
||||
let cache = FdCache::new();
|
||||
let entry = Arc::new(FdCacheEntry { file, len: 3 });
|
||||
let entry = Arc::new(FdCacheEntry {
|
||||
file,
|
||||
len: 3,
|
||||
invalidated: AtomicBool::new(false),
|
||||
});
|
||||
let old_generation = cache.generation();
|
||||
cache.invalidate_exact("v", "part").await;
|
||||
let current_generation = cache.generation();
|
||||
@@ -125,7 +130,7 @@ async fn uring_fd_cache_volume_and_clear_invalidation_cover_both_modes() {
|
||||
cache.insert(key("v", "part", direct), Arc::clone(&file)).await;
|
||||
assert!(
|
||||
cache.get(&key("v", "part", direct)).await.is_some(),
|
||||
"new entries must survive the earlier volume predicate"
|
||||
"new entries must survive the earlier volume invalidation"
|
||||
);
|
||||
}
|
||||
cache.clear();
|
||||
@@ -164,6 +169,7 @@ async fn uring_fd_cache_invalidation_keeps_borrowed_old_inode_alive_and_allows_r
|
||||
let replacement = Arc::new(FdCacheEntry {
|
||||
file: Arc::new(File::open(directory.path().join("part")).expect("open replacement inode")),
|
||||
len: 11,
|
||||
invalidated: AtomicBool::new(false),
|
||||
});
|
||||
for direct in [false, true] {
|
||||
let key = key("v", "part", direct);
|
||||
@@ -177,3 +183,459 @@ async fn uring_fd_cache_invalidation_keeps_borrowed_old_inode_alive_and_allows_r
|
||||
assert_eq!(fresh.len, 11);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fd_cache_tracker_skips_unmatched_invalidations_and_handles_deep_prefixes() {
|
||||
let (_directory, file) = fixture();
|
||||
let mut cache = FdCache::new();
|
||||
|
||||
// Unmatched invalidations on an empty cache must not spawn the maintenance task.
|
||||
cache.invalidate_under("v", "missing/prefix");
|
||||
cache.invalidate_volume("v");
|
||||
cache.invalidate_exact("v", "missing/prefix/part.1").await;
|
||||
cache.clear();
|
||||
assert!(
|
||||
cache.maintenance_task.get().is_none(),
|
||||
"empty-cache invalidations must not spawn maintenance"
|
||||
);
|
||||
|
||||
// Populate a normal entry and verify matched vs unmatched filter gates directly,
|
||||
// and ensure unmatched invalidations do not enqueue pending removals.
|
||||
let cached_key = key("v", "obj-1/0198a3f0/part.1", false);
|
||||
cache.insert(cached_key.clone(), Arc::clone(&file)).await;
|
||||
assert_eq!(cache.tracker.live_entries(), 1);
|
||||
assert!(cache.tracker.may_match_volume("v"));
|
||||
assert!(!cache.tracker.may_match_volume("other-volume"));
|
||||
assert!(cache.tracker.may_match_prefix("v", "obj-1"));
|
||||
assert!(cache.tracker.may_match_prefix("v", "obj-1/0198a3f0"));
|
||||
assert!(!cache.tracker.may_match_prefix("v", "obj-2/0198a3f0"));
|
||||
assert!(cache.tracker.may_match_exact("v", "obj-1/0198a3f0/part.1"));
|
||||
assert!(!cache.tracker.may_match_exact("v", "obj-1/0198a3f0/xl.meta"));
|
||||
|
||||
cache.invalidate_volume("other-volume");
|
||||
cache.invalidate_under("v", "obj-2/0198a3f0");
|
||||
cache.invalidate_exact("v", "obj-1/0198a3f0/xl.meta").await;
|
||||
assert!(
|
||||
!cache
|
||||
.tracker
|
||||
.has_pending_invalidations
|
||||
.load(std::sync::atomic::Ordering::SeqCst),
|
||||
"unmatched invalidations must not enqueue maintenance removals"
|
||||
);
|
||||
assert!(cache.get(&cached_key).await.is_some());
|
||||
|
||||
// Deeply nested path (> FD_CACHE_MAX_INDEXED_PREFIXES slashes) must fall back
|
||||
// to per-volume unindexed_counts without degrading other volumes.
|
||||
let other_vol_key = key("v2", "shallow/part.1", false);
|
||||
cache.insert(other_vol_key.clone(), Arc::clone(&file)).await;
|
||||
let deep_path = "c1/c2/c3/c4/c5/c6/c7/c8/c9/c10/c11/c12/c13/c14/part.1";
|
||||
let deep_key = key("v", deep_path, false);
|
||||
cache.insert(deep_key.clone(), Arc::clone(&file)).await;
|
||||
assert_eq!(
|
||||
cache
|
||||
.tracker
|
||||
.unindexed_counts
|
||||
.iter()
|
||||
.map(|c| c.load(std::sync::atomic::Ordering::Acquire))
|
||||
.sum::<u32>(),
|
||||
1
|
||||
);
|
||||
assert!(
|
||||
!cache.tracker.may_match_prefix("v2", "unmatched/prefix"),
|
||||
"deep key in volume v must not disable prefix filtering in volume v2"
|
||||
);
|
||||
cache.invalidate_under("v", "c1/c2/c3/c4/c5/c6/c7/c8/c9/c10/c11/c12/c13");
|
||||
assert!(cache.get(&deep_key).await.is_none());
|
||||
cache.invalidate_volume("v2");
|
||||
assert!(cache.get(&other_vol_key).await.is_none());
|
||||
|
||||
// Replacing an invalidated key before maintenance drains pending_invalidations
|
||||
// must release the old descriptor immediately and keep the replacement live
|
||||
// even when a strong reference to the old CachedFdEntry shell survives until
|
||||
// maintenance runs its Arc::ptr_eq check.
|
||||
let old_file = Arc::new(tempfile::tempfile().expect("create replaced descriptor"));
|
||||
let old_weak = Arc::downgrade(&old_file);
|
||||
cache.insert(cached_key.clone(), old_file).await;
|
||||
let held_old_shell = cache.cache.get(&cached_key).expect("cached shell must exist");
|
||||
cache.invalidate_under("v", "obj-1/0198a3f0");
|
||||
assert!(cache.get(&cached_key).await.is_none());
|
||||
cache.insert(cached_key.clone(), Arc::clone(&file)).await;
|
||||
assert!(
|
||||
old_weak.upgrade().is_none(),
|
||||
"replacing an entry must retire and drop the previous descriptor immediately"
|
||||
);
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
while cache
|
||||
.tracker
|
||||
.has_pending_invalidations
|
||||
.load(std::sync::atomic::Ordering::SeqCst)
|
||||
{
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("maintenance should drain pending invalidations");
|
||||
drop(held_old_shell);
|
||||
assert!(
|
||||
cache.get(&cached_key).await.is_some(),
|
||||
"stale pending invalidation must not evict a newer replacement"
|
||||
);
|
||||
assert_eq!(
|
||||
cache
|
||||
.tracker
|
||||
.unindexed_counts
|
||||
.iter()
|
||||
.map(|c| c.load(std::sync::atomic::Ordering::Acquire))
|
||||
.sum::<u32>(),
|
||||
0
|
||||
);
|
||||
|
||||
// Dropping a cache while a live non-invalidated entry is present (even if a
|
||||
// cloned Moka handle outlives FdCache and the maintenance task is aborted)
|
||||
// must retire the descriptor and release FdCacheTracker without cycles.
|
||||
let live_drop_file = Arc::new(tempfile::tempfile().expect("create live drop descriptor"));
|
||||
let live_drop_weak = Arc::downgrade(&live_drop_file);
|
||||
let shutdown_file = Arc::new(tempfile::tempfile().expect("create shutdown descriptor"));
|
||||
let shutdown_weak = Arc::downgrade(&shutdown_file);
|
||||
let tracker_weak = Arc::downgrade(&cache.tracker);
|
||||
let cloned_moka = cache.cache.clone();
|
||||
cache.insert(key("v", "live-on-drop/part.1", false), live_drop_file).await;
|
||||
cache.insert(key("v", "shutdown/part.1", false), shutdown_file).await;
|
||||
cache.invalidate_under("v", "shutdown");
|
||||
if let Some(handle) = cache.maintenance_task.take() {
|
||||
handle.abort();
|
||||
let _ = handle.await;
|
||||
}
|
||||
drop(cache);
|
||||
assert!(
|
||||
live_drop_weak.upgrade().is_none(),
|
||||
"Drop for FdCache must retire live descriptors even if a cloned Moka handle survives"
|
||||
);
|
||||
assert!(
|
||||
shutdown_weak.upgrade().is_none(),
|
||||
"dropping cache when maintenance is aborted must still release descriptors"
|
||||
);
|
||||
assert!(
|
||||
tracker_weak.upgrade().is_none(),
|
||||
"dropping cache must release FdCacheTracker without reference cycles"
|
||||
);
|
||||
drop(cloned_moka);
|
||||
|
||||
// Short-TTL expired entries (including mixed expired and fresh entries) must
|
||||
// be retired by invalidate_under and clear() even though Moka's iterator skips
|
||||
// expired entries.
|
||||
let ttl_cache = FdCache::with_ttl(std::time::Duration::from_millis(15));
|
||||
let expired_file = Arc::new(tempfile::tempfile().expect("create short-ttl descriptor"));
|
||||
let expired_weak = Arc::downgrade(&expired_file);
|
||||
ttl_cache.insert(key("v", "ttl-obj/part.1", false), expired_file).await;
|
||||
tokio::time::sleep(std::time::Duration::from_millis(30)).await;
|
||||
let fresh_file = Arc::new(tempfile::tempfile().expect("create fresh descriptor"));
|
||||
let fresh_weak = Arc::downgrade(&fresh_file);
|
||||
ttl_cache.insert(key("v", "ttl-obj/part.2", false), fresh_file).await;
|
||||
ttl_cache.invalidate_under("v", "ttl-obj");
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
while expired_weak.upgrade().is_some() || fresh_weak.upgrade().is_some() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("invalidate_under must retire both expired and fresh matching descriptors");
|
||||
assert_eq!(ttl_cache.tracker.live_entries(), 0);
|
||||
|
||||
let clear_expired_file = Arc::new(tempfile::tempfile().expect("create clear short-ttl descriptor"));
|
||||
let clear_expired_weak = Arc::downgrade(&clear_expired_file);
|
||||
ttl_cache
|
||||
.insert(key("v", "ttl-clear/part.1", false), clear_expired_file)
|
||||
.await;
|
||||
tokio::time::sleep(std::time::Duration::from_millis(30)).await;
|
||||
let clear_live_file = Arc::new(tempfile::tempfile().expect("create clear live descriptor"));
|
||||
let clear_live_weak = Arc::downgrade(&clear_live_file);
|
||||
ttl_cache.insert(key("v", "ttl-clear/part.2", false), clear_live_file).await;
|
||||
ttl_cache.clear();
|
||||
assert!(
|
||||
clear_expired_weak.upgrade().is_none() && clear_live_weak.upgrade().is_none(),
|
||||
"clear() must synchronously retire both TTL-expired and live descriptors"
|
||||
);
|
||||
assert_eq!(ttl_cache.tracker.live_entries(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fd_cache_lockstep_oracle_equivalence_and_counter_conservation() {
|
||||
use std::collections::HashMap;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
let (_directory, file) = fixture();
|
||||
let cache = FdCache::with_ttl(std::time::Duration::from_secs(3600));
|
||||
let mut oracle: HashMap<FdKey, Arc<FdCacheEntry>> = HashMap::new();
|
||||
|
||||
let volumes = ["vol-a", "vol-b", "vol-c", ".rustfs.sys/multipart"];
|
||||
let paths = [
|
||||
"a/b",
|
||||
"a/b/part.1",
|
||||
"a/b/dir/part.2",
|
||||
"a/bc",
|
||||
"a/bc/part.1",
|
||||
"prefix/obj-0001.bin/0198a3f0/part.1",
|
||||
"prefix/obj-0002.bin/0198a3f0/part.1",
|
||||
"d1/d2/d3/d4/d5/d6/d7/d8/d9/d10/d11/d12/d13/part.1",
|
||||
];
|
||||
let prefixes = [
|
||||
"a",
|
||||
"a/",
|
||||
"a/b",
|
||||
"a/b/",
|
||||
"a/b/dir",
|
||||
"a/bc",
|
||||
"prefix",
|
||||
"prefix/obj-0001.bin",
|
||||
"prefix/obj-0001.bin/0198a3f0",
|
||||
"d1/d2/d3/d4/d5/c_missing",
|
||||
"d1/d2/d3/d4/d5/d6/d7/d8/d9/d10/d11/d12/d13",
|
||||
"nonexistent/prefix",
|
||||
];
|
||||
|
||||
let mut candidate_keys = Vec::new();
|
||||
for v in volumes {
|
||||
for p in paths {
|
||||
for direct in [false, true] {
|
||||
candidate_keys.push(key(v, p, direct));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut rng = 0x9e37_79b9_7f4a_7c15u64;
|
||||
let mut next_u64 = || {
|
||||
rng ^= rng << 13;
|
||||
rng ^= rng >> 7;
|
||||
rng ^= rng << 17;
|
||||
rng
|
||||
};
|
||||
|
||||
for step in 0..1000 {
|
||||
let op = next_u64() % 100;
|
||||
if op < 45 {
|
||||
// Insert or replace (either fresh or stale generation)
|
||||
let k = candidate_keys[(next_u64() as usize) % candidate_keys.len()].clone();
|
||||
let entry = Arc::new(FdCacheEntry {
|
||||
file: Arc::clone(&file),
|
||||
len: step,
|
||||
invalidated: AtomicBool::new(false),
|
||||
});
|
||||
let cur_gen = cache.generation();
|
||||
let stale = (next_u64() % 5) == 0 && cur_gen > 0;
|
||||
let gen_to_use = if stale { cur_gen - 1 } else { cur_gen };
|
||||
cache.insert_if_fresh(k.clone(), Arc::clone(&entry), gen_to_use).await;
|
||||
if !stale {
|
||||
oracle.insert(k, entry);
|
||||
}
|
||||
} else if op < 65 {
|
||||
// Exact invalidation
|
||||
let v = volumes[(next_u64() as usize) % volumes.len()];
|
||||
let p = paths[(next_u64() as usize) % paths.len()];
|
||||
cache.invalidate_exact(v, p).await;
|
||||
oracle.remove(&key(v, p, false));
|
||||
oracle.remove(&key(v, p, true));
|
||||
} else if op < 88 {
|
||||
// Prefix invalidation
|
||||
let v = volumes[(next_u64() as usize) % volumes.len()];
|
||||
let raw_prefix = prefixes[(next_u64() as usize) % prefixes.len()];
|
||||
cache.invalidate_under(v, raw_prefix);
|
||||
let trimmed = raw_prefix.trim_end_matches('/');
|
||||
oracle.retain(|k, _| {
|
||||
!(k.volume == v && (k.path == trimmed || k.path.strip_prefix(trimmed).is_some_and(|r| r.starts_with('/'))))
|
||||
});
|
||||
} else if op < 97 {
|
||||
// Volume invalidation
|
||||
let v = volumes[(next_u64() as usize) % volumes.len()];
|
||||
cache.invalidate_volume(v);
|
||||
oracle.retain(|k, _| k.volume != v);
|
||||
} else {
|
||||
// Full clear
|
||||
cache.clear();
|
||||
oracle.clear();
|
||||
}
|
||||
|
||||
// Verify lockstep equivalence against the reference oracle after every operation.
|
||||
for k in &candidate_keys {
|
||||
let got = cache.get(k).await;
|
||||
let expected = oracle.get(k);
|
||||
match (got, expected) {
|
||||
(Some(g), Some(e)) => assert!(
|
||||
Arc::ptr_eq(&g, e),
|
||||
"step {step}: pointer mismatch for key ({}, {}, {})",
|
||||
k.volume,
|
||||
k.path,
|
||||
k.direct
|
||||
),
|
||||
(None, None) => {}
|
||||
(g, e) => panic!(
|
||||
"step {step}: oracle mismatch for key ({}, {}, {}): got={}, expected={}",
|
||||
k.volume,
|
||||
k.path,
|
||||
k.direct,
|
||||
g.is_some(),
|
||||
e.is_some()
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Clear and wait for maintenance to drain all entries, then verify exact counter conservation.
|
||||
cache.clear();
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
while cache.tracker.live_entries() != 0 || cache.tracker.has_pending_invalidations.load(Ordering::SeqCst) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("clear must drain all live entries and pending invalidations");
|
||||
|
||||
assert_eq!(cache.tracker.live_entries(), 0);
|
||||
assert!(cache.tracker.unindexed_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
|
||||
assert!(cache.tracker.volume_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
|
||||
assert!(cache.tracker.exact_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
|
||||
assert!(cache.tracker.prefix_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
|
||||
assert!(cache.tracker.prefix_shards.iter().all(|s| s.lock().is_empty()));
|
||||
assert!(cache.tracker.entry_shards.iter().all(|s| s.lock().is_empty()));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn fd_cache_concurrent_insert_and_invalidation_stress() {
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
let (_directory, file) = fixture();
|
||||
let cache = Arc::new(FdCache::new());
|
||||
|
||||
// Deterministic Dekker-partner interleaving check: pause `CachedFdEntry::new`
|
||||
// at `entry_shards[shard].lock()` after it has passed the initial generation
|
||||
// check and incremented `volume_counts`, bump generation via an unmatched
|
||||
// invalidation, and verify the post-`new()` recheck refuses publication and
|
||||
// preserves the existing entry.
|
||||
let seam_key = key("v-seam", "obj-1/part.1", false);
|
||||
cache.insert(seam_key.clone(), Arc::clone(&file)).await;
|
||||
let initial_entry = cache.get(&seam_key).await.expect("initial entry must exist");
|
||||
let (vol_hash, vol_slot) = super::fd_cache_volume_hash_and_slot("v-seam");
|
||||
let entry_shard = (vol_hash as usize) & (super::FD_CACHE_PREFIX_SHARDS - 1);
|
||||
let stale_gen = cache.generation();
|
||||
let stale_entry = Arc::new(FdCacheEntry {
|
||||
file: Arc::clone(&file),
|
||||
len: 999_999,
|
||||
invalidated: AtomicBool::new(false),
|
||||
});
|
||||
let blocked_insert = {
|
||||
let hold = cache.tracker.entry_shards[entry_shard].lock();
|
||||
let cache_clone = Arc::clone(&cache);
|
||||
let seam_key_clone = seam_key.clone();
|
||||
let task = tokio::spawn(async move {
|
||||
cache_clone.insert_if_fresh(seam_key_clone, stale_entry, stale_gen).await;
|
||||
});
|
||||
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
|
||||
while cache.tracker.volume_counts[usize::from(vol_slot)].load(Ordering::SeqCst) < 2 {
|
||||
assert!(
|
||||
std::time::Instant::now() < deadline,
|
||||
"insert never reached CachedFdEntry::new with the previous entry still live"
|
||||
);
|
||||
std::thread::yield_now();
|
||||
}
|
||||
cache.invalidate_under("v-seam", "unmatched-prefix");
|
||||
drop(hold);
|
||||
task
|
||||
};
|
||||
blocked_insert.await.expect("blocked insert task should complete");
|
||||
let preserved = cache
|
||||
.get(&seam_key)
|
||||
.await
|
||||
.expect("refused stale insert must not retire a valid previous entry");
|
||||
assert!(Arc::ptr_eq(&preserved, &initial_entry));
|
||||
cache.invalidate_volume("v-seam");
|
||||
|
||||
let floor_v1 = Arc::new(AtomicU64::new(0));
|
||||
let floor_v2 = Arc::new(AtomicU64::new(0));
|
||||
let mut handles = Vec::new();
|
||||
|
||||
for t in 0..8usize {
|
||||
let cache = Arc::clone(&cache);
|
||||
let file = Arc::clone(&file);
|
||||
let floor_v1 = Arc::clone(&floor_v1);
|
||||
let floor_v2 = Arc::clone(&floor_v2);
|
||||
handles.push(tokio::spawn(async move {
|
||||
for i in 0..250u64 {
|
||||
let is_v1 = (t + i as usize).is_multiple_of(2);
|
||||
let vol = if is_v1 { "v1" } else { "v2" };
|
||||
let path = format!("dir-{}/obj-{}/part.1", t % 4, i % 16);
|
||||
let k = key(vol, &path, i.is_multiple_of(2));
|
||||
let g = cache.generation();
|
||||
let entry = Arc::new(FdCacheEntry {
|
||||
file: Arc::clone(&file),
|
||||
len: g,
|
||||
invalidated: AtomicBool::new(false),
|
||||
});
|
||||
cache.insert_if_fresh(k.clone(), entry, g).await;
|
||||
let min_gen = if is_v1 {
|
||||
floor_v1.load(Ordering::SeqCst)
|
||||
} else {
|
||||
floor_v2.load(Ordering::SeqCst)
|
||||
};
|
||||
if let Some(hit) = cache.get(&k).await {
|
||||
assert!(
|
||||
hit.len >= min_gen,
|
||||
"stale descriptor served after volume invalidation: opened_gen={}, floor_gen={min_gen}",
|
||||
hit.len
|
||||
);
|
||||
}
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
for t in 0..4usize {
|
||||
let cache = Arc::clone(&cache);
|
||||
let floor_v1 = Arc::clone(&floor_v1);
|
||||
let floor_v2 = Arc::clone(&floor_v2);
|
||||
handles.push(tokio::spawn(async move {
|
||||
for i in 0..250u64 {
|
||||
let is_v1 = (t + i as usize).is_multiple_of(2);
|
||||
let vol = if is_v1 { "v1" } else { "v2" };
|
||||
match i % 4 {
|
||||
0 => cache.invalidate_under(vol, &format!("dir-{}/obj-{}", t % 4, i % 16)),
|
||||
1 => {
|
||||
let p = format!("dir-{}/obj-{}/part.1", t % 4, i % 16);
|
||||
cache.invalidate_exact(vol, &p).await;
|
||||
}
|
||||
2 => {
|
||||
let before = cache.generation();
|
||||
cache.invalidate_volume(vol);
|
||||
let floor = before + 1;
|
||||
if is_v1 {
|
||||
floor_v1.fetch_max(floor, Ordering::SeqCst);
|
||||
} else {
|
||||
floor_v2.fetch_max(floor, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
_ => cache.invalidate_under("uncached-vol", "uncached/prefix"),
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
for h in handles {
|
||||
h.await.expect("worker task should succeed");
|
||||
}
|
||||
|
||||
cache.clear();
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
while cache.tracker.live_entries() != 0 || cache.tracker.has_pending_invalidations.load(Ordering::SeqCst) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("clear after concurrent stress must drain all live entries");
|
||||
|
||||
assert_eq!(cache.tracker.live_entries(), 0);
|
||||
assert!(cache.tracker.unindexed_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
|
||||
assert!(cache.tracker.volume_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
|
||||
assert!(cache.tracker.exact_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
|
||||
assert!(cache.tracker.prefix_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
|
||||
assert!(cache.tracker.prefix_shards.iter().all(|s| s.lock().is_empty()));
|
||||
assert!(cache.tracker.entry_shards.iter().all(|s| s.lock().is_empty()));
|
||||
}
|
||||
|
||||
+2254
-335
File diff suppressed because it is too large
Load Diff
@@ -15,6 +15,8 @@
|
||||
//! Single-disk object rename publication and rollback. The shared execution core
|
||||
//! retains instrumentation, mutation leases, and commit guards through the syscall.
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
use super::has_rename_data_test_failpoints;
|
||||
#[cfg(all(test, windows))]
|
||||
use super::run_destination_commit_directory_preparation;
|
||||
#[cfg(any(not(windows), test))]
|
||||
@@ -96,7 +98,7 @@ struct RenameCommitContext<'a> {
|
||||
data_paths: Option<&'a (PathBuf, PathBuf)>,
|
||||
}
|
||||
|
||||
struct StagedRollbackBackup(PathBuf);
|
||||
pub(super) struct StagedRollbackBackup(PathBuf);
|
||||
|
||||
impl StagedRollbackBackup {
|
||||
fn path(&self) -> &Path {
|
||||
@@ -175,17 +177,30 @@ pub(super) fn create_local_inline_rollback_backup(
|
||||
dst_file_path: &Path,
|
||||
staging_file_path: &Path,
|
||||
old_metadata: &[u8],
|
||||
sync: bool,
|
||||
) -> std::io::Result<PathBuf> {
|
||||
let Some(staging_parent) = staging_file_path.parent() else {
|
||||
return Err(std::io::Error::new(ErrorKind::InvalidInput, "missing staging metadata parent"));
|
||||
};
|
||||
let backup_path = staging_parent.join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
remove_file_if_exists(&backup_path)?;
|
||||
if (should_fail_local_inline_rollback_hardlink(dst_file_path) || std::fs::hard_link(dst_file_path, &backup_path).is_err())
|
||||
&& let Err(err) = std::fs::write(&backup_path, old_metadata)
|
||||
{
|
||||
let _ = remove_file_if_exists(&backup_path);
|
||||
return Err(err);
|
||||
if should_fail_local_inline_rollback_hardlink(dst_file_path) || std::fs::hard_link(dst_file_path, &backup_path).is_err() {
|
||||
let write_res = (|| -> std::io::Result<()> {
|
||||
#[cfg(test)]
|
||||
run_owned_file_before_open(&backup_path);
|
||||
let mut file = std::fs::File::create(&backup_path)?;
|
||||
std::io::Write::write_all(&mut file, old_metadata)?;
|
||||
if sync && !old_metadata.is_empty() {
|
||||
file.sync_data()?;
|
||||
}
|
||||
Ok(())
|
||||
})();
|
||||
if let Err(err) = write_res {
|
||||
let _ = remove_file_if_exists(&backup_path);
|
||||
return Err(err);
|
||||
}
|
||||
} else if sync && !old_metadata.is_empty() {
|
||||
std::fs::File::open(&backup_path)?.sync_data()?;
|
||||
}
|
||||
Ok(backup_path)
|
||||
}
|
||||
@@ -220,11 +235,13 @@ fn rollback_inline_metadata_std(
|
||||
}
|
||||
}
|
||||
|
||||
fn fsync_inline_commit_std(
|
||||
pub(super) fn fsync_inline_commit_std(
|
||||
dst: &Path,
|
||||
dst_volume_dir: &Path,
|
||||
destination_was_absent: bool,
|
||||
rollback_data_dir: Option<Uuid>,
|
||||
local_backup: Option<&StagedRollbackBackup>,
|
||||
new_data_dir: Option<&Path>,
|
||||
commit_guard: &os::RenameCommitGuard,
|
||||
) -> std::io::Result<()> {
|
||||
if let Some(dst_parent) = dst.parent() {
|
||||
@@ -234,7 +251,10 @@ fn fsync_inline_commit_std(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC,
|
||||
fsync_started,
|
||||
);
|
||||
rollback_inline_metadata_std(dst, None, local_backup)?;
|
||||
rollback_inline_metadata_std(dst, rollback_data_dir, local_backup)?;
|
||||
if let Some(new_data_path) = new_data_dir {
|
||||
remove_dir_all_if_exists(new_data_path)?;
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
@@ -255,7 +275,10 @@ fn fsync_inline_commit_std(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC,
|
||||
fsync_started,
|
||||
);
|
||||
rollback_inline_metadata_std(dst, None, local_backup)?;
|
||||
rollback_inline_metadata_std(dst, rollback_data_dir, local_backup)?;
|
||||
if let Some(new_data_path) = new_data_dir {
|
||||
remove_dir_all_if_exists(new_data_path)?;
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
@@ -356,9 +379,12 @@ fn prepare_rename_commit_and_destination_metadata_std(
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
enum FusedCommitAttempt {
|
||||
Committed(RenameDataResp),
|
||||
Committed(Box<RenameDataResp>),
|
||||
CommitErr(DiskError),
|
||||
Fallback(os::RenameCommitGuard, Option<Bytes>, FileInfo),
|
||||
RestoreDataSource(os::RenameCommitGuard, DiskError),
|
||||
Fallback(os::RenameCommitGuard, Option<Bytes>, Box<FileInfo>),
|
||||
#[cfg(test)]
|
||||
ResumePanic(Box<dyn std::any::Any + Send + 'static>),
|
||||
}
|
||||
|
||||
async fn prepare_rename_commit_and_destination_metadata(
|
||||
@@ -470,6 +496,9 @@ impl LocalDisk {
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
})?;
|
||||
crate::hp_guard!("LocalDisk::rename_data");
|
||||
if dst_volume == super::super::RUSTFS_META_MULTIPART_BUCKET {
|
||||
self.multipart_index.record_info(dst_path, &fi).await?;
|
||||
}
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
if fi.is_healing() {
|
||||
wait_for_heal_commit_test_barrier(&self.root, dst_volume, dst_path).await?;
|
||||
@@ -540,7 +569,9 @@ impl LocalDisk {
|
||||
None
|
||||
};
|
||||
let multipart_publication = if let Some(staging) = multipart_staging.as_ref() {
|
||||
Some(pin_multipart_future(|| self.prepare_native_multipart_publication(src_path, &mut fi, staging)).await?)
|
||||
Some(Arc::new(
|
||||
pin_multipart_future(|| self.prepare_native_multipart_publication_deferred(src_path, &mut fi, staging)).await?,
|
||||
))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -741,11 +772,6 @@ impl LocalDisk {
|
||||
&& !fi_healing
|
||||
&& !os::dst_dir_fsync_group_commit_enabled()
|
||||
&& !os::file_fdatasync_group_commit_enabled()
|
||||
&& !super::has_rename_data_test_failpoints(dst_path)
|
||||
&& !crash_inject::is_armed(CrashPoint::RenameAfterDataRename, dst_path)
|
||||
&& !crash_inject::is_armed(CrashPoint::RenameAfterBackupBeforeMetaCommit, dst_path)
|
||||
&& !crash_inject::is_armed(CrashPoint::RenameAfterMetaCommit, dst_path)
|
||||
&& !super::has_delete_version_test_failpoints()
|
||||
{
|
||||
#[cfg(test)]
|
||||
if !no_inline {
|
||||
@@ -785,10 +811,11 @@ impl LocalDisk {
|
||||
&dst_volume_dir_owned,
|
||||
&publication_root_owned,
|
||||
)?;
|
||||
// An existing xl.meta can share versions or require rollback
|
||||
// restoration when a post-rename directory sync fails.
|
||||
if is_cancelled() || (sync_commit_metadata && has_dst_buf.is_some()) {
|
||||
return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, fi_moved));
|
||||
// An existing xl.meta that replaces an existing version slot
|
||||
// requires a rollback directory; appending a new version keeps
|
||||
// an inline rollback backup for post-rename directory sync errors.
|
||||
if is_cancelled() {
|
||||
return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, Box::new(fi_moved)));
|
||||
}
|
||||
let mut xlmeta = FileMeta::new();
|
||||
let mut dst_meta_unparsable = false;
|
||||
@@ -810,8 +837,20 @@ impl LocalDisk {
|
||||
None
|
||||
}
|
||||
});
|
||||
if rollback_data_dir.is_some() {
|
||||
return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, fi_moved));
|
||||
if rollback_data_dir.is_some() && has_rename_data_test_failpoints(&dst_path_str) {
|
||||
return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, Box::new(fi_moved)));
|
||||
}
|
||||
#[cfg(test)]
|
||||
let probe_active = os::file_sync_probe::is_active();
|
||||
#[cfg(not(test))]
|
||||
let probe_active = false;
|
||||
if no_inline
|
||||
&& sync_data_shards
|
||||
&& let Some((src_data_path, _)) = has_data_dir_path_owned.as_ref()
|
||||
&& (probe_active || !super::is_presynced_tmp_data_dir(src_data_path))
|
||||
&& os::regular_files(src_data_path).is_ok_and(|files| files.len() >= os::PARALLEL_FILE_SYNC_THRESHOLD)
|
||||
{
|
||||
return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, Box::new(fi_moved)));
|
||||
}
|
||||
let (old_current_size, old_current_source_checked, old_current_source) = observe_old_current(
|
||||
has_dst_buf.is_some(),
|
||||
@@ -821,6 +860,9 @@ impl LocalDisk {
|
||||
&dst_path_str,
|
||||
version_id,
|
||||
);
|
||||
if let Some(old_data_dir) = has_old_data_dir.as_ref() {
|
||||
let _ = xlmeta.data.remove_two(version_id, *old_data_dir);
|
||||
}
|
||||
xlmeta.add_version(fi_moved)?;
|
||||
let version_signature = rename_data_versions_signature(&xlmeta);
|
||||
let new_dst_buf = xlmeta.marshal_msg().map_err(std::io::Error::other)?;
|
||||
@@ -830,13 +872,32 @@ impl LocalDisk {
|
||||
skip_parent = dst_parent_owned.clone();
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
run_owned_file_before_open(&src_file_path_owned);
|
||||
let mut prepared_metadata_source =
|
||||
os::create_prepared_rename_source_with_commit_guard(&src_file_path_owned, &dst_file_path_owned, &guard)?;
|
||||
prepared_metadata_source.write_all(&new_dst_buf, sync_commit_metadata)?;
|
||||
prepared_metadata_source.write_all(&new_dst_buf, false)?;
|
||||
if !no_inline {
|
||||
run_inline_preparation_before_backup(&dst_path_str);
|
||||
}
|
||||
|
||||
let mut staged_rollback_path = None;
|
||||
if (!no_inline || rollback_data_dir.is_none())
|
||||
&& let Some(ref old_metadata) = has_dst_buf
|
||||
&& (rollback_data_dir.is_some() || sync_commit_metadata || cfg!(test))
|
||||
{
|
||||
let backup_path = create_local_inline_rollback_backup(
|
||||
&dst_file_path_owned,
|
||||
&src_file_path_owned,
|
||||
old_metadata,
|
||||
sync_commit_metadata,
|
||||
)?;
|
||||
staged_rollback_path = Some(StagedRollbackBackup(backup_path));
|
||||
}
|
||||
|
||||
if no_inline {
|
||||
if sync_data_shards && let Some((src_data_path, _)) = has_data_dir_path_owned.as_ref() {
|
||||
let sync_result = if super::take_presynced_tmp_data_dir(src_data_path) {
|
||||
let sync_result = if !probe_active && super::take_presynced_tmp_data_dir(src_data_path) {
|
||||
// The payload was synced, but its part.1 directory entry
|
||||
// must also be persisted before moving the data directory.
|
||||
os::fsync_dir_std(src_data_path)
|
||||
@@ -846,58 +907,212 @@ impl LocalDisk {
|
||||
if let Err(err) = sync_result
|
||||
&& err.kind() != ErrorKind::NotFound
|
||||
{
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
if sync_commit_metadata && let Err(err) = prepared_metadata_source.sync_data() {
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
} else if sync_commit_metadata && let Err(err) = prepared_metadata_source.sync_data() {
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
if is_cancelled() {
|
||||
drop(prepared_metadata_source);
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
return Err(std::io::Error::new(ErrorKind::Interrupted, "commit cancelled"));
|
||||
}
|
||||
|
||||
let guard = match super::remove_dst_base_before_commit_std(
|
||||
&dst_path_str,
|
||||
guard,
|
||||
&src_parent_owned,
|
||||
&dst_parent_owned,
|
||||
&dst_volume_dir_owned,
|
||||
&publication_root_owned,
|
||||
) {
|
||||
Ok(guard) => guard,
|
||||
Err(err) => {
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
return Ok(FusedCommitAttempt::CommitErr(err));
|
||||
}
|
||||
};
|
||||
if should_remove_staged_meta_before_commit(&dst_path_str) {
|
||||
drop(prepared_metadata_source);
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
std::fs::remove_file(&src_file_path_owned)?;
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::FileNotFound));
|
||||
}
|
||||
|
||||
if no_inline {
|
||||
if let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref()
|
||||
&& let Err(err) = os::rename_all_with_commit_guard_on_thread(src_data_path, dst_data_path, &guard)
|
||||
{
|
||||
return Ok(FusedCommitAttempt::CommitErr(err));
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
return Ok(FusedCommitAttempt::RestoreDataSource(guard, err));
|
||||
}
|
||||
#[cfg(test)]
|
||||
if has_data_dir_path_owned.is_some() {
|
||||
run_rename_data_after_first_publication(&root_owned, &dst_volume_str, &dst_path_str);
|
||||
if has_data_dir_path_owned.is_some()
|
||||
&& let Err(panic_payload) = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
|
||||
run_rename_data_after_first_publication(&root_owned, &dst_volume_str, &dst_path_str);
|
||||
}))
|
||||
{
|
||||
return Ok(FusedCommitAttempt::ResumePanic(panic_payload));
|
||||
}
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterDataRename, &dst_path_str) {
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
|
||||
}
|
||||
if should_fail_before_old_metadata_backup(&dst_path_str) {
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
return Ok(FusedCommitAttempt::RestoreDataSource(guard, DiskError::Unexpected));
|
||||
}
|
||||
if let (Some(old_data_dir), Some(dst_buf)) = (rollback_data_dir, has_dst_buf.as_ref()) {
|
||||
let backup_parent = dst_parent_owned.join(old_data_dir.to_string());
|
||||
if let Err(err) =
|
||||
os::mkdir_all_below_existing_base_std(&backup_parent, &skip_parent, &publication_root_owned)
|
||||
{
|
||||
return Ok(FusedCommitAttempt::RestoreDataSource(guard, to_file_error(err).into()));
|
||||
}
|
||||
let backup_path_guard = match guard.create_destination_directory_for_path_access(&backup_parent) {
|
||||
Ok(g) => g,
|
||||
Err(err) => return Ok(FusedCommitAttempt::RestoreDataSource(guard, to_file_error(err).into())),
|
||||
};
|
||||
let backup_path = backup_parent.join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
if let Err(err) = check_path_length(backup_path.to_string_lossy().as_ref()) {
|
||||
return Ok(FusedCommitAttempt::RestoreDataSource(guard, err));
|
||||
}
|
||||
#[cfg(test)]
|
||||
run_owned_file_before_open(&backup_path);
|
||||
if let Err(err) = backup_path_guard.write_file_for_path_access(
|
||||
&backup_path,
|
||||
dst_buf.as_ref(),
|
||||
sync_commit_metadata,
|
||||
sync_commit_metadata,
|
||||
) {
|
||||
return Ok(FusedCommitAttempt::RestoreDataSource(guard, to_file_error(err).into()));
|
||||
}
|
||||
}
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterBackupBeforeMetaCommit, &dst_path_str) {
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
|
||||
}
|
||||
} else if let (Some(old_data_dir), Some(staged_backup)) =
|
||||
(rollback_data_dir, staged_rollback_path.as_ref().map(StagedRollbackBackup::path))
|
||||
{
|
||||
let backup_parent = dst_parent_owned.join(old_data_dir.to_string());
|
||||
let backup_path = backup_parent.join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
if let Err(err) = os::rename_all_with_precondition_and_parent_sync_std(
|
||||
staged_backup,
|
||||
&backup_path,
|
||||
&dst_volume_dir_owned,
|
||||
&publication_root_owned,
|
||||
None,
|
||||
sync_commit_metadata,
|
||||
) {
|
||||
let _ = remove_file_if_exists(staged_backup);
|
||||
return Ok(FusedCommitAttempt::CommitErr(to_file_error(err).into()));
|
||||
}
|
||||
staged_rollback_path = None;
|
||||
}
|
||||
|
||||
if let Err(err) = os::rename_prepared_source_on_thread(
|
||||
prepared_metadata_source,
|
||||
&src_file_path_owned,
|
||||
&dst_file_path_owned,
|
||||
&skip_parent,
|
||||
&publication_root_owned,
|
||||
&guard,
|
||||
) {
|
||||
if no_inline && let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref() {
|
||||
let _ = std::fs::rename(dst_data_path, src_data_path);
|
||||
let rename_res = if should_fail_commit_rename(&dst_path_str) {
|
||||
drop(prepared_metadata_source);
|
||||
Err(DiskError::other("test fail during metadata commit rename"))
|
||||
} else {
|
||||
os::rename_prepared_source_on_thread(
|
||||
prepared_metadata_source,
|
||||
&src_file_path_owned,
|
||||
&dst_file_path_owned,
|
||||
&skip_parent,
|
||||
&publication_root_owned,
|
||||
&guard,
|
||||
)
|
||||
};
|
||||
if let Err(err) = rename_res {
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
if no_inline {
|
||||
return Ok(FusedCommitAttempt::RestoreDataSource(guard, err));
|
||||
}
|
||||
return Ok(FusedCommitAttempt::CommitErr(err));
|
||||
}
|
||||
|
||||
if sync_commit_metadata
|
||||
&& let Err(err) =
|
||||
fsync_inline_commit_std(&dst_file_path_owned, &dst_volume_dir_owned, destination_was_absent, None, &guard)
|
||||
{
|
||||
if no_inline && let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref() {
|
||||
let _ = std::fs::rename(dst_data_path, src_data_path);
|
||||
if should_fail_after_metadata_commit(&dst_path_str) {
|
||||
if no_inline {
|
||||
rollback_inline_metadata_std(&dst_file_path_owned, rollback_data_dir, staged_rollback_path.as_ref())?;
|
||||
if let Some((_, new_data_path)) = has_data_dir_path_owned.as_ref() {
|
||||
remove_dir_all_if_exists(new_data_path)?;
|
||||
}
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
|
||||
}
|
||||
return Err(err);
|
||||
let err =
|
||||
rollback_inline_metadata_std(&dst_file_path_owned, rollback_data_dir, staged_rollback_path.as_ref())
|
||||
.err()
|
||||
.unwrap_or_else(|| std::io::Error::other("test fail after metadata commit"));
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::from(err)));
|
||||
}
|
||||
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterMetaCommit, &dst_path_str) {
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
|
||||
}
|
||||
|
||||
if sync_commit_metadata
|
||||
&& let Err(err) = fsync_inline_commit_std(
|
||||
&dst_file_path_owned,
|
||||
&dst_volume_dir_owned,
|
||||
destination_was_absent,
|
||||
rollback_data_dir,
|
||||
staged_rollback_path.as_ref(),
|
||||
has_data_dir_path_owned
|
||||
.as_ref()
|
||||
.filter(|_| no_inline)
|
||||
.map(|(_, new_data_path)| new_data_path.as_path()),
|
||||
&guard,
|
||||
)
|
||||
{
|
||||
if no_inline {
|
||||
return Ok(FusedCommitAttempt::CommitErr(to_file_error(err).into()));
|
||||
}
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::from(err)));
|
||||
}
|
||||
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
|
||||
if let Some(parent) = src_file_path_owned.parent() {
|
||||
let _ = std::fs::remove_dir(parent);
|
||||
}
|
||||
|
||||
Ok(FusedCommitAttempt::Committed(RenameDataResp {
|
||||
Ok(FusedCommitAttempt::Committed(Box::new(RenameDataResp {
|
||||
multipart_source: None,
|
||||
old_data_dir: None,
|
||||
rollback_data_dir: None,
|
||||
cleanup_data_dir: None,
|
||||
old_data_dir: has_old_data_dir,
|
||||
rollback_data_dir,
|
||||
cleanup_data_dir: has_old_data_dir,
|
||||
sign: version_signature,
|
||||
old_current_size,
|
||||
old_current_source_checked,
|
||||
old_current_source,
|
||||
}))
|
||||
})))
|
||||
};
|
||||
|
||||
let attempt_res = if let Some(admission) = file_sync_admission.as_ref() {
|
||||
@@ -917,7 +1132,23 @@ impl LocalDisk {
|
||||
if let Some(receipt_mutation) = receipt_mutation.as_mut() {
|
||||
receipt_mutation.complete();
|
||||
}
|
||||
return Ok(resp);
|
||||
return Ok(*resp);
|
||||
}
|
||||
Ok(FusedCommitAttempt::RestoreDataSource(directory_guard, err)) => {
|
||||
let commit = RenameCommitContext {
|
||||
directory_guard,
|
||||
publication_root: &self.publication_root,
|
||||
mutation_lease,
|
||||
source_metadata: &src_file_path,
|
||||
destination_metadata: &dst_file_path,
|
||||
source_volume_dir: &src_volume_dir,
|
||||
data_paths: has_data_dir_path.as_ref(),
|
||||
};
|
||||
commit.restore_published_data_source().await?;
|
||||
for part_path in &invalidate_part_paths {
|
||||
self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
Ok(FusedCommitAttempt::CommitErr(err)) | Err(err) => {
|
||||
for part_path in &invalidate_part_paths {
|
||||
@@ -925,11 +1156,269 @@ impl LocalDisk {
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
#[cfg(test)]
|
||||
Ok(FusedCommitAttempt::ResumePanic(panic_payload)) => {
|
||||
std::panic::resume_unwind(panic_payload);
|
||||
}
|
||||
Ok(FusedCommitAttempt::Fallback(guard, dst_buf, restored_fi)) => {
|
||||
fi = restored_fi;
|
||||
fi = *restored_fi;
|
||||
(guard, dst_buf)
|
||||
}
|
||||
}
|
||||
} else if let Some(publication) = multipart_publication.as_ref()
|
||||
&& no_inline
|
||||
&& !preserve_upload_metadata
|
||||
&& !publication.legacy
|
||||
&& !fi_healing
|
||||
&& !os::dst_dir_fsync_group_commit_enabled()
|
||||
{
|
||||
enum MultipartFusedCommitOutcome {
|
||||
Committed(Box<RenameDataResp>),
|
||||
AbortBeforeDataRename(DiskError),
|
||||
RestoreDataSource(os::RenameCommitGuard, DiskError),
|
||||
RestoreTargetAfterError(DiskError, bool),
|
||||
}
|
||||
let publication_owned = Arc::clone(publication);
|
||||
let src_parent_owned = src_file_parent.to_path_buf();
|
||||
let dst_parent_owned = dst_file_parent.to_path_buf();
|
||||
let dst_file_path_owned = dst_file_path.clone();
|
||||
let src_file_path_owned = src_file_path.clone();
|
||||
let dst_volume_dir_owned = dst_volume_dir.clone();
|
||||
let publication_root_owned = self.publication_root.clone();
|
||||
let has_data_dir_path_owned = has_data_dir_path.clone();
|
||||
let sync_data_shards = durability.syncs_data_shards();
|
||||
let sync_commit_metadata = durability.syncs_commit_metadata();
|
||||
let staging_cleanup = multipart_staging.as_ref().map(|s| s.cleanup_handle());
|
||||
let dst_volume_str = dst_volume.to_string();
|
||||
let dst_path_str = dst_path.to_string();
|
||||
let fi_moved = fi;
|
||||
#[cfg(test)]
|
||||
let root_owned = self.root.clone();
|
||||
|
||||
let outcome = os::run_blocking_namespace_operation_with_sync(
|
||||
mutation_lease.clone(),
|
||||
true,
|
||||
move || -> std::io::Result<MultipartFusedCommitOutcome> {
|
||||
let (directory_guard_owned, has_dst_buf_owned) = match prepare_rename_commit_and_destination_metadata_std(
|
||||
false,
|
||||
&src_parent_owned,
|
||||
&dst_parent_owned,
|
||||
&dst_file_path_owned,
|
||||
&dst_volume_dir_owned,
|
||||
&publication_root_owned,
|
||||
) {
|
||||
Ok(res) => res,
|
||||
Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into())),
|
||||
};
|
||||
let mut xlmeta = FileMeta::new();
|
||||
let mut dst_meta_unparsable = false;
|
||||
if let Some(dst_buf) = has_dst_buf_owned.as_ref() {
|
||||
if FileMeta::is_xl2_v1_format(dst_buf)
|
||||
&& let Ok(nmeta) = FileMeta::load(dst_buf)
|
||||
{
|
||||
xlmeta = nmeta;
|
||||
} else {
|
||||
dst_meta_unparsable = true;
|
||||
}
|
||||
}
|
||||
let skip_parent_owned = if has_dst_buf_owned.is_some() {
|
||||
dst_parent_owned.clone()
|
||||
} else {
|
||||
dst_volume_dir_owned.clone()
|
||||
};
|
||||
let version_id = fi_moved.version_id.unwrap_or_default();
|
||||
let (old_current_size, old_current_source_checked, old_current_source) = observe_old_current(
|
||||
has_dst_buf_owned.is_some(),
|
||||
dst_meta_unparsable,
|
||||
&xlmeta,
|
||||
&dst_volume_str,
|
||||
&dst_path_str,
|
||||
version_id,
|
||||
);
|
||||
let has_old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id));
|
||||
if let Some(old_data_dir) = has_old_data_dir.as_ref() {
|
||||
let _ = xlmeta.data.remove_two(version_id, *old_data_dir);
|
||||
}
|
||||
if let Err(err) = xlmeta.add_version(fi_moved) {
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(err.into()));
|
||||
}
|
||||
let version_signature = rename_data_versions_signature(&xlmeta);
|
||||
let new_dst_buf_owned = match xlmeta.marshal_msg() {
|
||||
Ok(buf) => buf,
|
||||
Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(err.into())),
|
||||
};
|
||||
|
||||
let prepared_metadata_source = match publication_owned.prepare_and_install_head_on_thread(
|
||||
&new_dst_buf_owned,
|
||||
has_dst_buf_owned.as_deref(),
|
||||
&dst_file_path_owned,
|
||||
&directory_guard_owned,
|
||||
) {
|
||||
Ok(source) => source,
|
||||
Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(err)),
|
||||
};
|
||||
#[cfg(test)]
|
||||
let probe_active = os::file_sync_probe::is_active();
|
||||
#[cfg(not(test))]
|
||||
let probe_active = false;
|
||||
// When presynced, shard payloads were synced during UploadPart and
|
||||
// prepare_and_install_head_on_thread just fsynced src_data_path after
|
||||
// writing RETIREMENT_ANCHOR.
|
||||
if sync_data_shards
|
||||
&& let Some((src_data_path, _)) = has_data_dir_path_owned.as_ref()
|
||||
&& (probe_active || !super::take_presynced_tmp_data_dir(src_data_path))
|
||||
&& let Err(err) = os::sync_dir_files_std(src_data_path)
|
||||
&& err.kind() != ErrorKind::NotFound
|
||||
{
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into()));
|
||||
}
|
||||
let directory_guard_owned = match super::remove_dst_base_before_commit_std(
|
||||
&dst_path_str,
|
||||
directory_guard_owned,
|
||||
&src_parent_owned,
|
||||
&dst_parent_owned,
|
||||
&dst_volume_dir_owned,
|
||||
&publication_root_owned,
|
||||
) {
|
||||
Ok(guard) => guard,
|
||||
Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(err)),
|
||||
};
|
||||
if should_remove_staged_meta_before_commit(&dst_path_str) {
|
||||
drop(prepared_metadata_source);
|
||||
if let Err(err) = std::fs::remove_file(&src_file_path_owned) {
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into()));
|
||||
}
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::FileNotFound));
|
||||
}
|
||||
if let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref() {
|
||||
let Some(src_parent) = src_data_path.parent() else {
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::InvalidPath));
|
||||
};
|
||||
let payload_guard = match os::multipart_payload_commit_guard_std(&directory_guard_owned, src_parent) {
|
||||
Ok(guard) => guard,
|
||||
Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into())),
|
||||
};
|
||||
if let Err(err) = os::rename_all_with_commit_guard_on_thread(src_data_path, dst_data_path, &payload_guard)
|
||||
{
|
||||
return Ok(MultipartFusedCommitOutcome::RestoreDataSource(directory_guard_owned, err));
|
||||
}
|
||||
#[cfg(test)]
|
||||
run_rename_data_after_first_publication(&root_owned, &dst_volume_str, &dst_path_str);
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterDataRename, &dst_path_str) {
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::Unexpected));
|
||||
}
|
||||
if should_fail_before_old_metadata_backup(&dst_path_str) {
|
||||
return Ok(MultipartFusedCommitOutcome::RestoreDataSource(
|
||||
directory_guard_owned,
|
||||
DiskError::Unexpected,
|
||||
));
|
||||
}
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterBackupBeforeMetaCommit, &dst_path_str) {
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::Unexpected));
|
||||
}
|
||||
}
|
||||
let rename_res = if should_fail_commit_rename(&dst_path_str) {
|
||||
drop(prepared_metadata_source);
|
||||
Err(DiskError::other("test fail during metadata commit rename"))
|
||||
} else {
|
||||
os::rename_prepared_source_on_thread(
|
||||
prepared_metadata_source,
|
||||
&src_file_path_owned,
|
||||
&dst_file_path_owned,
|
||||
&skip_parent_owned,
|
||||
&publication_root_owned,
|
||||
&directory_guard_owned,
|
||||
)
|
||||
};
|
||||
if let Err(err) = rename_res {
|
||||
return Ok(MultipartFusedCommitOutcome::RestoreDataSource(directory_guard_owned, err));
|
||||
}
|
||||
if should_fail_after_metadata_commit(&dst_path_str) {
|
||||
return Ok(MultipartFusedCommitOutcome::RestoreTargetAfterError(DiskError::Unexpected, false));
|
||||
}
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterMetaCommit, &dst_path_str) {
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::Unexpected));
|
||||
}
|
||||
if sync_commit_metadata {
|
||||
if let Some(parent) = dst_file_path_owned.parent()
|
||||
&& let Err(err) = os::fsync_commit_directory_limited_std(parent, &directory_guard_owned)
|
||||
{
|
||||
return Ok(MultipartFusedCommitOutcome::RestoreTargetAfterError(to_file_error(err).into(), true));
|
||||
}
|
||||
if has_dst_buf_owned.is_none() {
|
||||
let mut ancestor = dst_file_path_owned.parent().and_then(|object_dir| object_dir.parent());
|
||||
while let Some(dir) = ancestor {
|
||||
if !dir.starts_with(&dst_volume_dir_owned) {
|
||||
break;
|
||||
}
|
||||
if let Err(err) = os::fsync_dir_limited_std(dir) {
|
||||
return Ok(MultipartFusedCommitOutcome::RestoreTargetAfterError(
|
||||
to_file_error(err).into(),
|
||||
true,
|
||||
));
|
||||
}
|
||||
if dir == dst_volume_dir_owned.as_path() {
|
||||
break;
|
||||
}
|
||||
ancestor = dir.parent();
|
||||
}
|
||||
}
|
||||
}
|
||||
match publication_owned.finish_on_thread(sync_commit_metadata, staging_cleanup) {
|
||||
Ok(result) => Ok(MultipartFusedCommitOutcome::Committed(Box::new(RenameDataResp {
|
||||
multipart_source: Some(Box::new(result)),
|
||||
old_data_dir: has_old_data_dir,
|
||||
rollback_data_dir: None,
|
||||
cleanup_data_dir: has_old_data_dir,
|
||||
sign: version_signature,
|
||||
old_current_size,
|
||||
old_current_source_checked,
|
||||
old_current_source,
|
||||
}))),
|
||||
Err(err) => Ok(MultipartFusedCommitOutcome::RestoreTargetAfterError(err, false)),
|
||||
}
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(to_file_error)
|
||||
.map_err(DiskError::from)?;
|
||||
|
||||
match outcome {
|
||||
MultipartFusedCommitOutcome::Committed(resp) => {
|
||||
for part_path in &invalidate_part_paths {
|
||||
self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
|
||||
}
|
||||
return Ok(*resp);
|
||||
}
|
||||
MultipartFusedCommitOutcome::AbortBeforeDataRename(err) => return Err(err),
|
||||
MultipartFusedCommitOutcome::RestoreDataSource(directory_guard, err) => {
|
||||
let commit = RenameCommitContext {
|
||||
directory_guard,
|
||||
publication_root: &self.publication_root,
|
||||
mutation_lease,
|
||||
source_metadata: &src_file_path,
|
||||
destination_metadata: &dst_file_path,
|
||||
source_volume_dir: &src_volume_dir,
|
||||
data_paths: has_data_dir_path.as_ref(),
|
||||
};
|
||||
commit.restore_published_data_source().await?;
|
||||
return Err(err);
|
||||
}
|
||||
MultipartFusedCommitOutcome::RestoreTargetAfterError(err, invalidate_before_return) => {
|
||||
if !invalidate_before_return {
|
||||
for part_path in &invalidate_part_paths {
|
||||
self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
|
||||
}
|
||||
}
|
||||
pin_multipart_future(|| publication.restore_target_after_error(mutation_lease.clone())).await?;
|
||||
if invalidate_before_return {
|
||||
for part_path in &invalidate_part_paths {
|
||||
self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
|
||||
}
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
prepare_rename_commit_and_destination_metadata(
|
||||
!no_inline && !preserve_upload_metadata,
|
||||
@@ -1053,71 +1542,82 @@ impl LocalDisk {
|
||||
// after the commit rename, and a crash before the rename means the
|
||||
// PUT was never acknowledged. A metadata commit: relaxed tiers
|
||||
// leave it to the page cache.
|
||||
if let Some(publication) = multipart_publication.as_ref() {
|
||||
pin_multipart_future(|| publication.prepare_metadata(&new_dst_buf, has_dst_buf.as_deref())).await?;
|
||||
}
|
||||
let tmp_meta_sync = if durability.syncs_commit_metadata() || multipart_publication.is_some() {
|
||||
SyncMode::FileOnly
|
||||
} else {
|
||||
SyncMode::None
|
||||
};
|
||||
// The tmp xl.meta write and the shard-file fdatasync are independent
|
||||
// (disjoint paths) and both only need to be durable before the commit
|
||||
// renames below, so run them concurrently to drop a blocking
|
||||
// round-trip from the PUT commit critical path (rustfs/backlog#922
|
||||
// step 2). The "contents durable -> rename -> dst dir fsync" ordering
|
||||
// is unchanged — both futures complete before any rename — which the
|
||||
// rename_data crash-consistency harness (backlog#935) exercises.
|
||||
//
|
||||
// Shard durability: once rename_data succeeds the write is
|
||||
// acknowledged, so data must not live only in the page cache.
|
||||
// Multipart parts were already synced during rename_part, so their
|
||||
// fdatasync here is a cheap no-op. A missing source dir is left for the
|
||||
// rename below to report through the existing rollback path. Payload
|
||||
// durability is kept by both strict and relaxed.
|
||||
let tmp_meta_write = {
|
||||
let src_file_path = multipart_publication
|
||||
.as_ref()
|
||||
.map_or_else(|| commit.source_metadata.to_path_buf(), |publication| publication.prepared_path.clone());
|
||||
let dst_file_path = dst_file_path.clone();
|
||||
let owned_directory_guard = commit.directory_guard.clone();
|
||||
let owned_mutation_lease = commit.mutation_lease.clone();
|
||||
async move {
|
||||
os::run_blocking_namespace_operation(owned_mutation_lease, move || {
|
||||
#[cfg(test)]
|
||||
run_owned_file_before_open(&src_file_path);
|
||||
let mut prepared_metadata_source = os::create_prepared_rename_source_with_commit_guard(
|
||||
&src_file_path,
|
||||
&dst_file_path,
|
||||
&owned_directory_guard,
|
||||
)?;
|
||||
prepared_metadata_source.write_all(&new_dst_buf, tmp_meta_sync != SyncMode::None)?;
|
||||
Ok(prepared_metadata_source)
|
||||
})
|
||||
.await
|
||||
.map_err(to_file_error)
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
};
|
||||
#[cfg(all(test, target_os = "linux"))]
|
||||
let probe_active = os::file_sync_probe::is_active();
|
||||
#[cfg(all(not(test), target_os = "linux"))]
|
||||
let probe_active = false;
|
||||
let shard_sync = async {
|
||||
if durability.syncs_data_shards()
|
||||
&& let Some((src_data_path, _)) = has_data_dir_path.as_ref()
|
||||
&& let Err(err) = os::sync_dir_files_with_limiter(src_data_path, self.file_sync_permits.clone()).await
|
||||
&& err.kind() != ErrorKind::NotFound
|
||||
{
|
||||
return Err::<(), DiskError>(to_file_error(err).into());
|
||||
#[cfg(target_os = "linux")]
|
||||
if !probe_active && super::take_presynced_tmp_data_dir(src_data_path) {
|
||||
if multipart_publication.is_none()
|
||||
&& let Err(err) = os::fsync_dir_with_owner(src_data_path, Some(commit.mutation_lease.clone())).await
|
||||
&& err.kind() != ErrorKind::NotFound
|
||||
{
|
||||
return Err::<(), DiskError>(to_file_error(err).into());
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
if let Err(err) = os::sync_dir_files_with_limiter(src_data_path, self.file_sync_permits.clone()).await
|
||||
&& err.kind() != ErrorKind::NotFound
|
||||
{
|
||||
return Err::<(), DiskError>(to_file_error(err).into());
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
};
|
||||
let (tmp_meta_res, shard_sync_res) = tokio::join!(tmp_meta_write, shard_sync);
|
||||
// Surface a tmp-meta failure first (its prior serial position), then a
|
||||
// shard-sync failure; either aborts before any rename, exactly as the
|
||||
// sequential version did.
|
||||
let prepared_metadata_source = tmp_meta_res?;
|
||||
shard_sync_res?;
|
||||
let prepared_metadata_source = match multipart_publication.as_ref() {
|
||||
Some(publication) => pin_multipart_future(|| publication.install_head(prepared_metadata_source)).await?,
|
||||
None => prepared_metadata_source,
|
||||
let prepared_metadata_source = if let Some(publication) = multipart_publication.as_ref() {
|
||||
let dst_file_path_ref = &dst_file_path;
|
||||
let dir_guard_ref = &commit.directory_guard;
|
||||
let lease_clone = commit.mutation_lease.clone();
|
||||
let prepare_fut = pin_multipart_future(|| {
|
||||
publication.prepare_and_install_head(
|
||||
&new_dst_buf,
|
||||
has_dst_buf.as_deref(),
|
||||
dst_file_path_ref,
|
||||
dir_guard_ref,
|
||||
lease_clone,
|
||||
)
|
||||
});
|
||||
let (tmp_meta_res, shard_sync_res) = tokio::join!(prepare_fut, shard_sync);
|
||||
let prepared_metadata_source = tmp_meta_res?;
|
||||
shard_sync_res?;
|
||||
prepared_metadata_source
|
||||
} else {
|
||||
let tmp_meta_sync = if durability.syncs_commit_metadata() {
|
||||
SyncMode::FileOnly
|
||||
} else {
|
||||
SyncMode::None
|
||||
};
|
||||
let tmp_meta_write = {
|
||||
let src_file_path = commit.source_metadata.to_path_buf();
|
||||
let dst_file_path = dst_file_path.clone();
|
||||
let owned_directory_guard = commit.directory_guard.clone();
|
||||
let owned_mutation_lease = commit.mutation_lease.clone();
|
||||
let sync = tmp_meta_sync != SyncMode::None;
|
||||
async move {
|
||||
os::run_blocking_namespace_operation_with_sync(owned_mutation_lease, sync, move || {
|
||||
#[cfg(test)]
|
||||
run_owned_file_before_open(&src_file_path);
|
||||
let mut prepared_metadata_source = os::create_prepared_rename_source_with_commit_guard(
|
||||
&src_file_path,
|
||||
&dst_file_path,
|
||||
&owned_directory_guard,
|
||||
)?;
|
||||
prepared_metadata_source.write_all(&new_dst_buf, sync)?;
|
||||
Ok(prepared_metadata_source)
|
||||
})
|
||||
.await
|
||||
.map_err(to_file_error)
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
};
|
||||
let (tmp_meta_res, shard_sync_res) = tokio::join!(tmp_meta_write, shard_sync);
|
||||
let prepared_metadata_source = tmp_meta_res?;
|
||||
shard_sync_res?;
|
||||
prepared_metadata_source
|
||||
};
|
||||
commit.directory_guard = remove_dst_base_before_commit(
|
||||
dst_path,
|
||||
@@ -1269,13 +1769,15 @@ impl LocalDisk {
|
||||
// sync finish. A detached spawn_blocking writer could survive
|
||||
// cancellation and later truncate a newer transaction's
|
||||
// deterministic rollback backup.
|
||||
let write_result = os::run_blocking_namespace_operation(commit.mutation_lease.clone(), move || {
|
||||
let lease = commit.mutation_lease.clone();
|
||||
let sync = backup_sync != SyncMode::None;
|
||||
let write_result = os::run_blocking_namespace_operation_with_sync(lease, sync, move || {
|
||||
#[cfg(test)]
|
||||
run_owned_file_before_open(&backup_path);
|
||||
backup_path_guard.write_file_for_path_access(
|
||||
&backup_path,
|
||||
backup_bytes.as_ref(),
|
||||
backup_sync != SyncMode::None,
|
||||
sync,
|
||||
backup_sync == SyncMode::FileAndDir,
|
||||
)
|
||||
})
|
||||
@@ -1457,13 +1959,22 @@ impl LocalDisk {
|
||||
}
|
||||
|
||||
let multipart_source = match multipart_publication.as_ref() {
|
||||
Some(publication) => match pin_multipart_future(|| publication.finish(commit.mutation_lease.clone())).await {
|
||||
Ok(result) => Some(Box::new(result)),
|
||||
Err(error) => {
|
||||
pin_multipart_future(|| publication.restore_target_after_error(commit.mutation_lease.clone())).await?;
|
||||
return Err(error);
|
||||
Some(publication) => {
|
||||
let target_already_synced = durability.syncs_commit_metadata();
|
||||
let staging_cleanup = multipart_staging.as_ref().map(|s| s.cleanup_handle());
|
||||
match pin_multipart_future(|| {
|
||||
publication.finish(commit.mutation_lease.clone(), target_already_synced, staging_cleanup)
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(result) => Some(Box::new(result)),
|
||||
Err(error) => {
|
||||
pin_multipart_future(|| publication.restore_target_after_error(commit.mutation_lease.clone()))
|
||||
.await?;
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
Ok(RenameDataResp {
|
||||
@@ -1663,13 +2174,8 @@ impl LocalDisk {
|
||||
#[cfg(not(windows))]
|
||||
let backup_path = {
|
||||
let _backup = os::StageTimer::new(native_timing, "pool_meta_native_backup_create");
|
||||
create_local_inline_rollback_backup(&dst, &src, old_metadata)?
|
||||
create_local_inline_rollback_backup(&dst, &src, old_metadata, sync)?
|
||||
};
|
||||
#[cfg(not(windows))]
|
||||
if sync {
|
||||
let _backup_sync = os::StageTimer::new(native_timing, "pool_meta_native_backup_file_sync");
|
||||
std::fs::File::open(&backup_path)?.sync_data()?;
|
||||
}
|
||||
staged_rollback_path = Some(StagedRollbackBackup(backup_path));
|
||||
}
|
||||
|
||||
@@ -1711,7 +2217,9 @@ impl LocalDisk {
|
||||
&dst,
|
||||
&dst_volume_dir_for_commit,
|
||||
destination_was_absent,
|
||||
rollback_data_dir,
|
||||
staged_rollback_path.as_ref(),
|
||||
None,
|
||||
&rename_commit_guard_for_preparation,
|
||||
)
|
||||
{
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -421,6 +421,7 @@ impl LocalDisk {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let permit = AbortDeletePermit::new(&store, &expected, certificate.clone())?;
|
||||
super::multipart_source_state::invalidate_active_multipart_source_head(&store.source);
|
||||
let destination = self.io_get_object_path(RUSTFS_META_TMP_DELETED_BUCKET, &Uuid::new_v4().to_string())?;
|
||||
let source = store.source.clone();
|
||||
let base = store.base.clone();
|
||||
@@ -618,6 +619,9 @@ pub(super) struct OrdinaryMultipartDeleteCheck {
|
||||
root: os::PublicationRoot,
|
||||
}
|
||||
impl OrdinaryMultipartDeleteCheck {
|
||||
pub(super) fn source(&self) -> &Path {
|
||||
&self.source
|
||||
}
|
||||
pub(super) fn validate(&self) -> Result<()> {
|
||||
let pinned = match os::pin_multipart_source_directory(&self.source, &self.base, &self.root) {
|
||||
Ok(pinned) => pinned,
|
||||
|
||||
@@ -117,6 +117,7 @@ pub(super) struct LoadedSource {
|
||||
}
|
||||
|
||||
pub(super) struct SourceStore {
|
||||
multipart_index: Arc<super::multipart_index::IndexOwner>,
|
||||
pub disk_id: Uuid,
|
||||
pub physical_slot: usize,
|
||||
pub disk_ids: Vec<Uuid>,
|
||||
@@ -186,6 +187,7 @@ impl LocalDisk {
|
||||
let logical = self.get_object_path(RUSTFS_META_MULTIPART_BUCKET, path)?;
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&logical, owner.clone()).await;
|
||||
Ok(SourceStore {
|
||||
multipart_index: self.multipart_index.clone(),
|
||||
disk_id,
|
||||
physical_slot,
|
||||
disk_ids,
|
||||
@@ -238,9 +240,177 @@ pub(super) fn read_optional_in_pinned(
|
||||
Err(error) => Err(error.into()),
|
||||
}
|
||||
}
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
struct FileStamp {
|
||||
len: u64,
|
||||
modified: Option<std::time::SystemTime>,
|
||||
#[cfg(unix)]
|
||||
ino: u64,
|
||||
}
|
||||
|
||||
impl FileStamp {
|
||||
fn from_path(path: &Path) -> Option<Self> {
|
||||
let meta = std::fs::symlink_metadata(path).ok()?;
|
||||
Some(Self {
|
||||
len: meta.len(),
|
||||
modified: meta.modified().ok(),
|
||||
#[cfg(unix)]
|
||||
ino: {
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
meta.ino()
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct ActiveMultipartSourceEntry {
|
||||
raw: Bytes,
|
||||
fi: FileInfo,
|
||||
head_stamp: FileStamp,
|
||||
unsettled_parts: std::collections::HashSet<usize>,
|
||||
parts: std::collections::HashMap<usize, (FileStamp, rustfs_filemeta::ObjectPartInfo)>,
|
||||
}
|
||||
|
||||
const ACTIVE_MULTIPART_SOURCE_HEADS_PER_SHARD: usize = 64;
|
||||
|
||||
static ACTIVE_MULTIPART_SOURCE_HEADS: std::sync::LazyLock<os::ShardedPathCache<ActiveMultipartSourceEntry>> =
|
||||
std::sync::LazyLock::new(|| os::ShardedPathCache::new(ACTIVE_MULTIPART_SOURCE_HEADS_PER_SHARD));
|
||||
|
||||
pub(super) fn put_active_multipart_source_head(source: impl AsRef<Path>, raw: Bytes, fi: FileInfo) {
|
||||
let source = source.as_ref();
|
||||
let Some(head_stamp) = FileStamp::from_path(&source.join(STORAGE_FORMAT_FILE)) else {
|
||||
return;
|
||||
};
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.insert(
|
||||
source,
|
||||
ActiveMultipartSourceEntry {
|
||||
raw,
|
||||
fi,
|
||||
head_stamp,
|
||||
unsettled_parts: std::collections::HashSet::new(),
|
||||
parts: std::collections::HashMap::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
fn stamp_matches_active_disk(source: &Path, expected_head_stamp: FileStamp) -> bool {
|
||||
FileStamp::from_path(&source.join(STORAGE_FORMAT_FILE)) == Some(expected_head_stamp)
|
||||
&& !source.join(STATE).exists()
|
||||
&& !source.join(crate::disk::MULTIPART_RECOVERY_DIR).exists()
|
||||
&& !source.join(".mpu-completions").exists()
|
||||
}
|
||||
|
||||
pub(super) fn get_active_multipart_source_head(source: &Path) -> Option<(Bytes, FileInfo)> {
|
||||
let (raw, fi, head_stamp) =
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| (entry.raw.clone(), entry.fi.clone(), entry.head_stamp))?;
|
||||
if !stamp_matches_active_disk(source, head_stamp) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
|
||||
return None;
|
||||
}
|
||||
Some((raw, fi))
|
||||
}
|
||||
|
||||
pub(super) fn has_active_multipart_source_head(source: &Path) -> bool {
|
||||
let Some(head_stamp) = ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| entry.head_stamp) else {
|
||||
return false;
|
||||
};
|
||||
if !stamp_matches_active_disk(source, head_stamp) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
|
||||
return false;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
pub(super) fn has_clean_active_multipart_source(source: &Path) -> bool {
|
||||
let Some((head_stamp, clean)) =
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| (entry.head_stamp, entry.unsettled_parts.is_empty()))
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
if !stamp_matches_active_disk(source, head_stamp) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
|
||||
return false;
|
||||
}
|
||||
clean
|
||||
}
|
||||
|
||||
pub(super) fn begin_active_multipart_part_transaction(source: &Path, part_number: usize) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.with_get_mut(source, |entry| {
|
||||
entry.unsettled_parts.insert(part_number);
|
||||
entry.parts.remove(&part_number);
|
||||
});
|
||||
}
|
||||
|
||||
pub(super) fn record_active_multipart_part_committed(source: &Path, part: rustfs_filemeta::ObjectPartInfo) {
|
||||
let Some(data_dir) = ACTIVE_MULTIPART_SOURCE_HEADS
|
||||
.with_get(source, |entry| entry.fi.data_dir)
|
||||
.flatten()
|
||||
.filter(|dir| !dir.is_nil())
|
||||
else {
|
||||
return;
|
||||
};
|
||||
let part_meta_path = source.join(data_dir.to_string()).join(format!("part.{}.meta", part.number));
|
||||
let Some(part_stamp) = FileStamp::from_path(&part_meta_path) else {
|
||||
return;
|
||||
};
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.with_get_mut(source, |entry| {
|
||||
entry.parts.insert(part.number, (part_stamp, part));
|
||||
});
|
||||
}
|
||||
|
||||
pub(super) fn finish_active_multipart_part_transaction(
|
||||
source: &Path,
|
||||
part_number: Option<usize>,
|
||||
committed: bool,
|
||||
retired_existing_txn: bool,
|
||||
) {
|
||||
let Some(part_number) = part_number.filter(|_| committed) else {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
|
||||
return;
|
||||
};
|
||||
if !retired_existing_txn {
|
||||
return;
|
||||
}
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.remove_if_mut(source, |entry| !entry.unsettled_parts.remove(&part_number));
|
||||
}
|
||||
|
||||
pub(super) fn get_active_multipart_source_part(source: &Path, part_number: usize) -> Option<rustfs_filemeta::ObjectPartInfo> {
|
||||
let (head_stamp, data_dir, expected_stamp, part) = ACTIVE_MULTIPART_SOURCE_HEADS
|
||||
.with_get(source, |entry| {
|
||||
if !entry.unsettled_parts.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let data_dir = entry.fi.data_dir.filter(|dir| !dir.is_nil())?;
|
||||
let (expected_stamp, part) = entry.parts.get(&part_number)?;
|
||||
Some((entry.head_stamp, data_dir, *expected_stamp, part.clone()))
|
||||
})
|
||||
.flatten()?;
|
||||
if !stamp_matches_active_disk(source, head_stamp) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
|
||||
return None;
|
||||
}
|
||||
let part_meta_path = source.join(data_dir.to_string()).join(format!("part.{part_number}.meta"));
|
||||
if FileStamp::from_path(&part_meta_path) != Some(expected_stamp) {
|
||||
return None;
|
||||
}
|
||||
Some(part)
|
||||
}
|
||||
|
||||
pub(super) fn invalidate_active_multipart_source_head(source: &Path) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
|
||||
}
|
||||
|
||||
pub(super) fn invalidate_active_multipart_sources_under(prefix: &Path) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.retain_outside_prefix(prefix);
|
||||
}
|
||||
|
||||
/// Canonical source mutations are admitted under SourceRW at the syscall,
|
||||
/// not on the earlier async caller that may have timed out.
|
||||
pub(super) fn validate_source_mutation(source: &Path, base: &Path, root: &os::PublicationRoot, initial: bool) -> Result<()> {
|
||||
if !initial && has_active_multipart_source_head(source) {
|
||||
return Ok(());
|
||||
}
|
||||
let pinned = match os::pin_multipart_source_directory(source, base, root) {
|
||||
Ok(pinned) => pinned,
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
|
||||
@@ -440,6 +610,7 @@ impl SourceStore {
|
||||
terminal: Option<&Path>,
|
||||
sync_source_before: bool,
|
||||
) -> std::io::Result<()> {
|
||||
invalidate_active_multipart_source_head(source);
|
||||
#[cfg(unix)]
|
||||
let mut pinned = os::pin_multipart_source_directory(source, base, root)?;
|
||||
#[cfg(not(unix))]
|
||||
@@ -470,7 +641,7 @@ impl SourceStore {
|
||||
let terminal = terminal_record_path(&source, state);
|
||||
let bytes = bytes.to_vec();
|
||||
os::run_blocking_namespace_operation_with_sync(self.lease.clone(), true, move || {
|
||||
Self::write_owned_backup_std(&source, &base, &root, owner, target, &bytes)?;
|
||||
Self::write_owned_backup_std(&source, &base, &root, owner, target, &bytes, false)?;
|
||||
Self::persist_state_std(&source, &base, &root, &raw, terminal.as_deref(), false)
|
||||
})
|
||||
.await
|
||||
@@ -539,7 +710,7 @@ impl SourceStore {
|
||||
let root = self.publication_root.clone();
|
||||
let bytes = bytes.to_vec();
|
||||
os::run_blocking_namespace_operation(self.lease.clone(), move || {
|
||||
Self::write_owned_backup_std(&source, &base, &root, owner, target, &bytes)
|
||||
Self::write_owned_backup_std(&source, &base, &root, owner, target, &bytes, true)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)
|
||||
@@ -551,6 +722,7 @@ impl SourceStore {
|
||||
owner: Uuid,
|
||||
target: bool,
|
||||
bytes: &[u8],
|
||||
sync_source_root: bool,
|
||||
) -> std::io::Result<()> {
|
||||
let path = if target {
|
||||
target_backup_path(source, owner)
|
||||
@@ -582,9 +754,13 @@ impl SourceStore {
|
||||
if !existed {
|
||||
os::replace_multipart_source_metadata(&path, bytes, base, root)?;
|
||||
}
|
||||
os::sync_multipart_owned_record_parents(source, parent, existed, base, root)
|
||||
os::sync_multipart_owned_record_parents_options(source, parent, existed, sync_source_root, base, root)
|
||||
}
|
||||
pub async fn replace_head(&self, bytes: &[u8]) -> Result<()> {
|
||||
self.multipart_index
|
||||
.record_head(&format!("{}/xl.meta", self.path), bytes)
|
||||
.await?;
|
||||
invalidate_active_multipart_source_head(&self.source);
|
||||
let path = self.source.join(STORAGE_FORMAT_FILE);
|
||||
let base = self.base.clone();
|
||||
let root = self.publication_root.clone();
|
||||
@@ -1097,6 +1273,82 @@ pub(super) fn load_source(
|
||||
})
|
||||
}
|
||||
|
||||
fn load_uninitialized_source(
|
||||
raw: Bytes,
|
||||
path: &str,
|
||||
disk_id: Uuid,
|
||||
physical_slot: usize,
|
||||
disk_ids: &[Uuid],
|
||||
) -> Result<LoadedSource> {
|
||||
let sha = raw_sha(&raw);
|
||||
let length = raw.len() as u64;
|
||||
let current = get_file_info(
|
||||
&raw,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
path,
|
||||
"",
|
||||
FileInfoOpts {
|
||||
data: false,
|
||||
include_free_versions: false,
|
||||
include_part_checksums: true,
|
||||
},
|
||||
)
|
||||
.map_err(Error::other)?;
|
||||
native_source_protocol(¤t)?;
|
||||
if source_status(¤t)?.is_some()
|
||||
|| publication_identity(¤t)?.is_some()
|
||||
|| SourceConditionV1::take(&mut current.clone())?.is_some()
|
||||
{
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
let bucket = rustfs_utils::http::metadata_compat::get_consistent_str(¤t.metadata, "multipart-bucket")
|
||||
.ok_or(DiskError::FileCorrupt)?;
|
||||
let object = rustfs_utils::http::metadata_compat::get_consistent_str(¤t.metadata, "multipart-object")
|
||||
.ok_or(DiskError::FileCorrupt)?;
|
||||
let condition = capture_source_condition(bucket, object, path, ¤t, current.data_dir.ok_or(DiskError::FileCorrupt)?)?;
|
||||
if condition.generation.epoch != 0 {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
let slot = condition
|
||||
.lineage
|
||||
.geometry
|
||||
.distribution
|
||||
.iter()
|
||||
.position(|index| *index == current.erasure.index)
|
||||
.ok_or(DiskError::FileCorrupt)?;
|
||||
if slot != physical_slot || condition.lineage.geometry.disks() != disk_ids.len() {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let inventory = raw_sha(&encode(&(None::<[u8; 32]>, Some(sha), sha, None::<[u8; 32]>, None::<[u8; 32]>))?);
|
||||
let snapshot = Some(MultipartSourceSnapshot {
|
||||
disk_id,
|
||||
physical_slot: slot,
|
||||
lineage: condition.lineage,
|
||||
generation: condition.generation,
|
||||
revision: 0,
|
||||
phase: MultipartSourcePhase::Active,
|
||||
state_sha256: [0; 32],
|
||||
head_sha256: Some(sha),
|
||||
inventory_sha256: inventory,
|
||||
capability_sha256: None,
|
||||
publication: None,
|
||||
});
|
||||
Ok(LoadedSource {
|
||||
root_missing: false,
|
||||
state: None,
|
||||
head: Some(SourceHead {
|
||||
sha256: sha,
|
||||
length,
|
||||
raw: Some(raw.clone()),
|
||||
}),
|
||||
original: Some(raw),
|
||||
info: Some(current),
|
||||
snapshot,
|
||||
head_matches_state: true,
|
||||
head_matches_activation: false,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) const RETIREMENT_ANCHOR: &str = ".mpu-source-retirement";
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
@@ -1323,41 +1575,6 @@ impl LocalDisk {
|
||||
}
|
||||
|
||||
impl SourceStore {
|
||||
/// Only native source validation may create provenance. Ordinary TMP/heal
|
||||
/// publication can retain a valid old local anchor, never issue/rebind one.
|
||||
pub async fn write_retirement_anchor(
|
||||
&self,
|
||||
state: &SourceState,
|
||||
publication: &PublicationIdentity,
|
||||
directory: &Path,
|
||||
lease: Arc<os::NamespaceMutationLease>,
|
||||
) -> Result<()> {
|
||||
if state.lineage != publication.lineage
|
||||
|| state.generation != publication.source_generation
|
||||
|| state.disk_id != self.disk_id
|
||||
|| state.physical_slot != self.physical_slot
|
||||
{
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let record = NativeRetirementAnchor {
|
||||
protocol_version: MULTIPART_SOURCE_PROTOCOL_VERSION,
|
||||
disk_id: self.disk_id,
|
||||
physical_slot: self.physical_slot,
|
||||
publication: publication.clone(),
|
||||
source_original_sha256: state.original_sha256,
|
||||
source_validation_receipt_id: state.backup_owner,
|
||||
};
|
||||
let bytes = encode(&record)?;
|
||||
let directory = directory.to_path_buf();
|
||||
let base = self.base.clone();
|
||||
let root = self.publication_root.clone();
|
||||
os::run_blocking_namespace_operation(lease, move || {
|
||||
let _parent = os::mkdir_all_below_existing_base_std(&directory, &base, &root)?;
|
||||
os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &bytes, &base, &root)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
/// This caller already owns SourceRW and destination metadata ownership.
|
||||
/// An Aborting survivor can confirm a historical target only with its
|
||||
/// independent, same-disk publication anchor. Closure never creates one.
|
||||
@@ -1456,6 +1673,8 @@ struct ValidatedPreparedPublication {
|
||||
length: u64,
|
||||
}
|
||||
|
||||
type DeferredInitialPublication = (Vec<u8>, PathBuf, Vec<u8>);
|
||||
|
||||
pub(super) struct NativeMultipartPublication {
|
||||
pub store: Arc<SourceStore>,
|
||||
pub publication: PublicationIdentity,
|
||||
@@ -1463,22 +1682,49 @@ pub(super) struct NativeMultipartPublication {
|
||||
state: parking_lot::Mutex<SourceState>,
|
||||
capability: parking_lot::Mutex<Option<MultipartUndoCapability>>,
|
||||
validated_prepared: parking_lot::Mutex<Option<ValidatedPreparedPublication>>,
|
||||
deferred_initial: parking_lot::Mutex<Option<DeferredInitialPublication>>,
|
||||
pub legacy: bool,
|
||||
}
|
||||
|
||||
impl LocalDisk {
|
||||
#[cfg(test)]
|
||||
pub(super) async fn prepare_native_multipart_publication(
|
||||
&self,
|
||||
path: &str,
|
||||
incoming: &mut FileInfo,
|
||||
staging: &MultipartCommitStaging,
|
||||
) -> Result<NativeMultipartPublication> {
|
||||
self.prepare_native_multipart_publication_inner(path, incoming, staging, false)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn prepare_native_multipart_publication_deferred(
|
||||
&self,
|
||||
path: &str,
|
||||
incoming: &mut FileInfo,
|
||||
staging: &MultipartCommitStaging,
|
||||
) -> Result<NativeMultipartPublication> {
|
||||
self.prepare_native_multipart_publication_inner(path, incoming, staging, true)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn prepare_native_multipart_publication_inner(
|
||||
&self,
|
||||
path: &str,
|
||||
incoming: &mut FileInfo,
|
||||
staging: &MultipartCommitStaging,
|
||||
defer_persist: bool,
|
||||
) -> Result<NativeMultipartPublication> {
|
||||
// Real Complete is parts-backed, including a zero-byte last part.
|
||||
if incoming.data.is_some() || incoming.inline_data() {
|
||||
return Err(DiskError::UnsupportedDisk);
|
||||
}
|
||||
let store = Arc::new(self.multipart_store_from_staging(path, staging).await?);
|
||||
let loaded = store.load().await?;
|
||||
let loaded = if defer_persist && let Some(raw) = staging.fresh_source_head() {
|
||||
load_uninitialized_source(raw, &store.path, store.disk_id, store.physical_slot, &store.disk_ids)?
|
||||
} else {
|
||||
store.load().await?
|
||||
};
|
||||
let snapshot = loaded.snapshot.as_ref().ok_or(DiskError::ConditionalDeleteConflict)?;
|
||||
let original = loaded
|
||||
.head
|
||||
@@ -1569,7 +1815,44 @@ impl LocalDisk {
|
||||
};
|
||||
// Copy the small source FI to an independent immutable inode. Its
|
||||
// canonical inode is never truncated to build the destination proxy.
|
||||
store.write_owned_backup_and_persist(owner, false, original, &state).await?;
|
||||
let directory = staging.private_directory(publication.final_data_dir);
|
||||
let anchor_record = NativeRetirementAnchor {
|
||||
protocol_version: MULTIPART_SOURCE_PROTOCOL_VERSION,
|
||||
disk_id: store.disk_id,
|
||||
physical_slot: store.physical_slot,
|
||||
publication: publication.clone(),
|
||||
source_original_sha256: state.original_sha256,
|
||||
source_validation_receipt_id: state.backup_owner,
|
||||
};
|
||||
let anchor_bytes = encode(&anchor_record)?;
|
||||
state.validate(store.disk_id, &store.path)?;
|
||||
let original_bytes = original.to_vec();
|
||||
let deferred_initial = if defer_persist {
|
||||
Some((original_bytes, directory, anchor_bytes))
|
||||
} else {
|
||||
let state_raw = encode(&state)?;
|
||||
let source_dir = store.source.clone();
|
||||
let base_dir = store.base.clone();
|
||||
let pub_root = store.publication_root.clone();
|
||||
let terminal = terminal_record_path(&source_dir, &state);
|
||||
#[cfg(target_os = "linux")]
|
||||
let can_mark_presynced = has_active_multipart_source_head(&source_dir)
|
||||
&& super::effective_durability(RUSTFS_META_MULTIPART_BUCKET).syncs_data_shards();
|
||||
os::run_blocking_namespace_operation_with_sync(staging.lease.clone(), true, move || {
|
||||
SourceStore::write_owned_backup_std(&source_dir, &base_dir, &pub_root, owner, false, &original_bytes, false)?;
|
||||
SourceStore::persist_state_std(&source_dir, &base_dir, &pub_root, &state_raw, terminal.as_deref(), false)?;
|
||||
let _parent = os::mkdir_all_below_existing_base_std(&directory, &base_dir, &pub_root)?;
|
||||
os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &anchor_bytes, &base_dir, &pub_root)?;
|
||||
#[cfg(target_os = "linux")]
|
||||
if can_mark_presynced {
|
||||
super::mark_presynced_tmp_data_dir(directory);
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?;
|
||||
None
|
||||
};
|
||||
let prepared_path = store.source.join(format!(".mpu-prepared-{owner}"));
|
||||
let context = NativeMultipartPublication {
|
||||
store,
|
||||
@@ -1578,14 +1861,9 @@ impl LocalDisk {
|
||||
state: parking_lot::Mutex::new(state),
|
||||
capability: parking_lot::Mutex::new(Some(capability)),
|
||||
validated_prepared: parking_lot::Mutex::new(None),
|
||||
deferred_initial: parking_lot::Mutex::new(deferred_initial),
|
||||
legacy,
|
||||
};
|
||||
let directory = staging.private_directory(context.publication.final_data_dir);
|
||||
let reserved = context.state.lock().clone();
|
||||
context
|
||||
.store
|
||||
.write_retirement_anchor(&reserved, &context.publication, &directory, staging.lease.clone())
|
||||
.await?;
|
||||
Ok(context)
|
||||
}
|
||||
}
|
||||
@@ -1594,18 +1872,15 @@ impl NativeMultipartPublication {
|
||||
pub fn canonical_head(&self) -> PathBuf {
|
||||
self.store.source.join(STORAGE_FORMAT_FILE)
|
||||
}
|
||||
#[cfg(test)]
|
||||
pub async fn prepare_metadata(&self, new_metadata: &[u8], old_metadata: Option<&[u8]>) -> Result<()> {
|
||||
let mut state = self.state.lock().clone();
|
||||
let current = self.store.load().await?;
|
||||
let current_state = current.state.as_ref().ok_or(DiskError::ConditionalDeleteConflict)?;
|
||||
if encode(current_state)? != encode(&state)? || !current.head_matches_state {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
// Validate the actual persisted representation once, after FileMeta's
|
||||
// canonicalization. No unvalidated input can install a source proxy.
|
||||
if !self.store.target_matches(&self.publication, new_metadata)? {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let expected_state_raw = encode(&state)?;
|
||||
let validated = ValidatedPreparedPublication {
|
||||
publication: self.publication.clone(),
|
||||
sha256: raw_sha(new_metadata),
|
||||
@@ -1618,17 +1893,298 @@ impl NativeMultipartPublication {
|
||||
state.prepared_sha256 = Some(validated.sha256);
|
||||
state.prepared_length = Some(validated.length);
|
||||
state.advance(MultipartSourcePhase::Prepared)?;
|
||||
if let Some(previous) = old_metadata {
|
||||
self.store
|
||||
.write_owned_backup_and_persist(state.backup_owner, true, previous, &state)
|
||||
.await?;
|
||||
} else {
|
||||
self.store.persist(&state).await?;
|
||||
}
|
||||
state.validate(self.store.disk_id, &self.store.path)?;
|
||||
let next_state_raw = encode(&state)?;
|
||||
let source = self.store.source.clone();
|
||||
let base = self.store.base.clone();
|
||||
let root = self.store.publication_root.clone();
|
||||
let path = self.store.path.clone();
|
||||
let disk_id = self.store.disk_id;
|
||||
let physical_slot = self.store.physical_slot;
|
||||
let disk_ids = self.store.disk_ids.clone();
|
||||
let backup_owner = state.backup_owner;
|
||||
let old_bytes = old_metadata.map(|b| b.to_vec());
|
||||
let terminal = terminal_record_path(&source, &state);
|
||||
os::run_blocking_namespace_operation_with_sync(self.store.lease.clone(), true, move || {
|
||||
let current = load_source(&source, &base, &root, &path, disk_id, physical_slot, &disk_ids).map_err(IoError::other)?;
|
||||
let current_state = current
|
||||
.state
|
||||
.as_ref()
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
let current_raw = encode(current_state).map_err(IoError::other)?;
|
||||
if current_raw != expected_state_raw || !current.head_matches_state {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
if let Some(previous) = old_bytes.as_deref() {
|
||||
SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, true, previous, false)?;
|
||||
}
|
||||
SourceStore::persist_state_std(&source, &base, &root, &next_state_raw, terminal.as_deref(), false)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?;
|
||||
*self.state.lock() = state;
|
||||
*self.validated_prepared.lock() = Some(validated);
|
||||
Ok(())
|
||||
}
|
||||
#[cfg(target_os = "linux")]
|
||||
pub(super) fn prepare_and_install_head_on_thread(
|
||||
&self,
|
||||
new_metadata: &[u8],
|
||||
old_metadata: Option<&[u8]>,
|
||||
dst_file_path: &Path,
|
||||
directory_guard: &os::RenameCommitGuard,
|
||||
) -> Result<os::PreparedRenameSource> {
|
||||
let mut state = self.state.lock().clone();
|
||||
if !self.store.target_matches(&self.publication, new_metadata)? {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let deferred_initial = self.deferred_initial.lock().take();
|
||||
let expected_state_raw = encode(&state)?;
|
||||
let expected_lineage = state.lineage.clone();
|
||||
let expected_generation = state.generation;
|
||||
let expected_revision = state.revision;
|
||||
let validated = ValidatedPreparedPublication {
|
||||
publication: self.publication.clone(),
|
||||
sha256: raw_sha(new_metadata),
|
||||
length: new_metadata.len() as u64,
|
||||
};
|
||||
if let Some(previous) = old_metadata {
|
||||
state.previous_target_sha256 = Some(raw_sha(previous));
|
||||
state.previous_target_length = Some(previous.len() as u64);
|
||||
}
|
||||
state.prepared_sha256 = Some(validated.sha256);
|
||||
state.prepared_length = Some(validated.length);
|
||||
state.advance(MultipartSourcePhase::Prepared)?;
|
||||
state.validate(self.store.disk_id, &self.store.path)?;
|
||||
let next_state_raw = encode(&state)?;
|
||||
let source = &self.store.source;
|
||||
let base = &self.store.base;
|
||||
let root = &self.store.publication_root;
|
||||
let path = &self.store.path;
|
||||
let disk_id = self.store.disk_id;
|
||||
let physical_slot = self.store.physical_slot;
|
||||
let disk_ids = &self.store.disk_ids;
|
||||
let backup_owner = state.backup_owner;
|
||||
#[cfg(target_os = "linux")]
|
||||
let can_mark_presynced = has_active_multipart_source_head(source)
|
||||
&& super::effective_durability(RUSTFS_META_MULTIPART_BUCKET).syncs_data_shards();
|
||||
let terminal = terminal_record_path(source, &state);
|
||||
let prepared_path = &self.prepared_path;
|
||||
let canonical = self.canonical_head();
|
||||
let prepared = (|| -> std::io::Result<os::PreparedRenameSource> {
|
||||
if let Some((ref original_bytes, _, _)) = deferred_initial {
|
||||
let current = load_source(source, base, root, path, disk_id, physical_slot, disk_ids).map_err(IoError::other)?;
|
||||
let snapshot = current
|
||||
.snapshot
|
||||
.as_ref()
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
let current_head = current
|
||||
.head
|
||||
.as_ref()
|
||||
.and_then(SourceHead::raw)
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
if !current.head_matches_state
|
||||
|| snapshot.phase != MultipartSourcePhase::Active
|
||||
|| snapshot.capability_sha256.is_some()
|
||||
|| snapshot.lineage != expected_lineage
|
||||
|| snapshot.generation != expected_generation
|
||||
|| snapshot.revision.checked_add(1) != Some(expected_revision)
|
||||
|| current_head != original_bytes.as_slice()
|
||||
{
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
} else {
|
||||
let current = load_source(source, base, root, path, disk_id, physical_slot, disk_ids).map_err(IoError::other)?;
|
||||
let current_state = current
|
||||
.state
|
||||
.as_ref()
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
let current_raw = encode(current_state).map_err(IoError::other)?;
|
||||
if current_raw != expected_state_raw || !current.head_matches_state {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
}
|
||||
#[cfg(test)]
|
||||
super::run_owned_file_before_open(prepared_path);
|
||||
let mut prepared_metadata_source =
|
||||
os::create_prepared_rename_source_with_commit_guard(prepared_path, dst_file_path, directory_guard)?;
|
||||
if let Err(err) = prepared_metadata_source.write_all(new_metadata, false) {
|
||||
drop(prepared_metadata_source);
|
||||
let _ = std::fs::remove_file(prepared_path);
|
||||
return Err(err);
|
||||
}
|
||||
let setup_res: std::io::Result<()> = (|| {
|
||||
if let Some((original_bytes, directory, anchor_bytes)) = deferred_initial {
|
||||
SourceStore::write_owned_backup_std(source, base, root, backup_owner, false, &original_bytes, false)?;
|
||||
if let Some(previous) = old_metadata {
|
||||
SourceStore::write_owned_backup_std(source, base, root, backup_owner, true, previous, false)?;
|
||||
}
|
||||
SourceStore::persist_state_std(source, base, root, &next_state_raw, terminal.as_deref(), false)?;
|
||||
let _parent = os::mkdir_all_below_existing_base_std(&directory, base, root)?;
|
||||
os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &anchor_bytes, base, root)?;
|
||||
#[cfg(target_os = "linux")]
|
||||
if can_mark_presynced {
|
||||
super::mark_presynced_tmp_data_dir(directory);
|
||||
}
|
||||
} else {
|
||||
if let Some(previous) = old_metadata {
|
||||
SourceStore::write_owned_backup_std(source, base, root, backup_owner, true, previous, false)?;
|
||||
}
|
||||
SourceStore::persist_state_std(source, base, root, &next_state_raw, terminal.as_deref(), false)?;
|
||||
}
|
||||
Ok(())
|
||||
})();
|
||||
if let Err(err) = setup_res {
|
||||
drop(prepared_metadata_source);
|
||||
let _ = std::fs::remove_file(prepared_path);
|
||||
return Err(err);
|
||||
}
|
||||
if let Err(err) = prepared_metadata_source.sync_data() {
|
||||
drop(prepared_metadata_source);
|
||||
let _ = std::fs::remove_file(prepared_path);
|
||||
return Err(err);
|
||||
}
|
||||
os::install_multipart_prepared_head(prepared_metadata_source, &canonical, base, root)
|
||||
})()
|
||||
.map_err(DiskError::from)?;
|
||||
*self.state.lock() = state;
|
||||
*self.validated_prepared.lock() = Some(validated);
|
||||
Ok(prepared)
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
pub async fn prepare_and_install_head(
|
||||
&self,
|
||||
new_metadata: &[u8],
|
||||
old_metadata: Option<&[u8]>,
|
||||
dst_file_path: &Path,
|
||||
directory_guard: &os::RenameCommitGuard,
|
||||
mutation_lease: Arc<os::NamespaceMutationLease>,
|
||||
) -> Result<os::PreparedRenameSource> {
|
||||
let mut state = self.state.lock().clone();
|
||||
if !self.store.target_matches(&self.publication, new_metadata)? {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let deferred_initial = self.deferred_initial.lock().take();
|
||||
let expected_state_raw = encode(&state)?;
|
||||
let expected_lineage = state.lineage.clone();
|
||||
let expected_generation = state.generation;
|
||||
let expected_revision = state.revision;
|
||||
let validated = ValidatedPreparedPublication {
|
||||
publication: self.publication.clone(),
|
||||
sha256: raw_sha(new_metadata),
|
||||
length: new_metadata.len() as u64,
|
||||
};
|
||||
if let Some(previous) = old_metadata {
|
||||
state.previous_target_sha256 = Some(raw_sha(previous));
|
||||
state.previous_target_length = Some(previous.len() as u64);
|
||||
}
|
||||
state.prepared_sha256 = Some(validated.sha256);
|
||||
state.prepared_length = Some(validated.length);
|
||||
state.advance(MultipartSourcePhase::Prepared)?;
|
||||
state.validate(self.store.disk_id, &self.store.path)?;
|
||||
let next_state_raw = encode(&state)?;
|
||||
let source = self.store.source.clone();
|
||||
let base = self.store.base.clone();
|
||||
let root = self.store.publication_root.clone();
|
||||
let path = self.store.path.clone();
|
||||
let disk_id = self.store.disk_id;
|
||||
let physical_slot = self.store.physical_slot;
|
||||
let disk_ids = self.store.disk_ids.clone();
|
||||
let backup_owner = state.backup_owner;
|
||||
#[cfg(target_os = "linux")]
|
||||
let can_mark_presynced = has_active_multipart_source_head(&source)
|
||||
&& super::effective_durability(RUSTFS_META_MULTIPART_BUCKET).syncs_data_shards();
|
||||
let old_bytes = old_metadata.map(|b| b.to_vec());
|
||||
let new_bytes = new_metadata.to_vec();
|
||||
let terminal = terminal_record_path(&source, &state);
|
||||
let prepared_path = self.prepared_path.clone();
|
||||
let canonical = self.canonical_head();
|
||||
let dst_file_path = dst_file_path.to_path_buf();
|
||||
let owned_directory_guard = directory_guard.clone();
|
||||
let prepared = os::run_blocking_namespace_operation_with_sync(mutation_lease, true, move || {
|
||||
let current = load_source(&source, &base, &root, &path, disk_id, physical_slot, &disk_ids).map_err(IoError::other)?;
|
||||
if let Some((original_bytes, directory, anchor_bytes)) = deferred_initial {
|
||||
let snapshot = current
|
||||
.snapshot
|
||||
.as_ref()
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
let current_head = current
|
||||
.head
|
||||
.as_ref()
|
||||
.and_then(SourceHead::raw)
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
if !current.head_matches_state
|
||||
|| snapshot.phase != MultipartSourcePhase::Active
|
||||
|| snapshot.capability_sha256.is_some()
|
||||
|| snapshot.lineage != expected_lineage
|
||||
|| snapshot.generation != expected_generation
|
||||
|| snapshot.revision.checked_add(1) != Some(expected_revision)
|
||||
|| current_head != original_bytes.as_slice()
|
||||
{
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, false, &original_bytes, false)?;
|
||||
if let Some(previous) = old_bytes.as_deref() {
|
||||
SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, true, previous, false)?;
|
||||
}
|
||||
SourceStore::persist_state_std(&source, &base, &root, &next_state_raw, terminal.as_deref(), false)?;
|
||||
let _parent = os::mkdir_all_below_existing_base_std(&directory, &base, &root)?;
|
||||
os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &anchor_bytes, &base, &root)?;
|
||||
#[cfg(target_os = "linux")]
|
||||
if can_mark_presynced {
|
||||
super::mark_presynced_tmp_data_dir(directory);
|
||||
}
|
||||
} else {
|
||||
let current_state = current
|
||||
.state
|
||||
.as_ref()
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
let current_raw = encode(current_state).map_err(IoError::other)?;
|
||||
if current_raw != expected_state_raw || !current.head_matches_state {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
if let Some(previous) = old_bytes.as_deref() {
|
||||
SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, true, previous, false)?;
|
||||
}
|
||||
SourceStore::persist_state_std(&source, &base, &root, &next_state_raw, terminal.as_deref(), false)?;
|
||||
}
|
||||
#[cfg(test)]
|
||||
super::run_owned_file_before_open(&prepared_path);
|
||||
let mut prepared_metadata_source =
|
||||
os::create_prepared_rename_source_with_commit_guard(&prepared_path, &dst_file_path, &owned_directory_guard)?;
|
||||
prepared_metadata_source.write_all(&new_bytes, true)?;
|
||||
os::install_multipart_prepared_head(prepared_metadata_source, &canonical, &base, &root)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?;
|
||||
*self.state.lock() = state;
|
||||
*self.validated_prepared.lock() = Some(validated);
|
||||
Ok(prepared)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
pub async fn prepare_and_install_head(
|
||||
self: &Arc<Self>,
|
||||
new_metadata: &[u8],
|
||||
old_metadata: Option<&[u8]>,
|
||||
dst_file_path: &Path,
|
||||
directory_guard: &os::RenameCommitGuard,
|
||||
mutation_lease: Arc<os::NamespaceMutationLease>,
|
||||
) -> Result<os::PreparedRenameSource> {
|
||||
let this = Arc::clone(self);
|
||||
let new_bytes = new_metadata.to_vec();
|
||||
let old_bytes = old_metadata.map(|b| b.to_vec());
|
||||
let dst_file_path = dst_file_path.to_path_buf();
|
||||
let owned_directory_guard = directory_guard.clone();
|
||||
os::run_blocking_namespace_operation_with_sync(mutation_lease, true, move || {
|
||||
this.prepare_and_install_head_on_thread(&new_bytes, old_bytes.as_deref(), &dst_file_path, &owned_directory_guard)
|
||||
.map_err(IoError::other)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
#[cfg(test)]
|
||||
pub async fn install_head(&self, prepared: os::PreparedRenameSource) -> Result<os::PreparedRenameSource> {
|
||||
{
|
||||
let guard = self.validated_prepared.lock();
|
||||
@@ -1699,13 +2255,13 @@ impl NativeMultipartPublication {
|
||||
// return an error after consuming the last claimable source heads.
|
||||
self.store.restore_exact_target_with_lease(&state, lease).await
|
||||
}
|
||||
pub async fn finish(&self, target_lease: Arc<os::NamespaceMutationLease>) -> Result<MultipartPublicationResult> {
|
||||
pub(super) fn finish_on_thread(
|
||||
&self,
|
||||
target_already_synced: bool,
|
||||
staging_cleanup: Option<(PathBuf, Arc<std::sync::atomic::AtomicBool>)>,
|
||||
) -> Result<MultipartPublicationResult> {
|
||||
let mut state = self.state.lock().clone();
|
||||
let current = self.store.load().await?;
|
||||
let current_state = current.state.as_ref().ok_or(DiskError::ConditionalDeleteConflict)?;
|
||||
if current.head.is_some() || encode(current_state)? != encode(&state)? {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let expected_state_raw = encode(&state)?;
|
||||
let (expected_sha256, expected_length) = {
|
||||
let guard = self.validated_prepared.lock();
|
||||
let validated = guard.as_ref().ok_or(DiskError::ConditionalDeleteConflict)?;
|
||||
@@ -1717,16 +2273,88 @@ impl NativeMultipartPublication {
|
||||
}
|
||||
(validated.sha256, validated.length)
|
||||
};
|
||||
self.store
|
||||
.read_and_sync_target_with_lease(&self.publication, target_lease, move |target| {
|
||||
if target.len() as u64 != expected_length || raw_sha(target) != expected_sha256 {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.await?;
|
||||
let target_dir = self.store.target_path(&self.publication)?;
|
||||
state.advance(MultipartSourcePhase::ConsumedUnsealed)?;
|
||||
let snapshot = self.store.persist_with_source_sync(&state, true).await?;
|
||||
state.validate(self.store.disk_id, &self.store.path)?;
|
||||
let next_state_raw = encode(&state)?;
|
||||
let snapshot = snapshot_for_state(self.store.disk_id, &state, &next_state_raw, None)?;
|
||||
let source = &self.store.source;
|
||||
let base = &self.store.base;
|
||||
let root = &self.store.publication_root;
|
||||
let path = &self.store.path;
|
||||
let disk_id = self.store.disk_id;
|
||||
let physical_slot = self.store.physical_slot;
|
||||
let disk_ids = &self.store.disk_ids;
|
||||
let terminal = terminal_record_path(source, &state);
|
||||
(|| -> std::io::Result<()> {
|
||||
let current = load_source(source, base, root, path, disk_id, physical_slot, disk_ids).map_err(IoError::other)?;
|
||||
let current_state = current
|
||||
.state
|
||||
.as_ref()
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
let current_raw = encode(current_state).map_err(IoError::other)?;
|
||||
if current.head.is_some() || current_raw != expected_state_raw {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
if target_already_synced {
|
||||
let pinned = os::pin_multipart_source_directory(&target_dir, base, root).map_err(|error| {
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
IoError::other(DiskError::ConditionalDeleteConflict)
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
let target = os::read_multipart_source_metadata_in_pinned(
|
||||
&pinned,
|
||||
&target_dir,
|
||||
std::ffi::OsStr::new(STORAGE_FORMAT_FILE),
|
||||
base,
|
||||
root,
|
||||
)
|
||||
.map_err(|error| {
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
IoError::other(DiskError::ConditionalDeleteConflict)
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
if target.len() as u64 != expected_length || raw_sha(&target) != expected_sha256 {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
} else {
|
||||
os::read_and_sync_multipart_target_metadata(
|
||||
&target_dir,
|
||||
std::ffi::OsStr::new(STORAGE_FORMAT_FILE),
|
||||
base,
|
||||
root,
|
||||
|target| {
|
||||
if target.len() as u64 != expected_length || raw_sha(target) != expected_sha256 {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.map_err(|error| {
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
IoError::other(DiskError::ConditionalDeleteConflict)
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
}
|
||||
SourceStore::persist_state_std(source, base, root, &next_state_raw, terminal.as_deref(), true)?;
|
||||
if let Some((io_path, cleaned)) = staging_cleanup {
|
||||
match std::fs::remove_dir(&io_path) {
|
||||
Ok(()) => cleaned.store(true, std::sync::atomic::Ordering::Release),
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
|
||||
cleaned.store(true, std::sync::atomic::Ordering::Release);
|
||||
}
|
||||
Err(_) => {}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
})()
|
||||
.map_err(DiskError::from)?;
|
||||
*self.state.lock() = state;
|
||||
let proof = self
|
||||
.store
|
||||
@@ -1738,6 +2366,21 @@ impl NativeMultipartPublication {
|
||||
undo_capability: capability,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn finish(
|
||||
self: &Arc<Self>,
|
||||
target_lease: Arc<os::NamespaceMutationLease>,
|
||||
target_already_synced: bool,
|
||||
staging_cleanup: Option<(PathBuf, Arc<std::sync::atomic::AtomicBool>)>,
|
||||
) -> Result<MultipartPublicationResult> {
|
||||
let this = Arc::clone(self);
|
||||
os::run_blocking_namespace_operation_with_sync(target_lease, true, move || {
|
||||
this.finish_on_thread(target_already_synced, staging_cleanup)
|
||||
.map_err(IoError::other)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
}
|
||||
|
||||
impl SourceStore {
|
||||
|
||||
@@ -207,6 +207,12 @@ pub(super) struct MultipartSourceAdmission {
|
||||
initial: bool,
|
||||
}
|
||||
impl MultipartSourceAdmission {
|
||||
pub(super) fn source(&self) -> &Path {
|
||||
&self.source
|
||||
}
|
||||
pub(super) fn is_initial(&self) -> bool {
|
||||
self.initial
|
||||
}
|
||||
pub(super) fn precondition(&self) -> Arc<dyn Fn() -> std::io::Result<()> + Send + Sync> {
|
||||
let admission = self.clone();
|
||||
Arc::new(move || admission.validate())
|
||||
@@ -250,6 +256,8 @@ pub(super) struct MultipartCommitStaging {
|
||||
logical_path: PathBuf,
|
||||
io_path: PathBuf,
|
||||
owner: Option<Arc<dyn Send + Sync>>,
|
||||
cleaned: Arc<std::sync::atomic::AtomicBool>,
|
||||
fresh_source_head: Option<bytes::Bytes>,
|
||||
}
|
||||
|
||||
impl MultipartCommitStaging {
|
||||
@@ -259,10 +267,19 @@ impl MultipartCommitStaging {
|
||||
pub(super) fn private_directory(&self, directory: Uuid) -> PathBuf {
|
||||
self.io_path.join(directory.to_string())
|
||||
}
|
||||
pub(super) fn cleanup_handle(&self) -> (PathBuf, Arc<std::sync::atomic::AtomicBool>) {
|
||||
(self.io_path.clone(), Arc::clone(&self.cleaned))
|
||||
}
|
||||
pub(super) fn fresh_source_head(&self) -> Option<bytes::Bytes> {
|
||||
self.fresh_source_head.clone()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for MultipartCommitStaging {
|
||||
fn drop(&mut self) {
|
||||
if self.cleaned.load(std::sync::atomic::Ordering::Acquire) {
|
||||
return;
|
||||
}
|
||||
let logical_path = self.logical_path.clone();
|
||||
let io_path = self.io_path.clone();
|
||||
let owner = self.owner.clone();
|
||||
@@ -285,6 +302,14 @@ impl Drop for MultipartCommitStaging {
|
||||
}
|
||||
|
||||
impl LocalDisk {
|
||||
pub(crate) fn has_clean_active_multipart_source(&self, upload_or_part_path: &str, part_path: bool) -> bool {
|
||||
let root = upload_scope_root(upload_or_part_path, part_path);
|
||||
let Ok(source) = self.io_get_object_path(RUSTFS_META_MULTIPART_BUCKET, root) else {
|
||||
return false;
|
||||
};
|
||||
super::multipart_source_state::has_clean_active_multipart_source(&source)
|
||||
}
|
||||
|
||||
pub(super) fn multipart_source_admission(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -356,12 +381,14 @@ impl LocalDisk {
|
||||
let io_path = self.io_get_object_path(RUSTFS_META_TMP_BUCKET, &attempt)?;
|
||||
// The attempt root is exactly the namespace used by the tmp sweeper.
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&logical_path, owner.clone()).await;
|
||||
let staging = MultipartCommitStaging {
|
||||
let mut staging = MultipartCommitStaging {
|
||||
path: attempt,
|
||||
lease,
|
||||
logical_path,
|
||||
io_path,
|
||||
owner,
|
||||
cleaned: Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||
fresh_source_head: None,
|
||||
};
|
||||
let canonical_metadata = self.io_get_object_path(RUSTFS_META_MULTIPART_BUCKET, &format!("{source_path}/xl.meta"))?;
|
||||
let source_object = self.io_get_object_path(RUSTFS_META_MULTIPART_BUCKET, source_path)?;
|
||||
@@ -371,31 +398,39 @@ impl LocalDisk {
|
||||
let source_path = source_path.to_owned();
|
||||
let incoming = incoming.clone();
|
||||
let sync_directory = effective_durability(destination_volume).syncs_data_shards();
|
||||
os::run_cancellable_blocking_namespace_operation(Arc::clone(&staging.lease), move |cancelled| {
|
||||
let sync_lane = sync_directory && incoming.data.is_none() && !incoming.is_remote();
|
||||
let lease = Arc::clone(&staging.lease);
|
||||
let fresh_head = os::run_cancellable_blocking_namespace_operation_with_sync(lease, sync_lane, move |cancelled| {
|
||||
if cancelled() {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "multipart staging cancelled"));
|
||||
}
|
||||
let raw = os::read_multipart_source_metadata(&canonical_metadata, &base, &publication_root).map_err(|error| {
|
||||
// A retired source has no authority for this delayed completion.
|
||||
// Preserve confinement and other I/O failures as their real errors.
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
DiskError::FileNotFound.into()
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
let source = get_file_info(
|
||||
&raw,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&source_path,
|
||||
"",
|
||||
FileInfoOpts {
|
||||
data: false,
|
||||
include_free_versions: false,
|
||||
include_part_checksums: true,
|
||||
},
|
||||
)
|
||||
.map_err(std::io::Error::other)?;
|
||||
let cached_head = super::multipart_source_state::get_active_multipart_source_head(&source_object);
|
||||
let (raw, source, from_active_cache) = if let Some((raw, source)) = cached_head {
|
||||
(raw, source, true)
|
||||
} else {
|
||||
let raw = os::read_multipart_source_metadata(&canonical_metadata, &base, &publication_root).map_err(|error| {
|
||||
// A retired source has no authority for this delayed completion.
|
||||
// Preserve confinement and other I/O failures as their real errors.
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
DiskError::FileNotFound.into()
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
let source = get_file_info(
|
||||
&raw,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&source_path,
|
||||
"",
|
||||
FileInfoOpts {
|
||||
data: false,
|
||||
include_free_versions: false,
|
||||
include_part_checksums: true,
|
||||
},
|
||||
)
|
||||
.map_err(std::io::Error::other)?;
|
||||
(bytes::Bytes::from(raw), source, false)
|
||||
};
|
||||
let source_dir = source
|
||||
.data_dir
|
||||
.filter(|dir| !dir.is_nil())
|
||||
@@ -412,23 +447,31 @@ impl LocalDisk {
|
||||
let _attempt = os::mkdir_all_below_existing_base_std(&private_object, &base, &publication_root)?;
|
||||
if incoming.data.is_none() && !incoming.is_remote() {
|
||||
let source_data_dir = source_object.join(source_dir.to_string());
|
||||
let pinned_source_data = (!incoming.parts.is_empty())
|
||||
.then(|| os::pin_multipart_source_directory(&source_data_dir, &base, &publication_root))
|
||||
.transpose()?;
|
||||
let mut pinned_source_data = None;
|
||||
let mut names = Vec::with_capacity(incoming.parts.len().saturating_mul(3));
|
||||
for part in &incoming.parts {
|
||||
if cancelled() {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "multipart staging cancelled"));
|
||||
}
|
||||
let meta_name = format!("part.{}.meta", part.number);
|
||||
let bytes = os::read_multipart_source_metadata_in_pinned(
|
||||
pinned_source_data.as_ref().expect("pinned source data directory"),
|
||||
&source_data_dir,
|
||||
std::ffi::OsStr::new(&meta_name),
|
||||
&base,
|
||||
&publication_root,
|
||||
)?;
|
||||
let actual = rustfs_filemeta::ObjectPartInfo::unmarshal(&bytes).map_err(std::io::Error::other)?;
|
||||
let actual = if let Some(cached_part) =
|
||||
super::multipart_source_state::get_active_multipart_source_part(&source_object, part.number)
|
||||
{
|
||||
cached_part
|
||||
} else {
|
||||
if pinned_source_data.is_none() {
|
||||
pinned_source_data =
|
||||
Some(os::pin_multipart_source_directory(&source_data_dir, &base, &publication_root)?);
|
||||
}
|
||||
let bytes = os::read_multipart_source_metadata_in_pinned(
|
||||
pinned_source_data.as_ref().expect("pinned source data directory"),
|
||||
&source_data_dir,
|
||||
std::ffi::OsStr::new(&meta_name),
|
||||
&base,
|
||||
&publication_root,
|
||||
)?;
|
||||
rustfs_filemeta::ObjectPartInfo::unmarshal(&bytes).map_err(std::io::Error::other)?
|
||||
};
|
||||
let normalized = i64::try_from(actual.size)
|
||||
.map_err(std::io::Error::other)?
|
||||
.max(actual.actual_size);
|
||||
@@ -464,10 +507,15 @@ impl LocalDisk {
|
||||
cancelled,
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
let has_state = !from_active_cache
|
||||
&& (source_object.join(".mpu-source-state").exists()
|
||||
|| source_object.join(".mpu-recovery").exists()
|
||||
|| source_object.join(".mpu-completions").exists());
|
||||
Ok((!has_state).then_some(raw))
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?;
|
||||
staging.fresh_source_head = fresh_head;
|
||||
Ok(staging)
|
||||
}
|
||||
}
|
||||
@@ -1219,4 +1267,211 @@ mod tests {
|
||||
.expect("non-part empty hash cleanup must remain supported");
|
||||
assert!(!directory.exists(), "a per-upload safety guard must not strand empty hash directories");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_source_active_head_cache_populates_and_invalidates_across_part_and_delete_paths() {
|
||||
use crate::disk::{DiskAPI as _, PartTransactionAction, ReadOptions};
|
||||
let (_root, disk, completed, _) = fixture().await;
|
||||
std::fs::create_dir_all(disk.io_get_bucket_path(RUSTFS_META_TMP_BUCKET).expect("TMP path")).expect("TMP bucket");
|
||||
let upload = format!("{}/{}", "c".repeat(64), Uuid::new_v4());
|
||||
let data_dir = Uuid::new_v4();
|
||||
let mut init_fi = FileInfo::new(&upload, 2, 2);
|
||||
init_fi.erasure.index = 1;
|
||||
init_fi.data_dir = Some(data_dir);
|
||||
init_fi.fresh = true;
|
||||
init_fi.mod_time = Some(time::OffsetDateTime::now_utc());
|
||||
|
||||
disk.write_metadata("", RUSTFS_META_MULTIPART_BUCKET, &upload, init_fi.clone())
|
||||
.await
|
||||
.expect("fresh multipart upload metadata");
|
||||
assert!(disk.has_clean_active_multipart_source(&upload, false));
|
||||
|
||||
let read_fi = disk
|
||||
.read_version("", RUSTFS_META_MULTIPART_BUCKET, &upload, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("cached read_version");
|
||||
assert_eq!(read_fi.data_dir, Some(data_dir));
|
||||
|
||||
let destination = format!("{upload}/{data_dir}/part.1");
|
||||
let part_meta = bytes::Bytes::from(completed.parts[0].marshal_msg().expect("part meta"));
|
||||
disk.write_all(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"cache-test-part/part.1",
|
||||
bytes::Bytes::from_static(b"immutable shard"),
|
||||
)
|
||||
.await
|
||||
.expect("stage tmp part");
|
||||
disk.prepare_part_transaction(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"cache-test-part/part.1",
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&destination,
|
||||
part_meta.clone(),
|
||||
)
|
||||
.await
|
||||
.expect("prepare part transaction");
|
||||
assert!(
|
||||
!disk.has_clean_active_multipart_source(&upload, false),
|
||||
"in-flight part transaction must mark active source as having unsettled part txn"
|
||||
);
|
||||
disk.rename_part(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"cache-test-part/part.1",
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&destination,
|
||||
part_meta,
|
||||
)
|
||||
.await
|
||||
.expect("rename part");
|
||||
disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, &destination, PartTransactionAction::Commit)
|
||||
.await
|
||||
.expect("settle part transaction");
|
||||
assert!(
|
||||
disk.has_clean_active_multipart_source(&upload, false),
|
||||
"settled commit must restore clean active multipart source status"
|
||||
);
|
||||
let read_parts = disk
|
||||
.read_parts(RUSTFS_META_MULTIPART_BUCKET, &[format!("{destination}.meta")])
|
||||
.await
|
||||
.expect("cached read_parts");
|
||||
assert_eq!(read_parts.len(), 1);
|
||||
assert_eq!(read_parts[0].number, 1);
|
||||
|
||||
// Prepare part 2, then replay Commit for already-settled part 1: part 2's
|
||||
// pending transaction must keep the upload marked as having unsettled part txns.
|
||||
let destination2 = format!("{upload}/{data_dir}/part.2");
|
||||
let mut part2_info = completed.parts[0].clone();
|
||||
part2_info.number = 2;
|
||||
let part2_meta = bytes::Bytes::from(part2_info.marshal_msg().expect("part 2 meta"));
|
||||
disk.write_all(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"cache-test-part/part.2",
|
||||
bytes::Bytes::from_static(b"immutable shard"),
|
||||
)
|
||||
.await
|
||||
.expect("stage tmp part 2");
|
||||
disk.prepare_part_transaction(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"cache-test-part/part.2",
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&destination2,
|
||||
part2_meta.clone(),
|
||||
)
|
||||
.await
|
||||
.expect("prepare part 2 transaction");
|
||||
assert!(!disk.has_clean_active_multipart_source(&upload, false));
|
||||
disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, &destination, PartTransactionAction::Commit)
|
||||
.await
|
||||
.expect("replayed part 1 commit settlement");
|
||||
assert!(
|
||||
!disk.has_clean_active_multipart_source(&upload, false),
|
||||
"replayed settlement for part 1 must not clear part 2's pending transaction"
|
||||
);
|
||||
disk.rename_part(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"cache-test-part/part.2",
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&destination2,
|
||||
part2_meta,
|
||||
)
|
||||
.await
|
||||
.expect("rename part 2");
|
||||
disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, &destination2, PartTransactionAction::Commit)
|
||||
.await
|
||||
.expect("settle part 2 transaction");
|
||||
assert!(disk.has_clean_active_multipart_source(&upload, false));
|
||||
|
||||
// Deleting {upload}/xl.meta through delete_paths must invalidate the canonical {upload} cache key.
|
||||
disk.delete_paths(RUSTFS_META_MULTIPART_BUCKET, &[format!("{upload}/xl.meta")])
|
||||
.await
|
||||
.expect("delete xl.meta via delete_paths");
|
||||
assert!(
|
||||
!disk.has_clean_active_multipart_source(&upload, false),
|
||||
"per-file head deletion must invalidate the canonical upload cache key"
|
||||
);
|
||||
assert!(matches!(
|
||||
disk.read_version("", RUSTFS_META_MULTIPART_BUCKET, &upload, "", &ReadOptions::default())
|
||||
.await,
|
||||
Err(DiskError::FileNotFound)
|
||||
));
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn multipart_source_cache_publication_holds_source_lock_against_concurrent_retirement() {
|
||||
use crate::disk::{DiskAPI as _, disk_store::LocalDiskWrapper, os::prepared_publication_test_hooks as hooks};
|
||||
|
||||
let (_root, disk, _, _) = fixture().await;
|
||||
std::fs::create_dir_all(disk.io_get_bucket_path(RUSTFS_META_TMP_BUCKET).expect("TMP path")).expect("TMP bucket");
|
||||
let disk = Arc::new(disk);
|
||||
let wrapper = LocalDiskWrapper::new(Arc::clone(&disk), false);
|
||||
let upload = format!("{}/{}", "d".repeat(64), Uuid::new_v4());
|
||||
let data_dir = Uuid::new_v4();
|
||||
let mut init_fi = FileInfo::new(&upload, 2, 2);
|
||||
init_fi.erasure.index = 1;
|
||||
init_fi.data_dir = Some(data_dir);
|
||||
init_fi.fresh = true;
|
||||
init_fi.mod_time = Some(time::OffsetDateTime::now_utc());
|
||||
|
||||
let source_dir = disk
|
||||
.io_get_object_path(RUSTFS_META_MULTIPART_BUCKET, &upload)
|
||||
.expect("source dir");
|
||||
let base_dir = disk
|
||||
.io_get_bucket_path(RUSTFS_META_MULTIPART_BUCKET)
|
||||
.expect("multipart bucket path");
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel();
|
||||
let _hook = hooks::install_at(hooks::Stage::DirFsync, &source_dir, move || {
|
||||
let _ = entered_tx.send(());
|
||||
let _ = release_rx.recv();
|
||||
});
|
||||
|
||||
let init_wrapper = wrapper.clone();
|
||||
let init_upload = upload.clone();
|
||||
let init_task = tokio::spawn(async move {
|
||||
init_wrapper
|
||||
.write_metadata("", RUSTFS_META_MULTIPART_BUCKET, &init_upload, init_fi)
|
||||
.await
|
||||
});
|
||||
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), entered_rx)
|
||||
.await
|
||||
.expect("initial write_metadata reaches parent dir fsync")
|
||||
.expect("entered channel");
|
||||
|
||||
let retire_wrapper = wrapper.clone();
|
||||
let retire_path = format!("{upload}/xl.meta");
|
||||
let mut retire_fut = Box::pin(async move {
|
||||
retire_wrapper
|
||||
.delete_paths(RUSTFS_META_MULTIPART_BUCKET, &[retire_path])
|
||||
.await
|
||||
});
|
||||
assert!(
|
||||
futures::poll!(tokio::task::unconstrained(retire_fut.as_mut())).is_pending(),
|
||||
"concurrent retirement must block on SourceRW while initial head publication is in flight"
|
||||
);
|
||||
|
||||
let _ = release_tx.send(());
|
||||
init_task.await.expect("init join").expect("initial write_metadata succeeds");
|
||||
retire_fut
|
||||
.await
|
||||
.expect("concurrent retirement completes after SourceRW release");
|
||||
|
||||
assert!(
|
||||
!super::super::multipart_source_state::has_active_multipart_source_head(&source_dir),
|
||||
"retired source must not remain active in cache after interleaved initial publication"
|
||||
);
|
||||
assert!(
|
||||
matches!(
|
||||
super::super::multipart_source_state::validate_source_mutation(
|
||||
&source_dir,
|
||||
&base_dir,
|
||||
&disk.publication_root,
|
||||
false
|
||||
),
|
||||
Err(DiskError::FileNotFound)
|
||||
),
|
||||
"validate_source_mutation must reject non-initial mutation on retired source"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,6 +27,7 @@ mod io_admission;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod lifecycle_test_io;
|
||||
pub mod local;
|
||||
pub mod multipart_index;
|
||||
pub mod multipart_source;
|
||||
pub mod os;
|
||||
|
||||
@@ -52,6 +53,10 @@ pub const RUSTFS_META_MULTIPART_BUCKET: &str = ".rustfs.sys/multipart";
|
||||
pub const RUSTFS_META_TMP_BUCKET: &str = ".rustfs.sys/tmp";
|
||||
pub const RUSTFS_META_TMP_DELETED_BUCKET: &str = ".rustfs.sys/tmp/.trash";
|
||||
pub const BUCKET_META_PREFIX: &str = "buckets";
|
||||
/// Directory under `BUCKET_META_PREFIX` that holds drive-replacement heal
|
||||
/// intents, seals and recovery proofs. It shares the bucket-metadata root with
|
||||
/// a user bucket of the same name, so bucket deletion must leave it in place.
|
||||
pub const HEAL_REPLACEMENT_RECOVERY_DIR: &str = "ahm-replacement";
|
||||
pub const FORMAT_CONFIG_FILE: &str = "format.json";
|
||||
/// Per-disk marker present while an erasure-set heal is rebuilding this disk.
|
||||
/// `LocalDisk::disk_info` reports `healing = true` while the file exists.
|
||||
@@ -401,16 +406,24 @@ impl Disk {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn has_clean_active_multipart_source(&self, upload_or_part_path: &str, part_path: bool) -> bool {
|
||||
match self {
|
||||
Self::Local(disk) => disk.has_clean_active_multipart_source(upload_or_part_path, part_path),
|
||||
Self::Remote(_) => false,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn try_publish_prepared_delete_local(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
|
||||
) -> Result<Option<(DeleteRecoveryJournal, bool, bool)>> {
|
||||
match self {
|
||||
Self::Local(disk) => {
|
||||
disk.try_publish_prepared_delete_local(volume, object, transaction, namespace_owner)
|
||||
disk.try_publish_prepared_delete_local(volume, object, transaction, namespace_owner, work_budget)
|
||||
.await
|
||||
}
|
||||
Self::Remote(_) => Ok(None),
|
||||
@@ -424,11 +437,19 @@ impl Disk {
|
||||
transaction: Uuid,
|
||||
journal: &DeleteRecoveryJournal,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
|
||||
) -> Result<bool> {
|
||||
match self {
|
||||
Self::Local(disk) => {
|
||||
disk.try_finish_and_finalize_prepared_delete_local(volume, object, transaction, journal, namespace_owner)
|
||||
.await
|
||||
disk.try_finish_and_finalize_prepared_delete_local(
|
||||
volume,
|
||||
object,
|
||||
transaction,
|
||||
journal,
|
||||
namespace_owner,
|
||||
work_budget,
|
||||
)
|
||||
.await
|
||||
}
|
||||
Self::Remote(_) => Ok(false),
|
||||
}
|
||||
@@ -505,6 +526,17 @@ impl Disk {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl DiskAPI for Disk {
|
||||
async fn list_multipart_index_roots(
|
||||
&self,
|
||||
bucket: &str,
|
||||
marker: Option<&str>,
|
||||
) -> Result<multipart_index::MultipartIndexPage> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.list_multipart_index_roots(bucket, marker).await,
|
||||
Self::Remote(disk) => disk.list_multipart_index_roots(bucket, marker).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn supports_guarded_delete_versions(&self) -> Result<()> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.supports_guarded_delete_versions().await,
|
||||
@@ -548,6 +580,17 @@ impl DiskAPI for Disk {
|
||||
Self::Remote(disk) => disk.has_delete_transaction_evidence(volume, object).await,
|
||||
}
|
||||
}
|
||||
async fn discover_delete_transactions_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
max_entries: u32,
|
||||
) -> Result<BoundedDeleteRecoveryDiscovery> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.discover_delete_transactions_bounded(volume, object, max_entries).await,
|
||||
Self::Remote(disk) => disk.discover_delete_transactions_bounded(volume, object, max_entries).await,
|
||||
}
|
||||
}
|
||||
async fn discover_delete_transactions(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -560,6 +603,35 @@ impl DiskAPI for Disk {
|
||||
Self::Remote(disk) => disk.discover_delete_transactions(volume, object, cursor, limit).await,
|
||||
}
|
||||
}
|
||||
async fn settle_delete_transaction_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
request: BoundedDeleteRecoverySettlement,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<BoundedDeleteRecoverySettlementReply> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.settle_delete_transaction_bounded(volume, request, namespace_owner).await,
|
||||
Self::Remote(disk) => disk.settle_delete_transaction_bounded(volume, request, namespace_owner).await,
|
||||
}
|
||||
}
|
||||
async fn inspect_delete_transaction_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
max_entries: u32,
|
||||
) -> Result<BoundedDeleteRecoveryInspection> {
|
||||
match self {
|
||||
Self::Local(disk) => {
|
||||
disk.inspect_delete_transaction_bounded(volume, object, transaction, max_entries)
|
||||
.await
|
||||
}
|
||||
Self::Remote(disk) => {
|
||||
disk.inspect_delete_transaction_bounded(volume, object, transaction, max_entries)
|
||||
.await
|
||||
}
|
||||
}
|
||||
}
|
||||
async fn inspect_delete_transaction(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -1506,6 +1578,15 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
|
||||
fn host_name(&self) -> String;
|
||||
fn endpoint(&self) -> Endpoint;
|
||||
async fn close(&self) -> Result<()>;
|
||||
/// Return candidates only while this disk maintains complete boot-local coverage.
|
||||
/// Unsupported or unready disks must never return an authoritative empty page.
|
||||
async fn list_multipart_index_roots(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_marker: Option<&str>,
|
||||
) -> Result<multipart_index::MultipartIndexPage> {
|
||||
Err(DiskError::MethodNotAllowed)
|
||||
}
|
||||
async fn get_disk_id(&self) -> Result<Option<Uuid>>;
|
||||
async fn set_disk_id(&self, id: Option<Uuid>) -> Result<()>;
|
||||
|
||||
@@ -1683,6 +1764,35 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
|
||||
) -> Result<DeleteRecoveryPage> {
|
||||
Err(DiskError::MethodNotAllowed)
|
||||
}
|
||||
/// Bounded foreground inventory owns all traversal state until return or cancellation.
|
||||
/// An implementation without the bounded protocol is an unsupported participant
|
||||
/// (the same verdict a remote peer without it receives); callers must never fall
|
||||
/// back to paginated discovery.
|
||||
async fn discover_delete_transactions_bounded(
|
||||
&self,
|
||||
_volume: &str,
|
||||
_object: &str,
|
||||
_max_entries: u32,
|
||||
) -> Result<BoundedDeleteRecoveryDiscovery> {
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
}
|
||||
async fn inspect_delete_transaction_bounded(
|
||||
&self,
|
||||
_volume: &str,
|
||||
_object: &str,
|
||||
_transaction: Uuid,
|
||||
_max_entries: u32,
|
||||
) -> Result<BoundedDeleteRecoveryInspection> {
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
}
|
||||
async fn settle_delete_transaction_bounded(
|
||||
&self,
|
||||
_volume: &str,
|
||||
_request: BoundedDeleteRecoverySettlement,
|
||||
_namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<BoundedDeleteRecoverySettlementReply> {
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
}
|
||||
/// A read-only capability check; unsupported implementations must fail closed.
|
||||
async fn supports_guarded_delete_versions(&self) -> Result<()> {
|
||||
Err(DiskError::GuardedDeleteUnsupported)
|
||||
@@ -2068,6 +2178,13 @@ pub struct WalkDirOptions {
|
||||
#[serde(default)]
|
||||
pub skip_hidden_prefix_check: bool,
|
||||
|
||||
/// Bound hidden metadata read while proving a non-recursive prefix visible.
|
||||
#[serde(default)]
|
||||
pub hidden_entry_budget: usize,
|
||||
/// A prefix whose visibility was not established before the preceding cut.
|
||||
#[serde(default)]
|
||||
pub pending_prefix: Option<String>,
|
||||
|
||||
// ReportNotFound will return errFileNotFound if all disks reports the BaseDir cannot be found.
|
||||
pub report_notfound: bool,
|
||||
|
||||
@@ -2400,9 +2517,62 @@ pub enum DeleteRecoveryAction {
|
||||
AdoptCommit { authority: Box<DeleteRecoveryJournal> },
|
||||
}
|
||||
|
||||
/// A complete inventory or a work-limit refusal. No public discovery cursor is allocated.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct BoundedDeleteRecoveryDiscovery {
|
||||
pub transactions: Vec<DeleteTransactionId>,
|
||||
pub visited: u32,
|
||||
pub complete: bool,
|
||||
}
|
||||
|
||||
pub const DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES: u32 = 4096;
|
||||
pub const DELETE_RECOVERY_MAX_INSPECTION_ENTRIES: u32 = 65536;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct BoundedDeleteRecoverySettlement {
|
||||
pub object: String,
|
||||
pub transaction: Uuid,
|
||||
pub expected_fingerprint: DeleteRecoveryDigest,
|
||||
pub action: DeleteRecoveryAction,
|
||||
pub max_entries: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub enum DeleteRecoverySettlementOutcome {
|
||||
Inspected(Box<DeleteRecoveryInspection>),
|
||||
ReceiptPublishedInspectionDeferred,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct BoundedDeleteRecoverySettlementReply {
|
||||
pub outcome: DeleteRecoverySettlementOutcome,
|
||||
pub visited: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct BoundedDeleteRecoveryInspection {
|
||||
pub inspection: DeleteRecoveryInspection,
|
||||
pub visited: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub enum DeleteRecoveryRpcOperation {
|
||||
SettleBounded(BoundedDeleteRecoverySettlement),
|
||||
InspectBounded {
|
||||
object: String,
|
||||
transaction: Uuid,
|
||||
max_entries: u32,
|
||||
},
|
||||
DiscoverBounded {
|
||||
object: String,
|
||||
max_entries: u32,
|
||||
},
|
||||
InspectOrphanDataDir {
|
||||
path: String,
|
||||
},
|
||||
@@ -2440,7 +2610,11 @@ impl DeleteRecoveryRpcOperation {
|
||||
/// Full metadata application and adoption share the existing mutation envelope.
|
||||
pub fn max_wire_payload_bytes(&self) -> usize {
|
||||
match self {
|
||||
Self::ApplyVersion { .. }
|
||||
Self::SettleBounded(BoundedDeleteRecoverySettlement {
|
||||
action: DeleteRecoveryAction::AdoptCommit { .. },
|
||||
..
|
||||
})
|
||||
| Self::ApplyVersion { .. }
|
||||
| Self::ApplyVersions { .. }
|
||||
| Self::Settle {
|
||||
action: DeleteRecoveryAction::AdoptCommit { .. },
|
||||
@@ -2456,8 +2630,11 @@ impl DeleteRecoveryRpcOperation {
|
||||
pub enum DeleteRecoveryRpcReply {
|
||||
OrphanDataDir { plain: bool },
|
||||
Discovered(DeleteRecoveryPage),
|
||||
DiscoveredBounded(BoundedDeleteRecoveryDiscovery),
|
||||
Inspected(DeleteRecoveryInspection),
|
||||
InspectedBounded(BoundedDeleteRecoveryInspection),
|
||||
Settled(DeleteRecoveryInspection),
|
||||
SettledBounded(BoundedDeleteRecoverySettlementReply),
|
||||
AppliedVersion,
|
||||
AppliedVersions(Vec<Option<u32>>),
|
||||
Evidence { present: bool },
|
||||
@@ -2935,6 +3112,8 @@ mod tests {
|
||||
recursive: true,
|
||||
incl_deleted: false,
|
||||
skip_hidden_prefix_check: false,
|
||||
hidden_entry_budget: 0,
|
||||
pending_prefix: None,
|
||||
report_notfound: false,
|
||||
filter_prefix: Some("prefix_".to_string()),
|
||||
forward_to: Some("object/path".to_string()),
|
||||
|
||||
@@ -0,0 +1,359 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! A candidate index never replaces multipart metadata or quorum validation.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use uuid::Uuid;
|
||||
|
||||
pub(crate) const RUSTFS_MULTIPART_BUCKET_KEY: &str = "x-rustfs-internal-multipart-bucket";
|
||||
pub(crate) const RUSTFS_MULTIPART_OBJECT_KEY: &str = "x-rustfs-internal-multipart-object";
|
||||
|
||||
pub(crate) const MULTIPART_INDEX_PAGE_SIZE: usize = 256;
|
||||
/// Pages one listing accepts from one disk. Pruned pages may carry few roots, so
|
||||
/// the cursor alone no longer bounds a peer's answer; a disk that keeps
|
||||
/// continuing past this many pages contributes no coverage.
|
||||
pub(crate) const MULTIPART_INDEX_MAX_PAGES_PER_DISK: usize = 4096;
|
||||
|
||||
/// A bounded page from one disk's continuously maintained, boot-local index.
|
||||
/// `epoch` changes whenever coverage must be rebuilt; callers must not combine
|
||||
/// pages from different epochs or physical disk identities.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct MultipartIndexPage {
|
||||
pub bucket: String,
|
||||
pub disk_id: Uuid,
|
||||
pub epoch: Uuid,
|
||||
pub roots: Vec<String>,
|
||||
pub next_marker: Option<String>,
|
||||
}
|
||||
|
||||
pub const MULTIPART_INDEX_RPC_MAX_BYTES: usize = 64 * 1024;
|
||||
pub const MULTIPART_INDEX_RPC_MAX_REQUEST_BYTES: usize = 4096;
|
||||
|
||||
/// Additive operation inside the legacy read-only DiskInfo JSON envelope.
|
||||
/// Old peers may ignore this field, but their ordinary response cannot decode
|
||||
/// as the required signed, challenge-bound MultipartIndexRpcReply.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct MultipartIndexRpcRequest {
|
||||
pub disk_id: String,
|
||||
pub metrics: bool,
|
||||
pub noop: bool,
|
||||
pub multipart_index: MultipartIndexQuery,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct MultipartIndexQuery {
|
||||
pub operation: String,
|
||||
pub version: u32,
|
||||
pub challenge: Uuid,
|
||||
pub bucket: String,
|
||||
pub marker: Option<String>,
|
||||
pub page_cap: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct MultipartIndexRpcReply {
|
||||
pub operation: String,
|
||||
pub version: u32,
|
||||
pub challenge: Uuid,
|
||||
pub page: MultipartIndexPage,
|
||||
pub proof: Vec<u8>,
|
||||
}
|
||||
|
||||
impl MultipartIndexRpcRequest {
|
||||
pub fn new(bucket: &str, marker: Option<&str>) -> Self {
|
||||
Self {
|
||||
disk_id: String::new(),
|
||||
metrics: false,
|
||||
noop: true,
|
||||
multipart_index: MultipartIndexQuery {
|
||||
operation: "list_multipart_index_roots".into(),
|
||||
version: 1,
|
||||
challenge: Uuid::new_v4(),
|
||||
bucket: bucket.to_owned(),
|
||||
marker: marker.map(str::to_owned),
|
||||
page_cap: MULTIPART_INDEX_PAGE_SIZE,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn validate(&self) -> crate::disk::error::Result<()> {
|
||||
let query = &self.multipart_index;
|
||||
if !self.disk_id.is_empty()
|
||||
|| self.metrics
|
||||
|| !self.noop
|
||||
|| query.operation != "list_multipart_index_roots"
|
||||
|| query.version != 1
|
||||
|| query.challenge.is_nil()
|
||||
|| query.bucket.is_empty()
|
||||
|| query.bucket.len() > 255
|
||||
|| query.bucket.contains('/')
|
||||
|| matches!(query.bucket.as_str(), "." | "..")
|
||||
|| query.page_cap != MULTIPART_INDEX_PAGE_SIZE
|
||||
|| query.marker.as_ref().is_some_and(|marker| !valid_root(marker))
|
||||
{
|
||||
return Err(crate::disk::error::DiskError::MethodNotAllowed);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn canonical_request(&self, disk: &str) -> crate::disk::error::Result<Vec<u8>> {
|
||||
self.validate()?;
|
||||
if disk.len() > MULTIPART_INDEX_RPC_MAX_REQUEST_BYTES {
|
||||
return Err(crate::disk::error::DiskError::FileCorrupt);
|
||||
}
|
||||
Ok(serde_json::to_vec(&("rustfs-multipart-index-request-v1", disk, self))?)
|
||||
}
|
||||
|
||||
pub fn canonical_response(&self, disk: &str, page: &MultipartIndexPage) -> crate::disk::error::Result<Vec<u8>> {
|
||||
self.validate()?;
|
||||
Ok(serde_json::to_vec(&("rustfs-multipart-index-response-v1", disk, self, page))?)
|
||||
}
|
||||
}
|
||||
|
||||
fn valid_root(root: &str) -> bool {
|
||||
root.len() == 64 && root.bytes().all(|byte| byte.is_ascii_hexdigit())
|
||||
}
|
||||
|
||||
impl MultipartIndexPage {
|
||||
pub fn validate(&self, bucket: &str, marker: Option<&str>, expected_disk: Uuid) -> crate::disk::error::Result<()> {
|
||||
if expected_disk.is_nil()
|
||||
|| self.disk_id != expected_disk
|
||||
|| self.epoch.is_nil()
|
||||
|| self.bucket != bucket
|
||||
|| self.roots.len() > MULTIPART_INDEX_PAGE_SIZE
|
||||
|| self
|
||||
.roots
|
||||
.iter()
|
||||
.any(|root| !valid_root(root) || marker.is_some_and(|marker| root.as_str() <= marker))
|
||||
|| self.roots.windows(2).any(|pair| pair[0] >= pair[1])
|
||||
// The cursor names the last examined marker; pruning may leave fewer
|
||||
// live roots than examined names, so it only has to bound them.
|
||||
|| self.next_marker.as_ref().is_some_and(|next| {
|
||||
!valid_root(next)
|
||||
|| marker.is_some_and(|marker| next.as_str() <= marker)
|
||||
|| self.roots.last().is_some_and(|last| last > next)
|
||||
})
|
||||
{
|
||||
return Err(crate::disk::error::DiskError::FileCorrupt);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl MultipartIndexRpcReply {
|
||||
/// This is structural validation only; the caller must also verify `proof`.
|
||||
pub fn validate(&self, request: &MultipartIndexRpcRequest, expected_disk: Uuid) -> crate::disk::error::Result<()> {
|
||||
let query = &request.multipart_index;
|
||||
request.validate()?;
|
||||
if self.operation != query.operation
|
||||
|| self.version != query.version
|
||||
|| self.challenge != query.challenge
|
||||
|| self.proof.len() != 32
|
||||
{
|
||||
return Err(crate::disk::error::DiskError::FileCorrupt);
|
||||
}
|
||||
self.page.validate(&query.bucket, query.marker.as_deref(), expected_disk)
|
||||
}
|
||||
}
|
||||
|
||||
/// A disk can attest coverage only after every page belongs to the same physical
|
||||
/// identity and index generation. An interrupted/changed stream contributes no
|
||||
/// partial candidates to the erasure set's coverage count.
|
||||
pub(crate) async fn collect_pages<F, Fut>(bucket: &str, identity: Uuid, mut fetch: F) -> crate::disk::error::Result<Vec<String>>
|
||||
where
|
||||
F: FnMut(Option<String>) -> Fut,
|
||||
Fut: std::future::Future<Output = crate::disk::error::Result<MultipartIndexPage>>,
|
||||
{
|
||||
let mut epoch = None;
|
||||
let mut marker = None;
|
||||
let mut roots = Vec::new();
|
||||
let mut pages = 0_usize;
|
||||
loop {
|
||||
if pages >= MULTIPART_INDEX_MAX_PAGES_PER_DISK {
|
||||
return Err(crate::disk::error::DiskError::FileCorrupt);
|
||||
}
|
||||
pages += 1;
|
||||
let page = fetch(marker.clone()).await?;
|
||||
page.validate(bucket, marker.as_deref(), identity)?;
|
||||
if epoch.is_some_and(|epoch| epoch != page.epoch) {
|
||||
return Err(crate::disk::error::DiskError::DiskNotFound);
|
||||
}
|
||||
epoch = Some(page.epoch);
|
||||
roots.extend(page.roots);
|
||||
marker = page.next_marker;
|
||||
if marker.is_none() {
|
||||
return Ok(roots);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn multipart_index_reply_rejects_challenge_disk_cursor_and_unbounded_pages() {
|
||||
let request = MultipartIndexRpcRequest::new("bucket", None);
|
||||
let id = Uuid::new_v4();
|
||||
let mut reply = MultipartIndexRpcReply {
|
||||
operation: request.multipart_index.operation.clone(),
|
||||
version: 1,
|
||||
challenge: request.multipart_index.challenge,
|
||||
page: MultipartIndexPage {
|
||||
bucket: "bucket".into(),
|
||||
disk_id: id,
|
||||
epoch: Uuid::new_v4(),
|
||||
roots: vec!["a".repeat(64)],
|
||||
next_marker: None,
|
||||
},
|
||||
proof: vec![0; 32],
|
||||
};
|
||||
reply.validate(&request, id).expect("valid structural envelope");
|
||||
assert!(
|
||||
reply.validate(&MultipartIndexRpcRequest::new("bucket", None), id).is_err(),
|
||||
"old challenge cannot replay"
|
||||
);
|
||||
assert!(reply.validate(&request, Uuid::new_v4()).is_err(), "physical disk identity cannot change");
|
||||
reply.page.next_marker = Some("9".repeat(64));
|
||||
assert!(reply.validate(&request, id).is_err(), "a cursor below the last root would re-list it");
|
||||
reply.page.next_marker = Some("a".repeat(64));
|
||||
reply
|
||||
.validate(&request, id)
|
||||
.expect("a pruned short page continues from its last examined name");
|
||||
reply.page.next_marker = None;
|
||||
reply.page.roots = vec!["a".repeat(64); MULTIPART_INDEX_PAGE_SIZE + 1];
|
||||
assert!(reply.validate(&request, id).is_err(), "peer cannot enlarge the fixed page cap");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multipart_index_response_proof_binds_the_complete_query_and_page() {
|
||||
crate::runtime::sources::ensure_test_rpc_secret();
|
||||
let mut request = MultipartIndexRpcRequest::new("bucket", None);
|
||||
let mut page = MultipartIndexPage {
|
||||
bucket: "bucket".into(),
|
||||
disk_id: Uuid::new_v4(),
|
||||
epoch: Uuid::new_v4(),
|
||||
roots: vec![],
|
||||
next_marker: None,
|
||||
};
|
||||
let canonical = request.canonical_response("disk", &page).expect("canonical response");
|
||||
let proof = crate::cluster::rpc::sign_tonic_rpc_response_proof(&canonical).expect("signed response");
|
||||
let verify = |request: &MultipartIndexRpcRequest, page: &MultipartIndexPage| {
|
||||
crate::cluster::rpc::verify_tonic_rpc_response_proof(
|
||||
&request.canonical_response("disk", page).expect("canonical"),
|
||||
&proof,
|
||||
)
|
||||
};
|
||||
verify(&request, &page).expect("exact response");
|
||||
request.multipart_index.marker = Some("0".repeat(64));
|
||||
assert!(verify(&request, &page).is_err(), "cursor is signed");
|
||||
request.multipart_index.marker = None;
|
||||
page.epoch = Uuid::new_v4();
|
||||
assert!(verify(&request, &page).is_err(), "boot-local coverage epoch is signed");
|
||||
}
|
||||
#[test]
|
||||
fn multipart_index_page_cursor_bounds_roots_without_requiring_a_full_page() {
|
||||
let disk = Uuid::new_v4();
|
||||
let page = |roots: Vec<String>, next_marker: Option<String>| MultipartIndexPage {
|
||||
bucket: "bucket".into(),
|
||||
disk_id: disk,
|
||||
epoch: Uuid::new_v4(),
|
||||
roots,
|
||||
next_marker,
|
||||
};
|
||||
// Pruning leaves fewer live roots than examined names; the cursor still advances.
|
||||
page(vec![format!("{:064x}", 1)], Some(format!("{:064x}", 5)))
|
||||
.validate("bucket", None, disk)
|
||||
.expect("short page with a cursor");
|
||||
page(vec![], Some(format!("{:064x}", 5)))
|
||||
.validate("bucket", Some(&format!("{:064x}", 2)), disk)
|
||||
.expect("all examined names pruned");
|
||||
assert!(
|
||||
page(vec![format!("{:064x}", 6)], Some(format!("{:064x}", 5)))
|
||||
.validate("bucket", None, disk)
|
||||
.is_err(),
|
||||
"a root beyond the cursor would be re-listed"
|
||||
);
|
||||
assert!(
|
||||
page(vec![], Some(format!("{:064x}", 2)))
|
||||
.validate("bucket", Some(&format!("{:064x}", 2)), disk)
|
||||
.is_err(),
|
||||
"the cursor must advance past the request marker"
|
||||
);
|
||||
assert!(
|
||||
page(vec![], Some("not-a-sha".to_owned()))
|
||||
.validate("bucket", None, disk)
|
||||
.is_err(),
|
||||
"the cursor is a marker name"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_index_page_collection_bounds_the_pages_one_disk_may_answer() {
|
||||
let id = Uuid::new_v4();
|
||||
let epoch = Uuid::new_v4();
|
||||
let mut served = 0_usize;
|
||||
let result = collect_pages("bucket", id, |marker| {
|
||||
served += 1;
|
||||
let next = marker.map_or(0_u64, |marker| u64::from_str_radix(&marker[48..], 16).expect("hex cursor") + 1);
|
||||
std::future::ready(Ok(MultipartIndexPage {
|
||||
bucket: "bucket".into(),
|
||||
disk_id: id,
|
||||
epoch,
|
||||
roots: vec![],
|
||||
next_marker: Some(format!("{next:064x}")),
|
||||
}))
|
||||
})
|
||||
.await;
|
||||
assert!(matches!(result, Err(crate::disk::error::DiskError::FileCorrupt)), "{result:?}");
|
||||
assert_eq!(served, MULTIPART_INDEX_MAX_PAGES_PER_DISK, "the peer is cut off at the page cap");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_index_page_collection_rejects_replaced_disk_or_restarted_epoch() {
|
||||
let id = Uuid::new_v4();
|
||||
let epoch = Uuid::new_v4();
|
||||
for replace_disk in [false, true] {
|
||||
let first = MultipartIndexPage {
|
||||
bucket: "bucket".into(),
|
||||
disk_id: id,
|
||||
epoch,
|
||||
roots: (0..MULTIPART_INDEX_PAGE_SIZE).map(|index| format!("{index:064x}")).collect(),
|
||||
next_marker: Some(format!("{:064x}", MULTIPART_INDEX_PAGE_SIZE - 1)),
|
||||
};
|
||||
let second = MultipartIndexPage {
|
||||
bucket: "bucket".into(),
|
||||
disk_id: if replace_disk { Uuid::new_v4() } else { id },
|
||||
epoch: if replace_disk { epoch } else { Uuid::new_v4() },
|
||||
roots: vec![],
|
||||
next_marker: None,
|
||||
};
|
||||
let mut replies = std::collections::VecDeque::from([first, second]);
|
||||
let result = collect_pages("bucket", id, |marker| {
|
||||
let page = replies.pop_front().expect("bounded fixture pages");
|
||||
if marker.is_some() {
|
||||
assert!(page.roots.is_empty());
|
||||
}
|
||||
std::future::ready(Ok(page))
|
||||
})
|
||||
.await;
|
||||
// A replaced disk fails the page's identity check; a restarted
|
||||
// epoch fails the cross-page continuity check.
|
||||
let expected_corrupt = replace_disk;
|
||||
assert!(
|
||||
match &result {
|
||||
Err(crate::disk::error::DiskError::FileCorrupt) => expected_corrupt,
|
||||
Err(crate::disk::error::DiskError::DiskNotFound) => !expected_corrupt,
|
||||
_ => false,
|
||||
},
|
||||
"changed physical source cannot contribute partial coverage: {result:?}"
|
||||
);
|
||||
assert!(replies.is_empty(), "exercise the continuation response");
|
||||
}
|
||||
}
|
||||
}
|
||||
+1257
-200
File diff suppressed because it is too large
Load Diff
@@ -326,6 +326,40 @@ impl Default for WriteProgressPolicy {
|
||||
}
|
||||
}
|
||||
|
||||
/// Caller-specific completion behavior for an erasure write.
|
||||
///
|
||||
/// A multipart upload must not discard a different healthy shard from each
|
||||
/// part merely because another quorum finished first: the final object needs
|
||||
/// a quorum of disks carrying every part. Pending writers still obey the
|
||||
/// existing progress and absolute deadlines and never count as completed.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub(crate) struct ShardWritePolicy {
|
||||
quorum: usize,
|
||||
post_quorum: PostQuorumPolicy,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum PostQuorumPolicy {
|
||||
Grace,
|
||||
ExistingBudget,
|
||||
}
|
||||
|
||||
impl ShardWritePolicy {
|
||||
pub(crate) fn object(quorum: usize) -> Self {
|
||||
Self {
|
||||
quorum,
|
||||
post_quorum: PostQuorumPolicy::Grace,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn multipart(quorum: usize) -> Self {
|
||||
Self {
|
||||
quorum,
|
||||
post_quorum: PostQuorumPolicy::ExistingBudget,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// After write quorum is already in hand, how long a slower shard may still
|
||||
/// finish before it is dropped. The per-shard stall budget (30s by default)
|
||||
/// is what a black-hole peer is allowed to consume *before* quorum exists.
|
||||
@@ -338,13 +372,15 @@ fn post_quorum_straggler_grace(stall_timeout: Option<std::time::Duration>) -> Op
|
||||
stall_timeout.map(|stall| stall.min(WRITE_QUORUM_STRAGGLER_GRACE))
|
||||
}
|
||||
|
||||
/// Drive shard ops until write quorum has succeeded and stragglers have had
|
||||
/// [`post_quorum_straggler_grace`] to finish. Returns `true` for each slot
|
||||
/// whose future completed; `false` means the caller must fail that slot.
|
||||
/// Drive shard operations within their existing deadlines. Ordinary object
|
||||
/// writes additionally stop after the post-quorum grace; multipart writes keep
|
||||
/// polling all candidates within the same operation budget. A `false` slot
|
||||
/// never completed and must be excluded from commit.
|
||||
async fn await_write_quorum<F>(
|
||||
write_quorum: usize,
|
||||
stall_timeout: Option<std::time::Duration>,
|
||||
slot_count: usize,
|
||||
post_quorum: PostQuorumPolicy,
|
||||
mut futures: FuturesUnordered<F>,
|
||||
) -> Vec<bool>
|
||||
where
|
||||
@@ -376,6 +412,7 @@ where
|
||||
if succeeded {
|
||||
successes = successes.saturating_add(1);
|
||||
if successes >= write_quorum
|
||||
&& post_quorum == PostQuorumPolicy::Grace
|
||||
&& grace.is_none()
|
||||
&& let Some(delay) = post_quorum_straggler_grace(stall_timeout)
|
||||
{
|
||||
@@ -404,6 +441,7 @@ pub(crate) struct MultiWriter<'a> {
|
||||
writers: &'a mut [Option<BitrotWriterWrapper>],
|
||||
integrity: Option<&'a mut IntegrityBuilder>,
|
||||
write_quorum: usize,
|
||||
post_quorum: PostQuorumPolicy,
|
||||
errs: Vec<Option<Error>>,
|
||||
policy: WriteProgressPolicy,
|
||||
/// Absolute deadline for the whole object, armed lazily on the first
|
||||
@@ -412,16 +450,21 @@ pub(crate) struct MultiWriter<'a> {
|
||||
}
|
||||
|
||||
impl<'a> MultiWriter<'a> {
|
||||
pub fn new(writers: &'a mut [Option<BitrotWriterWrapper>], write_quorum: usize) -> Self {
|
||||
Self::with_policy(writers, write_quorum, WriteProgressPolicy::default())
|
||||
pub fn new(writers: &'a mut [Option<BitrotWriterWrapper>], write_policy: ShardWritePolicy) -> Self {
|
||||
Self::with_policy(writers, write_policy, WriteProgressPolicy::default())
|
||||
}
|
||||
|
||||
pub fn with_policy(writers: &'a mut [Option<BitrotWriterWrapper>], write_quorum: usize, policy: WriteProgressPolicy) -> Self {
|
||||
pub fn with_policy(
|
||||
writers: &'a mut [Option<BitrotWriterWrapper>],
|
||||
write_policy: ShardWritePolicy,
|
||||
policy: WriteProgressPolicy,
|
||||
) -> Self {
|
||||
let length = writers.len();
|
||||
MultiWriter {
|
||||
writers,
|
||||
integrity: None,
|
||||
write_quorum,
|
||||
write_quorum: write_policy.quorum,
|
||||
post_quorum: write_policy.post_quorum,
|
||||
errs: vec![None; length],
|
||||
policy,
|
||||
absolute_deadline: None,
|
||||
@@ -506,9 +549,8 @@ impl<'a> MultiWriter<'a> {
|
||||
// failed and its disk dropped, so a stalled peer cannot pin an
|
||||
// otherwise-healthy write quorum (rustfs/backlog#1319). `budget`
|
||||
// is recomputed per block, so it bounds a stall — not the total
|
||||
// transfer time — and a slow-but-honest writer is never killed
|
||||
// while quorum is still open. Once quorum has succeeded, a
|
||||
// straggler only gets `WRITE_QUORUM_STRAGGLER_GRACE` more time.
|
||||
// transfer time. Only ordinary object writes add the shorter
|
||||
// post-quorum grace; multipart preserves this operation budget.
|
||||
futures.push(async move {
|
||||
match budget {
|
||||
Some(budget) => match tokio::time::timeout(budget, Self::write_shard(writer_opt, err, shard)).await {
|
||||
@@ -524,7 +566,7 @@ impl<'a> MultiWriter<'a> {
|
||||
(index, succeeded)
|
||||
});
|
||||
}
|
||||
await_write_quorum(write_quorum, stall_timeout, slot_count, futures).await
|
||||
await_write_quorum(write_quorum, stall_timeout, slot_count, self.post_quorum, futures).await
|
||||
};
|
||||
abandon_unfinished_writers(self.writers, &mut self.errs, &saw);
|
||||
|
||||
@@ -587,8 +629,8 @@ impl<'a> MultiWriter<'a> {
|
||||
// here forever for a small object whose bytes were fully buffered
|
||||
// (so `write` never blocked). Bound it with the same progress
|
||||
// budget and drop the stalled writer before the quorum check.
|
||||
// After quorum has shut down, do not keep waiting out the rest of
|
||||
// that budget for a peer that has stopped answering.
|
||||
// Ordinary object writes also apply their post-quorum grace;
|
||||
// multipart waits for all candidates within this same budget.
|
||||
futures.push(async move {
|
||||
match budget {
|
||||
Some(budget) => match tokio::time::timeout(budget, Self::shutdown_writer(writer_opt, err)).await {
|
||||
@@ -604,7 +646,7 @@ impl<'a> MultiWriter<'a> {
|
||||
(index, succeeded)
|
||||
});
|
||||
}
|
||||
await_write_quorum(write_quorum, stall_timeout, slot_count, futures).await
|
||||
await_write_quorum(write_quorum, stall_timeout, slot_count, self.post_quorum, futures).await
|
||||
};
|
||||
abandon_unfinished_writers(self.writers, &mut self.errs, &saw);
|
||||
|
||||
@@ -639,7 +681,7 @@ impl Erasure {
|
||||
self: Arc<Self>,
|
||||
reader: R,
|
||||
writers: &mut [Option<BitrotWriterWrapper>],
|
||||
quorum: usize,
|
||||
write_policy: ShardWritePolicy,
|
||||
part_number: usize,
|
||||
mode: IntegrityEncodeMode,
|
||||
protected: bool,
|
||||
@@ -668,19 +710,19 @@ impl Erasure {
|
||||
}
|
||||
IntegrityEncodeMode::SingleBlock(hint) => {
|
||||
let (reader, size) = self
|
||||
.encode_small_direct(reader, writers, quorum, true, hint, integrity.as_mut())
|
||||
.encode_small_direct(reader, writers, write_policy, true, hint, integrity.as_mut())
|
||||
.await?;
|
||||
(reader, size, None)
|
||||
}
|
||||
IntegrityEncodeMode::Streaming => {
|
||||
let (reader, size) = self
|
||||
.encode_with_ingest_mode(reader, writers, quorum, use_bytesmut_ingest(), integrity.as_mut())
|
||||
.encode_with_ingest_mode(reader, writers, write_policy, use_bytesmut_ingest(), integrity.as_mut())
|
||||
.await?;
|
||||
(reader, size, None)
|
||||
}
|
||||
IntegrityEncodeMode::Batched => {
|
||||
let (reader, size) = self
|
||||
.encode_batched_with_integrity(reader, writers, quorum, integrity.as_mut())
|
||||
.encode_batched_with_integrity(reader, writers, write_policy, integrity.as_mut())
|
||||
.await?;
|
||||
(reader, size, None)
|
||||
}
|
||||
@@ -745,7 +787,7 @@ impl Erasure {
|
||||
self: Arc<Self>,
|
||||
mut reader: R,
|
||||
writers: &mut [Option<BitrotWriterWrapper>],
|
||||
quorum: usize,
|
||||
write_policy: ShardWritePolicy,
|
||||
require_single_block: bool,
|
||||
size_hint: usize,
|
||||
integrity: Option<&mut IntegrityBuilder>,
|
||||
@@ -780,7 +822,7 @@ impl Erasure {
|
||||
}
|
||||
|
||||
let block = self.encode_data_owned_block(buf)?;
|
||||
let mut mw = MultiWriter::new(writers, quorum);
|
||||
let mut mw = MultiWriter::new(writers, write_policy);
|
||||
mw.integrity = integrity;
|
||||
mw.write_block(&block).await?;
|
||||
mw.shutdown().await?;
|
||||
@@ -847,7 +889,7 @@ impl Erasure {
|
||||
R: AsyncRead + Send + Sync + Unpin + 'static,
|
||||
{
|
||||
let use_bytesmut_ingest = use_bytesmut_ingest();
|
||||
self.encode_with_ingest_mode(reader, writers, quorum, use_bytesmut_ingest, None)
|
||||
self.encode_with_ingest_mode(reader, writers, ShardWritePolicy::object(quorum), use_bytesmut_ingest, None)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -859,7 +901,7 @@ impl Erasure {
|
||||
self: Arc<Self>,
|
||||
mut reader: R,
|
||||
writers: &mut [Option<BitrotWriterWrapper>],
|
||||
quorum: usize,
|
||||
write_policy: ShardWritePolicy,
|
||||
use_bytesmut_ingest: bool,
|
||||
integrity: Option<&mut IntegrityBuilder>,
|
||||
) -> std::io::Result<(R, usize)>
|
||||
@@ -961,7 +1003,7 @@ impl Erasure {
|
||||
Ok((reader, total))
|
||||
}));
|
||||
|
||||
let mut writers = MultiWriter::new(writers, quorum);
|
||||
let mut writers = MultiWriter::new(writers, write_policy);
|
||||
writers.integrity = integrity;
|
||||
|
||||
let mut write_err = None;
|
||||
@@ -1017,14 +1059,15 @@ impl Erasure {
|
||||
where
|
||||
R: AsyncRead + Send + Sync + Unpin + 'static,
|
||||
{
|
||||
self.encode_batched_with_integrity(reader, writers, quorum, None).await
|
||||
self.encode_batched_with_integrity(reader, writers, ShardWritePolicy::object(quorum), None)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn encode_batched_with_integrity<R>(
|
||||
self: Arc<Self>,
|
||||
mut reader: R,
|
||||
writers: &mut [Option<BitrotWriterWrapper>],
|
||||
quorum: usize,
|
||||
write_policy: ShardWritePolicy,
|
||||
integrity: Option<&mut IntegrityBuilder>,
|
||||
) -> std::io::Result<(R, usize)>
|
||||
where
|
||||
@@ -1105,7 +1148,7 @@ impl Erasure {
|
||||
Ok((reader, total))
|
||||
}));
|
||||
|
||||
let mut writers = MultiWriter::new(writers, quorum);
|
||||
let mut writers = MultiWriter::new(writers, write_policy);
|
||||
writers.integrity = integrity;
|
||||
let mut write_err = None;
|
||||
|
||||
@@ -1165,7 +1208,7 @@ impl Erasure {
|
||||
R: AsyncRead + Send + Sync + Unpin,
|
||||
{
|
||||
let size_hint = self.block_size;
|
||||
self.encode_small_direct(reader, writers, quorum, false, size_hint, None)
|
||||
self.encode_small_direct(reader, writers, ShardWritePolicy::object(quorum), false, size_hint, None)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -1182,7 +1225,7 @@ impl Erasure {
|
||||
where
|
||||
R: AsyncRead + Send + Sync + Unpin,
|
||||
{
|
||||
self.encode_small_direct(reader, writers, quorum, false, size_hint, None)
|
||||
self.encode_small_direct(reader, writers, ShardWritePolicy::object(quorum), false, size_hint, None)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -1199,7 +1242,8 @@ impl Erasure {
|
||||
R: AsyncRead + Send + Sync + Unpin,
|
||||
{
|
||||
let size_hint = self.block_size;
|
||||
self.encode_small_direct(reader, writers, quorum, true, size_hint, None).await
|
||||
self.encode_small_direct(reader, writers, ShardWritePolicy::object(quorum), true, size_hint, None)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Size-aware single-block fast path. `size_hint` only controls the bounded
|
||||
@@ -1215,7 +1259,8 @@ impl Erasure {
|
||||
where
|
||||
R: AsyncRead + Send + Sync + Unpin,
|
||||
{
|
||||
self.encode_small_direct(reader, writers, quorum, true, size_hint, None).await
|
||||
self.encode_small_direct(reader, writers, ShardWritePolicy::object(quorum), true, size_hint, None)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1632,6 +1677,219 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
enum DelayedPhase {
|
||||
Write,
|
||||
Flush,
|
||||
Shutdown,
|
||||
}
|
||||
|
||||
struct PhaseDelayedWriter {
|
||||
inner: DeferredCommitWriter,
|
||||
phase: DelayedPhase,
|
||||
delay: Duration,
|
||||
timer: Option<Pin<Box<tokio::time::Sleep>>>,
|
||||
ready: bool,
|
||||
}
|
||||
|
||||
impl PhaseDelayedWriter {
|
||||
fn poll_delay(&mut self, cx: &mut Context<'_>) -> Poll<()> {
|
||||
if self.ready {
|
||||
return Poll::Ready(());
|
||||
}
|
||||
let timer = self.timer.get_or_insert_with(|| Box::pin(tokio::time::sleep(self.delay)));
|
||||
match timer.as_mut().poll(cx) {
|
||||
Poll::Pending => Poll::Pending,
|
||||
Poll::Ready(()) => {
|
||||
self.ready = true;
|
||||
Poll::Ready(())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncWrite for PhaseDelayedWriter {
|
||||
fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, bytes: &[u8]) -> Poll<std::io::Result<usize>> {
|
||||
if matches!(self.phase, DelayedPhase::Write) && self.poll_delay(cx).is_pending() {
|
||||
return Poll::Pending;
|
||||
}
|
||||
Pin::new(&mut self.inner).poll_write(cx, bytes)
|
||||
}
|
||||
|
||||
fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
|
||||
if matches!(self.phase, DelayedPhase::Flush) && self.poll_delay(cx).is_pending() {
|
||||
return Poll::Pending;
|
||||
}
|
||||
Pin::new(&mut self.inner).poll_flush(cx)
|
||||
}
|
||||
|
||||
fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
|
||||
if matches!(self.phase, DelayedPhase::Shutdown) && self.poll_delay(cx).is_pending() {
|
||||
return Poll::Pending;
|
||||
}
|
||||
Pin::new(&mut self.inner).poll_shutdown(cx)
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn multipart_budget_keeps_slow_writers_in_every_encode_mode() {
|
||||
const BLOCK: usize = 64;
|
||||
for make_mode in [
|
||||
|| IntegrityEncodeMode::SingleBlock(BLOCK),
|
||||
|| IntegrityEncodeMode::Streaming,
|
||||
|| IntegrityEncodeMode::Batched,
|
||||
] {
|
||||
for phase in [DelayedPhase::Write, DelayedPhase::Flush, DelayedPhase::Shutdown] {
|
||||
for delay in [Duration::from_secs(2), Duration::from_secs(5)] {
|
||||
let committed: Vec<_> = (0..4).map(|_| Arc::new(Mutex::new(Vec::new()))).collect();
|
||||
let mut writers = committed
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(slot, data)| {
|
||||
if slot == 0 {
|
||||
Some(bitrot_writer(
|
||||
PhaseDelayedWriter {
|
||||
inner: DeferredCommitWriter::new(data.clone()),
|
||||
phase,
|
||||
delay,
|
||||
timer: None,
|
||||
ready: false,
|
||||
},
|
||||
BLOCK / 2,
|
||||
))
|
||||
} else {
|
||||
Some(bitrot_writer(DeferredCommitWriter::new(data.clone()), BLOCK / 2))
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let payload = vec![0x5a; BLOCK];
|
||||
let erasure = Arc::new(Erasure::new(2, 2, BLOCK));
|
||||
let started = tokio::time::Instant::now();
|
||||
erasure
|
||||
.encode_with_shard_integrity(
|
||||
Cursor::new(payload.clone()),
|
||||
&mut writers,
|
||||
ShardWritePolicy::multipart(3),
|
||||
1,
|
||||
make_mode(),
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.expect("multipart must keep finite slow writers within the existing budget");
|
||||
assert!(started.elapsed() >= delay, "must await the slow phase: {phase:?}");
|
||||
assert!(writers.iter().all(Option::is_some), "all four completed shards must survive");
|
||||
for data in &committed {
|
||||
let bytes = data.lock().expect("committed bytes").clone();
|
||||
crate::erasure::coding::bitrot_verify(
|
||||
Cursor::new(bytes.clone()),
|
||||
bytes.len(),
|
||||
BLOCK / 2,
|
||||
HashAlgorithm::HighwayHash256S,
|
||||
BLOCK / 2,
|
||||
)
|
||||
.await
|
||||
.expect("each completed shard must retain valid bitrot framing");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn multipart_budget_expires_blackholes_without_restarting_deadlines() {
|
||||
for shutdown in [false, true] {
|
||||
for blackholes in [1, 2] {
|
||||
for absolute in [Duration::ZERO, Duration::from_secs(3)] {
|
||||
let mut writers = (0..4)
|
||||
.map(|slot| {
|
||||
if slot < blackholes {
|
||||
Some(bitrot_writer_plain(
|
||||
if shutdown {
|
||||
StallingWriter::stalls_on_shutdown()
|
||||
} else {
|
||||
StallingWriter::stalls_on_write(0)
|
||||
},
|
||||
64,
|
||||
))
|
||||
} else {
|
||||
Some(bitrot_writer_plain(DeferredCommitWriter::new(Arc::new(Mutex::new(Vec::new()))), 64))
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let started = tokio::time::Instant::now();
|
||||
let result = {
|
||||
let mut writer = MultiWriter::with_policy(
|
||||
&mut writers,
|
||||
ShardWritePolicy::multipart(3),
|
||||
WriteProgressPolicy::new(Duration::from_secs(7), absolute),
|
||||
);
|
||||
if shutdown {
|
||||
writer.write(four_shards()).await.expect("buffered writes should complete");
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
writer.shutdown().await
|
||||
} else {
|
||||
writer.write(four_shards()).await
|
||||
}
|
||||
};
|
||||
assert_eq!(result.is_ok(), blackholes == 1, "only a confirmed quorum may succeed");
|
||||
let expected = if absolute.is_zero() {
|
||||
Duration::from_secs(7 + u64::from(shutdown))
|
||||
} else {
|
||||
absolute
|
||||
};
|
||||
assert_eq!(started.elapsed(), expected, "budget must not restart at quorum or shutdown");
|
||||
assert!(writers[..blackholes].iter().all(Option::is_none));
|
||||
assert!(writers[blackholes..].iter().all(Option::is_some));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn multipart_budget_cancellation_drops_pending_writers() {
|
||||
struct DropSignal(Option<oneshot::Sender<()>>);
|
||||
impl Drop for DropSignal {
|
||||
fn drop(&mut self) {
|
||||
if let Some(tx) = self.0.take() {
|
||||
let _ = tx.send(());
|
||||
}
|
||||
}
|
||||
}
|
||||
impl AsyncWrite for DropSignal {
|
||||
fn poll_write(self: Pin<&mut Self>, _: &mut Context<'_>, bytes: &[u8]) -> Poll<std::io::Result<usize>> {
|
||||
Poll::Ready(Ok(bytes.len()))
|
||||
}
|
||||
fn poll_flush(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<std::io::Result<()>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
fn poll_shutdown(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<std::io::Result<()>> {
|
||||
Poll::Pending
|
||||
}
|
||||
}
|
||||
let (tx, rx) = oneshot::channel();
|
||||
let (entered_tx, entered) = oneshot::channel();
|
||||
let task = tokio::spawn(async move {
|
||||
let mut writers = vec![Some(bitrot_writer_plain(DropSignal(Some(tx)), 64))];
|
||||
writers.extend(
|
||||
(0..3).map(|_| Some(bitrot_writer_plain(DeferredCommitWriter::new(Arc::new(Mutex::new(Vec::new()))), 64))),
|
||||
);
|
||||
let mut writer = MultiWriter::new(&mut writers, ShardWritePolicy::multipart(3));
|
||||
writer.write(four_shards()).await.expect("buffered writes should succeed");
|
||||
let _ = entered_tx.send(());
|
||||
writer.shutdown().await
|
||||
});
|
||||
entered.await.expect("write must reach shutdown");
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!task.is_finished());
|
||||
task.abort();
|
||||
assert!(
|
||||
task.await
|
||||
.expect_err("caller cancellation should cancel the task")
|
||||
.is_cancelled()
|
||||
);
|
||||
rx.await.expect("pending writer must be dropped with its owning task");
|
||||
}
|
||||
|
||||
/// A writer that always makes forward progress but each `poll_write` first
|
||||
/// waits `delay` (on the runtime timer). Under a paused virtual clock this
|
||||
/// deterministically models a slow-but-honest peer: it must never be killed
|
||||
@@ -1704,12 +1962,16 @@ mod tests {
|
||||
let erasure = Arc::new(Erasure::new(1, 0, BLOCK_SIZE));
|
||||
|
||||
let encode = match pipeline {
|
||||
EncodePipeline::Vec => {
|
||||
tokio::spawn(async move { erasure.encode_with_ingest_mode(reader, &mut writers, 1, false, None).await })
|
||||
}
|
||||
EncodePipeline::BytesMut => {
|
||||
tokio::spawn(async move { erasure.encode_with_ingest_mode(reader, &mut writers, 1, true, None).await })
|
||||
}
|
||||
EncodePipeline::Vec => tokio::spawn(async move {
|
||||
erasure
|
||||
.encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(1), false, None)
|
||||
.await
|
||||
}),
|
||||
EncodePipeline::BytesMut => tokio::spawn(async move {
|
||||
erasure
|
||||
.encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(1), true, None)
|
||||
.await
|
||||
}),
|
||||
EncodePipeline::Batched => tokio::spawn(async move { erasure.encode_batched(reader, &mut writers, 1).await }),
|
||||
};
|
||||
|
||||
@@ -1767,8 +2029,16 @@ mod tests {
|
||||
))];
|
||||
|
||||
let result = match pipeline {
|
||||
EncodePipeline::Vec => erasure.encode_with_ingest_mode(reader, &mut writers, 1, false, None).await,
|
||||
EncodePipeline::BytesMut => erasure.encode_with_ingest_mode(reader, &mut writers, 1, true, None).await,
|
||||
EncodePipeline::Vec => {
|
||||
erasure
|
||||
.encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(1), false, None)
|
||||
.await
|
||||
}
|
||||
EncodePipeline::BytesMut => {
|
||||
erasure
|
||||
.encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(1), true, None)
|
||||
.await
|
||||
}
|
||||
EncodePipeline::Batched => erasure.encode_batched(reader, &mut writers, 1).await,
|
||||
};
|
||||
|
||||
@@ -1841,8 +2111,16 @@ mod tests {
|
||||
let mut writers = vec![Some(bitrot_writer(FailAfterReaderErrorWriter { reader_failed }, BLOCK_SIZE))];
|
||||
|
||||
let result = match pipeline {
|
||||
EncodePipeline::Vec => erasure.encode_with_ingest_mode(reader, &mut writers, 1, false, None).await,
|
||||
EncodePipeline::BytesMut => erasure.encode_with_ingest_mode(reader, &mut writers, 1, true, None).await,
|
||||
EncodePipeline::Vec => {
|
||||
erasure
|
||||
.encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(1), false, None)
|
||||
.await
|
||||
}
|
||||
EncodePipeline::BytesMut => {
|
||||
erasure
|
||||
.encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(1), true, None)
|
||||
.await
|
||||
}
|
||||
EncodePipeline::Batched => erasure.encode_batched(reader, &mut writers, 1).await,
|
||||
};
|
||||
|
||||
@@ -1879,7 +2157,7 @@ mod tests {
|
||||
let erasure_for_task = erasure.clone();
|
||||
let encode = tokio::spawn(async move {
|
||||
erasure_for_task
|
||||
.encode_with_ingest_mode(reader, &mut writers, 1, false, None)
|
||||
.encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(1), false, None)
|
||||
.await
|
||||
});
|
||||
|
||||
@@ -2048,8 +2326,16 @@ mod tests {
|
||||
let erasure = Arc::new(Erasure::new(2, 2, 64));
|
||||
let result = match path {
|
||||
"direct" => erasure.encode_single_block_non_inline(reader, &mut writers, 2).await,
|
||||
"vec" => erasure.encode_with_ingest_mode(reader, &mut writers, 2, false, None).await,
|
||||
"bytesmut" => erasure.encode_with_ingest_mode(reader, &mut writers, 2, true, None).await,
|
||||
"vec" => {
|
||||
erasure
|
||||
.encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(2), false, None)
|
||||
.await
|
||||
}
|
||||
"bytesmut" => {
|
||||
erasure
|
||||
.encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(2), true, None)
|
||||
.await
|
||||
}
|
||||
"batched" => erasure.encode_batched(reader, &mut writers, 2).await,
|
||||
_ => unreachable!(),
|
||||
};
|
||||
@@ -2094,7 +2380,7 @@ mod tests {
|
||||
async fn multi_writer_short_write_fails_before_shutdown() {
|
||||
let mut writers = vec![Some(bitrot_writer(ShortWriteWriter, 32))];
|
||||
let err = {
|
||||
let mut writer = MultiWriter::new(&mut writers, 1);
|
||||
let mut writer = MultiWriter::new(&mut writers, ShardWritePolicy::object(1));
|
||||
writer
|
||||
.write(vec![Bytes::from_static(b"short-write payload")])
|
||||
.await
|
||||
@@ -2111,7 +2397,7 @@ mod tests {
|
||||
async fn multi_writer_reports_fallback_summary_when_only_offline_writers_remain() {
|
||||
let mut writers = vec![None, None];
|
||||
let (err, summary) = {
|
||||
let mut writer = MultiWriter::new(&mut writers, 1);
|
||||
let mut writer = MultiWriter::new(&mut writers, ShardWritePolicy::object(1));
|
||||
let err = writer
|
||||
.write(vec![Bytes::from_static(b"offline-a"), Bytes::from_static(b"offline-b")])
|
||||
.await
|
||||
@@ -2131,7 +2417,7 @@ mod tests {
|
||||
assert!(summary.contains("required=1"));
|
||||
|
||||
let (shutdown_err, summary) = {
|
||||
let mut writer = MultiWriter::new(&mut writers, 1);
|
||||
let mut writer = MultiWriter::new(&mut writers, ShardWritePolicy::object(1));
|
||||
let err = writer
|
||||
.shutdown()
|
||||
.await
|
||||
@@ -2155,7 +2441,7 @@ mod tests {
|
||||
async fn multi_writer_reports_quorum_failure_when_quorum_exceeds_writer_count() {
|
||||
let committed = Arc::new(Mutex::new(Vec::new()));
|
||||
let mut writers = vec![Some(bitrot_writer(DeferredCommitWriter::new(committed), 32))];
|
||||
let mut writer = MultiWriter::new(&mut writers, 2);
|
||||
let mut writer = MultiWriter::new(&mut writers, ShardWritePolicy::object(2));
|
||||
|
||||
let err = writer
|
||||
.write(vec![Bytes::from_static(b"quorum impossible")])
|
||||
@@ -2244,7 +2530,7 @@ mod tests {
|
||||
|
||||
{
|
||||
let policy = WriteProgressPolicy::new(Duration::from_secs(5), Duration::ZERO);
|
||||
let mut mw = MultiWriter::with_policy(&mut writers, 3, policy);
|
||||
let mut mw = MultiWriter::with_policy(&mut writers, ShardWritePolicy::object(3), policy);
|
||||
mw.write(four_shards())
|
||||
.await
|
||||
.expect("one stalled writer must still satisfy a 3/4 write quorum without hanging");
|
||||
@@ -2271,7 +2557,7 @@ mod tests {
|
||||
let started = tokio::time::Instant::now();
|
||||
{
|
||||
let policy = WriteProgressPolicy::new(Duration::from_secs(5), Duration::ZERO);
|
||||
let mut mw = MultiWriter::with_policy(&mut writers, 3, policy);
|
||||
let mut mw = MultiWriter::with_policy(&mut writers, ShardWritePolicy::object(3), policy);
|
||||
mw.write(four_shards())
|
||||
.await
|
||||
.expect("quorum must succeed without waiting out the black-hole stall");
|
||||
@@ -2304,7 +2590,7 @@ mod tests {
|
||||
|
||||
{
|
||||
let policy = WriteProgressPolicy::new(Duration::from_secs(5), Duration::ZERO);
|
||||
let mut mw = MultiWriter::with_policy(&mut writers, 3, policy);
|
||||
let mut mw = MultiWriter::with_policy(&mut writers, ShardWritePolicy::object(3), policy);
|
||||
mw.write(four_shards())
|
||||
.await
|
||||
.expect("a straggler inside the grace window must still satisfy the write");
|
||||
@@ -2329,7 +2615,7 @@ mod tests {
|
||||
];
|
||||
|
||||
let policy = WriteProgressPolicy::new(Duration::from_secs(5), Duration::ZERO);
|
||||
let mut mw = MultiWriter::with_policy(&mut writers, 3, policy);
|
||||
let mut mw = MultiWriter::with_policy(&mut writers, ShardWritePolicy::object(3), policy);
|
||||
let err = mw
|
||||
.write(four_shards())
|
||||
.await
|
||||
@@ -2358,7 +2644,7 @@ mod tests {
|
||||
|
||||
{
|
||||
let policy = WriteProgressPolicy::new(Duration::from_secs(5), Duration::ZERO);
|
||||
let mut mw = MultiWriter::with_policy(&mut writers, 3, policy);
|
||||
let mut mw = MultiWriter::with_policy(&mut writers, ShardWritePolicy::object(3), policy);
|
||||
mw.write(four_shards()).await.expect("all writers accept the buffered shard");
|
||||
mw.shutdown()
|
||||
.await
|
||||
@@ -2379,7 +2665,7 @@ mod tests {
|
||||
];
|
||||
|
||||
let policy = WriteProgressPolicy::new(Duration::from_secs(5), Duration::ZERO);
|
||||
let mut mw = MultiWriter::with_policy(&mut writers, 3, policy);
|
||||
let mut mw = MultiWriter::with_policy(&mut writers, ShardWritePolicy::object(3), policy);
|
||||
mw.write(four_shards()).await.expect("all writers accept the buffered shard");
|
||||
let err = mw
|
||||
.shutdown()
|
||||
@@ -2408,7 +2694,7 @@ mod tests {
|
||||
|
||||
{
|
||||
let policy = WriteProgressPolicy::new(Duration::from_secs(5), Duration::ZERO);
|
||||
let mut mw = MultiWriter::with_policy(&mut writers, 3, policy);
|
||||
let mut mw = MultiWriter::with_policy(&mut writers, ShardWritePolicy::object(3), policy);
|
||||
for _ in 0..8 {
|
||||
mw.write(four_full_shards(64))
|
||||
.await
|
||||
@@ -2438,7 +2724,7 @@ mod tests {
|
||||
// stall (10s) never fires for an 8s-per-block writer, but the 15s
|
||||
// absolute cap does once its aggregate engagement crosses the cap.
|
||||
let policy = WriteProgressPolicy::new(Duration::from_secs(10), Duration::from_secs(15));
|
||||
let mut mw = MultiWriter::with_policy(&mut writers, 3, policy);
|
||||
let mut mw = MultiWriter::with_policy(&mut writers, ShardWritePolicy::object(3), policy);
|
||||
for _ in 0..4 {
|
||||
mw.write(four_full_shards(64))
|
||||
.await
|
||||
@@ -2583,7 +2869,7 @@ mod tests {
|
||||
let erasure = Arc::new(Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE));
|
||||
let reader = tokio::io::BufReader::new(Cursor::new(payload.clone()));
|
||||
let (_reader, written) = erasure
|
||||
.encode_with_ingest_mode(reader, &mut writers, DATA_SHARDS, true, None)
|
||||
.encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(DATA_SHARDS), true, None)
|
||||
.await
|
||||
.expect("BytesMut ingest path should encode the streaming payload");
|
||||
|
||||
@@ -2612,7 +2898,9 @@ mod tests {
|
||||
let reader = tokio::io::BufReader::new(Cursor::new(vec![0x5a; block_size * 2]));
|
||||
|
||||
rustfs_io_metrics::set_put_stage_metrics_enabled(true);
|
||||
let result = erasure.encode_with_ingest_mode(reader, &mut writers, 1, true, None).await;
|
||||
let result = erasure
|
||||
.encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(1), true, None)
|
||||
.await;
|
||||
rustfs_io_metrics::set_put_stage_metrics_enabled(false);
|
||||
|
||||
let (_reader, written) = result.expect("bytesmut streaming encode should complete");
|
||||
@@ -3230,12 +3518,12 @@ mod tests {
|
||||
let (_reader, total) = match pipeline {
|
||||
EncodePipeline::Vec => {
|
||||
erasure
|
||||
.encode_with_ingest_mode(reader, &mut writers, DATA_SHARDS, false, None)
|
||||
.encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(DATA_SHARDS), false, None)
|
||||
.await
|
||||
}
|
||||
EncodePipeline::BytesMut => {
|
||||
erasure
|
||||
.encode_with_ingest_mode(reader, &mut writers, DATA_SHARDS, true, None)
|
||||
.encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(DATA_SHARDS), true, None)
|
||||
.await
|
||||
}
|
||||
EncodePipeline::Batched => erasure.encode_batched(reader, &mut writers, DATA_SHARDS).await,
|
||||
|
||||
@@ -205,7 +205,7 @@ impl super::Erasure {
|
||||
// the survivors. Reconstruction correctness is still guaranteed by the
|
||||
// read-side quorum check and parity cross-checks below, which are unaffected.
|
||||
let write_quorum = 1;
|
||||
let mut writers = MultiWriter::new(writers, write_quorum);
|
||||
let mut writers = MultiWriter::new(writers, super::encode::ShardWritePolicy::object(write_quorum));
|
||||
let read_timeout = get_object_disk_read_timeout();
|
||||
let shard_file_size = self.shard_file_size(total_length as i64) as usize;
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user